scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-21 00:21:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 73

Merge remote-tracking branch 'origin/main' into litellm_transcribe_passthrough

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

# Conflicts:
#	litellm/proxy/_lazy_features.py
#	litellm/proxy/_lazy_openapi_snapshot.json
#	litellm/proxy/pass_through_endpoints/success_handler.py
#	ui/litellm-dashboard/src/lib/http/schema.d.ts
This commit is contained in:
yassin 2026-09-18 00:17:47 +00:00
commit 4f8a5b5e12
50 changed files with 7988 additions and 735 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

614
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_1/grafana_dashboard.json
View file

@ -1,614 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 2039,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum(rate(litellm_self_latency_bucket{self=\"self\"}[1m])) by (le))",
"legendFormat": "Time to first token",
"range": true,
"refId": "A"
}
],
"title": "Time to first token (latency)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "currencyUSD"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "7e4b0627fd32efdd2313c846325575808aadcf2839f0fde90723aab9ab73c78f"
},
"properties": [
{
"id": "displayName",
"value": "Translata"
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 11,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(increase(litellm_spend_metric_total[30d])) by (hashed_api_key)",
"legendFormat": "{{team}}",
"range": true,
"refId": "A"
}
],
"title": "Spend by team",
"transformations": [],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 16
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum by (model) (increase(litellm_requests_metric_total[5m]))",
"legendFormat": "{{model}}",
"range": true,
"refId": "A"
}
],
"title": "Requests by model",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 3,
"x": 0,
"y": 25
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.4.17",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(increase(litellm_llm_api_failed_requests_metric_total[1h]))",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Faild Requests",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "currencyUSD"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 3,
"x": 3,
"y": 25
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(increase(litellm_spend_metric_total[30d])) by (model)",
"legendFormat": "{{model}}",
"range": true,
"refId": "A"
}
],
"title": "Spend",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 6,
"x": 6,
"y": 25
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(increase(litellm_total_tokens_total[5m])) by (model)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Tokens",
"type": "timeseries"
}
],
"refresh": "1m",
"revision": 1,
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "prometheus",
"value": "edx8memhpd9tsa"
},
"hide": 0,
"includeAll": false,
"label": "datasource",
"multi": false,
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "LLM Proxy",
"uid": "rgRrHxESz",
"version": 15,
"weekStart": ""
}

6
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_1/readme.md
View file

@ -1,6 +0,0 @@
## This folder contains the `json` for creating the following Grafana Dashboard
### Pre-Requisites
- Setup LiteLLM Proxy Prometheus Metrics https://docs.litellm.ai/docs/proxy/prometheus
![1716623265684](https://github.com/BerriAI/litellm/assets/29436595/0e12c57e-4a2d-4850-bd4f-e4294f87a814)

6312
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json Normal file
View file

File diff suppressed because it is too large Load diff

11
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md Normal file
View file

@ -0,0 +1,11 @@
# LiteLLM All Prometheus Metrics dashboard
Every `litellm_*` metric family the proxy can expose on `/metrics` (134 families across 95 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
The first eleven rows need only `callbacks: ["prometheus"]`. The last three rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected
## Pre-requisites
Prometheus metrics on the proxy: https://docs.litellm.ai/docs/proxy/prometheus

4
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_v2/grafana_dashboard.json
View file

@ -476,7 +476,7 @@
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "topk(5, sort(litellm_remaining_requests))",
"expr": "topk(5, sort(litellm_remaining_requests_metric))",
"legendFormat": "__auto",
"range": true,
"refId": "A"
@ -573,7 +573,7 @@
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "topk(5, sort(litellm_remaining_tokens))",
"expr": "topk(5, sort(litellm_remaining_tokens_metric))",
"legendFormat": "__auto",
"range": true,
"refId": "A"

6
cookbook/litellm_proxy_server/grafana_dashboard/readme.md
View file

@ -6,8 +6,14 @@ This folder contains the `json` for creating Grafana Dashboards
Charts the `gen_ai.*` metrics from the OpenTelemetry v2 integration: spend, tokens, request rate, and latency percentiles by model. Separate from the dashboards below, which chart the `litellm_*` Prometheus metrics.
## [LiteLLM All Prometheus Metrics dashboard](./dashboard_all_metrics)
Every `litellm_*` Prometheus metric family the proxy can emit (134 families, 95 panels) grouped by theme: traffic, latency, spend and tokens, cache, deployments, rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, plus the Redis circuit breaker, spend log cleanup and `prometheus_system` service metrics. Start here if you want everything on one screen; see its [readme](./dashboard_all_metrics/readme.md) for import steps and which panels need a feature enabled before they show data
## [LiteLLM v2 Dashboard](./dashboard_v2)
A compact view of proxy request rate, failures, latency and the top remaining-request / remaining-token gauges per model group
<img width="1316" alt="grafana_1" src="https://github.com/user-attachments/assets/d0df802d-0cb9-4906-a679-941c547789ab">
<img width="1289" alt="grafana_2" src="https://github.com/user-attachments/assets/b11f755f-e113-42ab-b21d-83f91f451a28">
<img width="1323" alt="grafana_3" src="https://github.com/user-attachments/assets/cb29ffdb-477d-4be1-a5cd-c3f7f2cb21c5">

1
gateway/routes/allowlist.py
View file

@ -97,6 +97,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/langfuse/",
"/vllm/",
"/mistral/",
"/typesafe/",
"/nvidia_nim/",
"/groq/",
"/voyage/",

34
litellm/litellm_core_utils/litellm_logging.py
View file

@ -90,6 +90,7 @@ from litellm.litellm_core_utils.logging_utils import (
truncate_base64_in_messages_async,
)
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request
from litellm.litellm_core_utils.redact_messages import (
redact_message_input_output_from_custom_logger,
redact_message_input_output_from_logging,
@ -1746,8 +1747,14 @@ class Logging(LiteLLMLoggingBaseClass):
if transformed_result is not None:
result = transformed_result
result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({})
result_additional_headers: Final = (
result_hidden_params.get("additional_headers")
if isinstance(result_hidden_params, dict)
else getattr(result_hidden_params, "additional_headers", None)
)
if isinstance(result, (BaseModel, HttpxBinaryResponseContent)) and hasattr(result, "_hidden_params"):
hidden_params: Final = getattr(result, "_hidden_params", {})
hidden_params: Final = result_hidden_params
if (
"response_cost" in hidden_params and hidden_params["response_cost"] is not None
): # use cost if already calculated
@ -1762,8 +1769,17 @@ class Logging(LiteLLMLoggingBaseClass):
router_model_id = self.get_router_model_id()
## RESPONSE COST ##
custom_pricing: Final = use_custom_pricing_for_model(
litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None)
spilled_over: Final = is_spilled_over_ptu_request(
model_info=_deployment_model_info(self.litellm_params if hasattr(self, "litellm_params") else None),
response_headers=self.model_call_details.get("response_headers"),
additional_headers=result_additional_headers,
)
custom_pricing: Final = (
False
if spilled_over
else use_custom_pricing_for_model(
litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None)
)
)
prompt = self._prompt_for_cost_calculation()
@ -5257,6 +5273,18 @@ def _get_custom_logger_settings_from_proxy_server(callback_name: str) -> dict:
return {}
def _deployment_model_info(litellm_params: dict | None) -> Mapping[str, object]:
"""The router-stamped deployment model_info from whichever metadata field carries it."""
if litellm_params is None:
return MappingProxyType({})
for metadata_key in ("metadata", "litellm_metadata"):
if not isinstance(metadata := litellm_params.get(metadata_key), Mapping):
continue
if model_info := metadata.get("model_info"):
return model_info
return MappingProxyType({})
def use_custom_pricing_for_model(litellm_params: dict | None) -> bool:
"""
Check if the model uses custom pricing

34
litellm/litellm_core_utils/ptu_pricing.py
View file

@ -14,9 +14,11 @@ from typing import Final
from litellm.secret_managers.main import get_secret_bool
from litellm.types.router import ModelInfo
from litellm.types.utils import CustomPricingLiteLLMParams, MirroredPricingParams
from litellm.types.utils import AzureSpillover, CustomPricingLiteLLMParams, MirroredPricingParams
PTU_COST_ATTRIBUTION_ENV_VAR: Final = "LITELLM_ENABLE_PTU_COST_ATTRIBUTION"
AZURE_SPILLOVER_HEADER: Final = "x-ms-is-spilled-over"
AZURE_SPILLOVER_FROM_HEADER: Final = "x-ms-spillover-from-deployment"
def is_ptu_cost_attribution_enabled() -> bool:
@ -235,3 +237,33 @@ def zeroed_ptu_pricing(
),
}
)
def is_spilled_over_ptu_request(
model_info: Mapping[str, object],
response_headers: Mapping[str, object] | None,
additional_headers: Mapping[str, object] | None,
) -> bool:
"""Whether Azure served this request from pay-as-you-go capacity, so the zeroed PTU rates must not apply."""
if ptu_terms(model_info) is None:
return False
if not is_ptu_cost_attribution_enabled():
return False
return azure_spillover(response_headers, additional_headers) is not None
def azure_spillover(
response_headers: Mapping[str, object] | None,
additional_headers: Mapping[str, object] | None,
) -> AzureSpillover | None:
"""The spillover Azure reports in the response headers, else None."""
for headers, prefix in (
(response_headers, ""),
(additional_headers, "llm_provider-"),
):
if headers is None or str(headers.get(f"{prefix}{AZURE_SPILLOVER_HEADER}")).lower() != "true":
continue
return AzureSpillover(
from_deployment=str(v) if (v := headers.get(f"{prefix}{AZURE_SPILLOVER_FROM_HEADER}")) is not None else None
)
return None

1
litellm/llms/azure/azure.py
View file

@ -561,6 +561,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
headers, response = self.make_sync_azure_openai_chat_completion_request(
azure_client=azure_client, data=data, timeout=timeout
)
logging_obj.model_call_details["response_headers"] = headers
streamwrapper: Final = CustomStreamWrapper(
completion_stream=response,
model=model,

25
litellm/model_prices_and_context_window_backup.json
View file

@ -23788,7 +23788,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
"supports_vision": true
},
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
"input_cost_per_token": 1.2e-06,
@ -24114,7 +24114,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
"supports_vision": true
},
"fireworks_ai/qwen3p7-plus": {
"cache_read_input_token_cost": 8e-08,
@ -69194,6 +69194,27 @@
"supports_reasoning": true,
"supports_vision": true
},
"typesafe/jev-1.13.0": {
"input_cost_per_token": 4.2e-08,
"litellm_provider": "typesafe",
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
},
"typesafe/jev-latest": {
"input_cost_per_token": 4.2e-08,
"litellm_provider": "typesafe",
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
},
"typesafe/jev-preview": {
"input_cost_per_token": 4.2e-08,
"litellm_provider": "typesafe",
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
},
"wandb/zai-org/GLM-5.3-Flash": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 1.5e-07,

1
litellm/proxy/_lazy_features.py
View file

@ -209,6 +209,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
"/openai/",
"/openai_passthrough/",
"/transcribe",
"/typesafe/",
"/vertex-ai/",
"/vertex_ai/",
"/vllm/",

90
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -20444,6 +20444,96 @@
]
}
},
"/typesafe/{endpoint}": {
"get": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/typesafe)",
"operationId": "typesafe_proxy_route_typesafe__endpoint__get",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Typesafe Proxy Route",
"tags": [
"llm_passthrough"
]
},
"post": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/typesafe)",
"operationId": "typesafe_proxy_route_typesafe__endpoint__post",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Typesafe Proxy Route",
"tags": [
"llm_passthrough"
]
}
},
"/vertex_ai/discovery/{endpoint}": {
"delete": {
"description": "Call any vertex discovery endpoint using the proxy.\n\nJust use `{PROXY_BASE_URL}/vertex_ai/discovery/{endpoint:path}`\n\nTarget url: `https://discoveryengine.googleapis.com`",

3
litellm/proxy/_types.py
View file

@ -51,6 +51,7 @@ from litellm.types.router import RouterErrors, UpdateRouterConfig
from litellm.types.router_weights import validate_router_settings_dict
from litellm.types.secret_managers.main import KeyManagementSystem
from litellm.types.utils import (
AzureSpillover,
CallTypes,
CostBreakdown,
EmbeddingResponse,
@ -484,6 +485,7 @@ class LiteLLMRoutes(enum.Enum):
"/eu.assemblyai",
"/vllm",
"/mistral",
"/typesafe",
"/milvus",
"/gigachat",
"/watsonx",
@ -3897,6 +3899,7 @@ class SpendLogsMetadata(TypedDict):
autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed
litellm_gateway_injected_cache: ReadOnly[str | None]
router_metadata: ReadOnly[SpendLogsRouterMetadata | None] # None = deployment not flagged internal_router_model
azure_spillover: ReadOnly[AzureSpillover | None] # None = Azure did not report spillover
class SpendLogsPayload(TypedDict):

19
litellm/proxy/auth/litellm_license.py
View file

@ -17,6 +17,7 @@ if TYPE_CHECKING:
AUTO_ROUTER_LICENSE_FEATURE: Final = "auto_router"
LICENSE_ALL_FEATURES: Final = "*"
AUTO_ROUTER_LICENSE_REMEDY: Final = "A LiteLLM license with the 'auto_router' feature lifts the limit."
@ -153,17 +154,21 @@ class LicenseCheck:
return False
return team_count > _max_teams_in_license
def grants_feature(self, feature: str) -> bool:
if self.airgapped_license_data is None:
return False
allowed_features: Final = self.airgapped_license_data.get("allowed_features")
granted: Final = allowed_features if isinstance(allowed_features, list) else (allowed_features,)
return feature in granted or LICENSE_ALL_FEATURES in granted
def auto_router_capability_limit(self) -> int | None:
"""
How many auto-routers may claim each gated classifier or customization capability:
unlimited (None) only when the signed license lists the auto_router
feature, otherwise one per capability. A license verified through the API carries no
feature list, so it does not lift the limit either.
unlimited (None) only when the signed license lists the auto_router feature or the
"*" wildcard that grants every feature, otherwise one per capability. A license verified
through the API carries no feature list, so it does not lift the limit either.
"""
if self.airgapped_license_data is None:
return 1
allowed_features: Final = self.airgapped_license_data.get("allowed_features")
if isinstance(allowed_features, list) and AUTO_ROUTER_LICENSE_FEATURE in allowed_features:
if self.grants_feature(AUTO_ROUTER_LICENSE_FEATURE):
return None
return 1

7
litellm/proxy/hooks/responses_id_security.py
View file

@ -22,7 +22,7 @@ from litellm.types.llms.openai import (
BaseLiteLLMOpenAIResponseObject,
ResponsesAPIResponse,
)
from litellm.types.utils import CallTypesLiteral, LLMResponseTypes, SpecialEnums
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypesLiteral, LLMResponseTypes, SpecialEnums
if TYPE_CHECKING:
from litellm.caching.caching import DualCache
@ -32,7 +32,6 @@ if TYPE_CHECKING:
_RESPONSES_API_PROVIDER_PREFIX: Final = "/openai"
_RESPONSES_API_CREATE_ROUTES: Final = frozenset({"/v1/responses", "/responses"})
_ADDRESSED_RESPONSE_ID_KEY: Final = "_litellm_addressed_response_id"
_UNMANAGED_RESPONSE_ID_DETAIL: Final = (
"Forbidden. This response id was not issued by this proxy, so the proxy cannot tell who owns it. "
"To let keys address responses this proxy did not issue, set "
@ -132,7 +131,7 @@ class ResponsesIDSecurity(CustomLogger):
if call_type not in responses_api_call_types:
return None
addressed_id_field: Final = "previous_response_id" if call_type == "aresponses" else "response_id"
retained_id: Final = data.get(_ADDRESSED_RESPONSE_ID_KEY)
retained_id: Final = data.get(ADDRESSED_RESPONSE_ID_FIELD)
addressed_id: Final = (
retained_id if isinstance(retained_id, str) and retained_id else data.get(addressed_id_field)
)
@ -140,7 +139,7 @@ class ResponsesIDSecurity(CustomLogger):
return data
authorized_id: Final = self._authorize_response_id(addressed_id, user_api_key_dict)
data[addressed_id_field] = authorized_id
data[_ADDRESSED_RESPONSE_ID_KEY] = addressed_id
data[ADDRESSED_RESPONSE_ID_FIELD] = addressed_id
return data
def _authorize_response_id(

30
litellm/proxy/management_endpoints/common_utils.py
View file

@ -582,25 +582,25 @@ async def _upsert_budget_and_membership(
)
return
create_data: Final[dict[str, Any]] = {
source_row: Final = (
await tx.litellm_budgettable.find_unique(where={"budget_id": existing_budget_id}) if is_shared_default else None
)
source: Final[Mapping[str, Any]] = source_row.model_dump() if source_row is not None else MappingProxyType({})
create_data: Final[dict[str, Any]] = { # mutable-ok: Prisma create payloads are dict-shaped
"created_by": user_api_key_dict.user_id or "",
"updated_by": user_api_key_dict.user_id or "",
**MappingProxyType(
{f: source[f] for f in _TEAM_MEMBER_BUDGET_LIMIT_FIELDS if _is_set_budget_value(source.get(f))}
),
**write_data,
}
if is_shared_default:
default_budget_row: Final = await tx.litellm_budgettable.find_unique(where={"budget_id": existing_budget_id})
if default_budget_row is not None:
default_budget_dict: Final = default_budget_row.model_dump()
for field in _TEAM_MEMBER_BUDGET_LIMIT_FIELDS:
value = default_budget_dict.get(field)
if _is_set_budget_value(value):
create_data[field] = value
create_data.update(write_data)
if create_data.get("budget_duration") is not None:
create_data["budget_reset_at"] = get_budget_reset_time(budget_duration=create_data["budget_duration"])
else:
# Restarting an inherited window on an unrelated edit hands the member a free period.
carried: Final = source.get("budget_reset_at") if "budget_duration" not in budget_patch else None
if carried is not None:
create_data["budget_reset_at"] = carried
if create_data.get("budget_reset_at") is None:
create_data.pop("budget_reset_at", None)
if not _has_meaningful_budget_limit(create_data):

12
litellm/proxy/management_endpoints/management_v1/teams.py
View file

@ -2,7 +2,7 @@
from typing import Annotated, Final
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, Header
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
@ -108,6 +108,12 @@ async def bulk_update_team_member_budgets_action(
team_id: str,
data: BulkTeamMemberBudgetUpdateRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
litellm_changed_by: Annotated[
str | None,
Header(
description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability",
),
] = None,
) -> BulkTeamMemberBudgetUpdateResponse:
"""
Set per-member limits for up to 500 members of one team in one call. Same
@ -135,7 +141,7 @@ async def bulk_update_team_member_budgets_action(
```
"""
try:
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
from litellm.proxy.proxy_server import litellm_proxy_admin_name, prisma_client, user_api_key_cache
if prisma_client is None:
raise ManagementProblem(
@ -153,6 +159,8 @@ async def bulk_update_team_member_budgets_action(
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
litellm_proxy_admin_name=litellm_proxy_admin_name,
litellm_changed_by=litellm_changed_by,
)
return BulkTeamMemberBudgetUpdateResponse(data=results)

77
litellm/proxy/management_helpers/bulk_team_member_budgets.py
View file

@ -7,11 +7,20 @@ cap never moves another member's.
"""
from collections.abc import Sequence
from datetime import timedelta
from datetime import datetime, timedelta
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, UserAPIKeyAuth
from pydantic import BaseModel, ConfigDict
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import (
LiteLLM_TeamTable,
LitellmTableNames,
LitellmUserRoles,
Member,
UserAPIKeyAuth,
)
from litellm.proxy.auth.auth_checks import invalidate_team_member_spend_state
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
@ -21,6 +30,7 @@ from litellm.proxy.management_endpoints.common_utils import (
_upsert_budget_and_membership, # pyright: ignore[reportPrivateUsage] # the single-member write, shared so the two surfaces cannot drift
member_budget_patch,
)
from litellm.proxy.management_helpers.audit_logs import create_object_audit_log
from litellm.proxy.management_helpers.bulk_user_deletion import (
_duplicate_member_indexes, # pyright: ignore[reportPrivateUsage] # same duplicate rule as members/bulk_delete
_eq_filter, # pyright: ignore[reportPrivateUsage] # same prisma filter shape as members/bulk_delete
@ -77,6 +87,54 @@ async def _shared_budget_ids(tx: "Prisma", budget_ids: frozenset[str]) -> frozen
return frozenset(budget_id for budget_id in budget_ids if sum(1 for row in rows if row.budget_id == budget_id) > 1)
class _AuditedMemberBudget(BaseModel):
"""One member's limits as the audit log's before/after values record them."""
model_config = ConfigDict(frozen=True)
user_id: str
budget_id: str | None = None
max_budget: float | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
budget_duration: str | None = None
budget_reset_at: datetime | None = None
allowed_models: tuple[str, ...] | None = None
class _AuditedMemberBudgets(BaseModel):
"""The audit-log columns hold a JSON object, so the per-member list is nested under a key."""
model_config = ConfigDict(frozen=True)
team_member_budgets: tuple[_AuditedMemberBudget, ...]
def _audited_member_budget(row: "prisma_models.LiteLLM_TeamMembership") -> _AuditedMemberBudget:
budget: Final = row.litellm_budget_table
if budget is None:
return _AuditedMemberBudget(user_id=row.user_id, budget_id=row.budget_id)
return _AuditedMemberBudget(
user_id=row.user_id,
budget_id=row.budget_id,
max_budget=budget.max_budget,
tpm_limit=budget.tpm_limit,
rpm_limit=budget.rpm_limit,
budget_duration=budget.budget_duration,
budget_reset_at=budget.budget_reset_at,
allowed_models=tuple(budget.allowed_models),
)
def _limits_audit_value(rows: "Sequence[prisma_models.LiteLLM_TeamMembership]") -> str:
"""Serialize the members' limits for an audit-log value, dropping the limits they do not set."""
return safe_dumps(
_AuditedMemberBudgets(
team_member_budgets=tuple(_audited_member_budget(row) for row in sorted(rows, key=lambda row: row.user_id))
).model_dump(exclude_none=True, mode="json")
)
def _result(
member: TeamMemberBudgetPatch,
user_id: str | None,
@ -114,6 +172,8 @@ async def bulk_update_team_member_budgets(
user_api_key_dict: UserAPIKeyAuth,
prisma_client: PrismaClient,
user_api_key_cache: UserApiKeyCache,
litellm_proxy_admin_name: str,
litellm_changed_by: str | None = None,
) -> tuple[TeamMemberBudgetUpdateResult, ...]:
"""Apply one merge patch of per-member limits per requested member, in one transaction."""
team: Final = await TeamRepository(WriterPinnedClient(prisma_client.db)).find_by_id(team_id)
@ -151,7 +211,7 @@ async def bulk_update_team_member_budgets(
team_members_filter: Final = _team_users_filter(team_id, user_ids)
async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx:
memberships: Final = await _membership_tx_db(tx).find_many(where=team_members_filter)
memberships: Final = await _membership_tx_db(tx).find_many(where=team_members_filter, include=_WITH_BUDGET)
budget_id_of: Final = MappingProxyType({m.user_id: m.budget_id for m in memberships})
shared: Final = await _shared_budget_ids(
tx, frozenset(budget_id for budget_id in budget_id_of.values() if budget_id is not None)
@ -179,6 +239,17 @@ async def bulk_update_team_member_budgets(
user_id=user_id, team_id=team_id, user_api_key_cache=user_api_key_cache
)
await create_object_audit_log(
object_id=team_id,
action="updated",
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
table_name=LitellmTableNames.TEAM_TABLE_NAME,
before_value=_limits_audit_value(memberships),
after_value=_limits_audit_value(written),
)
budget_of: Final = MappingProxyType({m.user_id: m.litellm_budget_table for m in written})
return tuple(
_result(

36
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -525,6 +525,42 @@ async def mistral_proxy_route(
return received_value
@router.api_route(
"/typesafe/{endpoint:path}",
methods=["GET", "POST"], # mutable-ok: FastAPI route metadata requires a list
tags=["TypeSafe AI Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list
)
async def typesafe_proxy_route(
endpoint: str,
request: Request,
fastapi_response: Response,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
"""[Docs](https://docs.litellm.ai/docs/pass_through/typesafe)"""
base_target_url: Final = get_secret_str("TYPESAFE_API_BASE") or "https://api.typesafe.ai"
encoded_endpoint: Final = httpx.URL(endpoint).path
normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}"
base_url: Final = httpx.URL(base_target_url)
updated_url: Final = base_url.copy_with(
path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint),
)
typesafe_api_key: Final = passthrough_endpoint_router.get_credentials(
custom_llm_provider="typesafe",
region_name=None,
)
endpoint_func: Final = create_pass_through_route(
endpoint=endpoint,
target=str(updated_url),
custom_headers={ # mutable-ok: pass-through request headers require a mutable mapping
"Authorization": f"Bearer {typesafe_api_key}",
"Content-Type": "application/json",
},
custom_llm_provider="typesafe",
is_streaming_request=False,
)
return await endpoint_func(request, fastapi_response, user_api_key_dict)
@router.api_route(
"/milvus/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],

117
litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py Normal file
View file

@ -0,0 +1,117 @@
from collections.abc import Mapping
from datetime import datetime
from typing import Final
import httpx
from pydantic import BaseModel, TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import (
get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature
)
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage
class _TypeSafeUsage(BaseModel):
input_tokens: int = 0
output_tokens: int = 0
class _TypeSafeResponse(BaseModel):
model: str | None = None
usage: _TypeSafeUsage | None = None
class _RegistryPricing(BaseModel):
input_cost_per_token: float = 0.0
output_cost_per_token: float = 0.0
_TYPESAFE_RESPONSE_ADAPTER: Final = TypeAdapter(_TypeSafeResponse)
_REGISTRY_PRICING_ADAPTER: Final = TypeAdapter(_RegistryPricing)
def _parse_typesafe_response(response_body: Mapping[str, object]) -> _TypeSafeResponse:
try:
return _TYPESAFE_RESPONSE_ADAPTER.validate_python(response_body)
except ValidationError:
return _TypeSafeResponse()
def _pricing_for(model_keys: tuple[str, ...]) -> _RegistryPricing:
for model_key in model_keys:
if model_key not in litellm.model_cost: # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed
continue
try:
return _REGISTRY_PRICING_ADAPTER.validate_python(
litellm.model_cost[model_key] # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed
)
except ValidationError:
continue
return _RegistryPricing()
class TypeSafePassthroughLoggingHandler:
@staticmethod
def typesafe_passthrough_handler(
httpx_response: httpx.Response,
response_body: Mapping[str, object],
logging_obj: LiteLLMLoggingObj,
url_route: str,
result: str,
start_time: datetime,
end_time: datetime,
cache_hit: bool,
request_body: Mapping[str, object],
**kwargs: object,
) -> PassThroughEndpointLoggingTypedDict:
response: Final = _parse_typesafe_response(response_body)
response_model: Final = response.model
request_model_value: Final = request_body.get("model")
request_model: Final = request_model_value if isinstance(request_model_value, str) else None
logged_model: Final = response_model or request_model or "unknown"
model_name: Final = f"typesafe/{logged_model}"
usage: Final = response.usage or _TypeSafeUsage()
input_tokens: Final = usage.input_tokens
output_tokens: Final = usage.output_tokens
candidate_model_keys: Final = tuple(
f"typesafe/{model}" for model in (response_model, request_model) if model is not None
)
pricing: Final = _pricing_for(candidate_model_keys)
response_cost: Final = (
input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
)
usage_object: Final = Usage(
prompt_tokens=input_tokens,
completion_tokens=output_tokens,
total_tokens=input_tokens + output_tokens,
)
updated_kwargs: Final = { # mutable-ok: pass-through logging contract requires mutable kwargs
**kwargs,
"model": model_name,
"custom_llm_provider": "typesafe",
"response_cost": response_cost,
"combined_usage_object": usage_object,
}
logging_obj.model_call_details.update(
model=model_name,
custom_llm_provider="typesafe",
response_cost=response_cost,
)
standard_logging_object: Final = get_standard_logging_object_payload(
kwargs=updated_kwargs,
init_response_obj=ModelResponse(model=model_name, usage=usage_object),
start_time=start_time,
end_time=end_time,
logging_obj=logging_obj,
status="success",
)
return { # mutable-ok: pass-through logging contract requires mutable result
"result": StandardPassThroughResponseObject(response=result),
"kwargs": { # mutable-ok: pass-through logging contract requires mutable kwargs
**updated_kwargs,
"standard_logging_object": standard_logging_object,
},
}

23
litellm/proxy/pass_through_endpoints/success_handler.py
View file

@ -1,5 +1,6 @@
import json
from datetime import datetime
from types import MappingProxyType
from typing import Any, Final
from urllib.parse import urlparse
@ -287,6 +288,25 @@ class PassThroughEndpointLogging:
)
standard_logging_response_object = transcribe_handler_result["result"] # rebind-ok: elif-chain
kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract
elif self.is_typesafe_route(custom_llm_provider):
from .llm_provider_handlers.typesafe_passthrough_logging_handler import (
TypeSafePassthroughLoggingHandler,
)
typesafe_handler_result: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
httpx_response=httpx_response,
response_body=response_body if isinstance(response_body, dict) else MappingProxyType({}),
logging_obj=logging_obj,
url_route=url_route,
result=result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
request_body=request_body,
**kwargs,
)
standard_logging_response_object = typesafe_handler_result["result"]
kwargs = typesafe_handler_result["kwargs"]
elif self.is_vertex_ai_live_route(url_route):
from .llm_provider_handlers.vertex_ai_live_passthrough_logging_handler import (
VertexAILivePassthroughLoggingHandler,
@ -440,6 +460,9 @@ class PassThroughEndpointLogging:
def is_transcribe_route(self, custom_llm_provider: str | None) -> bool:
return custom_llm_provider == TRANSCRIBE_CUSTOM_LLM_PROVIDER
def is_typesafe_route(self, custom_llm_provider: str | None) -> bool:
return custom_llm_provider == "typesafe"
def is_langfuse_route(self, url_route: str):
parsed_url: Final = urlparse(url_route)
for route in self.TRACKED_LANGFUSE_ROUTES:

19
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -39,6 +39,7 @@ from litellm.litellm_core_utils.litellm_logging import (
is_valid_sha256_hash,
request_model_access_groups_from_litellm_params,
)
from litellm.litellm_core_utils.ptu_pricing import azure_spillover
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes
from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload, SpendLogsRouterMetadata
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
@ -47,6 +48,7 @@ from litellm.proxy.utils import PrismaClient, hash_token
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params
from litellm.types.utils import (
PROMPT_CARRYING_GUARDRAIL_FIELDS,
AzureSpillover,
CallTypes,
CostBreakdown,
LlmProviders,
@ -133,6 +135,9 @@ def _get_router_metadata_for_spend_log(
)
_STAMPED_METADATA_KEYS: Final = frozenset(("router_metadata", "azure_spillover"))
def _get_spend_logs_metadata(
metadata: dict | None,
applied_guardrails: list[str] | None = None,
@ -150,6 +155,7 @@ def _get_spend_logs_metadata(
litellm_call_id: str | None = None,
autorouter_savings: float | None = None,
router_metadata: SpendLogsRouterMetadata | None = None,
azure_spillover: AzureSpillover | None = None,
) -> SpendLogsMetadata:
if metadata is None:
return SpendLogsMetadata(
@ -191,6 +197,7 @@ def _get_spend_logs_metadata(
litellm_gateway_injected_cache=None,
litellm_call_id=litellm_call_id,
router_metadata=router_metadata,
azure_spillover=azure_spillover,
)
verbose_proxy_logger.debug(
"getting payload for SpendLogs, available keys in metadata: " + str(list(metadata.keys()))
@ -198,8 +205,9 @@ def _get_spend_logs_metadata(
# Filter the metadata dictionary to include only the specified keys
clean_metadata: Final = SpendLogsMetadata(
**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__ if key != "router_metadata"},
**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__ if key not in _STAMPED_METADATA_KEYS},
router_metadata=router_metadata,
azure_spillover=azure_spillover,
)
_raw_key: Final = clean_metadata.get("user_api_key")
_trusted_hash: Final = metadata.get("user_api_key_hash")
@ -573,6 +581,15 @@ def get_logging_payload(
selected_provider=custom_llm_provider,
router_correlation_id=litellm_call_id,
),
azure_spillover=azure_spillover(
response_headers=kwargs.get("response_headers")
if isinstance(kwargs.get("response_headers"), Mapping)
else None,
additional_headers=standard_logging_payload["hidden_params"].get("additional_headers")
if standard_logging_payload is not None
and isinstance(standard_logging_payload.get("hidden_params"), Mapping)
else None,
),
)
special_usage_fields: Final = ["completion_tokens", "prompt_tokens", "total_tokens"]

11
litellm/types/utils.py
View file

@ -348,6 +348,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
"audio_transcription",
"audio_speech",
"responses",
"evaluation",
"ocr",
"realtime",
]
@ -3075,6 +3076,12 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata):
team_id: str | None
class AzureSpillover(TypedDict):
"""Spillover Azure reports in its response headers for a request it served from pay-as-you-go capacity."""
from_deployment: ReadOnly[str | None]
class StandardLoggingAdditionalHeaders(TypedDict, total=False):
x_ratelimit_limit_requests: int
x_ratelimit_limit_tokens: int
@ -3754,6 +3761,8 @@ agentic_loop_internal_litellm_params: Final = [
# the provider.
TRUSTED_CALLBACK_VARS_FIELD: Final = "litellm_trusted_callback_vars"
ADDRESSED_RESPONSE_ID_FIELD: Final = "_litellm_addressed_response_id"
# Bedrock managed-batch deployment config, read from litellm_params by the batch and
# files transformations. Listed for the same reason as the fields above: these sit on
# a deployment that also serves chat, so leaking them into extra_body makes Bedrock
@ -3768,7 +3777,7 @@ bedrock_batch_litellm_params: Final = (
all_litellm_params = (
agentic_loop_internal_litellm_params
+ [TRUSTED_CALLBACK_VARS_FIELD, *bedrock_batch_litellm_params]
+ [TRUSTED_CALLBACK_VARS_FIELD, ADDRESSED_RESPONSE_ID_FIELD, *bedrock_batch_litellm_params]
+ [
"metadata",
"litellm_metadata",

25
model_prices_and_context_window.json
View file

@ -23788,7 +23788,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
"supports_vision": true
},
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
"input_cost_per_token": 1.2e-06,
@ -24114,7 +24114,7 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
"supports_vision": true
},
"fireworks_ai/qwen3p7-plus": {
"cache_read_input_token_cost": 8e-08,
@ -69194,6 +69194,27 @@
"supports_reasoning": true,
"supports_vision": true
},
"typesafe/jev-1.13.0": {
"input_cost_per_token": 4.2e-08,
"litellm_provider": "typesafe",
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
},
"typesafe/jev-latest": {
"input_cost_per_token": 4.2e-08,
"litellm_provider": "typesafe",
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
},
"typesafe/jev-preview": {
"input_cost_per_token": 4.2e-08,
"litellm_provider": "typesafe",
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
},
"wandb/zai-org/GLM-5.3-Flash": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 1.5e-07,

1
model_prices_and_context_window.schema.json
View file

@ -427,6 +427,7 @@
"chat",
"completion",
"embedding",
"evaluation",
"guardrail",
"image_edit",
"image_generation",

1
tests/e2e/batches/COVERAGE.md
View file

@ -20,6 +20,7 @@ failures are hard test failures (see `tests/e2e/CLAUDE.md`).
| Azure | yes | yes | yes | yes | yes (byte-verbatim) | Azure Files |
| Vertex AI | yes | yes | yes | yes | yes (provider-transformed) | GCS (`gcs_bucket_name` / `GCS_BUCKET_NAME` on model) |
| Bedrock | yes (unified only) | yes | yes | yes (unfiltered managed list) | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) |
| Bedrock GovCloud (`us-gov-west-1`) | yes (unified only) | yes | no | no | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` on model, resolved from `AWS_GOVCLOUD_ACCESS_KEY_ID` / `AWS_GOVCLOUD_SECRET_ACCESS_KEY` / `AWS_GOVCLOUD_BATCH_S3_BUCKET` / `AWS_GOVCLOUD_BATCH_ROLE_ARN`) |
Bedrock cancel maps to `StopModelInvocationJob` and comes back `cancelling`; the
lifecycle asserts it the same way it does for OpenAI (`_CANCEL_ASSERTED_PROVIDERS`).

113
tests/e2e/batches/test_batches_e2e.py
View file

@ -21,21 +21,18 @@ import os
import re
import time
from datetime import datetime, timedelta, timezone
from typing import Final
import pytest
from pydantic import BaseModel
from e2e_config import MASTER_KEY, PROXY_BASE_URL, unique_marker
from batch_cleanup import cleanup_batch, cleanup_file
from batch_client import (
AZURE_FILE_EXPIRY_SECONDS,
batch_upload_form,
UPLOAD_FILENAME,
BatchClient,
BatchCreateBody,
BatchObject,
FileObject,
batch_upload_form,
is_model_access_denied,
is_result_access_denied,
)
@ -57,6 +54,7 @@ from capabilities import (
openai_batch_params,
raw_id_matches_provider,
)
from e2e_config import MASTER_KEY, PROXY_BASE_URL, unique_marker
from e2e_http import (
FileUploadForm,
Result,
@ -68,6 +66,7 @@ from e2e_http import (
)
from lifecycle import ResourceManager
from models import KeyGenerateBody, KeyMetadata, LiteLLMParamsBody, SpendLogRow
from pydantic import BaseModel, Field
pytestmark = pytest.mark.e2e
@ -75,6 +74,25 @@ CREATED_BATCH_STATUSES = {"validating", "in_progress", "finalizing"}
BATCH_CANCEL_DELAY_SECONDS = 2
BATCH_TERMINAL_BEFORE_CANCEL = {"failed", "cancelled", "expired"}
BATCH_OP_RETRIES = 5
class _GovCloudBedrockContent(BaseModel):
text: str
class _GovCloudBedrockMessage(BaseModel):
content: tuple[_GovCloudBedrockContent, ...]
class _GovCloudBedrockInput(BaseModel):
messages: tuple[_GovCloudBedrockMessage, ...]
class _GovCloudBedrockRecord(BaseModel):
record_id: str = Field(alias="recordId")
model_input: _GovCloudBedrockInput = Field(alias="modelInput")
# Azure / Vertex cancel and the pre-cancel re-retrieve are provider-side flakes
# (connection refused, brief 500s) and the registry only has one basic cell per
# provider (shared across scenarios). Create + retrieve already prove routing;
@ -1006,6 +1024,91 @@ class TestBedrockBatchAssumeRole:
assert fetched.id == batch.id
GOVCLOUD_REGION: Final = "us-gov-west-1"
GOVCLOUD_RAW_MODEL: Final = "bedrock/amazon.nova-lite-v1:0"
def _govcloud_params() -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=GOVCLOUD_RAW_MODEL,
aws_access_key_id="os.environ/AWS_GOVCLOUD_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_GOVCLOUD_SECRET_ACCESS_KEY",
aws_region_name=GOVCLOUD_REGION,
s3_region_name=GOVCLOUD_REGION,
s3_bucket_name="os.environ/AWS_GOVCLOUD_BATCH_S3_BUCKET",
s3_access_key_id="os.environ/AWS_GOVCLOUD_ACCESS_KEY_ID",
s3_secret_access_key="os.environ/AWS_GOVCLOUD_SECRET_ACCESS_KEY",
aws_batch_role_arn="os.environ/AWS_GOVCLOUD_BATCH_ROLE_ARN",
)
class TestBedrockBatchGovCloud:
"""Bedrock batch lifecycle in the AWS GovCloud partition (us-gov-west-1).
The deployment carries a GovCloud region for both Bedrock and S3, so the proxy has to
sign the file upload against the us-gov S3 endpoint and submit the job to the us-gov
Bedrock endpoint. Commercial-partition hostnames or arn:aws: ARNs reject the GovCloud
key, so a partition regression fails the upload instead of passing silently.
"""
@pytest.mark.covers(
"llm.batches.bedrock.govcloud_partition.nonstream.works",
"llm.files.bedrock.govcloud_partition.nonstream.works",
exercised_on=["batches", "files"],
)
def test_unified_file_upload_and_batch_create_in_govcloud(
self, client: BatchClient, resources: ResourceManager
) -> None:
model_name: Final = batch_model_name("bedrock-govcloud-batch")
model_id: Final = client.create_model(model_name, _govcloud_params())
resources.defer(lambda: client.delete_model(model_id))
key: Final = resources.key()
file: Final = unwrap(
client.upload_file(
content=render_jsonl(GOVCLOUD_RAW_MODEL),
form=FileUploadForm(purpose="batch", target_model_names=model_name),
key=key,
)
)
resources.defer(lambda: cleanup_file(client, file.id, key=key))
assert_file_object(file, provider="bedrock")
downloaded: Final = client.proxy.transport.download(
f"/v1/files/{file.id}/content",
headers=client.proxy.transport.bearer(key),
)
assert downloaded.status_code == 200, (
f"GovCloud file content must be 200, got {downloaded.status_code}: {downloaded.body[:300]}"
)
downloaded_lines: Final = downloaded.body.strip().splitlines()
assert len(downloaded_lines) == 1, (
f"GovCloud file content download must contain one JSONL record, got {len(downloaded_lines)}"
)
downloaded_record: Final = _GovCloudBedrockRecord.model_validate(json.loads(downloaded_lines[0]))
assert downloaded_record.record_id == "req-1", (
f"GovCloud file content must preserve the uploaded custom_id, got {downloaded_record.record_id!r}"
)
assert downloaded_record.model_input.messages[0].content[0].text == "ping", (
"GovCloud file content must preserve the uploaded message text"
)
created: Final = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key)
require_successful_call(created)
batch: Final = BatchObject.model_validate_json(created.body)
resources.defer(lambda: cleanup_batch(client, batch.id, key=key))
assert is_managed_id(batch.id), (
f"GovCloud create via target_model_names must return a managed batch id, got {batch.id!r}"
)
assert batch.status in CREATED_BATCH_STATUSES, (
f"GovCloud batch has non-transitional status {batch.status!r}"
)
assert_batch_object(batch)
fetched: Final = unwrap(client.retrieve_batch(batch.id, key=key))
assert fetched.id == batch.id
GEMINI_FILES_RAW_MODEL = "gemini-2.5-flash"

2
tests/e2e/coverage_registry/llm_nonconversational.yaml
View file

@ -23,6 +23,7 @@
- {id: llm.batches.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Vertex batches"}
- {id: llm.batches.bedrock.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Bedrock batches (encoded/unified only)"}
- {id: llm.batches.bedrock.assume_role.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: assume_role, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch create under STS assume-role credentials"}
- {id: llm.batches.bedrock.govcloud_partition.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: govcloud_partition, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch create in the us-gov-west-1 partition"}
- {id: llm.batches.bedrock.cancel.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch cancel (StopModelInvocationJob) returns the same id with a cancelling/cancelled status"}
- {id: llm.batches.bedrock.list.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "A Bedrock managed batch is present in the GET /v1/batches list envelope"}
- {id: llm.batches.hosted_vllm.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible batch create"}
@ -45,6 +46,7 @@
- {id: llm.files.azure_openai.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:45", rationale: "Azure file upload managed backend"}
- {id: llm.files.vertex.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:52", rationale: "Vertex file upload to GCS"}
- {id: llm.files.bedrock.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:59", rationale: "Bedrock file upload to S3"}
- {id: llm.files.bedrock.govcloud_partition.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: govcloud_partition, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock file upload to an S3 bucket in the us-gov-west-1 partition"}
- {id: llm.files.gemini.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Gemini Files API upload via proxy"}
- {id: llm.files.hosted_vllm.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible file upload"}
- {id: llm.files.openai.require_managed_files_upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_managed_files_enforcement_e2e.py / LIT-5902", rationale: "With require_managed_files enabled, an upload without target_model_names and an upload carrying a model param are both rejected 400; runs only in the sequential managed-files stack phase (E2E_MANAGED_FILES_STACK)"}

1
tests/e2e/coverage_registry/schema.py
View file

@ -64,6 +64,7 @@ LlmCapability = Literal[
"assume_role",
"basic",
"count_tokens",
"govcloud_partition",
"input_validation",
"long_context_1m",
"mid_conversation_system",

177
tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py
View file

@ -8,9 +8,184 @@ configuration works correctly.
Related issue: https://github.com/BerriAI/litellm/issues/18221
"""
from typing import get_args
import json
import re
from collections.abc import Iterator
from contextlib import contextmanager
from pathlib import Path
from types import MappingProxyType
from typing import Final, get_args
import pytest
from prometheus_client import REGISTRY, Gauge
from prometheus_client.registry import Collector
import litellm
from litellm.caching.redis_cache import _breaker_metrics
from litellm.integrations.prometheus import PrometheusLogger
from litellm.integrations.prometheus_services import PrometheusServicesLogger
from litellm.proxy.db.db_transaction_queue.spend_log_cleanup_metrics import SpendLogCleanupMetrics
from litellm.proxy.middleware.admission_control_middleware import create_prometheus_admission_metrics
from litellm.proxy.middleware.in_flight_requests_middleware import InFlightRequestsMiddleware
_GRAFANA_DIR: Final = Path(__file__).parents[3] / "cookbook" / "litellm_proxy_server" / "grafana_dashboard"
_ALL_METRICS_DASHBOARD: Final = _GRAFANA_DIR / "dashboard_all_metrics" / "grafana_dashboard.json"
_LITELLM_DASHBOARDS: Final = (_ALL_METRICS_DASHBOARD, _GRAFANA_DIR / "dashboard_v2" / "grafana_dashboard.json")
_METRIC_TOKEN_RE: Final = re.compile(r"\blitellm_[a-z0-9_]+")
_BY_CLAUSE_RE: Final = re.compile(r"\bby\s*\([^)]*\)")
_EXPOSITION_SUFFIXES: Final = ("", "_total", "_bucket", "_sum", "_count", "_created")
def _registered_collectors() -> MappingProxyType[Collector, tuple[str, ...]]:
return MappingProxyType({collector: tuple(names) for collector, names in REGISTRY._collector_to_names.items()})
def _unregister_everything() -> None:
for collector in tuple(REGISTRY._collector_to_names):
REGISTRY.unregister(collector)
def _register_if_absent(collectors: tuple[Collector, ...]) -> None:
for collector in collectors:
if collector not in REGISTRY._collector_to_names and not any(
name in REGISTRY._names_to_collectors for name in REGISTRY._get_names(collector)
):
REGISTRY.register(collector)
def _lazy_owner_collectors() -> tuple[Collector, ...]:
SpendLogCleanupMetrics._ensure_initialized()
assert SpendLogCleanupMetrics.rows_deleted is not None
assert SpendLogCleanupMetrics.batch_duration is not None
assert SpendLogCleanupMetrics.rows_remaining is not None
assert SpendLogCleanupMetrics.batch_failures is not None
assert SpendLogCleanupMetrics.runs is not None
in_flight: Final = InFlightRequestsMiddleware._get_gauge()
assert in_flight is not None
breaker: Final = _breaker_metrics()
assert breaker._state_gauge is not None
assert breaker._transitions is not None
assert breaker._failures is not None
return (
SpendLogCleanupMetrics.rows_deleted,
SpendLogCleanupMetrics.batch_duration,
SpendLogCleanupMetrics.rows_remaining,
SpendLogCleanupMetrics.batch_failures,
SpendLogCleanupMetrics.runs,
in_flight,
breaker._state_gauge,
breaker._transitions,
breaker._failures,
)
def _fresh_admission_collectors() -> tuple[Collector, ...]:
admission: Final = create_prometheus_admission_metrics()
assert admission is not None
return (admission.admitted_gauge, admission.queued_gauge, admission.rejected_counter)
@contextmanager
def _isolated_litellm_metric_families(monkeypatch: pytest.MonkeyPatch) -> Iterator[frozenset[str]]:
previous: Final = _registered_collectors()
_unregister_everything()
monkeypatch.setattr(litellm, "prometheus_metrics_config", None)
PrometheusLogger()
PrometheusServicesLogger()
lazy_owned: Final = _lazy_owner_collectors()
_register_if_absent(lazy_owned)
_fresh_admission_collectors()
try:
yield frozenset(metric.name for metric in REGISTRY.collect())
finally:
_unregister_everything()
for collector in previous:
REGISTRY.register(collector)
_register_if_absent(lazy_owned)
@pytest.fixture
def emitted_metric_families(monkeypatch: pytest.MonkeyPatch) -> Iterator[frozenset[str]]:
with _isolated_litellm_metric_families(monkeypatch) as families:
yield families
@pytest.fixture
def gauges_registered_by_an_earlier_test() -> Iterator[tuple[Collector, Collector]]:
sentinel: Final = Gauge("litellm_unrelated_sentinel", "registered by a test outside the isolated block")
already_registered: Final = REGISTRY._names_to_collectors.get("litellm_admission_admitted_requests")
admission: Final = already_registered or Gauge(
"litellm_admission_admitted_requests", "registered directly, bypassing admission_control_state"
)
yield (sentinel, admission)
for gauge in (sentinel,) if already_registered is not None else (sentinel, admission):
if gauge in REGISTRY._collector_to_names:
REGISTRY.unregister(gauge)
def test_isolated_metric_families_restore_the_registry_and_keep_lazy_owners_live(
monkeypatch: pytest.MonkeyPatch, gauges_registered_by_an_earlier_test: tuple[Collector, Collector]
):
before: Final = _registered_collectors()
with _isolated_litellm_metric_families(monkeypatch) as families:
assert "litellm_unrelated_sentinel" not in families
assert "litellm_admission_admitted_requests" in families
assert "litellm_in_flight_requests" in families
assert not any(gauge in REGISTRY._collector_to_names for gauge in gauges_registered_by_an_earlier_test)
after: Final = _registered_collectors()
assert all(after[collector] == names for collector, names in before.items())
lazy_owned: Final = _lazy_owner_collectors()
assert frozenset(after) - frozenset(before) <= frozenset(lazy_owned)
assert all(collector in after for collector in lazy_owned)
def _dashboard_expressions(path: Path) -> tuple[str, ...]:
dashboard: Final = json.loads(path.read_text())
return tuple(target["expr"] for panel in dashboard["panels"] for target in panel.get("targets", ()))
def _referenced_metric_tokens(path: Path) -> frozenset[str]:
return frozenset(
token
for expr in _dashboard_expressions(path)
for token in _METRIC_TOKEN_RE.findall(_BY_CLAUSE_RE.sub("", expr))
)
def _family_of(token: str, families: frozenset[str]) -> str | None:
candidates: Final = (token.removesuffix(suffix) for suffix in _EXPOSITION_SUFFIXES if token.endswith(suffix))
return next((candidate for candidate in candidates if candidate in families), None)
def test_all_metrics_dashboard_charts_every_emitted_metric_family(emitted_metric_families: frozenset[str]):
referenced: Final = _referenced_metric_tokens(_ALL_METRICS_DASHBOARD)
charted: Final = frozenset(
family for token in referenced for family in (_family_of(token, emitted_metric_families),) if family
)
assert emitted_metric_families - charted == frozenset()
@pytest.mark.parametrize("dashboard_path", _LITELLM_DASHBOARDS, ids=lambda p: p.parent.name)
def test_dashboards_only_reference_emitted_metrics(dashboard_path: Path, emitted_metric_families: frozenset[str]):
dead: Final = frozenset(
token
for token in _referenced_metric_tokens(dashboard_path)
if _family_of(token, emitted_metric_families) is None
)
assert dead == frozenset()
@pytest.mark.parametrize("dashboard_path", _LITELLM_DASHBOARDS, ids=lambda p: p.parent.name)
def test_dashboards_use_templated_prometheus_datasource(dashboard_path: Path):
dashboard: Final = json.loads(dashboard_path.read_text())
datasource_variables: Final = tuple(
variable["name"] for variable in dashboard["templating"]["list"] if variable["type"] == "datasource"
)
assert datasource_variables == ("DS_PROMETHEUS",)
panel_datasource_uids: Final = frozenset(
panel["datasource"]["uid"] for panel in dashboard["panels"] if panel["type"] != "row"
)
assert panel_datasource_uids == frozenset({"${DS_PROMETHEUS}"})
def test_remaining_requests_metric_name_in_defined_metrics():

152
tests/test_litellm/litellm_core_utils/test_litellm_logging.py
View file

@ -7229,3 +7229,155 @@ def test_add_dynamic_callback_registers_once_per_list_without_touching_the_calle
assert logging_obj.dynamic_async_failure_callbacks == [callback]
assert LitellmLogging._with_dynamic_callback(None, callback) == [callback]
assert LitellmLogging._with_dynamic_callback((callback,), callback) == [callback]
class TestAzurePTUSpilloverCost:
"""Azure PTU deployments price tokens at zero because the reservation is billed flat.
A request Azure spills onto pay-as-you-go capacity must bill per token instead, so
the zeroed custom pricing has to be skipped when the provider reports spillover.
"""
ROUTER_MODEL_ID: Final = "ptu-spill-router-model-id"
SERVED_MODEL: Final = "azure/spill-served-model-ptu"
PTU_MODEL_INFO: Final = {
"id": ROUTER_MODEL_ID,
"team_id": "team-1",
"ptu_count": 100,
"cost_per_ptu_per_hour": 1.0,
"ptu_effective_from": "2026-01-01",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
}
EXPECTED_SPILL_COST: Final = 100 * 2e-6 + 50 * 8e-6
@staticmethod
def _register_models() -> None:
litellm.register_model(
model_cost={
TestAzurePTUSpilloverCost.ROUTER_MODEL_ID: {
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"litellm_provider": "azure",
"mode": "chat",
},
TestAzurePTUSpilloverCost.SERVED_MODEL: {
"input_cost_per_token": 2e-6,
"output_cost_per_token": 8e-6,
"litellm_provider": "azure",
"mode": "chat",
},
}
)
@staticmethod
def _unregister_models() -> None:
litellm.model_cost.pop(TestAzurePTUSpilloverCost.ROUTER_MODEL_ID, None)
litellm.model_cost.pop(TestAzurePTUSpilloverCost.SERVED_MODEL, None)
def _logging_obj(self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch) -> LitellmLogging:
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", flag)
obj = LitellmLogging(
model=self.SERVED_MODEL,
messages=[{"role": "user", "content": "Hi"}],
stream=False,
call_type="completion",
start_time=time.time(),
litellm_call_id="ptu-spill-1",
function_id="f",
)
obj.update_environment_variables(
model=self.SERVED_MODEL,
user="",
optional_params={},
litellm_params={
"api_base": "",
"metadata": {"model_info": model_info},
"input_cost_per_token": litellm_rate,
"output_cost_per_token": litellm_rate,
},
custom_llm_provider="azure",
)
return obj
@staticmethod
def _response() -> ModelResponse:
from litellm.types.utils import Usage
return ModelResponse(
id="chatcmpl-spill-1",
created=1234567890,
model="spill-served-model-ptu",
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
)
def test_spillover_via_response_additional_headers_bills_per_token(self, monkeypatch) -> None:
self._register_models()
try:
obj = self._logging_obj(dict(self.PTU_MODEL_INFO), flag="True", litellm_rate=0.0, monkeypatch=monkeypatch)
response = self._response()
response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"}
assert obj._response_cost_calculator(result=response) == pytest.approx(self.EXPECTED_SPILL_COST)
finally:
self._unregister_models()
def test_spillover_via_streaming_response_headers_bills_per_token(self, monkeypatch) -> None:
self._register_models()
try:
obj = self._logging_obj(dict(self.PTU_MODEL_INFO), flag="True", litellm_rate=0.0, monkeypatch=monkeypatch)
obj.model_call_details["response_headers"] = {
"x-ms-is-spilled-over": "true",
"x-ms-spillover-from-deployment": "ptu-dep",
}
assert obj._response_cost_calculator(result=self._response()) == pytest.approx(self.EXPECTED_SPILL_COST)
finally:
self._unregister_models()
def test_non_spilled_ptu_request_stays_zero_priced(self, monkeypatch) -> None:
self._register_models()
try:
obj = self._logging_obj(dict(self.PTU_MODEL_INFO), flag="True", litellm_rate=0.0, monkeypatch=monkeypatch)
assert obj._response_cost_calculator(result=self._response()) == 0.0
finally:
self._unregister_models()
def test_spillover_header_without_the_flag_stays_zero_priced(self, monkeypatch) -> None:
self._register_models()
try:
obj = self._logging_obj(dict(self.PTU_MODEL_INFO), flag="", litellm_rate=0.0, monkeypatch=monkeypatch)
response = self._response()
response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"}
assert obj._response_cost_calculator(result=response) == 0.0
finally:
self._unregister_models()
def test_spillover_header_does_not_touch_non_ptu_custom_pricing(self, monkeypatch) -> None:
self._register_models()
custom_model_id: Final = "non-ptu-custom-router-model-id"
litellm.model_cost[custom_model_id] = {
"input_cost_per_token": 1e-6,
"output_cost_per_token": 1e-6,
"litellm_provider": "azure",
"mode": "chat",
}
try:
model_info: Final = {"id": custom_model_id, "input_cost_per_token": 1e-6}
obj = self._logging_obj(model_info, flag="True", litellm_rate=1e-6, monkeypatch=monkeypatch)
response = self._response()
response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"}
assert obj._response_cost_calculator(result=response) == pytest.approx(150 * 1e-6)
finally:
litellm.model_cost.pop(custom_model_id, None)
self._unregister_models()

66
tests/test_litellm/litellm_core_utils/test_ptu_pricing.py
View file

@ -7,13 +7,15 @@ from unittest.mock import patch
import pytest
from litellm.litellm_core_utils.ptu_pricing import (
ptu_config_error,
ptu_identity_error,
CUSTOM_PRICING_FIELDS,
PTU_EMPTIED_PRICING_FIELDS,
PTU_ZEROED_PRICING_FIELDS,
PTU_ZEROED_TABLE_FIELDS,
SEARCH_CONTEXT_SIZES,
azure_spillover,
is_spilled_over_ptu_request,
ptu_config_error,
ptu_identity_error,
ptu_terms,
zeroed_ptu_pricing,
)
@ -294,3 +296,63 @@ def test_an_empty_id_is_no_id():
assert error is not None
assert error.startswith("model_info.id is required")
def test_the_spillover_header_marks_the_request_as_pay_as_you_go():
with patch.dict(os.environ, {"LITELLM_ENABLE_PTU_COST_ATTRIBUTION": "True"}, clear=False):
assert (
is_spilled_over_ptu_request(
model_info=_VALID,
response_headers={"x-ms-is-spilled-over": "True"},
additional_headers=None,
)
is True
)
def test_no_spillover_marker_keeps_the_zeroed_ptu_rates():
with patch.dict(os.environ, {"LITELLM_ENABLE_PTU_COST_ATTRIBUTION": "True"}, clear=False):
assert (
is_spilled_over_ptu_request(
model_info=_VALID,
response_headers={"x-ms-is-spilled-over": "false"},
additional_headers=None,
)
is False
)
assert (
is_spilled_over_ptu_request(
model_info=_VALID,
response_headers=None,
additional_headers={"llm_provider-x-ms-is-spilled-over": "absent"},
)
is False
)
def test_azure_spillover_carries_the_source_deployment_from_raw_headers():
assert azure_spillover(
response_headers={
"x-ms-is-spilled-over": "true",
"x-ms-spillover-from-deployment": "my-ptu",
},
additional_headers=None,
) == {"from_deployment": "my-ptu"}
def test_azure_spillover_from_processed_headers_has_no_source_when_absent():
assert azure_spillover(
response_headers=None,
additional_headers={"llm_provider-x-ms-is-spilled-over": "true"},
) == {"from_deployment": None}
def test_no_spillover_marker_returns_none():
assert (
azure_spillover(
response_headers={"x-ms-is-spilled-over": "false"},
additional_headers=None,
)
is None
)
assert azure_spillover(response_headers=None, additional_headers=None) is None

54
tests/test_litellm/llms/azure/test_azure.py Normal file
View file

@ -0,0 +1,54 @@
"""Tests for litellm/llms/azure/azure.py AzureChatCompletion handler behaviour."""
import time
from typing import Final
from openai import AzureOpenAI
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.azure.azure import AzureChatCompletion
class _FakeRawResponse:
headers: Final = {"x-ms-is-spilled-over": "true"}
def parse(self):
return iter(())
class _FakeRawCompletions:
def create(self, **kwargs):
return _FakeRawResponse()
def test_sync_streaming_stamps_response_headers_on_the_logging_obj() -> None:
"""Sync streaming must mirror async_streaming and record the provider response
headers on model_call_details, or downstream consumers (spillover-aware cost
calculation) cannot see them."""
client = AzureOpenAI(api_key="fake", api_version="2024-02-01", azure_endpoint="https://fake.openai.azure.com")
client.chat.completions.with_raw_response = _FakeRawCompletions()
logging_obj = LiteLLMLoggingObj(
model="azure/gpt-4o-spill-test",
messages=[{"role": "user", "content": "Hi"}],
stream=True,
call_type="completion",
start_time=time.time(),
litellm_call_id="spill-sync-1",
function_id="f",
)
AzureChatCompletion().streaming(
logging_obj=logging_obj,
api_base="https://fake.openai.azure.com",
api_key="fake",
api_version="2024-02-01",
dynamic_params=False,
data={"messages": [{"role": "user", "content": "Hi"}], "stream": True},
model="gpt-4o-spill-test",
timeout=30.0,
max_retries=0,
client=client,
)
assert logging_obj.model_call_details["response_headers"] == {"x-ms-is-spilled-over": "true"}

2
tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py
View file

@ -978,6 +978,8 @@ def test_llama_vision_supports_vision_from_model_map():
for model in [
"fireworks_ai/accounts/fireworks/models/llama-v3p2-11b-vision-instruct",
"fireworks_ai/accounts/fireworks/models/minimax-m3",
"fireworks_ai/minimax-m3",
]:
assert supports_vision(model=model, custom_llm_provider="fireworks_ai") is True
assert config.get_provider_info(model)["supports_vision"] is True

35
tests/test_litellm/proxy/auth/test_litellm_license.py
View file

@ -35,8 +35,8 @@ def test_is_over_limit():
def test_auto_router_capability_limit() -> None:
"""Only the signed license's auto_router feature lifts the one-router limit; an API-verified
license (no airgapped data) and an airgapped license without the feature keep it."""
"""The signed license's auto_router feature or its "*" wildcard lifts the one-router limit; an
API-verified license (no airgapped data) and an airgapped license without either keep it."""
license_check = LicenseCheck()
license_check.airgapped_license_data = {"expiration_date": "2999-01-01", "allowed_features": ["auto_router"]}
assert license_check.auto_router_capability_limit() is None
@ -47,9 +47,18 @@ def test_auto_router_capability_limit() -> None:
}
assert license_check.auto_router_capability_limit() is None
license_check.airgapped_license_data = {"expiration_date": "2999-01-01", "allowed_features": ["*"]}
assert license_check.auto_router_capability_limit() is None
license_check.airgapped_license_data = {"expiration_date": "2999-01-01", "allowed_features": ["sso", "*"]}
assert license_check.auto_router_capability_limit() is None
license_check.airgapped_license_data = {"expiration_date": "2999-01-01", "allowed_features": ["sso"]}
assert license_check.auto_router_capability_limit() == 1
license_check.airgapped_license_data = {"expiration_date": "2999-01-01", "allowed_features": "*"}
assert license_check.auto_router_capability_limit() is None
license_check.airgapped_license_data = {"expiration_date": "2999-01-01"}
assert license_check.auto_router_capability_limit() == 1
@ -57,7 +66,9 @@ def test_auto_router_capability_limit() -> None:
assert license_check.auto_router_capability_limit() == 1
def _signed_license(expiration_date: str) -> tuple[RSAPublicKey, str]:
def _signed_license(
expiration_date: str, allowed_features: tuple[str, ...] = ("auto_router",)
) -> tuple[RSAPublicKey, str]:
import base64
from cryptography.hazmat.primitives import hashes
@ -65,7 +76,7 @@ def _signed_license(expiration_date: str) -> tuple[RSAPublicKey, str]:
private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
message = json.dumps(
{"expiration_date": expiration_date, "user_id": "u", "allowed_features": ["auto_router"]}
{"expiration_date": expiration_date, "user_id": "u", "allowed_features": list(allowed_features)}
).encode()
signature = private_key.sign(
message,
@ -99,3 +110,19 @@ def test_valid_signed_license_with_auto_router_lifts_the_limit() -> None:
assert license_check.verify_license_without_api_request(public_key=public_key, license_key=license_key) is True
assert license_check.auto_router_capability_limit() is None
def test_valid_signed_wildcard_license_lifts_the_limit() -> None:
"""The license generator defaults allowed_features to ["*"], meaning every feature, so a wildcard
license grants auto_router the same way a license that names it does."""
license_check = LicenseCheck()
public_key, license_key = _signed_license("2999-01-01", allowed_features=("*",))
assert license_check.verify_license_without_api_request(public_key=public_key, license_key=license_key) is True
assert license_check.grants_feature("auto_router") is True
assert license_check.auto_router_capability_limit() is None
named_public_key, named_key = _signed_license("2999-01-01", allowed_features=("sso", "audit_logs"))
assert license_check.verify_license_without_api_request(public_key=named_public_key, license_key=named_key) is True
assert license_check.grants_feature("auto_router") is False
assert license_check.auto_router_capability_limit() == 1

26
tests/test_litellm/proxy/common_utils/test_upsert_budget_membership.py
View file

@ -1,6 +1,6 @@
# tests/litellm/proxy/common_utils/test_upsert_budget_membership.py
import types
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -27,9 +27,7 @@ def mock_tx():
budget = MagicMock()
budget.update = AsyncMock()
budget.find_unique = AsyncMock(return_value=None)
budget.create = AsyncMock(
return_value=types.SimpleNamespace(budget_id="new-budget-123")
)
budget.create = AsyncMock(return_value=types.SimpleNamespace(budget_id="new-budget-123"))
tx = MagicMock()
tx.litellm_teammembership = membership
@ -83,9 +81,7 @@ async def test_empty_patch_is_noop(mock_tx, fake_user):
# member falls back to the team default instead of keeping an empty private row.
@pytest.mark.asyncio
async def test_clearing_all_limits_disconnects(mock_tx, fake_user):
mock_tx.litellm_budgettable.find_unique = AsyncMock(
return_value=budget_row(max_budget=100.0)
)
mock_tx.litellm_budgettable.find_unique = AsyncMock(return_value=budget_row(max_budget=100.0))
await _upsert_budget_and_membership(
mock_tx,
@ -136,9 +132,7 @@ async def test_clear_one_field_keeps_others(mock_tx, fake_user):
# budget_reset_at, so the budget rolls over without waiting for the reset cron.
@pytest.mark.asyncio
async def test_update_in_place_seeds_reset_at(mock_tx, fake_user):
mock_tx.litellm_budgettable.find_unique = AsyncMock(
return_value=budget_row(max_budget=20.0)
)
mock_tx.litellm_budgettable.find_unique = AsyncMock(return_value=budget_row(max_budget=20.0))
await _upsert_budget_and_membership(
mock_tx,
@ -163,9 +157,7 @@ async def test_update_in_place_seeds_reset_at(mock_tx, fake_user):
# budget_duration must not get a (re)computed reset time.
@pytest.mark.asyncio
async def test_update_in_place_single_field_leaves_reset_at_alone(mock_tx, fake_user):
mock_tx.litellm_budgettable.find_unique = AsyncMock(
return_value=budget_row(max_budget=50.0)
)
mock_tx.litellm_budgettable.find_unique = AsyncMock(return_value=budget_row(max_budget=50.0))
await _upsert_budget_and_membership(
mock_tx,
@ -225,6 +217,7 @@ async def test_create_seeds_reset_at_and_links(mock_tx, fake_user):
@pytest.mark.asyncio
async def test_clone_on_write_from_shared_default(mock_tx, fake_user):
shared_default_id = "team-default-budget-1"
shared_reset_at = datetime.now(timezone.utc) + timedelta(hours=3)
mock_tx.litellm_budgettable.find_unique = AsyncMock(
return_value=budget_row(
budget_id=shared_default_id,
@ -235,6 +228,7 @@ async def test_clone_on_write_from_shared_default(mock_tx, fake_user):
rpm_limit=None,
model_max_budget=None,
budget_duration="1d",
budget_reset_at=shared_reset_at,
allowed_models=[],
)
)
@ -252,7 +246,7 @@ async def test_clone_on_write_from_shared_default(mock_tx, fake_user):
mock_tx.litellm_budgettable.update.assert_not_called()
mock_tx.litellm_budgettable.create.assert_awaited_once()
create_data = mock_tx.litellm_budgettable.create.await_args.kwargs["data"]
assert_future_reset_time(create_data.pop("budget_reset_at"))
assert create_data.pop("budget_reset_at") == shared_reset_at
assert create_data == {
"created_by": fake_user.user_id,
"updated_by": fake_user.user_id,
@ -318,9 +312,7 @@ async def test_clone_on_write_clears_duration(mock_tx, fake_user):
# team default), we update it in place rather than forking another row.
@pytest.mark.asyncio
async def test_private_budget_updates_in_place(mock_tx, fake_user):
mock_tx.litellm_budgettable.find_unique = AsyncMock(
return_value=budget_row(max_budget=10.0)
)
mock_tx.litellm_budgettable.find_unique = AsyncMock(return_value=budget_row(max_budget=10.0))
await _upsert_budget_and_membership(
mock_tx,

93
tests/test_litellm/proxy/management_endpoints/management_v1/test_teams.py
View file

@ -7,6 +7,7 @@ table and the membership/budget relation the bulk budget writer needs.
"""
import copy
import json
from collections.abc import Mapping, Sequence
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
@ -244,6 +245,7 @@ def _budget(
tpm_limit: int | None = None,
rpm_limit: int | None = None,
budget_duration: str | None = None,
budget_reset_at: datetime | None = None,
) -> _BudgetRow:
return _BudgetRow(
budget_id=budget_id,
@ -251,6 +253,7 @@ def _budget(
tpm_limit=tpm_limit,
rpm_limit=rpm_limit,
budget_duration=budget_duration,
budget_reset_at=budget_reset_at,
)
@ -267,6 +270,7 @@ async def _bulk_update(
user_api_key_dict=caller,
prisma_client=prisma, # pyright: ignore[reportArgumentType] # fake stands in for PrismaClient
user_api_key_cache=cache or UserApiKeyCache(),
litellm_proxy_admin_name="default_user_id",
)
@ -631,6 +635,95 @@ async def test_the_roster_authz_read_runs_on_the_writer_so_a_lagging_replica_can
assert writer.db.litellm_budgettable.rows["priv-m1"].max_budget == 1.0
@pytest.mark.asyncio
async def test_the_batch_writes_one_audit_entry_carrying_every_written_members_limits_before_and_after(monkeypatch):
import litellm
from litellm.proxy._types import LitellmTableNames
monkeypatch.setattr(litellm, "store_audit_logs", True)
captured: list[object] = []
async def capture(request_data):
captured.append(request_data)
monkeypatch.setattr("litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", capture)
prisma = _FakePrisma(
teams=[_team("m1", "m2")],
memberships=[_membership("m1", "priv-m1"), _membership("m2", "priv-m2")],
budgets=[_budget("priv-m1", max_budget=1.0), _budget("priv-m2", max_budget=2.0)],
)
await _bulk_update(prisma, [{"user_id": "m1", "max_budget_in_team": 10}])
assert len(captured) == 1
entry = captured[0]
assert (entry.object_id, entry.action, entry.table_name) == (
TEAM_ID,
"updated",
LitellmTableNames.TEAM_TABLE_NAME,
)
before = {row["user_id"]: row for row in json.loads(entry.before_value)["team_member_budgets"]}
after = {row["user_id"]: row for row in json.loads(entry.updated_values)["team_member_budgets"]}
assert (before["m1"]["max_budget"], after["m1"]["max_budget"]) == (1.0, 10.0)
assert "m2" not in before and "m2" not in after
@pytest.mark.asyncio
async def test_no_audit_entry_is_written_when_audit_logging_is_off(monkeypatch):
import litellm
monkeypatch.setattr(litellm, "store_audit_logs", False)
captured: list[object] = []
async def capture(request_data):
captured.append(request_data)
monkeypatch.setattr("litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", capture)
prisma = _FakePrisma(
teams=[_team("m1")],
memberships=[_membership("m1", "priv-m1")],
budgets=[_budget("priv-m1", max_budget=1.0)],
)
await _bulk_update(prisma, [{"user_id": "m1", "max_budget_in_team": 10}])
assert captured == []
assert _budget_of(prisma, "m1").max_budget == 10.0
@pytest.mark.asyncio
async def test_forking_a_shared_row_keeps_its_reset_window_so_an_unrelated_limit_edit_grants_no_free_period():
shared_reset_at = datetime.now(timezone.utc) + timedelta(days=3)
prisma = _FakePrisma(
teams=[_team("m1", "m2")],
memberships=[_membership("m1", "shared-b"), _membership("m2", "shared-b")],
budgets=[_budget("shared-b", max_budget=100.0, budget_duration="30d", budget_reset_at=shared_reset_at)],
)
results = await _bulk_update(prisma, [{"user_id": "m1", "tpm_limit": 9}])
assert [(r.success, r.budget_duration) for r in results] == [(True, "30d")]
assert _budget_id_of(prisma, "m1") not in (None, "shared-b")
assert _budget_of(prisma, "m1").budget_reset_at == shared_reset_at
assert prisma.db.litellm_budgettable.rows["shared-b"].budget_reset_at == shared_reset_at
@pytest.mark.asyncio
async def test_forking_a_shared_row_does_restart_the_window_when_the_patch_sets_a_new_duration():
shared_reset_at = datetime.now(timezone.utc) + timedelta(days=3)
prisma = _FakePrisma(
teams=[_team("m1", "m2")],
memberships=[_membership("m1", "shared-b"), _membership("m2", "shared-b")],
budgets=[_budget("shared-b", max_budget=100.0, budget_duration="30d", budget_reset_at=shared_reset_at)],
)
await _bulk_update(prisma, [{"user_id": "m1", "budget_duration": "1d"}])
forked = _budget_of(prisma, "m1").budget_reset_at
assert forked is not None and forked != shared_reset_at
assert forked <= datetime.now(timezone.utc) + timedelta(days=1)
app = FastAPI()

134
tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py Normal file
View file

@ -0,0 +1,134 @@
from datetime import datetime
from unittest.mock import MagicMock
import httpx
import pytest
import litellm
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.typesafe_passthrough_logging_handler import (
TypeSafePassthroughLoggingHandler,
)
from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndpointLogging
@pytest.fixture(autouse=True)
def local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
def _response() -> httpx.Response:
return httpx.Response(
200,
request=httpx.Request("POST", "https://api.typesafe.ai/v1/systemone"),
json={"model": "jev-1.13.0"},
)
def _logging_obj() -> MagicMock:
logging_obj = MagicMock()
logging_obj.model_call_details = {}
return logging_obj
def _handler_result(response_body: dict, request_body: dict) -> dict:
return TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
httpx_response=_response(),
response_body=response_body,
logging_obj=_logging_obj(),
url_route="https://api.typesafe.ai/v1/systemone",
result='{"answers": {}}',
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body=request_body,
)
def test_uses_registry_pricing_and_standard_usage():
logging_obj = _logging_obj()
model_key = "typesafe/jev-1.13.0"
model_cost = litellm.model_cost[model_key]
response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
httpx_response=_response(),
response_body={"model": "jev-1.13.0", "usage": {"input_tokens": 312, "output_tokens": 48}},
logging_obj=logging_obj,
url_route="https://api.typesafe.ai/v1/systemone",
result='{"answers": {}}',
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={"model": "jev-latest"},
)
expected_cost = 312 * model_cost["input_cost_per_token"] + 48 * model_cost["output_cost_per_token"]
assert response["kwargs"]["response_cost"] == pytest.approx(expected_cost)
assert response["kwargs"]["combined_usage_object"].prompt_tokens == 312
assert response["kwargs"]["combined_usage_object"].completion_tokens == 48
assert response["kwargs"]["combined_usage_object"].total_tokens == 360
def test_falls_back_to_request_model_when_response_model_is_missing():
result = _handler_result(
{"usage": {"input_tokens": 10, "output_tokens": 2}},
{"model": "jev-latest"},
)
model_cost = litellm.model_cost["typesafe/jev-latest"]
expected_cost = 10 * model_cost["input_cost_per_token"] + 2 * model_cost["output_cost_per_token"]
assert result["kwargs"]["model"] == "typesafe/jev-latest"
assert result["kwargs"]["response_cost"] == pytest.approx(expected_cost)
def test_call_naming_no_model_is_logged_as_unknown_and_never_priced_as_a_registry_model():
result = _handler_result({"usage": {"input_tokens": 10, "output_tokens": 2}}, {})
assert result["kwargs"]["model"] == "typesafe/unknown"
assert result["kwargs"]["response_cost"] == 0.0
def test_missing_usage_is_zero_cost():
result = _handler_result({"model": "jev-1.13.0"}, {"model": "jev-latest"})
assert result["kwargs"]["response_cost"] == 0.0
def test_records_model_provider_and_cost_on_logging_details():
logging_obj = _logging_obj()
result = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
httpx_response=_response(),
response_body={"model": "jev-1.13.0", "usage": {"input_tokens": 1, "output_tokens": 0}},
logging_obj=logging_obj,
url_route="https://api.typesafe.ai/v1/systemone",
result="{}",
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={"model": "jev-latest"},
)
assert result["kwargs"]["model"] == "typesafe/jev-1.13.0"
assert result["kwargs"]["custom_llm_provider"] == "typesafe"
assert result["kwargs"]["response_cost"] > 0
assert logging_obj.model_call_details["model"] == "typesafe/jev-1.13.0"
assert logging_obj.model_call_details["custom_llm_provider"] == "typesafe"
assert logging_obj.model_call_details["response_cost"] == result["kwargs"]["response_cost"]
def test_success_handler_dispatches_to_typesafe_handler():
logging_obj = _logging_obj()
normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
httpx_response=_response(),
response_body={"model": "jev-1.13.0", "usage": {"input_tokens": 1, "output_tokens": 0}},
request_body={"model": "jev-latest"},
logging_obj=logging_obj,
url_route="https://api.typesafe.ai/v1/systemone",
result="{}",
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
custom_llm_provider="typesafe",
)
assert normalized["kwargs"]["custom_llm_provider"] == "typesafe"
assert normalized["kwargs"]["model"] == "typesafe/jev-1.13.0"

50
tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
View file

@ -9,6 +9,7 @@ from types import MappingProxyType, SimpleNamespace
from typing import Final
from unittest import mock
from unittest.mock import AsyncMock, MagicMock, Mock, patch
from urllib.parse import parse_qs
import httpx
import pytest
@ -43,6 +44,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
mistral_proxy_route,
relay_nvidia_nim_request,
openai_proxy_route,
typesafe_proxy_route,
vertex_discovery_proxy_route,
vertex_proxy_route,
vllm_proxy_route,
@ -6387,3 +6389,51 @@ class TestAzureRelayDeploymentSegment:
)
assert [call["model"] for call in captured] == ["gpt", "gpt"]
class TestTypeSafePassthroughRoute:
@staticmethod
def _request(body: object, query_params: Mapping[str, str] | None = None) -> MagicMock:
request = MagicMock(spec=Request)
request.method = "POST"
request.query_params = query_params or {}
request.json = AsyncMock(return_value=body)
return request
@pytest.mark.asyncio
async def test_forwards_target_auth_headers_provider_and_query(self, monkeypatch):
monkeypatch.setenv("TYPESAFE_API_KEY", "typesafe-test-key")
monkeypatch.setenv("TYPESAFE_API_BASE", "https://typesafe.example/base")
async def fake_upstream(request, *_args):
target: Final = create_route.call_args.kwargs["target"]
upstream_url: Final = httpx.URL(target).copy_merge_params(request.query_params)
return {"upstream_query": parse_qs(upstream_url.query.decode())}
endpoint_func = AsyncMock(side_effect=fake_upstream)
create_route = Mock(return_value=endpoint_func)
monkeypatch.setattr(
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route",
create_route,
)
request = self._request({"state": "x"}, {"trace": "yes"})
result = await typesafe_proxy_route(
endpoint="v1/systemone",
request=request,
fastapi_response=MagicMock(spec=Response),
user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"),
)
assert result == {"upstream_query": {"trace": ["yes"]}}
endpoint_func.assert_awaited_once()
create_route.assert_called_once_with(
endpoint="v1/systemone",
target="https://typesafe.example/base/v1/systemone",
custom_headers={
"Authorization": "Bearer typesafe-test-key",
"Content-Type": "application/json",
},
custom_llm_provider="typesafe",
is_streaming_request=False,
)

6
tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -3745,7 +3745,7 @@ class TestSpendLogsPayload:
"model": "gpt-4o",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "azure_spillover": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,
@ -3841,7 +3841,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "azure_spillover": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,
@ -3935,7 +3935,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "azure_spillover": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,

55
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -4850,3 +4850,58 @@ def test_spend_log_request_id_is_the_response_id_a_bridged_messages_caller_recei
)
== "resp_01Lit6806Bridged"
)
def test_azure_spillover_stamped_from_response_headers():
"""Raw provider response headers on the logging kwargs mark the request as spilled."""
kwargs: Final = {
**_routed_call_kwargs({"id": "mi-1"}),
"response_headers": {
"x-ms-is-spilled-over": "true",
"x-ms-spillover-from-deployment": "my-ptu",
},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj=litellm.ModelResponse(id="chatcmpl-spill-raw", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["azure_spillover"] == {"from_deployment": "my-ptu"}
def test_azure_spillover_stamped_from_standard_logging_additional_headers():
"""Streaming requests carry the processed llm_provider- headers on the standard payload."""
kwargs: Final = {
**_routed_call_kwargs({"id": "mi-1"}),
"standard_logging_object": {
"hidden_params": {
"additional_headers": {
"llm_provider-x-ms-is-spilled-over": "true",
"llm_provider-x-ms-spillover-from-deployment": "my-ptu",
}
},
"metadata": {},
"model_map_information": None,
},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj=litellm.ModelResponse(id="chatcmpl-spill-sl", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["azure_spillover"] == {"from_deployment": "my-ptu"}
def test_azure_spillover_absent_without_spillover_headers():
payload = get_logging_payload(
kwargs=_routed_call_kwargs({"id": "mi-1"}),
response_obj=litellm.ModelResponse(id="chatcmpl-no-spill", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["azure_spillover"] is None

55
tests/test_litellm/responses/litellm_completion_transformation/test_handler.py
View file

@ -12,14 +12,21 @@ capture the forwarded kwargs; if the flag-setting line is removed the captured
kwargs lack the flag and these tests fail.
"""
import json
from collections.abc import Mapping
from typing import Final
from unittest.mock import patch
import httpx
import pytest
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.responses.litellm_completion_transformation.handler import (
LiteLLMCompletionTransformationHandler,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD
class _StopForwarding(Exception):
@ -170,3 +177,49 @@ async def test_async_fallback_returns_hoisted_nested_custom_tool_call_as_custom_
tool_calls = [(item.type, item.name, item.input) for item in response.output if item.type == "custom_tool_call"]
assert tool_calls == [("custom_tool_call", "exec", "ls")]
class _RecordingAnthropicHandler:
def __init__(self, reply: Mapping[str, object]) -> None:
self.reply: Final = reply
self.request_body: Mapping[str, object] | None = None
def __call__(self, request: httpx.Request) -> httpx.Response:
self.request_body = json.loads(request.content)
return httpx.Response(200, json=dict(self.reply), request=request)
_ANTHROPIC_MESSAGE_PAYLOAD: Final = {
"id": "msg_turn_two",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-6",
"content": [{"type": "text", "text": "14"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 1},
}
@pytest.mark.asyncio
async def test_bridged_follow_up_turn_keeps_the_addressed_response_id_off_the_provider_body():
provider: Final = _RecordingAnthropicHandler(_ANTHROPIC_MESSAGE_PAYLOAD)
client: Final = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider))
response = await litellm.aresponses(
model="azure_ai/claude-sonnet-4-6",
api_base="https://fake-foundry-resource.services.ai.azure.com",
api_key="fake-api-key",
input="Double it",
previous_response_id="resp_turn_one",
client=client,
**{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"},
)
assert provider.request_body is not None, "the bridged turn never reached the provider"
assert ADDRESSED_RESPONSE_ID_FIELD not in provider.request_body, (
f"the addressed response id reached the provider body: {sorted(provider.request_body)}"
)
assert isinstance(response, ResponsesAPIResponse)
assert [item.type for item in response.output] == ["message"]

17
tests/test_litellm/test_typesafe_model_metadata.py Normal file
View file

@ -0,0 +1,17 @@
import pytest
import litellm
@pytest.fixture(autouse=True)
def local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
def test_typesafe_models_share_pricing_and_provider_metadata():
entries = [litellm.model_cost[f"typesafe/{model}"] for model in ("jev-1.13.0", "jev-latest", "jev-preview")]
assert {entry["input_cost_per_token"] for entry in entries} == {entries[0]["input_cost_per_token"]}
assert {entry["output_cost_per_token"] for entry in entries} == {entries[0]["output_cost_per_token"]}
assert {entry["litellm_provider"] for entry in entries} == {"typesafe"}

21
tests/test_litellm/test_utils.py
View file

@ -46,6 +46,7 @@ from litellm.types.utils import (
PromptTokensDetailsWrapper,
StreamingChoices,
Usage,
ADDRESSED_RESPONSE_ID_FIELD,
all_litellm_params,
bedrock_batch_litellm_params,
)
@ -818,6 +819,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"container",
"image_edit",
"embedding",
"evaluation",
"guardrail",
"image_generation",
"video_generation",
@ -3546,7 +3548,7 @@ _FIREWORKS_MODELS = [
"accounts/fireworks/models/minimax-m3",
512000,
512000,
None,
True,
True,
),
(
@ -3654,8 +3656,7 @@ def _assert_fireworks_entry(
assert info["supports_tool_choice"] is True
assert info["supports_reasoning"] is expected_reasoning
assert info["supports_response_schema"] is True
if expected_vision is not None:
assert info["supports_vision"] is expected_vision
assert info["supports_vision"] is expected_vision
@pytest.fixture
@ -4787,6 +4788,20 @@ def test_get_litellm_params_keys_never_reach_the_provider():
)
def test_addressed_response_id_never_reaches_the_provider():
kwargs = {
"a_real_provider_specific_param": 1,
ADDRESSED_RESPONSE_ID_FIELD: "resp_addressed-by-the-client",
}
non_default = get_non_default_completion_params(kwargs)
assert non_default == {"a_real_provider_specific_param": 1}, (
"the addressed response id leaked into the provider params: "
f"{sorted(set(non_default) - {'a_real_provider_specific_param'})}"
)
def test_bedrock_batch_params_never_reach_the_provider():
"""A Bedrock managed-batch deployment carries aws_batch_role_arn / s3_* /
bedrock_tags in its litellm_params, and the same deployment also serves chat.

6
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx
View file

@ -123,11 +123,11 @@ describe("CacheLeakageCard", () => {
expect(firstDataRow()).toHaveTextContent("alpha");
});
it("switches to the model view and lists only Anthropic models", () => {
it("switches to the model view and lists models from every provider", () => {
renderWith([
dayWithModels("2026-07-12", {
"claude-sonnet-5": { prompt_tokens: 5000, cache_read_input_tokens: 0 },
"gpt-4o": { prompt_tokens: 8000, cache_read_input_tokens: 0 },
"vertex_ai/gemini-2.5-pro": { prompt_tokens: 8000, cache_read_input_tokens: 2000 },
}),
]);
@ -135,7 +135,7 @@ describe("CacheLeakageCard", () => {
expect(screen.getByText("Cache leakage by model")).toBeInTheDocument();
expect(screen.getByText("claude-sonnet-5")).toBeInTheDocument();
expect(screen.queryByText("gpt-4o")).not.toBeInTheDocument();
expect(screen.getByText("vertex_ai/gemini-2.5-pro")).toBeInTheDocument();
});
it("shows an empty state when no key used tokens in the range", () => {

36
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts
View file

@ -10,7 +10,6 @@ import {
classificationRatePer1kTurns,
computeCacheLeakage,
formatRangeLabel,
isAnthropicModel,
localIsoDay,
savingsSeriesOf,
toCumulative,
@ -209,20 +208,21 @@ describe("computeCacheLeakage", () => {
});
describe("computeCacheLeakage by model", () => {
it("aggregates only Anthropic models and ignores other providers", () => {
it("lists every provider's models, not only Anthropic", () => {
const models: Record<string, Partial<SpendMetrics>> = {
"claude-sonnet-5": { prompt_tokens: 10000, cache_read_input_tokens: 0 },
"anthropic/claude-haiku-4-5": { prompt_tokens: 4000, cache_read_input_tokens: 0 },
"bedrock/anthropic.claude-3-5-sonnet": { prompt_tokens: 2000, cache_read_input_tokens: 0 },
"gpt-4o": { prompt_tokens: 9000, cache_read_input_tokens: 0 },
"deepseek-chat": { prompt_tokens: 8000, cache_read_input_tokens: 0 },
"vertex_ai/gemini-2.5-pro": { prompt_tokens: 9000, cache_read_input_tokens: 3000 },
"bedrock/openai.gpt-5.6-luna": { prompt_tokens: 8000, cache_read_input_tokens: 0 },
"deepseek-chat": { prompt_tokens: 4000, cache_read_input_tokens: 0 },
};
const { rows } = computeCacheLeakage([modelDay("2026-07-01", models)], "model");
expect(rows.map((r) => r.id)).toEqual([
"claude-sonnet-5",
"anthropic/claude-haiku-4-5",
"bedrock/anthropic.claude-3-5-sonnet",
"bedrock/openai.gpt-5.6-luna",
"vertex_ai/gemini-2.5-pro",
"deepseek-chat",
]);
expect(rows.find((r) => r.id === "vertex_ai/gemini-2.5-pro")?.cacheHitRatio).toBeCloseTo(1 / 3, 6);
});
it("labels model rows by model name with no sublabel", () => {
@ -232,34 +232,20 @@ describe("computeCacheLeakage by model", () => {
expect(rows[0].sublabel).toBeNull();
});
it("prices model leakage at the Anthropic realized cache-read discount", () => {
it("prices model leakage at the realized cache-read discount across providers", () => {
const results = [
modelDay("2026-07-01", {
"claude-sonnet-5": { prompt_tokens: 1000, cache_read_input_tokens: 1000, prompt_caching_savings_spend: 2.0 },
"claude-haiku-4-5": { prompt_tokens: 500 },
"gemini-2.5-flash": { prompt_tokens: 500 },
}),
];
const { rows, netSavingsPerCachedToken } = computeCacheLeakage(results, "model");
expect(netSavingsPerCachedToken).toBeCloseTo(0.002, 6);
expect(rows.map((r) => r.id)).toEqual(["claude-haiku-4-5"]);
expect(rows.map((r) => r.id)).toEqual(["gemini-2.5-flash"]);
expect(rows[0].potentialSavings).toBeCloseTo(1.0, 6);
});
});
describe("isAnthropicModel", () => {
it("matches Claude-family models across providers and rejects others", () => {
const anthropic = [
"claude-sonnet-5",
"anthropic/claude-haiku-4-5",
"bedrock/anthropic.claude-3-5-sonnet",
"vertex_ai/claude-opus-4-8",
];
const others = ["gpt-4o", "deepseek-chat", "gemini-2.5-pro", "mistral-large"];
expect(anthropic.every(isAnthropicModel)).toBe(true);
expect(others.some(isAnthropicModel)).toBe(false);
});
});
describe("buildDailyToolSeries", () => {
const daily: ToolSpendDailyEntry[] = [
{ date: "2026-07-01", tool_name: "search", spend: 1.0, call_count: 1 },

3
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts
View file

@ -44,8 +44,6 @@ export interface CacheLeakageResult {
netSavingsPerCachedToken: number | null;
}
export const isAnthropicModel = (model: string): boolean => /claude|anthropic/i.test(model);
interface LeakageAccumulator {
alias: string | null;
teamId: string | null;
@ -96,7 +94,6 @@ const aggregateByModel = (results: readonly DailyData[]): Map<string, LeakageAcc
const byModel = new Map<string, LeakageAccumulator>();
for (const day of results) {
for (const [model, entry] of Object.entries(day.breakdown?.models ?? {})) {
if (!isAnthropicModel(model)) continue;
const acc = byModel.get(model) ?? emptyAccumulator();
byModel.set(model, addMetrics(acc, entry.metrics, null, null));
}

91
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -16529,6 +16529,30 @@ export interface paths {
patch?: never;
trace?: never;
};
"/typesafe/{endpoint}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Typesafe Proxy Route
* @description [Docs](https://docs.litellm.ai/docs/pass_through/typesafe)
*/
get: operations["typesafe_proxy_route_typesafe__endpoint__get"];
put?: never;
/**
* Typesafe Proxy Route
* @description [Docs](https://docs.litellm.ai/docs/pass_through/typesafe)
*/
post: operations["typesafe_proxy_route_typesafe__endpoint__post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/update/default_team_settings": {
parameters: {
query?: never;
@ -52239,7 +52263,10 @@ export interface operations {
bulk_update_team_member_budgets_action_management_v1_teams__team_id__members_bulk_update_post: {
parameters: {
query?: never;
header?: never;
header?: {
/** @description The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability */
"litellm-changed-by"?: string | null;
};
path: {
team_id: string;
};
@ -61686,6 +61713,68 @@ export interface operations {
};
};
};
typesafe_proxy_route_typesafe__endpoint__get: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
typesafe_proxy_route_typesafe__endpoint__post: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
update_default_team_settings_update_default_team_settings_patch: {
parameters: {
query?: never;

8
uv.lock generated
View file

@ -10,7 +10,7 @@ resolution-markers = [
]
[options]
exclude-newer = "2026-09-14T20:32:38.482736111Z"
exclude-newer = "2026-09-14T23:55:55.024292355Z"
exclude-newer-span = "P3D"
[manifest]
@ -9262,11 +9262,11 @@ wheels = [
[[package]]
name = "soupsieve"
version = "2.8.4"
version = "2.9.2"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/47/2c/0a5f6f8ee0d5589e48c7640213ed5175d52cf540a06725b628cc1a45d6ce/soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e", size = 121110, upload-time = "2026-05-24T13:55:57.154Z" }
sdist = { url = "https://files.pythonhosted.org/packages/69/99/a6ca3beb3ccacb41fb3321d8a60e5566f9e6467601ef8eba6a17e1b89778/soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74", size = 122445, upload-time = "2026-08-07T00:57:24.801Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" },
{ url = "https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823", size = 37370, upload-time = "2026-08-07T00:57:23.524Z" },
]
[[package]]