scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_model_edit_form_delta

Browse source
This commit is contained in:
ryan-crabbe-berri 2026-07-02 14:09:31 -07:00
commit 03032c65e6
115 changed files with 5598 additions and 972 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.git-blame-ignore-revs
View file

@ -13,7 +13,7 @@
7edf3a9cb55548b143df1692f4ed7c4681d7fcf7
# style: reformat litellm/ with ruff format (#31317)
430b5b8f1b12dc261a49fda99ac5d1b22381a428
17bfd415aeb5a57fb646b5cc67da1c730aa7c50b
# style: unify ruff format width on 120 (#31518)
3dfbeabe626d203ac9de86024519d9a96c484ce4
48b5a5a0cc5a694a11219416ee0b6eb6e620e74e

2
CLAUDE.md
View file

@ -33,6 +33,8 @@ If you ever make public-facing PR descriptions, comments, issues, commit message
Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
Python max line length is 120, not 88
Run tests before you commit. Also, run `make pre-commit` right before each commit, which generates types (as needed) and formats/lints your code. Any errors found must be fixed. It only runs when there are staged frontend and/or backend changes and calculates violations, generates types, etc. based on the worktree, so stage what you need or stash/delete unwanted files in litellm/ or ui/ (where backend and frontend lint run, respectively) before running it. If it fails because dashboard api types are stale, it already regenerated them for you. You just need to stage the schema.d.ts, re-run `make pre-commit` to confirm it passes, and commit
When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing

2
Makefile
View file

@ -88,7 +88,7 @@ install-hooks:
# Formatting
# Wrap width is ruff.toml's single source of truth (line-length = 120), shared by the
# formatter, E501, and the import sorter so there's no 88-vs-120 split to reconcile.
# formatter and the import sorter so there's no 88-vs-120 split to reconcile.
format: install-dev
cd litellm && $(UV_RUN) ruff format --exclude '/enterprise/' . && cd ..

233
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -13,6 +13,8 @@ from litellm.constants import (
)
if TYPE_CHECKING:
from litellm.integrations.prometheus import PrometheusLogger
from litellm.proxy._types import LiteLLM_ManagedObjectTable
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
@ -26,6 +28,7 @@ class CheckBatchCost:
proxy_logging_obj: "ProxyLogging",
prisma_client: "PrismaClient",
llm_router: "Router",
track_unmanaged_vertex_batch_cost: bool = False,
):
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
@ -33,6 +36,7 @@ class CheckBatchCost:
self.proxy_logging_obj: ProxyLogging = proxy_logging_obj
self.prisma_client: PrismaClient = prisma_client
self.llm_router: Router = llm_router
self._track_unmanaged_vertex_batch_cost = track_unmanaged_vertex_batch_cost
# Cached after the first poll cycle. Once we know the column is absent we skip
# the guaranteed-failing primary query on every subsequent cycle.
self._has_batch_processed_column: bool = True
@ -97,6 +101,182 @@ class CheckBatchCost:
order={"created_at": "asc"},
)
@staticmethod
def _record_error(
prom_logger: Optional["PrometheusLogger"], error_type: str
) -> None:
if prom_logger is not None:
prom_logger.record_check_batch_cost_error(error_type)
def _resolve_job_routing(
self,
job: "LiteLLM_ManagedObjectTable",
prom_logger: Optional["PrometheusLogger"],
) -> Optional[Tuple[str, str]]:
"""
Resolve (model_id, batch_id) for a managed-object row, where model_id is a router
deployment id and batch_id is the raw provider batch id.
Managed batches encode both in a base64 unified id. Unmanaged Vertex batches, created with
a raw gs:// input_file_id, store the raw provider job id as unified_object_id; when
track_unmanaged_vertex_batch_cost is enabled the model is derived from the gs:// path and
mapped to a configured vertex_ai deployment. Returns None (recording a metric) when the row
can't be routed.
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
get_batch_id_from_unified_batch_id,
get_model_id_from_unified_batch_id,
)
unified_object_id = job.unified_object_id
decoded = _is_base64_encoded_unified_file_id(unified_object_id)
if decoded:
model_id = get_model_id_from_unified_batch_id(decoded)
if model_id is None:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid model id"
)
self._record_error(prom_logger, "invalid_model_id")
return None
return model_id, get_batch_id_from_unified_batch_id(decoded)
if self._track_unmanaged_vertex_batch_cost:
return self._resolve_unmanaged_vertex_routing(job, prom_logger)
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid unified object id"
)
self._record_error(prom_logger, "invalid_unified_id")
return None
def _resolve_unmanaged_vertex_routing(
self,
job: "LiteLLM_ManagedObjectTable",
prom_logger: Optional["PrometheusLogger"],
) -> Optional[Tuple[str, str]]:
from litellm.llms.vertex_ai.batches.transformation import (
VertexAIBatchTransformation,
)
input_file_id = self._get_input_file_id(job)
if not VertexAIBatchTransformation.is_unmanaged_gcs_batch_input_file_id(
input_file_id
):
verbose_proxy_logger.info(
f"Skipping job {job.unified_object_id}: not an unmanaged vertex batch "
"(no gs:// input_file_id with a publishers/ model path)"
)
self._record_error(prom_logger, "invalid_unified_id")
return None
assert input_file_id is not None # narrowed by is_unmanaged_gcs_batch_input_file_id
bare_model_name = VertexAIBatchTransformation.get_bare_model_name_from_gcs_file(
input_file_id
)
deployment_id = self._get_vertex_ai_deployment_id_for_bare_model(
bare_model_name
)
if deployment_id is None:
verbose_proxy_logger.info(
f"Skipping unmanaged vertex batch {job.unified_object_id}: no vertex_ai "
f"deployment configured for model {bare_model_name}"
)
self._record_error(prom_logger, "unmanaged_no_matching_deployment")
return None
return deployment_id, job.unified_object_id
def _get_vertex_ai_deployment_id_for_bare_model(
self, bare_model_name: str
) -> Optional[str]:
model_group = self.llm_router.resolve_model_name_from_model_id(bare_model_name)
deployment_id = (
self._get_vertex_ai_deployment_id(model_group) if model_group else None
)
if deployment_id is not None:
return deployment_id
return self._get_vertex_ai_deployment_id_from_matching_deployments(
bare_model_name
)
def _get_vertex_ai_deployment_id_from_matching_deployments(
self, bare_model_name: str
) -> Optional[str]:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
for deployment in self.llm_router.get_model_list(model_name=None) or []:
litellm_params = deployment.get("litellm_params") or {}
actual_model = litellm_params.get("model")
if not isinstance(actual_model, str):
continue
if not self._is_bare_model_match(actual_model, bare_model_name):
continue
try:
_, llm_provider, _, _ = get_llm_provider(
model=actual_model,
custom_llm_provider=litellm_params.get("custom_llm_provider"),
)
except Exception:
continue
if llm_provider != "vertex_ai":
continue
model_info = deployment.get("model_info") or {}
deployment_id = model_info.get("id")
if isinstance(deployment_id, str):
return deployment_id
return None
@staticmethod
def _is_bare_model_match(actual_model: str, bare_model_name: str) -> bool:
return (
actual_model == bare_model_name
or actual_model.endswith(f"/{bare_model_name}")
or actual_model.endswith(f":{bare_model_name}")
)
def _get_vertex_ai_deployment_id(self, model_group: str) -> Optional[str]:
"""
Returns the first deployment id for `model_group` whose provider is vertex_ai,
skipping deployments from other providers that happen to share the model group name.
"""
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
for deployment_id in self.llm_router.get_model_ids(model_name=model_group):
deployment_info = self.llm_router.get_deployment(model_id=deployment_id)
if deployment_info is None:
continue
try:
_, llm_provider, _, _ = get_llm_provider(
model=deployment_info.litellm_params.model,
custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
)
except Exception:
continue
if llm_provider == "vertex_ai":
return deployment_id
return None
@staticmethod
def _get_input_file_id(job: "LiteLLM_ManagedObjectTable") -> Optional[str]:
import json
from litellm.types.utils import LiteLLMBatch
file_object = job.file_object
if isinstance(file_object, str):
try:
file_object = json.loads(file_object)
except (json.JSONDecodeError, ValueError):
return None
if not isinstance(file_object, dict):
return None
try:
return LiteLLMBatch.model_validate(file_object).input_file_id
except Exception:
return None
async def check_batch_cost(self):
"""
Check if the batch JOB has been tracked.
@ -114,8 +294,6 @@ class CheckBatchCost:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
get_batch_id_from_unified_batch_id,
get_model_id_from_unified_batch_id,
)
try:
@ -172,31 +350,10 @@ class CheckBatchCost:
else:
jobs = await self._fallback_find_jobs()
for job in jobs:
# get the model from the job
unified_object_id = job.unified_object_id
decoded_unified_object_id = _is_base64_encoded_unified_file_id(
unified_object_id
)
if not decoded_unified_object_id:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid unified object id"
)
if prom_logger:
prom_logger.record_check_batch_cost_error("invalid_unified_id")
continue
else:
unified_object_id = decoded_unified_object_id
model_id = get_model_id_from_unified_batch_id(unified_object_id)
batch_id = get_batch_id_from_unified_batch_id(unified_object_id)
if model_id is None:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid model id"
)
if prom_logger:
prom_logger.record_check_batch_cost_error("invalid_model_id")
routing = self._resolve_job_routing(job, prom_logger)
if routing is None:
continue
model_id, batch_id = routing
verbose_proxy_logger.info(
f"Querying model ID: {model_id} for cost and usage of batch ID: {batch_id}"
@ -213,7 +370,7 @@ class CheckBatchCost:
)
except Exception as e:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because of error querying model ID: {model_id} for cost and usage of batch ID: {batch_id}: {e}"
f"Skipping job {job.unified_object_id} because of error querying model ID: {model_id} for cost and usage of batch ID: {batch_id}: {e}"
)
if prom_logger:
prom_logger.record_check_batch_cost_error("provider_retrieval_error")
@ -287,7 +444,7 @@ class CheckBatchCost:
deployment_info = self.llm_router.get_deployment(model_id=model_id)
if deployment_info is None:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} because it is not a valid deployment info"
f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
)
if prom_logger:
prom_logger.record_check_batch_cost_error("deployment_not_found")
@ -413,6 +570,26 @@ class CheckBatchCost:
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)
elif response.status in ("failed", "expired", "cancelled"):
try:
update_data = {
"status": response.status,
"file_object": response.model_dump_json(),
}
if self._has_batch_processed_column:
update_data["batch_processed"] = True
await self.prisma_client.db.litellm_managedobjecttable.update(
where={"id": job.id},
data=update_data,
)
verbose_proxy_logger.info(
f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} as {response.status} in DB: {db_err}"
)
# Record polling run metrics (always, even if nothing was processed)
if prom_logger:
prom_logger.record_check_batch_cost_run(

16
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -125,23 +125,33 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
}
update_data = {
"model_mappings": json.dumps(model_mappings),
"flat_model_file_ids": list(model_mappings.values()),
"updated_by": user_api_key_dict.user_id,
}
if file_object is not None:
db_data["file_object"] = file_object.model_dump_json()
file_object_json = file_object.model_dump_json()
db_data["file_object"] = file_object_json
update_data["file_object"] = file_object_json
# Extract storage metadata from hidden params if present
hidden_params = getattr(file_object, "_hidden_params", {}) or {}
if "storage_backend" in hidden_params:
db_data["storage_backend"] = hidden_params["storage_backend"]
update_data["storage_backend"] = hidden_params["storage_backend"]
if "storage_url" in hidden_params:
db_data["storage_url"] = hidden_params["storage_url"]
update_data["storage_url"] = hidden_params["storage_url"]
verbose_logger.debug(
f"Storage metadata: storage_backend={db_data.get('storage_backend')}, "
f"storage_url={db_data.get('storage_url')}"
)
result = await self.prisma_client.db.litellm_managedfiletable.create(
data=db_data
result = await self.prisma_client.db.litellm_managedfiletable.upsert(
where={"unified_file_id": file_id},
data={"create": db_data, "update": update_data},
)
verbose_logger.debug(
f"LiteLLM Managed File object with id={file_id} stored in db: {result}"

65
litellm/integrations/prometheus.py
View file

@ -593,6 +593,21 @@ class PrometheusLogger(CustomLogger):
labelnames=[],
)
########################################
# MCP Tool Call Metrics
########################################
self.litellm_mcp_tool_calls_total = self._counter_factory(
name="litellm_mcp_tool_calls_total",
documentation="Total MCP tool calls, segmented by tool and server name",
labelnames=self.get_labels_for_metric("litellm_mcp_tool_calls_total"),
)
self.litellm_mcp_tool_call_spend_metric = self._counter_factory(
name="litellm_mcp_tool_call_spend_metric",
documentation="Total spend on MCP tool calls, segmented by tool and server name",
labelnames=self.get_labels_for_metric("litellm_mcp_tool_call_spend_metric"),
)
except Exception as e:
print_verbose(f"Got exception on init prometheus client {str(e)}")
raise e
@ -1300,6 +1315,13 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
# MCP tool call metrics
self._increment_mcp_tool_call_metrics(
standard_logging_payload=standard_logging_payload,
enum_values=enum_values,
response_cost=response_cost,
)
# increment litellm_proxy_total_requests_metric for all successful requests
# (both streaming and non-streaming) in this single location to prevent
# double-counting that occurs when async_post_call_success_hook also increments
@ -1521,6 +1543,49 @@ class PrometheusLogger(CustomLogger):
amount=float(provider_cache_creation_tokens),
)
def _increment_mcp_tool_call_metrics(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
response_cost: float,
) -> None:
metadata = standard_logging_payload.get("metadata")
if not isinstance(metadata, dict):
return
mcp_meta = metadata.get("mcp_tool_call_metadata")
if not isinstance(mcp_meta, dict):
return
mcp_enum_values = UserAPIKeyLabelValues(
mcp_tool_name=mcp_meta.get("name"),
mcp_server_name=mcp_meta.get("mcp_server_name"),
hashed_api_key=enum_values.hashed_api_key,
api_key_alias=enum_values.api_key_alias,
team=enum_values.team,
team_alias=enum_values.team_alias,
user=enum_values.user,
end_user=enum_values.end_user,
)
mcp_label_context = PrometheusLabelFactoryContext(mcp_enum_values)
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_mcp_tool_calls_total,
"litellm_mcp_tool_calls_total",
mcp_enum_values,
label_context=mcp_label_context,
)
if response_cost > 0:
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_mcp_tool_call_spend_metric,
"litellm_mcp_tool_call_spend_metric",
mcp_enum_values,
label_context=mcp_label_context,
amount=response_cost,
)
async def _increment_remaining_budget_metrics(
self,
user_api_team: Optional[str],

9
litellm/litellm_core_utils/litellm_logging.py
View file

@ -4722,7 +4722,7 @@ class StandardLoggingPayloadSetup:
api_base: Optional[str] = None,
) -> StandardLoggingModelInformation:
model_cost_name = _select_model_name_for_cost_calc(
model=None,
model=base_model if custom_pricing else None,
completion_response=init_response_obj, # type: ignore
base_model=base_model,
custom_pricing=custom_pricing,
@ -5268,6 +5268,11 @@ def get_standard_logging_object_payload(
## Get model cost information ##
base_model = _get_base_model_from_metadata(model_call_details=kwargs)
# The router overrides completion_response.model to the model-group alias before
# this payload is built, so cost-map lookup via that alias always misses.
# Fall back to the actual deployment model set by the router in metadata.
if base_model is None:
base_model = metadata.get("deployment")
custom_pricing = use_custom_pricing_for_model(litellm_params=litellm_params)
raw_response_cost = kwargs.get("response_cost")
response_cost: float = raw_response_cost or 0.0
@ -5389,7 +5394,7 @@ def get_standard_logging_object_payload(
def emit_standard_logging_payload(payload: StandardLoggingPayload):
if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"):
print(json.dumps(payload, indent=4)) # noqa: T201
print(json.dumps(payload, indent=4), flush=True) # noqa: T201
def get_standard_logging_metadata(

9
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -5035,15 +5035,18 @@ def _bedrock_tools_pt(tools: List, model: Optional[str] = None) -> List[BedrockT
]
"""
from litellm.llms.bedrock.common_utils import (
get_bedrock_base_model,
bedrock_converse_supports_strict_tools,
normalize_json_schema_custom_types_to_object,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs
_valid_json_schema_root_types = frozenset(("array", "boolean", "integer", "null", "number", "object", "string"))
# Only Claude on Bedrock honours strict tool schemas; other families
# (Nova, Llama, GPT-OSS) reject the strict field outright.
supports_strict_tools = bool(model and get_bedrock_base_model(model).startswith("anthropic"))
# (Nova, Llama, GPT-OSS) reject the strict field outright. Opus 4.7/4.8
# also reject `strict` on Bedrock Converse (see #31582) — their validator
# maps toolSpec to the native Anthropic tool shape, which has no strict
# field, even though Anthropic's native API accepts it as a top-level key.
supports_strict_tools = bool(model and bedrock_converse_supports_strict_tools(model))
tool_block_list: List[BedrockToolBlock] = []
for tool_idx, tool in enumerate(tools):
# Check if tool is already a BedrockToolBlock (e.g., systemTool for Nova grounding)

6
litellm/litellm_core_utils/prompt_templates/image_handling.py
View file

@ -72,6 +72,9 @@ def _process_image_response(response: Response, url: str) -> str:
async def async_convert_url_to_base64(url: str) -> str:
if url.startswith("data:") and ";base64," in url:
return url
# If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
raise litellm.ImageFetchError(
@ -95,6 +98,9 @@ async def async_convert_url_to_base64(url: str) -> str:
def convert_url_to_base64(url: str) -> str:
if url.startswith("data:") and ";base64," in url:
return url
# If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
raise litellm.ImageFetchError(

47
litellm/llms/bedrock/common_utils.py
View file

@ -4,9 +4,11 @@ from __future__ import annotations
Common utilities used across bedrock chat/embedding/image generation
"""
import contextlib
import functools
import json
import os
import re
from typing import (
TYPE_CHECKING,
Any,
@ -718,6 +720,51 @@ def is_claude_4_5_on_bedrock(model: str) -> bool:
return any(pattern in model_lower for pattern in claude_4_5_patterns)
_BEDROCK_MODEL_VERSION_SUFFIX_RE = re.compile(r"-v\d+(?::\d+)?$")
def bedrock_converse_supports_strict_tools(model: str) -> bool:
"""
Whether ``toolSpec.strict`` can be forwarded to Bedrock Converse for ``model``.
Non-Anthropic Bedrock families (Nova, Llama, GPT-OSS) reject the field
outright. Anthropic models forward it unless their entry in
``model_prices_and_context_window.json`` sets
``bedrock_converse_supports_strict_tools: false`` — Bedrock routes those
(Opus 4.7/4.8, see #31582) through a stricter validator that rejects the
``strict`` key on ``toolSpec`` even though Anthropic's native API accepts
it as a top-level tool field.
"""
base = get_bedrock_base_model(model)
if not base.startswith("anthropic"):
return False
flag = _get_bedrock_converse_strict_tools_flag(base)
return flag if flag is not None else True
def _get_bedrock_converse_strict_tools_flag(base_model: str) -> Optional[bool]:
candidates = dict.fromkeys((base_model, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base_model)))
for candidate in candidates:
with contextlib.suppress(Exception):
model_info = get_cached_model_info()(
model=candidate,
custom_llm_provider="bedrock",
)
flag = model_info.get("bedrock_converse_supports_strict_tools")
if isinstance(flag, bool):
return flag
model_cost_key = model_info.get("key")
if isinstance(model_cost_key, str):
local_flag = (
_get_local_model_cost_map().get(model_cost_key, {}).get("bedrock_converse_supports_strict_tools")
)
if isinstance(local_flag, bool):
return local_flag
return None
def normalize_bedrock_opus_output_config_effort(model: str, output_config: Any) -> None:
"""
Normalize Anthropic ``output_config.effort`` values for Bedrock Opus ids.

37
litellm/llms/bedrock/realtime/handler.py
View file

@ -5,6 +5,7 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic.
"""
import asyncio
import contextlib
import json
from typing import Any, Optional
@ -156,12 +157,19 @@ class BedrockRealtime(BaseAWSLLM):
session_state: dict,
):
"""Forward messages from client WebSocket to Bedrock stream."""
try:
from aws_sdk_bedrock_runtime.models import (
BidirectionalInputPayloadPart,
InvokeModelWithBidirectionalStreamInputChunk,
)
from aws_sdk_bedrock_runtime.models import (
BidirectionalInputPayloadPart,
InvokeModelWithBidirectionalStreamInputChunk,
)
async def send_to_bedrock(bedrock_message: str) -> None:
event = InvokeModelWithBidirectionalStreamInputChunk(
value=BidirectionalInputPayloadPart(bytes_=bedrock_message.encode("utf-8"))
)
await bedrock_stream.input_stream.send(event)
verbose_proxy_logger.debug(f"Bedrock Realtime: Sent to Bedrock: {bedrock_message[:200]}")
try:
while True:
# Receive message from client
message = await client_ws.receive_text()
@ -176,19 +184,15 @@ class BedrockRealtime(BaseAWSLLM):
# Send transformed messages to Bedrock
for bedrock_message in transformed_messages:
event = InvokeModelWithBidirectionalStreamInputChunk(
value=BidirectionalInputPayloadPart(bytes_=bedrock_message.encode("utf-8"))
)
await bedrock_stream.input_stream.send(event)
verbose_proxy_logger.debug(f"Bedrock Realtime: Sent to Bedrock: {bedrock_message[:200]}")
await send_to_bedrock(bedrock_message)
except Exception as e:
verbose_proxy_logger.debug(f"Client to Bedrock forwarding ended: {e}", exc_info=True)
# Close the Bedrock stream input
try:
for close_message in transformation_config.session_close_messages():
with contextlib.suppress(Exception):
await send_to_bedrock(close_message)
with contextlib.suppress(Exception):
await bedrock_stream.input_stream.close()
except Exception:
pass
async def _forward_bedrock_to_client(
self,
@ -206,6 +210,10 @@ class BedrockRealtime(BaseAWSLLM):
output = await bedrock_stream.await_output()
result = await output[1].receive()
if result is None:
verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended")
break
if result.value and result.value.bytes_:
bedrock_response = result.value.bytes_.decode("utf-8")
verbose_proxy_logger.debug(f"Bedrock Realtime: Received from Bedrock: {bedrock_response[:200]}")
@ -252,6 +260,7 @@ class BedrockRealtime(BaseAWSLLM):
except Exception as e:
verbose_proxy_logger.debug(f"Bedrock to client forwarding ended: {e}", exc_info=True)
finally:
# Close the client WebSocket
try:
await client_ws.close()

146
litellm/llms/bedrock/realtime/transformation.py
View file

@ -4,14 +4,18 @@ This file contains the transformation logic for Bedrock Nova Sonic realtime API.
Transforms between OpenAI Realtime API format and Bedrock Nova Sonic format.
"""
import base64
import json
import uuid as uuid_lib
from typing import Any, List, Optional, Union
from pydantic import BaseModel
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
from litellm.types.llms.openai import (
OpenAIRealtimeContentPartDone,
OpenAIRealtimeDoneEvent,
@ -35,6 +39,17 @@ from litellm.types.realtime import (
from litellm.utils import get_empty_usage
class BedrockContentEnd(BaseModel):
stopReason: Optional[str] = None
TRIGGER_AUDIO_SAMPLE_RATE_HERTZ = 16000
TRIGGER_AUDIO_BYTES_PER_SECOND = TRIGGER_AUDIO_SAMPLE_RATE_HERTZ * 2
TRIGGER_LEADING_SILENCE = bytes(TRIGGER_AUDIO_BYTES_PER_SECOND // 2)
TRIGGER_TRAILING_SILENCE = bytes(TRIGGER_AUDIO_BYTES_PER_SECOND * 3)
TRIGGER_AUDIO_CHUNK_SIZE = 1024
class BedrockRealtimeConfig(BaseRealtimeConfig):
"""Configuration for Bedrock Nova Sonic realtime transformations."""
@ -43,6 +58,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
self.prompt_name = str(uuid_lib.uuid4())
self.content_name = str(uuid_lib.uuid4())
self.audio_content_name = str(uuid_lib.uuid4())
self.prompt_started = False
self.client_audio_streamed = False
# Default configuration values
# Inference configuration
@ -247,6 +264,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
prompt_start = {"event": {"promptStart": prompt_start_config}}
messages.append(json.dumps(prompt_start))
self.prompt_started = True
# Send system prompt if provided
instructions = session_config.get("instructions")
@ -304,8 +322,22 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
List of Bedrock format messages (JSON strings)
"""
verbose_logger.debug("Handling input_audio_buffer.append")
self.client_audio_streamed = True
messages: List[str] = []
if hasattr(self, "_audio_content_started") and self._audio_content_sample_rate != self.input_sample_rate_hertz:
mismatched_content_end = {
"event": {
"contentEnd": {
"promptName": self.prompt_name,
"contentName": self.audio_content_name,
}
}
}
messages.append(json.dumps(mismatched_content_end))
delattr(self, "_audio_content_started")
self.audio_content_name = str(uuid_lib.uuid4())
# Check if we need to start audio content
if not hasattr(self, "_audio_content_started"):
audio_content_start = {
@ -329,6 +361,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
}
messages.append(json.dumps(audio_content_start))
self._audio_content_started = True
self._audio_content_sample_rate = self.input_sample_rate_hertz
# Send audio chunk
audio_data = json_message.get("audio", "")
@ -383,7 +416,6 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
List of Bedrock format messages (JSON strings)
"""
verbose_logger.debug("Handling conversation.item.create")
messages: List[str] = []
item = json_message.get("item", {})
item_type = item.get("type")
@ -392,6 +424,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
if item_type == "function_call_output":
return self.transform_conversation_item_create_tool_result_event(json_message)
messages: list[str] = []
# Handle regular message
if item_type == "message":
content = item.get("content", [])
@ -443,6 +477,12 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
"""
Transform response.create event to Bedrock format.
Nova Sonic only starts generating after it detects user speech, so text-only
sessions never get a response on their own. Injecting a short spoken "ready"
utterance (followed by silence) makes the model respond to the pending
interactive text input. Sessions where the client streams its own audio rely
on Nova Sonic's built-in turn detection instead.
Args:
json_message: OpenAI response.create message
@ -450,8 +490,53 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
List of Bedrock format messages (JSON strings)
"""
verbose_logger.debug("Handling response.create")
# Bedrock starts generating automatically, no explicit trigger needed
return []
if not self.prompt_started or self.client_audio_streamed:
return []
messages: list[str] = []
if not hasattr(self, "_audio_content_started"):
trigger_content_start = {
"event": {
"contentStart": {
"promptName": self.prompt_name,
"contentName": self.audio_content_name,
"type": "AUDIO",
"interactive": True,
"role": "USER",
"audioInputConfiguration": {
"mediaType": self.input_media_type,
"sampleRateHertz": TRIGGER_AUDIO_SAMPLE_RATE_HERTZ,
"sampleSizeBits": self.input_sample_size_bits,
"channelCount": self.input_channel_count,
"audioType": self.input_audio_type,
"encoding": self.input_encoding,
},
}
}
}
messages.append(json.dumps(trigger_content_start))
self._audio_content_started = True
self._audio_content_sample_rate = TRIGGER_AUDIO_SAMPLE_RATE_HERTZ
messages.extend(self._response_trigger_audio_messages())
return messages
def _response_trigger_audio_messages(self) -> list[str]:
pcm = TRIGGER_LEADING_SILENCE + ready_trigger_pcm() + TRIGGER_TRAILING_SILENCE
return [
json.dumps(
{
"event": {
"audioInput": {
"promptName": self.prompt_name,
"contentName": self.audio_content_name,
"content": base64.b64encode(pcm[offset : offset + TRIGGER_AUDIO_CHUNK_SIZE]).decode(),
}
}
}
)
for offset in range(0, len(pcm), TRIGGER_AUDIO_CHUNK_SIZE)
]
def transform_response_cancel_event(self, json_message: dict) -> List[str]:
"""
@ -467,6 +552,35 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
# Send interrupt signal if needed
return []
def session_close_messages(self) -> list[str]:
"""
Build the Bedrock events that gracefully close the session
(contentEnd for any open audio content, promptEnd, sessionEnd).
Returns:
List of Bedrock format messages (JSON strings)
"""
if not self.prompt_started:
return []
messages: list[str] = []
if hasattr(self, "_audio_content_started"):
audio_content_end = {
"event": {
"contentEnd": {
"promptName": self.prompt_name,
"contentName": self.audio_content_name,
}
}
}
messages.append(json.dumps(audio_content_end))
delattr(self, "_audio_content_started")
messages.append(json.dumps({"event": {"promptEnd": {"promptName": self.prompt_name}}}))
messages.append(json.dumps({"event": {"sessionEnd": {}}}))
self.prompt_started = False
return messages
def transform_realtime_request(
self,
message: str,
@ -837,10 +951,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
Optional[ALL_DELTA_TYPES],
]:
"""
Transform Bedrock promptEnd event to OpenAI response.done.
Transform a Bedrock end-of-response event (promptEnd, completionEnd, or an
END_TURN contentEnd) to OpenAI response.done.
Args:
event: Bedrock promptEnd event
event: Bedrock event that ends the response
current_response_id: Current response ID
current_conversation_id: Current conversation ID
@ -848,7 +963,18 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
Tuple of (events, reset_output_item_id, reset_response_id, reset_delta_type)
"""
verbose_logger.debug("Handling promptEnd")
return self._response_done_events(current_response_id, current_conversation_id)
def _response_done_events(
self,
current_response_id: Optional[str],
current_conversation_id: Optional[str],
) -> tuple[
List[OpenAIRealtimeEvents],
Optional[str],
Optional[str],
Optional[ALL_DELTA_TYPES],
]:
if not current_response_id or not current_conversation_id:
return [], None, None, None
@ -1084,6 +1210,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
current_delta_chunks,
)
returned_messages.extend(events)
if BedrockContentEnd.model_validate(event["contentEnd"]).stopReason == "END_TURN":
(
done_events,
current_output_item_id,
current_response_id,
current_delta_type,
) = self._response_done_events(current_response_id, current_conversation_id)
returned_messages.extend(done_events)
elif "toolUse" in event:
events, tool_call_id, tool_name = self.transform_tool_use_event(
@ -1093,7 +1227,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
# Store tool call info for potential use
verbose_logger.debug(f"Tool use event: {tool_name} (ID: {tool_call_id})")
elif "promptEnd" in event:
elif "promptEnd" in event or "completionEnd" in event:
(
events,
current_output_item_id,

208
litellm/llms/bedrock/realtime/trigger_audio.py Normal file
View file

@ -0,0 +1,208 @@
"""
Pre-rendered spoken "ready" trigger audio (16kHz, 16-bit, mono PCM), generated with Amazon Polly.
Amazon Nova Sonic v1 only starts generating after it hears the user speak, so text-only realtime
sessions inject this short utterance to trigger a response (same approach as Pipecat's
AWSNovaSonicLLMService assistant-response trigger).
"""
import base64
import gzip
from functools import lru_cache
READY_TRIGGER_PCM_16KHZ_MONO_GZIP_B64 = (
"H4sIANGpRWoC/517dXQcR9Bnw+Duis3MzMwkc8xsxxTZMTMzM0PMmJhBjpmZmWKQZSaxFrQ80H0lJXf3vXf/nev17ExPQ3Xh"
"r2wPQv8/f/D/uMP/zxv8f64YkSyiWU1AIpCcRSqyICtQCApF4SgSRaHsKCfKjfKhgqgIKo5KobKoPKqEqqFaqC5qgBqjFqg1"
"ao86o+6oF+qDYtAgNBSNQKPQODQeTUZT0Uw0B81D89ECtAgtRcvRMrQCrQRahVajNWgtWo/+QBvQJrQRbQbagraibWg70Da4"
"2/rf7zbozxyxCcaug1krYZXFaCGai2ajGWgKmgi7jYFdh6ABqB/qjXqgLqgd8NUcNQUe66CaqDJwXQaVhDMUgNNkh7NZ4bSc"
"69zLXTyVJ/Kv/BN/z9/w5/w+0B1+g1/kZ/gpfoL/zQ/xA/wvvovv5Jv5Fmjr+Tq+lq/iS/kioPl8Fp+eRRP4KKBhfBDvw3vw"
"Lrw978jb8Fa8JW8B1JQ34PV5Q2iZ1wa8Ea/Da/Ja0Gpm9TTmzXgT3hzGt+a/wNz2vBPM78TbwtyW0NuIV+fVeCWgCrwyUHmg"
"yll9NWClOrwurNA0a42GMLYerFoN3pf7j8rwYkBFeWFehOcHKshz8dw8Ow/nUTyCh/IwaDYuc4mrPATIBs8WeJMNxmTnOWF0"
"Tp4DnsK4lYscc8405mNulsFc0FKYg9lZKjQX3KXDcxJck+E5haWxRPYd2k/2hX2G9gkonr1nX6H3C7RkGOmB1QjsGckLAK+1"
"QAqdeQwfD7Jdz3fzk/we/4f/5D4uoVyoGOizOdjbMLCw+WBBO1Esuo7uoZfoE/qJnCgARq7iUByGs+G8OGcWReAQbMUCDqBk"
"FI9eoGvoGFjVSrCdAWAr9VBhsHsHf8EP87m8F8gV88dsBxvFajKJvTS3mjFmOdNnXDMWG12NUoZPf6jv0afpnfRKejadaena"
"B+219kqL0xK0oBamV9E761P1I3qcLhhVjL7GEuOckWAUMHube02nWZ/NY7dYBFjJZR4JNnsRqbgr3o7f4WykNZlJDpEH5AfR"
"SRgtQMvTarQJbUZb0JZA7Wjr/1o7+gs8N6K1aRmal9qoSRLJC3KF7CHLyUTSl7QhtUkxkptkIyEklOQghUgF0oh0IINg/WVk"
"O9lLjsI+u8kfZC4ZQ/qQZqQ6jM9DchIbkQglfpyKP+Cb+DDejBfiEbgDrogx/gftQWNRbaTx23wBjwYbuMwmsfLsq7nZ7Ghi"
"84IxHs7r1A/rA/Wiery2UeukRWiPg4uD0UEeuBSYFWgQEAPP/dv9k/xd/K387fy9/DP9x/2p/mqB1QEjMCvIg/M1m75Tr2m8"
"NuaZ5dhrNoaHQnzIDpzUIzdJLXqS5hEmCVcFJpQXu4gjxelAk8XBYgexgiiLP4XLwmphkNBAyCkk0yt0Be1HK1JCX5JdZDSp"
"D2d7htfhLmAXX9FR0H5XVAPlQDp4/xN+BezsCHj6Xrge4cfA9y/wS/w6nPUxWN97/h2ihIObXISoWASiXwxEtDPIjRriTTiA"
"B5N3pD19RH8RbgilxWXiZ7GI1EUaLU2QxkojpcHSUGmGtEO6KX2UXkvbpYaSQ7wq/iXuEleKQ8XS4gdhsVBXEISjoNd3ZBzJ"
"S27hCbgMfgeRTUGHeXfuY6tZFfbQ7G+mGmOMJH2UTvRNWnZtfjA8eBjk+srfx+/x7fQ18/m8+739veW8yPvec8nzl2ejZzXQ"
"Ns9pz2dPiLeld4+3iO+pb6t/RWBr8IL2UY8wm7Pp/AC6huPIF/pOuC1ul4bKuZVDSnF1kfpUDbPUs/xq6WmpaZEtD9TJaqi6"
"QymlHJAj5KnSM7GYOF64SYvTtSSC7MB1cALE6WbIC/GyG8QaESJKAfDm2TyWx3EZInAryA9LQfZvwOor4c54PF4B8juGn+A0"
"HEWqgAf0JkPIeDKFTCOryA5yiaQSG42mI+hf9DvNLvQXrgg2sZ+4V/wilpH6SFulT1JReYJ8WU6QsynFlbxASPkiX5S3yNPk"
"0fJkebm8RO4ox0lNpbNia/Gj0BEsYxxYhZ1cJOdJE3IWM8h0vfgpFsVGmaeNaOOL3kK/ohXV2gfnBL74R/q/+rr57N6V3ibe"
"gGevp4fnm3uuu7T7bcaJjJkZTTMsGfddm11jXF1dfVyzXBdcpTOeZFx2f/HU8j33rww207Obz9l0VJqcoBXEQ1KyXEgtYgla"
"TlgL2ZbZ/rGl2GwheUKUkHhbN9t1a6S1iWWM2kepJxeTUoQLdD5pgWuiWnwWe282Ac3vNhZBNBllTDBWGDuNNwYzmpixZgTb"
"ysqBhJtDDOyL43E0nCw/HUPP0jTwlipCKaGQQIR4eotupcNoPcrJY3KP3CEmaUKnUZ22F24KMeItcbKUXf5DrqUsUxxKe3Wc"
"+lBNVQVLW8twy2kLs2S3NrY2tFqsryytLN/VieoLpaEyV/ZLG6VGUlDcKo4VO4uR4mOhs3CI/iQyyY3t/CmLNfcaF/TT2sXg"
"+0BUoLd/r4/4fvde80ieTu6/M1Jc0a5LzvzOBY4P9qH2bPaE9Pj0T+mivaZ9sv20XbfncNR3vHY8dHbPWOQp6v9VSzXX4wpi"
"HWWK5Zw1xtbd9t46yeq0PLHkt26y9rXlDukSEh1yz5ZgPWwppw6VN4qKsIL0wQPQJH6WtQH9Kmwge8em8ZFoNP6NtKDXAf5E"
"Co/AunLQzPiZi26gFYTzgiSGiYeEdHqKNMKneW0WahY2Ruo+7Yi2SzsBkT6P3lZfol/Tdb2+McLYblw2TNDEA/MhU5FANgnD"
"5BWWniEnw6ZHDIqcGZk7ck7EzfAe4ZXDK4WvCufhvSPORFyJGBnxPJyHpYW+DEm1mdYm1o2WoDpCjVOqKmvlEvJtaaqUXWom"
"NhLiSS5cmzczF+rZtBzBsYEE/xV/G/8nXz/feu87j+zp5l6TkTfjlquFq4DrunOUs6bzo2OsQ3Vst3ezlwdJNrBXg2uUXU8v"
"YRccp5293WX85fSd/C4tIt9UW1jnW7mljWWk2k5NV1Msw21KaL6wa2F/hrUPXWOba/lFmShlE2/SPeQNfoZsaDS/xtYyCRAS"
"Q71IN+qhbYRswl+0Ab1N/OQcfSa0l47Km5X3ynilh/xSjBIakivoL/6aDWcCm21+MmoYO/WC+m1tnrZUO6p91urq8/THeg8j"
"YHw28/Ih6DO+TW+KcfJE4KVw6O2wHeHFImIi9PDocCNsadiksPSwjLA3YTfCqoZ1Cz0fcsrmsi6ydrXOtn6z5rAxqwJetNBa"
"w/qPpYflndoRYtVpeYpUW/TRu2QktqKebJtxR7se8Phue85nNHGZjuoOh32MvYy9kP1D+iOgielqesv0kukF0wdCs6ZPT7+V"
"Xsf+1Z5gz+ko6Ryc8dHr19aiuaKpbrC1BaA02rpcfaNUV7tbGtkehFQJ9YSk2aKsK9SH8hWxibCFzEM9WG/Dqbm02UYCy0W+"
"CbOlmfJ4eZf0U7whrpAaKVUsH6wOW/mQt7bBtlfWn5azqqjYxao0G+pu1tK7a39px/U4Y7hZzaxvJGoDg7kCp/yJ/seBGE00"
"jplTeE9cmhYSp0mfpXjJK54RHgmzxDxyjCU8pFrYwvBLYUbIV2tXdYjslC5IyZJPPqUsVwOWYrZFIQmhxcECK4d+tcVa21kr"
"2mJDDobVi4iNiIooGUZCcljnqcWUSPmBGCGEkv68jtle7x787v/V53UnuD44CtuT0r+m77B3dMQ4tzvDnDmcC+3v01ukR6Tt"
"TU1NiUlpnKKkzk4bm97GUdn5wFHa2cQx3jnE0zJoQy7Rp26x9rP+Zsmh7lJKKHOVEeoPJUbpLn8SL4rVwffO0R0oYAwPCv6u"
"vqf+/noV/ogchlKFC7eFH+JB+YCabmliZdZctiW2DiH3Q1qFjg2Js2ZT44VpaKJRVOOBtYGMwG/BaUF74GwgIrg9uF7bodcF"
"9HZTb2M8Z+GkjlhfGaJ6lRVyZbEEnYC9fBZCdKT4WZYtsyDnzVRtymepqFQTOLog51Z3WIbZzoa8CG0YViwsLLRqyGXbUlux"
"0Blh9cIPRoRHvo14H/4+9KXtpkVTHssnpcLSafEvYSe1ksm8tnldEwN7vf3d91yHXYMztrtPex54FnoKu486v9u/p81Lq2Zf"
"51jhDHfddaxOb5w2K+Vz0vDk2sk1UtypQ1MrplZKr5ihBYeRnmopm2RrrtaX5otLxFNSYbWvdZ/NbZmjZJPzilNoE4xMR6CJ"
"d7a7jO+e9pYdJctETTwtxNB89K6QJEdaZ4WsCWsUPjl8emT5qKURR0K3qIwW5tX1ZsHHwdMQ15jWMVjMH+mb7X3oUwLvA9O1"
"3kYF3pRUFB/LLdVcahl5s5CPHAL0nIRvkXByGM1k7dkhXhDF4TkkkgSIRNcLprRV2WM5bb1pfWxZot5TvZarNlfIr6GvQ3rb"
"zlrXWivbqobst32wVrW+VAurMapb3Q8Svw6RppVYXahEz5JpaIy5WGvk/8tb1fsNUEwzb0fvBPdJl+o8YGfpetqxlE7JD5OV"
"ZHeyJd2bylJbptLEbgkzEub8nJrUPqUa2F2T9J0Zn/RE8aytctigkHVyfmrisUKI8kW9Z+mulpSGkXGsvDnUSAi6vH9nrHYd"
"zNjjL8C60fFSXmm+MFxoJc6QhlpWhA4Mt0SWiwyNWBs+MnS+db9yQtjO8xjN9ViwpSFGV+2Cv5D3c0ZOT/NAV8PkB/FG8oHO"
"EHaJQ+W2ynO5gtiQLIQKWoUKeTeqz18YX4I5AgW110YaTyd++kpoK42RXyjvLZutO61rbKlwHWfbZntqq2UbaTmtDJDPSivE"
"Z+Im+Z1yXm2reuWncmu5lrxIvqx8VzR5tXwEUO99uh3FsbV6seBNX3VvVY+asTVjbUaM+45nrPuOa7Kjpb1C6viU96lbU3+H"
"bLAqvaj9YdqQVG9S++SMpEpJ+ZOvJfZPupU8zLFBmyuWCskRut26Rub4LkvlbmGuckC9r1SUDpJu5qLACt8Zz09XlPOKc73n"
"czBonsPRtBO5C7V/FD0rb7A5w/dGXc/2W9THsDO2CEsZOU24Txriv9EP/oYdMO/q6wKPvfc8m7xlghfMmmSdWFdC0kBxn9hf"
"6iwtkBLFPmJNcZjgpnXJYtSLbdE2Bry+rv7DOkVf6BbpqhQQZ0tHpRRptdoSLOpL6PzQyyFNbYssJRVBuihECcWFC5Ku/KME"
"lFFKafmlRKSOUi55plLUkmLxWH9YTXWaPFdcCRmpBA81rgU/Brr48/sEr83TMyObe2LGtYyKGdQlgVfOs/9m/9W+1RnpLOTa"
"6yzuLOOw2KemD087n5or7XJqVNrxlI2OzcEmwjXLGltTWz7rXsmHR6D2Qg85XGmu9FduSHVJvLbYW9Q9OKOFu6N/oLGTj0WD"
"+RjeEkXjz2SfOF5Ntk0NC4scEtUj8n3IA6Wl+E4whfeCIrYQPuKnrKYR0P4MouDvwWjdyk+SseIieaKsgLzaiGWl3+UcgL/t"
"4kmxoxCPu3HNGKiN97cEVHAv8IFtIa3EpVJuqb+0Uv6hlLXeCu0R3i78aHjZ8KuhD6wB5Z3UX2wijhFVmaiXFEWeJM2VqskN"
"lGfqaUtXywBLJ8tjS0OrWx0sj4DYeAQ/Rk9YUb2Xv41HdI/wxPk3BqoHznkPuG46cju+2rO7Jrlfuld5Orh7uMY6UtNcKcVT"
"XqRMs/d2fAFLy5WamPwlMT4h2rmcpakTbLct9eQUWhg9QOWkebarIdstneUV4lK0UkvwXM8o5U7y7AycMP38lrkkuMbPgkvM"
"R/ik/MV6JvRC2OKw26HlQ2pZtsrvpMUKs3yw3Fbmih5eTTvvzevt6m+vjTFGQs1o5fXYXwyjFrgHnSp2lgeotS0n1HxKfekG"
"nYldPC8aiUrxDuYzbV1wj57MMvA3GifIYkF5gtJC/R3QbnnrGdvvIYNCRoSsty2zrJEjxDHCNGGcmENi4haxkVhQbCkmin+K"
"1aQSck8lQemozlOGSSWFFqQCNnkzNkDPHsjh++p54tnofe4p7NnnphkhzucO2RHpPORIcrRycecl++r0hmkd0i6nXUvLbk+2"
"b007n9wv0f+z18/eSdczTqCnwEV1eSjNiYuwYby1MMYSb8WW5/IF8Q46qKV6DrivePr79wSjjFxGpDbKN80bHqhp3ECquFZ5"
"py5XO6uJ6lJLH6jdHljKWT+ABMrL+ehD9lgrGiyiLTT2mR+Nq3p+QJUHg80NiU/A2+lyyCo7lA9ST7G4oJFtpAytJ8wX2pN1"
"bKNeWauqt2Nf8HlaR9SEUmJJKb/cTj6gRFjyWX22waHZQh/aBloqKWmARLLJY+Rlsk1OETeLzaTfpUZyqNxdKg4I5Kh0XEwX"
"2gr5hFPUiVvxFMMZXO2zeX96Xnp+95z0rPBU8sx1j3P5HGccVR0THCcc+ZzvHbvBzkTHqfS/0lhq7rRHqS9SHiUfT56WJCVO"
"TinnsbM4saFcU/ydKLgl7k5LyonKCbWepaKlitKcxurEl99d2X3Jt16/ZlbVW/o2ua+5a/h6Bp+bY8k46aO62/YjZKtttGW0"
"ck8pad0Y2jS8avhj2yC5Iz5ohhvT9C/aJL2u3kZfpGXTbgS1wOBAC20qy0PGCNehIh1J+9PG9D3dJBQUDNwY70EneF+eyErx"
"bmgk/5v7UVN8nRQUZovD5EbKW6WQpT6g7t8tFdWDUGVvV+up1dRR6gGlkdJPOSvHS3OkKOmq6Befi7PEOKjLiJhIV5BsuBeP"
"NQO6qB/S2gUf+Gv5Zd8nTzvPM3fpjDEZbd0lM+yueq5RjvaOk/Yj6eUcOR2b06um9Ujbl/otJZAyJmVHct/k8Ylm6iNfPL8m"
"vBaX0p1ordnabEZqWVjotfAjEQNDk6RkZg9sDEzQn7JCZCfpx5r4H7lyu2q7b/mi9EW8HDWB1gq9hWV0jlBL2WcLhPYM/xg+"
"OmyRdbtUmJbDyxGF+nMiyYcKGPmCBX2LQKtj/cP10eCPV9l9VpdNNZuzs+gQHSzekcbJ/SQH7Yr38VH8Mxol7FBmWpLU29IY"
"KuMfqBn9JtUCpDrBgi1vlbbyWOkPaay8Us1u7R1SPjTNWly9IX2Vsit31G7WrdZfLS3Ur1I5KVaYT0fSKjSO7MLvmUNvHGwW"
"OOO/4nvtb+Xv7D/rO+dxZSxxDnDUdPZ3+jOKeVhGWsZOxxzHLOcNRzVHFfu+tI72nvYSzvN2nLYh+U5i/bRX3oJsjbBOGMP3"
"aFv9/Q2H8FvInPDVYTVsWP1NmihZAOk9JUfJCuGm+EkYgmP1OP8dL/Jv1xawW7gj+Qc/RJfZIfYnukvbyolqYVunkCe2RdZK"
"lpxqbzla2gtoNsz6QE0QFdzD1PQocx27z/2skDlK9wQHBI8Ex2pD9InGZvMMX49aoVb8EPOa7dkPvpjWlG+rpS3V1WxyEbCT"
"dKmv+tLa1bbF2tkSq4yS8ggzSXVynJwRFkh15GxSSZqEuqC9uK2wTFovj5SvSjOlN2JRsZ+whBakaaQSbUkvkTr4BDtrtDRK"
"GgWMMMOr/2oc02toAwMj/O39NPA60Dp4NoACjfzcN8g/33/eX9Pfz7/LPzJQIdDN/8i71PvIq/na+6f6fb7FvrK+VpDB6wXL"
"6T2NXUayXkq36KqxyVzBf6ApuC9ug/fg2aQY7Uo70qm0mNBHHCqFyPelUCm/+EYYLo6Ucsqq3FzaJVYRK4klxThhidBYaCa0"
"FiYLc+HuJq1I15NhZBcJp9VobjqHfMF/4K34LN6HS+JDaACajY4CXv0FHectuMRdzMds/CNY4ho2iDVi9Vg1VpR9M/+B6h2z"
"7mw2W88WsXXsFrPwxfwHb4I+o214FqlOT9FywkqhipgkXpNmyW65PlS4awFreJXcaj91rjpWbawSNZtaDjDtVSW7ckPeJJ+Q"
"H8lz5ZbyFSlacon7xV/FIuJrYa1QWfhEe9PDgJhHYILvAzYcjt7zXHwy1MjXzbNmZ/O2Ud9I0LkeYvyp59evaCF6V/2HVl+7"
"FFS1odo+bZWWR3MGfcHcWh+oD19oH7Uw3dSi9G76Dv2+Lhn9jM9GTXOYudzMxY6z3jyBt0NnUGmcgJeTNyQDNL2KZhOmCjeF"
"/cJzIVVIFz5BZXUCetqAVOsKOj0K8hxKB9ESdCqpSsaTg2Qy0fGvuDy+jQlZgp+hTegaKoPD8E70htfgE3gldB81Rvl5GkM8"
"hl/ij/g4OFNP/op3Qh3Rc76fS+hPFIeeo1j0BMWji8BREfwPfoR34qW4J/6EB0HkOIcL4J7ER6uLTQERHRMKiUfF/NI06ZM0"
"UE6Rdfmn7JQny4XlgDRIVpQIpZscKp0SPdJguZvwinQVeymX5FhiomJ0txAmuPAEEi3cEeNJB3bfIKiqsFFYgzqbETw77sH7"
"m3HGbUZxU15AfxJYbj4jt9DcQE1Pu8AEQA+GVs/XxP3I89pX3dfWc8izxVfQ9ynjJkTWFs4Qd3bPNvfbjJ0ZroyJbuQu7C7p"
"669PRXdISTyC1TNv80pCbmWEJcaaCnk3TGksL5AXKw3UfOohiGOfxIOCHw9jHY1nxjeWD6Ww7sbrYOFAw0CJYIhe1jzDCnCF"
"tTE/mJv5e9yUWoQVgiFsFpk4TmwmCDQnrSnMEE+Jd6gPcLLf+M2MZTX4HLbTKA45aoB2XCuiT9ZT9KrGZOOm0d7MzZqxsUzh"
"VxChX+gGmkSO0q/SHMs72y3bQ8vfag+L23Y47GB4IIyF/hXSPWSibYxtrO2sLbstzXJWraO+VLyyJNcBlBgqD5BbSNuEUPqV"
"nKIvwVZa0f34CLrOh/BjLM50QcyYpY/TfIFFvl7e7p47npKemu5Ql+z43V4rfaq9rn2m470jxWHY+9gD6Rn2ac69rnoZ21yf"
"nE8dCY7Rjp7OEMc654yMyoFpbANgg799Yd5xwSn4jFLWFmodofwqrKbJQnO1ZsjQsCahf1vbK1ukW2J1MVY4KcwFO9kudMOD"
"zc6Qp8ELeAymGHGXPkILaJfN3XiesEDcKRSnPch0spdeEqNkopyXndJj0SL0IrPwc2yjOv1OP+EdbIveReun7zQT2F2WYTzT"
"OgVZ4FZwmE7NLhA5Nppe44KBWU/+FY0kPehPuoLaqIcspZWFdWKsfMyywXbEdkBdIbeVN1h+C8uIbBDVOqJ9SJp1sG1m2KjI"
"6KgGkRfCZodE2wxrrDVo/cP6wTrXGmrppNjhLKOEN0J7abTUWcT0JW/DnpqLWCLfxAexNvqXwFH/BH/dAA52CTz2j/HW8XQG"
"lLnD293bx1PD43V/gadmnuvuhxnRbu555Z3pu+BZ7lrveuh65on3fvTMz3DZ36UfcO7z3te7Glv09to37atxlX8lqXK0Jdq6"
"3XLOEms9YIsIuRA6I+x0+NDwAaHRNq5slffJR2Qmq1I+YQr6aAzQl+kB46d5wLip19RcgRrB6dpdoxl7xfbwKMRRbfwb/pPs"
"p5XFNGmjlCB2E24SCy1KI0FTC2lu3IntN44bb4x044QxXN+rXQxUDZQIJPsbBotqHfW7ekv9oD7COGpW4xfRX+QSXSh4BK9Q"
"XzwjSvI4Ja/6UPkgf5RqSM2lF1DlfJcaSYOkgdI0eZTUWxorFZEHKK3VDLWL5ara1nLU8tmaIyQyJNVW0NbaehckftzaDk7K"
"1dNKXXmrVFBKEqPEAO1FC5ICOMh/5cO4G07iN8eZd40YyIHVjdzGMWO1EW6YelP9qH5Hf6/XNsrqY7SZmk+rb9zTYoJ3/N99"
"Df2T/C8Dq7SPgak+X0bLjK/uEr6R/k3+Hd5ZGcmO2/aSzrYZZXzegKwtDxjeBb6+2p/sJCkF3nxTXCMa4i35rGVZSPbQ1qGX"
"bLr1hWWv2lTNrf6i3lZ+in+SQzzNzM5izR1mE7OF/jTo9yf6EwOdtK96DVMzC7IE8xY7juKJKnaUvksDZIu8Vm6jvFA2qyfU"
"HGpRpadUV4wWooU8wgTaj1RBSWyYed/IYVY0b5g5TMmw6o81ppc3j7E/+GP+FOJ1EFUnOWgd4bQ4X5okHYT4c0s8DDX1KDmn"
"PA+qyGNCUaGTUFvYQJNJKGSFEXgjqoVa8OmQOynqie6iUtiNp5Ph9BJUtyOl7vJ+ZaQ6Sa1guWIRrUnWN9bxttfW65aClli1"
"r+pRHsjT5ZJSM/EFrUQfkglkAN6B7vETPJ2/52/5CI54bV6GV0P90Xv0GvVG1RFFQV4V18VLcF4UZAvZUraJSXwcY6YXrOpc"
"cF+wmjZYuwvect230fvYG+W96W3stXuXeZO9g72jfdhnerODx2ieGH/LwIGA7o13L3ZJbp/ns39y8F6wTGCd76gvEDxi1kHh"
"uBtksHD8lZhCWfmJMkph8g95qDJFWaeUV+Yrk2SvtFE8S+/i+ugyyG8f97L3RiftRKBioFPwnvZOf6pP174G04O59QqmzNei"
"vWgMesGroI54EJ0LFv6LcJdmh5PXhXiiCDPpFvIJsmsRXodnwNkqsjNmS3Oi4dSHAzZYaNSGDH/Q/N2MMFeb81lfNA5bSDw+"
"iC/j4WQlHSLUESPF08JZ+gQQcZTAaFPhDBVpA9IJf4As3BNvwX6UyntwkS/lW/gD/pKf5PlQTkTQRJDtZfwDHyOlSVFSjvwk"
"u2gh+ZY63BqrtBRvCLklXW1giwm5Z30ia/QsLS19scyw7lW3i4/xKvQNmbSJ1EJsSbsjJ3sC2cxEGvbgDngef8Pqs3GsJMqJ"
"LbQVOYiK8aGsNHei4kSmS/BBHg8WWYZ15dPRH3yXWc7YqjXWFL2+cc7w6D+D4wO5/B/8CwNbg9eDawNvfMu8y71nfZ/8yYGQ"
"YJVAuH+dL+hL9r8OSNru4NZAG8Cgyb7K/iHglaavju+NZ61nmfeQ/0bgQ4AGogNnA5u0O0ZDbgHc4uQd0QXcX2gnXZRySylC"
"vBAQfpME+bX0u5hBF5I5+Djglt7oGy/MT5qTjIL6ccizTfUJxndjmPGrflOL0IdB/OsAqKY72sdX8UkgwdV0hDBAuEcdZB/J"
"DnKaKPygfahIOuI/0G20D61Cy3gO/sP8Zo40k81TZln203SZO808LIZl4624xsuiQ+gn6g54NgeZSl4Dzi1L29OF9A96gQ6m"
"PWkZWp6eIevJIjKQNCTpeAh+DzY1DC1Dh9F2NBhFoIN8B//ER6FkZMWv0Snwyc+oLt6N47EXNFyG5APMlRcyZRM8F88B9DUN"
"VUWf+Tn0lGymi2htfBAsoh3ZQI+LE0VVmItr4MWkgrhPOif+IswhGjbwG1oBcvAq6sFVcB2chseSAnQJ4fgFmouao79RNL6B"
"r+LO+BmvxPvBiS5Bj4pdfAnbY3KzIH/Bz4NdTDaHGSWMQ8ZaswfrYYYZT7VnWjc9ylhotDEW6wO1O5AZLmiadlcrpcnatuCc"
"YKvg8uCtYB1tp7Zfq6sVB0RcWOurubRhejP9kvY4GAjm1dZoKcHdwa+B9oE431jfLX/rYDDYJZgW6BS4F6ivXTR6A25dzXez"
"KrwsjhS+iYa0X7oIOL28RJQNUJ8fl03psLhfKCu8A83NJs/xO/SUt2A1TQm8DvKfqZjTjRCjqNHT/MEOg9ev5PNA3m9REO8E"
"NFBHWAzzFtDFQmnxunhPdAoXaCnamJ6morALPH0XyLcxmoI+ITvUAU4+lecDC1vMG6NnaD9YksZrQ3XQAktkA7lOupAK5DeC"
"6T+0uTBCoLBCY1qf7qC1AIlFCvkA2dwFNC2CHTzG1fFKpPNlwM02fpd7uJcfBPlrrBTE0rloCrlLCN2FV+MreAf9IVSVTHGU"
"aNI0ekEoJFWXR0itxLpCPL0D9b5T2C10oJOIieuQaWQumUU+YahV0W9oLWIoDb1EfVB+tIB35Im8NVqCBiEVjeBNAft/56Mh"
"lqSDN3xjn1kDfhsqkY38LcvDyrOS7G+WDFSUXTTzmQ3NoeYnM948aYabF4zZxjJjr+ExrhlrjMZGFcOlNzfyGtkhC/cGSRcy"
"DF3XX+g39WqGYUjmPWORscAoYqwyJhjbjQ+Aek9BpXkCaiMXRH9Zp8ZnvYeeDlXSDn2LKfEcqCiKBT4eo5NkidBa7CEmCYOE"
"CUJZMZdUVfoilhJLCoOpi1yCk4aSebgfeKmfLWBlWS3WkxVgDyDmTjefm9HMyabwhXw7j+V+QNeHcQhpCxIqC1X3JFIeasZH"
"oOupJCdxQUSeDh6ajNvh2agMYmDzSZDVXvO2/ALgjHdmPkZ5V96Mn4PqcxeL5PGQFRag0Wge1K+FoaJ9g29BDLgD9ZJK+pGl"
"pDv47Rt8AE/HM3B9XBG/RVsgOiXye4ATN4D91AWPSmSPGeEvWSrLz/vw8lCZKVCpreU/uZ1f5M/5bIjLR8FXn5G+OD9+hr/R"
"cDFB/CKMoflJFdpPWC01kz4Jg+hkUonE0F1CmnCbRpGf6ADEj1nATUWcG5Xgw1lr9hL2ymCbWV7GjarGF+Oc2Zc1YGvNw0Zl"
"o7MRajrNtuyC6TdqGaVAQwUg0iWZVcwWRub/kSpnjDBaGYn6bEBWrfWaemn9ofZKOwTe30JbrH3VtukrdFnvpaUHPwYfapHG"
"buO23l+bERS1QfpOI8FoYVzRNgUnB4fq1c3b5gTDDk8tgh20WP2AEWkc0ioHjwRqB84ECgdzabX0aVBHm8FhwU3BNvpHcw/4"
"wmIWYdYyv7OLWBDaCXPJSbCNENyZVhIThLx0HqrH/+RV8ESSg0xGvVl1UzZfQ4Um8pHsF7OYEdSnGtPMOnD2juwzVO5fzSfs"
"B98OmX0haohKodVoEG5F/oQa+ig2UDbcEGrwAmQcXgQV8Vq+HuJFgJ/mRfmvLNlMNXuy2+wHm8/c5iPTY66ASroxj+ClOGdN"
"wFIqo2gUwKWon06k+0k0vSdYAH80VprLJQBhF5WvKbvUi8ouaZyYU2wAttxYGiEuojvxLaizk7CJo/FRHsZGm6/MBawo38QO"
"md+N341NRjtTZArbbJrGTyPJKGHOMLeaG8w+ZoaxzmhvdDfyg34GgCY362+1G1qaVhF0lK7N1eppWCukjdUua7u07oCE/9Yu"
"aY+04lB7RulYb6iv1f36Ez1WZxDF8xtljUHGTqODEWF81csY0fB8yzhvtATNn9bP68eMp8ZRw69P1wfoY2BORV3TdmvttSda"
"DtD/St2mH9WG6E0MK1TFPVgFVpltZFd5PDqJd+D7+DTW8ABikmq0CN1EOpM+pAy5jffjXTgP3gY5qAh6D9k0yA6yADvPtrAB"
"rB3ryiaxdNYKUGVR3gAiZHPIyHlxBdwCb8Yy+YMECKI7yQVyk1Sko2lniml7cgnk2BM8OTeJxhPAj1uDd37OnIcW8t58HQ/y"
"pqgGikQPOEJ7kIBDwC/forKQM7dDbGgC8fsFROcA3g94oL+gizHSXvEBDRO4sEQ+qvRRo5RYQFfHxXrKR7WKmiAFhexCTaG/"
"9Ctg+TNCfTIMFyHT6Sc6nwZxKJoBEaQmPoXjkIx+5QIgiWdZ/z9tOrcD6nrKPjAXa80L8sesGgsCpkgzK7LZLJrNM+1GqpHd"
"3GdeNqeapc32Rro+3GgMz83NP4yuoPPFRnmwv1um1exnLDGY8c58b640J5oPjUuGag4yR5uFoAYZZa6A3wwjxGxh7jYlFsZe"
"mH+aE0wCWPU31oodBQ4UvoddAzk35GtAxicgwiWznFAfVASMUZUfYe1ZDXYKeJ3F3piNzApmN/O8mWJeMTuayGxjPgZLvccO"
"sBOsMh8Lc74AbmwNES832Uzy0xi6mt6nNkDN+4UzQpywWTgEGNovHBGWC4uEacIGYYXQWsgvvIEs5qTRQlXhDh0A6GkT/Uwj"
"hDDhHO1LR9A9lIKMswkfICcvAmT1lHrobcjZeagVcFs8OUQWkpLkOF6J/8Br8C84ATDUTPQraomaoCoomZ+BU+3kY/gQPpoP"
"4Pn5a+ZmYZBn4yD6XmYzWQfWh/0JVrcSbG4Q6wTeHcdk7mGxEOcNFs3nQBwewpvwipBHdJ4TneLt+Ax+E3Lpab6cl+aJIL/T"
"7B82mjnN2SCjbmZuwBxTjOZGTfCmTB/abQwxphnfjJVQHeaFODLf3GN2gsxaAmTdgSUCqlzDMmP/FPan2d8caNrYNjaDOSBa"
"5THXmB3YWeBvBLsPntWTPWCD4BTPYN5A0GIhfoyf4934PqaZ7VhuyC4ePoEfg3eH2G98EFqBHMD9L5BNjqJeeBXuiL+gE6g5"
"5KjF5CHIrT5g0t+JQAfR6bQLIJVcgFcH0aX0OX0GuptBp9FdoEFF8NFXUKXsAVzzmJo0lxAuIEDK72gcdVMXzSOUFpLpXnqc"
"vqcBKgspNJ4mgo7O0GP0Mj0H1610Dm1NI6lB/iEXyRrItDlJCkSFvoCCnOgaZNHeKDs6C3KtDNXObfYXG8PysX/MhWZb8zez"
"u1nHDBiPjT+NbXANMweDV3SGqjjaXG5WAinMYb8wysLheoF9Z59YP3bHRCChC2wR42Cru83rUEHbANEkmTpg+UKAbxLN/FDn"
"bGI3IDdeYW1YX9DgHL4L9D6DTQdrqMj/gHqrFlSqMvjqKf6RL+CxLDfkkO6sC8g0lV0xx5onzPqgncfA7XKzgfnSfAkYys2G"
"slxgSRfZYO7ml3l2/gkqqFjI6REoFOLcfrivgOYDTUTd0Ep0ErBlFTwI98R9cCwOJ+1ISzIOsA0nrcAb9tBVdBlIuq1wVYgR"
"GgrFhYrCDuG94BO2CNWFdFpDOCDsgTc24Ss9CXrjtKTwnY6lw0GjW0CTo0GfPWgL2px2h5o9B5UgTn4mxyBG9iZNSSo+D1G4"
"J7ZAvmuOCiGM4gCbNOGP2E6wycxqPYaVY5/MJ6YXrLQAE1kEK8JGwpt1bDf7BtF5KoxvxkuAbxREf6HJyAOy6QxW+BdKRfkQ"
"Bvm9ZHY+BCLgHrbRPGyeZD7+hhfgCeZ4sybrwXsgO7/BcrLGbDHvDRK5xW+x9Swc8Noy9JaH8f3sKqvAu/OZPIT3Y7VZL5bE"
"7oDH5mdW0Fpb8N+hrAlzmcUga5wDPxwPmf4j+MV4sKUVIPvi/A5vC7IuDRX/Rn4B4v8OyP3loHpdwHOh8+g71HgR6AO3gh4a"
"44E4HY0EnRTEx7AbqqKWuDTuhr/g/qQu8eG1gPFU0gFQZyNSk+g4P0S5O+Qc5LPGgPoXkPPQJpI2gP0WgH/NIc2BOoB/7YAZ"
"MaQaaUbGQN5aQlqRPKQErNiF1CGYnMDH8Vf8Cf+N22IRJ0EV2BgXww4UA/KrApzfBn7qgd084gSeqwO37yDuzIOYdgEs9A6c"
"sy54+C98OOS3pTyGV+HR0NOfb+Vn+QNAv1P5Zs4ArRZBtVBhVAxVg6ozFs6dAtcZaCj6A71AOXADQD/zwQZi0D9QzXaAKDoD"
"0FFtFI/64SlYwNXQd94eEdwf94J3ZaFyqYycaDzOh7cgBSE0AOJKdvwTdUZfuJN3hrwbD/VtEcTBBzqDhb8EVDoD6qVBaCfS"
"kIyj8DW0FWT+DZXANjivHR0DPrzoF1wV18YI5JAT+gfgybg3rgnv2wFPeyBLL4GePhDJNuFrED+24EOABg7B3TnQzVl4vxIw"
"xd+AfXeCL+2E380wFmo4sPIL+B6+iY/i61B7fcXpMP4ESP0baPcFfgn19k5A6/vwZbwY/4qbAw4cj3/HkwBDExyGa+HfQEIN"
"sQR6aYy742kQSfNhDjh7CnjuQOBPxSWhuh6AO8HJ7qPHyAZzauMfYGFX4akYrgZzn0DN74c5Q2F9N5xVwiNhz93AcyWwtNX4"
"FQ7CGWbiIVBP3McZ2AOn7I1H47v4LT6Jx+GyIIVecP4FuD2OAC1UwIPh7S+YAo/FYeQ6OPkUsNhieAR+CLXINuCtLm4C1xP4"
"COyVDeuoAIxsB/uXwCnoOHoEVpADZP8CUNAZdAc9QXHIA56bBHyGg9YzUC7cFFaoB3G6GlBFeOoBZy8Ke/qRgq1YQ270APR4"
"Al1Cr9EhqHJmgkedh7sD4Ge7AGUdQgkoACgsDr1D6VmaL4Jz47xYhhq1DkgtJ1h9KPSaIJGnMOYruoXuocuQxcehsWAzM9Bc"
"NAZNh+sk8M3xsGLm10br0Z/oAljwKRj7Cj2E2BMLfFyD5+3ob5h9Aew4HnoWoo5gZ3HAwQPwoxhA6KPA0r6g56g9VHkR8HQQ"
"bHUVaoUk8KwY4Pwi6oWyoVyoDRoIz0MAL3ohducCPFEc+fg/PJVbsr7LskFzAJ7rhPqiquAPAioP/tMcVYZxAYhxGbw8zHXy"
"q4BAbvJX/BvEtDi+B/LzTOi5Dv47iQ/iXSDa9uUT+TTeizeCyDkK3o4E/LUavPYoP86vwHUt5KVrsO9tvgXmDOBzwdO381kw"
"PhQwc33o6Qy45ht7zt6yCMApeXgG5P9tkJdUHgkY4x92GCiOBSGuMvYOMtMz5oPa2M88gPzesIcQo++xF5AHr8P1NvzugAz4"
"AOgji4fK8SWsYGcCz8utXIV6pwjE+/K8NuxUCCJsLdi9HUScXLBvDog+9Xh96E/KmpON5wSk9Ra4OQVR28841Lqn2XZATKks"
"FNZKhVG3YQcJcFQYd7LPEN3DYJ98gJZdgNMMVpiXhCedpQDHCE4j8gCMT8j6gkmHmvYDcPg16ykF8lECy/wXZD9gqcx/S9Zh"
"fjjwLMFqHDCdwA0WAs0LY3U4T6YMMr+I4jDHBOlk7v+NMajJ3bDmOzj9VxhpByl9YjdBMq9h/ZfsFrsE0jnFjgBCPMk2sFWA"
"HLextWw5mwd36yB7zQe0PIdNAyy/AbLRQshQw9kU6N/H/mDLoKoZAW0zzN4I2Ws0GwLzFrAlMHoYIJe+bCqMmQH9fWHWDMjA"
"2wCH7oCxu2DXJ+wV6O4+aOwVcPMJatFPwNFd6HsE7Snc3QVOrwCufQTvnwLPz2HOY/ae/QSJp8GJ3kCzg4R0kIcGJwsyK+gw"
"G2iVQZ+Y9YWZyhFPgLVTs74ZSwRJPIP93rMvgFlT4PoeUFYCzMWcgEw1kLKPabCeAjoUuBfeuOGdBTK2BTSX+RUagvUj4UmC"
"0UF4p4AeEcjfAfIPQnOBTtJhxwSgL6DTz7DHBzhB5lnuw1nuAd1kf4PMTwFie5aF2U6BPG7AGd8D3YOnk5D/n8Cch9B/HGqb"
"83DuNzD7HNh+LEjkJTxfh/7tgFxOgA7Psr2AGdYB0jkNY87CmL9g5EUY9Qxs9Ro7A5b5AlZ4AatfgBkXofch+ETmzifg/Y0s"
"zu5D73XofZxlH09h/AewoR8gtUzJpcPZvf/5WVrWd3SMUZCaAXWuASdnIBkdbA+BNEx4MqE/CM0DErHDDCfc+0G6Bsz8l4ws"
"+8xcReaZI3WwXpYlvUSQpi9rTS9I8SM0b9a85CzNpWRpiAK+/Q7c2WFU5peCOoxxQKOgEQY7uWCcEzTmyfKLn2BdicBFpgc4"
"4ZoEazmybMYH751gHQ6Yk8m9BnNSss6dBL0JWfvHAb2DnTNl8SVrpWSYw4CLf79htIK1RPIoiFyleTGoggtlURmIKqXA54tA"
"1V8BYkwT3hjiSQ2oResDzmzD2/JWvDn8doZ4GcMH8t+zrkP5CIiM0wHrTAGaDL9zAN8tgjYPouVivgqq240QRxfyJXC/EbDR"
"Zoihq/kGiMKxgJNO8YOAy49CXL4FNdRtoJtQDb/g7wD3v4Xfp0AvIYa/4E/+o8fQnkM9/5rH8w8Q239Cre+CPGEHvOOG2O/M"
"ujqA0uDXBe8/8s8w7itP4Mk8Ba6JWTM8PIn/gJYIfZlfun6AFd/Avp8z/44aKB5mpMCoVOj5B3Z9Du/igJeHwN8d4OE1PL3L"
"4vMb7O3nBmA9g5tQqQazyOAUWSFXyUgEnGaDKisfygF1iIwsKArlRjmhT0SZowXokSEneoBnDZ4yv0PWss4QhDU1HgB+0+HZ"
"CyfL/NtqB/xqWd/oemBfjWd+s4wgQ2a+C3IF8imGJy/wYoV9MneKhL0iUQHA4uWBiqKCgDCrZH2pXAdQaUVUKeu75frQakNG"
"rQitMqoAWLMkUGkYWzyLMr91LoTyojyQo3PBunlh5QhYNxKFoXDYQYLz/PvVNAb+vf/x6wPOU7Iknwb3buA+Fe7c8NYP5M3i"
"WoNz/vsHAW4lsIqUJbd/v8KWQH4yXKX/VpeyrlAI//elNs36Jf/d/89vuDMl+b+/7/73G+/MHTL34P9jv/97/ffd/3369/d/"
"AYxHlHJ2PgAA"
)
@lru_cache(maxsize=1)
def ready_trigger_pcm() -> bytes:
return gzip.decompress(base64.b64decode(READY_TRIGGER_PCM_16KHZ_MONO_GZIP_B64))

20
litellm/llms/vertex_ai/batches/transformation.py
View file

@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any, Dict, Optional
from litellm._uuid import uuid
from litellm.llms.vertex_ai.common_utils import (
@ -47,7 +47,7 @@ class VertexAIBatchTransformation:
) -> LiteLLMBatch:
return LiteLLMBatch(
id=cls._get_batch_id_from_vertex_ai_batch_response(response),
completion_window="24hrs",
completion_window="24h",
created_at=_convert_vertex_datetime_to_openai_datetime(vertex_datetime=response.get("createTime", "")),
endpoint="",
input_file_id=cls._get_input_file_id_from_vertex_ai_batch_response(response),
@ -207,3 +207,19 @@ class VertexAIBatchTransformation:
parts = model_path.split("/")
model = f"publishers/{'/'.join(parts[:3])}"
return model
@classmethod
def is_unmanaged_gcs_batch_input_file_id(cls, input_file_id: Optional[str]) -> bool:
"""
Returns True if `input_file_id` is a raw gs:// Vertex batch input file (i.e. not a
LiteLLM-managed unified file id) with a `publishers/` model path that
`_get_model_from_gcs_file` can parse.
"""
return input_file_id is not None and input_file_id.startswith("gs://") and "publishers/" in input_file_id
@classmethod
def get_bare_model_name_from_gcs_file(cls, gcs_file_uri: str) -> str:
"""
Extracts the bare model name (e.g. "gemini-1.5-flash-001") from a gcs file uri.
"""
return cls._get_model_from_gcs_file(gcs_file_uri).rsplit("/", 1)[-1]

16
litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py
View file

@ -1,6 +1,8 @@
import os
from typing import TYPE_CHECKING, Any, Dict, List, Optional
from litellm._logging import verbose_logger
import httpx
import litellm
@ -52,6 +54,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
return [
"n",
"size",
"imageConfig",
"aspectRatio",
"aspect_ratio",
"imageSize",
@ -83,7 +86,12 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
mapped_params["aspectRatio"] = v
elif k in ("imageSize", "image_size"):
mapped_params["imageSize"] = v
elif k not in ("tools", "web_search_options"):
elif k == "imageConfig":
if isinstance(v, dict):
mapped_params["imageConfig"] = v
else:
verbose_logger.warning("imageConfig must be a dict, got %s — ignoring.", type(v).__name__)
elif k not in ("tools", "web_search_options", "imageConfig"):
mapped_params[k] = v
mapped_params = map_gemini_image_tools_params(non_default_params, mapped_params)
@ -211,16 +219,14 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
# Prepare generation config
generation_config: Dict[str, Any] = {"responseModalities": ["IMAGE"]}
# Handle image-specific config parameters
image_config: Dict[str, Any] = {}
# Seed from user-supplied imageConfig dict; flat params are overlaid for backward compat.
image_config: Dict[str, Any] = dict(optional_params.get("imageConfig") or {})
# Map aspectRatio
if "aspectRatio" in optional_params:
image_config["aspectRatio"] = optional_params["aspectRatio"]
elif "aspect_ratio" in optional_params:
image_config["aspectRatio"] = optional_params["aspect_ratio"]
# Map imageSize (for Gemini 3 Pro)
if "imageSize" in optional_params:
image_config["imageSize"] = optional_params["imageSize"]
elif "image_size" in optional_params:

126
litellm/model_prices_and_context_window_backup.json
View file

@ -1154,6 +1154,7 @@
"bedrock_output_config_effort_ceiling": "max"
},
"anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -1203,6 +1204,7 @@
"supports_output_config": true
},
"global.anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -1237,6 +1239,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"us.anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1271,6 +1274,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"eu.anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1305,6 +1309,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"au.anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1471,6 +1476,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -1505,6 +1511,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"global.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -1539,6 +1546,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"us.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1573,6 +1581,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"eu.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1607,6 +1616,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"au.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1641,6 +1651,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"jp.anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 6.875e-06,
"cache_read_input_token_cost": 5.5e-07,
"input_cost_per_token": 5.5e-06,
@ -1672,16 +1683,16 @@
"supports_minimal_reasoning_effort": true
},
"anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -1705,16 +1716,16 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"global.anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -1738,16 +1749,16 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"us.anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_1hr": 4.4e-06,
"cache_read_input_token_cost": 2.2e-07,
"input_cost_per_token": 2.2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -1771,16 +1782,16 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"eu.anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_1hr": 4.4e-06,
"cache_read_input_token_cost": 2.2e-07,
"input_cost_per_token": 2.2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -1804,16 +1815,16 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"au.anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_1hr": 4.4e-06,
"cache_read_input_token_cost": 2.2e-07,
"input_cost_per_token": 2.2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -1837,16 +1848,16 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"jp.anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_1hr": 4.4e-06,
"cache_read_input_token_cost": 2.2e-07,
"input_cost_per_token": 2.2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -2082,7 +2093,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"bedrock_converse_supports_strict_tools": false
},
"anthropic.claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 3.75e-06,
@ -2409,7 +2421,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"bedrock_converse_supports_strict_tools": false
},
"assemblyai/best": {
"input_cost_per_second": 3.333e-05,
@ -2710,16 +2723,16 @@
"supports_vision": true
},
"azure_ai/claude-sonnet-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -10474,16 +10487,16 @@
"supports_web_search": true
},
"claude-sonnet-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -14813,7 +14826,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"bedrock_converse_supports_strict_tools": false
},
"eu.anthropic.claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
@ -20074,7 +20088,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"bedrock_converse_supports_strict_tools": false
},
"global.anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.25e-06,
@ -33158,7 +33173,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"bedrock_converse_supports_strict_tools": false
},
"us.deepseek.r1-v1:0": {
"input_cost_per_token": 1.35e-06,
@ -35210,16 +35226,16 @@
"supports_vision": true
},
"vertex_ai/claude-sonnet-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -42677,16 +42693,16 @@
}
},
"vertex_ai/claude-sonnet-5@default": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,

17
litellm/proxy/_experimental/mcp_server/db.py
View file

@ -1228,6 +1228,23 @@ def _remaining_token_seconds(expires_at: str | None) -> int | None:
return remaining if remaining > 0 else None
async def get_active_submitted_mcp_server_ids_for_user(
prisma_client: PrismaClient,
user_id: str,
) -> list[str]:
"""Return active BYOM servers submitted by this user (creator visibility)."""
if not user_id:
return []
rows = await MCPServerRepository(prisma_client).table.find_many(
where={
"submitted_by": user_id,
"approval_status": MCPApprovalStatus.active,
},
)
return [row.server_id for row in rows]
async def approve_mcp_server(
prisma_client: PrismaClient,
server_id: str,

47
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -390,6 +390,46 @@ async def _store_per_user_token_server_side(
)
def _raise_if_not_oauth2(mcp_server: MCPServer) -> None:
"""Reject a non-oauth2 server from the gateway's OAuth authorize/token/register flow."""
if mcp_server.auth_type == MCPAuth.oauth2:
return
raise HTTPException(
status_code=400,
detail={
"error": "server_not_oauth2",
"message": (
f"MCP server '{mcp_server.server_name or mcp_server.name}' does not use OAuth "
f"(auth_type={mcp_server.auth_type}). This server does not support the authorization-code "
"flow; it has no client_id, authorize, token, or registration endpoint. "
"Access is controlled by the server's configured auth_type and access groups"
),
},
)
def _raise_unless_oauth2_discovery_server(
mcp_server: Optional[MCPServer],
mcp_server_name: Optional[str],
description: str,
) -> None:
"""404 a NAMED discovery request unless it resolves to an oauth2 server.
A named server that is unknown (or hidden from the caller) and one that exists
but is non-oauth2 both return the same 404, so the well-known discovery paths
cannot be used to enumerate non-OAuth server names. Root discovery (no name) is
unaffected, and pass-through servers are resolved by the caller before this runs.
"""
if mcp_server_name is None:
return
if mcp_server is not None and mcp_server.auth_type == MCPAuth.oauth2:
return
raise HTTPException(
status_code=404,
detail=f"MCP server '{mcp_server_name}' is {description}",
)
async def authorize_with_server(
request: Request,
mcp_server: MCPServer,
@ -457,6 +497,7 @@ async def exchange_token_with_server(
refresh_token: Optional[str] = None,
scope: Optional[str] = None,
):
_raise_if_not_oauth2(mcp_server)
if grant_type not in ("authorization_code", "refresh_token"):
raise HTTPException(status_code=400, detail="Unsupported grant_type")
@ -582,6 +623,7 @@ async def register_client_with_server(
token_endpoint_auth_method: Optional[str],
fallback_client_id: Optional[str] = None,
):
_raise_if_not_oauth2(mcp_server)
request_base_url = get_request_base_url(request)
dummy_return = {
"client_id": fallback_client_id or mcp_server.server_name,
@ -655,6 +697,7 @@ async def authorize(
mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
if mcp_server is None:
raise HTTPException(status_code=404, detail="MCP server not found")
_raise_if_not_oauth2(mcp_server)
# Use server's stored client_id when caller doesn't supply one.
# Raise a clear error instead of passing an empty string — an empty
# client_id would silently produce a broken authorization URL.
@ -1063,6 +1106,8 @@ async def _build_oauth_protected_resource_response(
detail=(f"Upstream oauth-protected-resource metadata unavailable for MCP server {mcp_server.name!r}"),
)
_raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth-protected resource")
return {
"authorization_servers": [
(f"{request_base_url}/{mcp_server_name}" if mcp_server_name else f"{request_base_url}")
@ -1149,6 +1194,8 @@ def _build_oauth_authorization_server_response(
if mcp_server_name:
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name, client_ip=client_ip)
_raise_unless_oauth2_discovery_server(mcp_server, mcp_server_name, "not an OAuth authorization server")
return {
"issuer": request_base_url, # point to your proxy
"authorization_endpoint": authorization_endpoint,

107
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -1246,6 +1246,67 @@ class MCPServerManager:
"""Return server IDs that bypass per-key restrictions."""
return [server.server_id for server in self.get_registry().values() if server.allow_all_keys is True]
@staticmethod
def get_byom_submitted_servers_cache_key(user_id: str) -> str:
return f"byom_submitted_servers:{user_id}"
async def invalidate_byom_submitted_servers_cache(self, user_id: str | None) -> None:
if not user_id:
return
try:
from litellm.proxy.proxy_server import user_api_key_cache
await user_api_key_cache.async_delete_cache(key=self.get_byom_submitted_servers_cache_key(user_id))
except Exception as e: # noqa: BLE001
verbose_logger.warning(f"Failed to invalidate BYOM submitted MCP server cache: {str(e)}")
async def _get_active_submitted_mcp_server_ids_for_user(
self, user_api_key_auth: UserAPIKeyAuth | None
) -> list[str]:
submitter_user_id = getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None
if not submitter_user_id:
return []
try:
from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415
get_active_submitted_mcp_server_ids_for_user,
)
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
except Exception as e: # noqa: BLE001
verbose_logger.warning(f"Failed to load BYOM submitted MCP server cache dependencies: {str(e)}")
return []
byom_cache_key = self.get_byom_submitted_servers_cache_key(submitter_user_id)
submitted_server_ids: list[str] | None = None
try:
cached_submitted_server_ids = await user_api_key_cache.async_get_cache(key=byom_cache_key)
if cached_submitted_server_ids is not None:
submitted_server_ids = cast(list[str], cached_submitted_server_ids)
except Exception as e: # noqa: BLE001
verbose_logger.warning(f"Failed to read BYOM submitted MCP server cache: {str(e)}")
if submitted_server_ids is None:
if prisma_client is None:
submitted_server_ids = []
else:
try:
submitted_server_ids = await get_active_submitted_mcp_server_ids_for_user(
prisma_client, submitter_user_id
)
except Exception as e: # noqa: BLE001
verbose_logger.warning(f"Failed to read BYOM submitted MCP servers from database: {str(e)}")
submitted_server_ids = []
try:
await user_api_key_cache.async_set_cache(
key=byom_cache_key,
value=submitted_server_ids,
ttl=60,
)
except Exception as e: # noqa: BLE001
verbose_logger.warning(f"Failed to write BYOM submitted MCP server cache: {str(e)}")
return [server_id for server_id in submitted_server_ids if self.get_mcp_server_by_id(server_id) is not None]
async def get_allowed_mcp_servers(self, user_api_key_auth: Optional[UserAPIKeyAuth] = None) -> List[str]:
"""
Get the allowed MCP Servers for the user.
@ -1259,25 +1320,30 @@ class MCPServerManager:
allow_all_server_ids = self.get_allow_all_keys_server_ids()
# The key explicitly opted out of every MCP server. Return zero before
# layering on allow_all_keys or submitted servers so the opt-out is absolute.
key_object_permission = user_api_key_auth.object_permission if user_api_key_auth else None
if key_object_permission is not None and (
SpecialMCPServerNames.no_mcp_servers.value in (key_object_permission.mcp_servers or [])
):
return []
# Check if object_permission.mcp_servers is explicitly set (not None, empty list is valid)
has_explicit_object_permission = key_object_permission is not None and (
key_object_permission.mcp_servers is not None
)
if has_explicit_object_permission:
verbose_logger.debug(f"Object permission mcp_servers explicitly set: {key_object_permission.mcp_servers}")
# BYOM creator visibility never widens a key that was explicitly scoped:
# only keys without their own mcp_servers list get submitted servers unioned in.
submitted_server_ids = (
[]
if has_explicit_object_permission
else await self._get_active_submitted_mcp_server_ids_for_user(user_api_key_auth)
)
try:
# The key explicitly opted out of every MCP server. Return zero before
# layering on allow_all_keys servers so the opt-out is absolute.
key_object_permission = user_api_key_auth.object_permission if user_api_key_auth else None
if key_object_permission is not None and (
SpecialMCPServerNames.no_mcp_servers.value in (key_object_permission.mcp_servers or [])
):
return []
# Check if object_permission.mcp_servers is explicitly set
has_explicit_object_permission = False
if user_api_key_auth and user_api_key_auth.object_permission:
# Check if mcp_servers is explicitly set (not None, empty list is valid)
if user_api_key_auth.object_permission.mcp_servers is not None:
has_explicit_object_permission = True
verbose_logger.debug(
f"Object permission mcp_servers explicitly set: {user_api_key_auth.object_permission.mcp_servers}"
)
# If admin but NO explicit object permission, get all servers
if user_api_key_auth and _user_has_admin_view(user_api_key_auth) and not has_explicit_object_permission:
verbose_logger.debug("Admin user without explicit object_permission - returning all servers")
@ -1299,6 +1365,7 @@ class MCPServerManager:
in_toolset_scope = _mcp_active_toolset_id.get() is not None
if not in_toolset_scope:
combined_servers.update(allow_all_server_ids)
combined_servers.update(submitted_server_ids)
# For anonymous callers (no user_id, no role), also surface any
# servers the operator has opted into upstream-delegated auth.
@ -1331,9 +1398,9 @@ class MCPServerManager:
except Exception: # noqa: BLE001
verbose_logger.exception(
"Failed to get allowed MCP servers; team-level object_permission "
"grants may be dropped. Falling back to global servers only."
"grants may be dropped. Falling back to global and submitted servers."
)
return allow_all_server_ids
return list(dict.fromkeys(allow_all_server_ids + submitted_server_ids))
async def resolve_toolset_tool_permissions(
self,

28
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -77,6 +77,7 @@ if MCP_AVAILABLE:
ListMCPToolsRestAPIResponseObject,
MCPInfo,
MCPServer,
_fire_mcp_success_logging,
_tool_name_matches,
execute_mcp_tool,
filter_tools_by_allowed_tools,
@ -84,6 +85,24 @@ if MCP_AVAILABLE:
########################################################
############ MCP Server REST API Routes #################
async def _safe_fire_mcp_success_logging(
logging_obj: Optional[Any],
result: Any,
start_time: datetime,
end_time: datetime,
) -> None:
if logging_obj is None:
return
logging_results = await asyncio.gather(
_fire_mcp_success_logging(logging_obj, result, start_time, end_time),
return_exceptions=True,
)
logging_error = logging_results[0]
if isinstance(logging_error, asyncio.CancelledError):
raise logging_error
if isinstance(logging_error, BaseException):
verbose_logger.warning("MCP tool success logging failed (continuing): %s", logging_error)
def _get_server_auth_header(
server,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]],
@ -798,7 +817,8 @@ if MCP_AVAILABLE:
proxy_logging_obj=proxy_logging_obj,
general_settings=general_settings,
)
return await handle_mcp_tool_call(
_tool_start_time = datetime.now()
result = await handle_mcp_tool_call(
tool_name=tool_arguments.get("tool_name", ""),
arguments=tool_arguments.get("arguments") or {},
user_api_key_dict=user_api_key_dict,
@ -809,6 +829,8 @@ if MCP_AVAILABLE:
raw_headers=virtual_raw_headers,
litellm_logging_obj=virtual_logging_obj,
)
await _safe_fire_mcp_success_logging(virtual_logging_obj, result, _tool_start_time, datetime.now())
return result
# Validate required parameters early
server_id = data.get("server_id")
@ -876,11 +898,12 @@ if MCP_AVAILABLE:
user_oauth_extra_headers = await _get_user_oauth_extra_headers(target_server, user_api_key_dict)
# Call execute_mcp_tool directly (permission checks already done)
_tool_start_time = datetime.now()
result = await execute_mcp_tool(
name=tool_name,
arguments=tool_arguments,
allowed_mcp_servers=allowed_mcp_servers,
start_time=datetime.now(),
start_time=_tool_start_time,
user_api_key_auth=data.get("user_api_key_auth"),
mcp_auth_header=data.get("mcp_auth_header"),
mcp_server_auth_headers=data.get("mcp_server_auth_headers"),
@ -889,6 +912,7 @@ if MCP_AVAILABLE:
litellm_logging_obj=data.get("litellm_logging_obj"),
requested_server_id=canonical_server_id,
)
await _safe_fire_mcp_success_logging(logging_obj, result, _tool_start_time, datetime.now())
return result
except MCPMissingUserEnvVarsError as e:
verbose_logger.info(

33
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -1681,6 +1681,7 @@ if MCP_AVAILABLE:
log_list_tools_to_spendlogs: bool = False,
list_tools_log_source: Optional[str] = None,
litellm_trace_id: Optional[str] = None,
request_tags: Optional[list[str]] = None,
client_ip: Optional[str] = None,
) -> List[MCPTool]:
"""
@ -1724,6 +1725,7 @@ if MCP_AVAILABLE:
"litellm_trace_id": effective_litellm_trace_id,
"metadata": {
"spend_logs_metadata": spend_logs_metadata,
**({"tags": request_tags} if request_tags else {}),
},
# Provide a small input payload for standard logging
"input": [
@ -1899,7 +1901,9 @@ if MCP_AVAILABLE:
end_time = datetime.now()
try:
await litellm_logging_obj.async_success_handler(
result=all_tools,
result=[
tool.model_dump(mode="json") if isinstance(tool, MCPTool) else tool for tool in all_tools
],
start_time=list_tools_start_time,
end_time=end_time,
)
@ -2741,6 +2745,22 @@ if MCP_AVAILABLE:
return response
async def _fire_mcp_success_logging(
logging_obj: LiteLLMLoggingObj,
result: Any,
start_time: datetime,
end_time: datetime,
) -> None:
logging_obj.post_call(original_response=result)
await logging_obj.async_post_mcp_tool_call_hook(
kwargs=logging_obj.model_call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
)
logging_obj.call_type = CallTypes.call_mcp_tool.value
await logging_obj.async_success_handler(result=result, start_time=start_time, end_time=end_time)
@client
async def call_mcp_tool(
name: str,
@ -2812,16 +2832,7 @@ if MCP_AVAILABLE:
raise
if litellm_logging_obj:
litellm_logging_obj.post_call(original_response=response)
end_time = datetime.now()
await litellm_logging_obj.async_post_mcp_tool_call_hook(
kwargs=litellm_logging_obj.model_call_details,
response_obj=response,
start_time=start_time,
end_time=end_time,
)
litellm_logging_obj.call_type = CallTypes.call_mcp_tool.value
await litellm_logging_obj.async_success_handler(result=response, start_time=start_time, end_time=end_time)
await _fire_mcp_success_logging(litellm_logging_obj, response, start_time, datetime.now())
return response
async def mcp_get_prompt(

2
litellm/proxy/batches_endpoints/endpoints.py
View file

@ -625,7 +625,7 @@ async def list_batches(
route_type="alist_batches",
)
# Try to use managed objects table for listing batches (returns encoded IDs)
# Try to use managed objects table for listing batches (returns encoded IDs).
managed_files_obj = proxy_logging_obj.get_proxy_hook("managed_files")
if managed_files_obj is not None and hasattr(managed_files_obj, "list_user_batches"):
verbose_proxy_logger.debug("Using managed objects table for batch listing")

17
litellm/proxy/common_utils/callback_utils.py
View file

@ -611,10 +611,17 @@ def _transform_callback_vars(metadata: Any, transform: Callable[[str, Any], Any]
return out
def _is_sensitive_callback_var(key: str) -> bool:
"""Match codebase precedent: only credential-bearing fields get encrypted;
routing/identifier fields (host, base_url, project, region) stay plain."""
if key in _EXTRA_SENSITIVE_CALLBACK_KEYS:
def is_sensitive_callback_key(
key: str,
extra: Optional[set[str]] = None,
) -> bool:
"""Return ``True`` if ``key`` is present in ``extra`` (checked as-is), or
if its lowercase form is in ``_EXTRA_SENSITIVE_CALLBACK_KEYS``, or if
``_CALLBACK_VAR_MASKER.is_sensitive_key`` matches it.
"""
if extra and key in extra:
return True
if key.lower() in _EXTRA_SENSITIVE_CALLBACK_KEYS:
return True
return _CALLBACK_VAR_MASKER.is_sensitive_key(key)
@ -622,7 +629,7 @@ def _is_sensitive_callback_var(key: str) -> bool:
def _encrypt_if_plaintext(key: str, value: Any) -> Any:
if not isinstance(value, str) or not value:
return value
if not _is_sensitive_callback_var(key):
if not is_sensitive_callback_key(key):
return value
if value.startswith(_CALLBACK_VAR_ENCRYPTED_PREFIX):
# Already encrypted — round-tripping ciphertext (e.g. UI Edit Settings

7
litellm/proxy/dev_config.yaml
View file

@ -28,6 +28,10 @@ model_list:
litellm_params:
model: anthropic/claude-opus-4-8
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: anthropic-sonnet-5
litellm_params:
model: anthropic/claude-sonnet-5
api_key: os.environ/ANTHROPIC_API_KEY
# ---------- Bedrock Invoke ----------
- model_name: bedrock-invoke-haiku-4-5
@ -189,6 +193,9 @@ model_list:
general_settings:
master_key: sk-1234
# Opt-in: let CheckBatchCost track cost for unmanaged Vertex batches created with a raw gs:// input_file_id.
# Requires a vertex_ai deployment configured for the batched model. Defaults to false.
# track_unmanaged_vertex_batch_cost: true
sandbox_tools:
- sandbox_tool_name: e2b_sandbox

19
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -190,6 +190,11 @@ class _ProxyDBLogger(CustomLogger):
litellm_params = kwargs.get("litellm_params", {}) or {}
end_user_id = get_end_user_id_for_cost_tracking(litellm_params)
metadata = get_litellm_metadata_from_kwargs(kwargs=kwargs)
# Only fetch key details when user_id wasn't already populated (e.g. direct MCP REST calls).
# Avoids a cache/DB lookup on every normal LLM request.
if metadata.get("user_api_key") and not metadata.get("user_api_key_user_id"):
metadata = await _ProxyDBLogger._enrich_failure_metadata_with_key_info(metadata=metadata)
_write_spend_metadata_to_kwargs(kwargs=kwargs, metadata=metadata)
budget_reservation = _get_budget_reservation_from_metadata(metadata=metadata)
user_id = cast(Optional[str], metadata.get("user_api_key_user_id", None))
team_id = cast(Optional[str], metadata.get("user_api_key_team_id", None))
@ -388,6 +393,20 @@ class _ProxyDBLogger(CustomLogger):
return
def _write_spend_metadata_to_kwargs(kwargs: dict, metadata: dict) -> None:
patch = {k: v for k, v in metadata.items() if (k.startswith("user_api_key") or k == "tags") and v is not None}
if not patch:
return
litellm_params = kwargs.setdefault("litellm_params", {})
for bucket_name in ("litellm_metadata", "metadata"):
bucket = litellm_params.get(bucket_name)
if isinstance(bucket, dict):
for key, value in patch.items():
if bucket.get(key) is None:
bucket[key] = value
def _should_track_cost_callback(
user_api_key: Optional[str],
user_id: Optional[str],

21
litellm/proxy/management_endpoints/mcp_management_endpoints.py
View file

@ -132,6 +132,7 @@ if MCP_AVAILABLE:
update_mcp_server,
)
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
_raise_if_not_oauth2,
authorize_with_server,
exchange_token_with_server,
get_request_base_url,
@ -148,7 +149,6 @@ if MCP_AVAILABLE:
LitellmUserRoles,
MakeMCPServersPublicRequest,
MCPApprovalStatus,
MCPEnvVarScope,
MCPOAuthUserCredentialRequest,
MCPOAuthUserCredentialStatus,
MCPSubmissionsSummary,
@ -460,18 +460,6 @@ if MCP_AVAILABLE:
) -> List[LiteLLM_MCPServerTable]:
return [_redact_mcp_credentials(server) for server in mcp_servers]
def _redact_global_env_var_values(mcp_server: LiteLLM_MCPServerTable) -> None:
"""Blank admin-supplied ``scope="global"`` env var secrets in place.
Global entries hold the admin's plaintext credential (API key,
password, ...) and must never reach non-admin callers. Per-user
entries only carry a placeholder the user fills in themselves, so
their value is left intact.
"""
for env_var in mcp_server.env_vars or []:
if env_var.scope == MCPEnvVarScope.global_:
env_var.value = ""
def _user_is_full_admin(user_api_key_dict: UserAPIKeyAuth) -> bool:
"""True only for ``PROXY_ADMIN``; ``PROXY_ADMIN_VIEW_ONLY`` returns False.
@ -1114,9 +1102,9 @@ if MCP_AVAILABLE:
prisma_client = get_prisma_client_or_throw("Database not connected. Connect a database to your proxy")
submissions = await get_mcp_submissions(prisma_client)
submissions.items = _redact_mcp_credentials_list(submissions.items)
if not _user_is_full_admin(user_api_key_dict):
for item in submissions.items:
_redact_global_env_var_values(item)
submissions.items = _sanitize_mcp_server_list_for_non_admin(submissions.items)
return submissions
@router.put(
@ -1158,6 +1146,7 @@ if MCP_AVAILABLE:
server_id,
touched_by=user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME,
)
await global_mcp_server_manager.invalidate_byom_submitted_servers_cache(approved.submitted_by)
await global_mcp_server_manager.reload_servers_from_database()
return _redact_mcp_credentials(approved)
@ -1623,6 +1612,7 @@ if MCP_AVAILABLE:
scope: Optional[str] = None,
):
mcp_server = await _get_cached_temporary_mcp_server_or_404(server_id, user_api_key_dict, request=request)
_raise_if_not_oauth2(mcp_server)
# Use the server's stored client_id when the caller doesn't supply one
resolved_client_id = mcp_server.client_id or client_id or ""
if not resolved_client_id:
@ -1667,6 +1657,7 @@ if MCP_AVAILABLE:
scope: Optional[str] = Form(None),
):
mcp_server = await _get_cached_temporary_mcp_server_or_404(server_id, user_api_key_dict, request=request)
_raise_if_not_oauth2(mcp_server)
resolved_client_id = mcp_server.client_id or client_id or ""
if not resolved_client_id:
raise HTTPException(

69
litellm/proxy/proxy_server.py
View file

@ -102,6 +102,7 @@ from litellm.proxy._types import (
)
from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
from litellm.proxy.common_utils.callback_utils import (
is_sensitive_callback_key,
normalize_callback_names,
process_callback,
)
@ -7626,6 +7627,7 @@ class ProxyStartupEvent:
proxy_logging_obj=proxy_logging_obj,
prisma_client=prisma_client,
llm_router=llm_router,
track_unmanaged_vertex_batch_cost=general_settings.get("track_unmanaged_vertex_batch_cost", False),
)
scheduler.add_job(
check_batch_cost_job.check_batch_cost,
@ -14310,6 +14312,50 @@ async def create_config_audit_log(
)
_EXTRA_SECRET_CALLBACK_ENV_VARS = frozenset(
{
"GALILEO_USERNAME",
"GENERIC_LOGGER_HEADERS",
"OTEL_HEADERS",
"SLACK_WEBHOOK_URL",
"SMTP_USERNAME",
}
)
def _redact_callback_env_vars(env_vars: dict[str, Optional[str]]) -> dict[str, Optional[str]]:
"""Return a copy of ``env_vars`` with values for keys classified as
sensitive by ``is_sensitive_callback_key`` replaced with ``"REDACTED"``.
``None`` values pass through unchanged.
"""
return {
key: (
"REDACTED"
if value is not None and is_sensitive_callback_key(key, extra=_EXTRA_SECRET_CALLBACK_ENV_VARS)
else value
)
for key, value in env_vars.items()
}
def _apply_callback_role_gate(entries: list, is_full_admin: bool) -> list:
if is_full_admin:
return entries
return [{**entry, "variables": _redact_callback_env_vars(entry.get("variables") or {})} for entry in entries]
def _apply_alerting_env_role_gate(env_vars: dict, is_full_admin: bool) -> dict:
if is_full_admin:
return mask_sensitive_keys(env_vars, _ALERTING_SENSITIVE_VARS)
return _redact_callback_env_vars(env_vars)
def _apply_webhook_role_gate(webhook_map, is_full_admin: bool):
if is_full_admin or not isinstance(webhook_map, dict):
return webhook_map
return {alert_type: "REDACTED" for alert_type in webhook_map}
@router.get(
"/config/field/info",
tags=["config.yaml"],
@ -14720,7 +14766,9 @@ async def delete_callback(
include_in_schema=False,
dependencies=[Depends(user_api_key_auth)],
)
async def get_config():
async def get_config(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
For Admin UI - allows admin to view config via UI
# return the callbacks and the env variables for the callback
@ -14735,6 +14783,8 @@ async def get_config():
_general_settings = config_data.get("general_settings", {})
environment_variables = config_data.get("environment_variables", {})
is_full_admin = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
_success_callbacks = _litellm_settings.get("success_callback", [])
_failure_callbacks = _litellm_settings.get("failure_callback", [])
_success_and_failure_callbacks = _litellm_settings.get("callbacks", [])
@ -14776,6 +14826,8 @@ async def get_config():
for _callback in _success_and_failure_callbacks:
_data_to_return.append(process_callback(_callback, "success_and_failure", environment_variables))
_data_to_return = _apply_callback_role_gate(_data_to_return, is_full_admin)
# Check if slack alerting is on
_alerting = _general_settings.get("alerting", [])
alerting_data = []
@ -14787,11 +14839,13 @@ async def get_config():
_var: (value if (value := environment_variables.get(_var)) is not None else os.getenv(_var))
for _var in _slack_vars
}
_slack_env_vars = mask_sensitive_keys(_slack_env_vars, _ALERTING_SENSITIVE_VARS)
_slack_env_vars = _apply_alerting_env_role_gate(_slack_env_vars, is_full_admin)
_alerting_types = proxy_logging_obj.slack_alerting_instance.alert_types
_all_alert_types = proxy_logging_obj.slack_alerting_instance._all_possible_alert_types()
_alerts_to_webhook = proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url
_alerts_to_webhook = _apply_webhook_role_gate(
proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url, is_full_admin
)
alerting_data.append(
{
"name": "slack",
@ -14811,8 +14865,9 @@ async def get_config():
"EMAIL_LOGO_URL",
"EMAIL_SUPPORT_CONTACT",
]
_email_env_vars = {_var: environment_variables.get(_var) for _var in _email_vars}
_email_env_vars = mask_sensitive_keys(_email_env_vars, _ALERTING_SENSITIVE_VARS)
_email_env_vars = _apply_alerting_env_role_gate(
{_var: environment_variables.get(_var) for _var in _email_vars}, is_full_admin
)
alerting_data.append(
{
@ -15566,9 +15621,10 @@ app.include_router(search_router)
app.include_router(image_router)
app.include_router(fine_tuning_router)
app.include_router(credential_router)
app.include_router(batches_router)
app.include_router(openai_files_router)
app.include_router(llm_passthrough_router)
app.include_router(pass_through_router)
app.include_router(batches_router)
app.include_router(health_router)
app.include_router(key_management_router)
app.include_router(internal_user_router)
@ -15583,7 +15639,6 @@ app.include_router(callback_management_endpoints_router)
app.include_router(debugging_endpoints_router)
app.include_router(rust_control_plane_router)
app.include_router(ui_crud_endpoints_router)
app.include_router(openai_files_router)
app.include_router(team_callback_router)
app.include_router(budget_management_router)
app.include_router(model_management_router)

47
litellm/proxy/spend_tracking/spend_management_endpoints.py
View file

@ -3444,12 +3444,59 @@ async def _build_ui_spend_logs_response(
)
count_map = {r["session_id"]: r["_count"]["session_id"] for r in counts if r.get("session_id")}
mcp_spend_map: dict[str, dict[str, Union[int, float]]] = {}
if enrich_session_counts and session_ids:
from prisma.errors import PrismaError
try:
# Collect api_keys already present in the authorized page rows so the
# aggregate is scoped to the same ownership as the main query — prevents
# cross-tenant disclosure via a colliding session_id.
authorized_api_keys = list(
{
(row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None))
for row in data
if (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None))
}
)
rows = await prisma_client.db.query_raw(
"""
SELECT session_id,
COUNT(*)::int AS mcp_tool_call_count,
COALESCE(SUM(spend), 0)::double precision AS mcp_tool_call_spend
FROM "LiteLLM_SpendLogs"
WHERE session_id = ANY($1::text[])
AND api_key = ANY($2::text[])
AND call_type IN ('call_mcp_tool', 'list_mcp_tools')
GROUP BY session_id
""",
session_ids,
authorized_api_keys,
)
mcp_spend_map = {
row["session_id"]: {
"mcp_tool_call_count": int(row.get("mcp_tool_call_count") or 0),
"mcp_tool_call_spend": float(row.get("mcp_tool_call_spend") or 0.0),
}
for row in rows
if row.get("session_id")
}
except PrismaError:
verbose_proxy_logger.debug(
"Failed to enrich MCP session spend aggregates for spend logs UI",
exc_info=True,
)
if enrich_session_counts:
enriched: List[dict] = []
for row in data:
row_dict = dict(row) if isinstance(row, dict) else row.model_dump()
sid = row_dict.get("session_id")
row_dict["session_total_count"] = count_map.get(sid, 1) if sid else 1
mcp_stats = mcp_spend_map.get(sid) if sid else None
if mcp_stats:
row_dict["mcp_tool_call_count"] = mcp_stats["mcp_tool_call_count"]
row_dict["mcp_tool_call_spend"] = mcp_stats["mcp_tool_call_spend"]
enriched.append(row_dict)
response_data: list = enriched
else:

6
litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
View file

@ -1160,6 +1160,12 @@ async def update_mcp_semantic_filter_settings(
Update MCP semantic filter settings in database.
Settings will be picked up by all pods within approximately 10 seconds via background polling.
"""
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
raise HTTPException(
status_code=403,
detail="Only proxy admins can update MCP semantic filter settings.",
)
result = await _update_litellm_setting(
settings=settings,
settings_key="mcp_semantic_tool_filter",

3
litellm/responses/main.py
View file

@ -212,6 +212,7 @@ async def aresponses_api_with_mcp(
litellm_trace_id=kwargs.get("litellm_trace_id"),
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs),
)
openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(original_mcp_tools)
@ -327,6 +328,7 @@ async def aresponses_api_with_mcp(
raw_headers=raw_headers_from_request,
litellm_call_id=kwargs.get("litellm_call_id"),
litellm_trace_id=kwargs.get("litellm_trace_id"),
request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs),
)
if tool_results:
@ -382,6 +384,7 @@ async def aresponses_api_with_mcp(
mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs),
)
final_response = LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response(
response=final_response,

29
litellm/responses/mcp/chat_completions_handler.py
View file

@ -1,5 +1,6 @@
"""Helpers for handling MCP-aware `/chat/completions` requests."""
import logging
from typing import (
Any,
List,
@ -115,6 +116,7 @@ async def acompletion_with_mcp(
# Extract user_api_key_auth from metadata or kwargs
user_api_key_auth = kwargs.get("user_api_key_auth") or ((kwargs.get("metadata", {}) or {}).get("user_api_key_auth"))
request_tags = LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs)
# Extract MCP auth headers before fetching tools (needed for dynamic auth)
(
@ -137,6 +139,7 @@ async def acompletion_with_mcp(
litellm_trace_id=kwargs.get("litellm_trace_id"),
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
request_tags=request_tags,
)
openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(
@ -218,6 +221,7 @@ async def acompletion_with_mcp(
litellm_trace_id,
openai_tools,
base_call_args,
request_tags,
):
self.stream_wrapper = stream_wrapper
self.messages = messages
@ -231,6 +235,7 @@ async def acompletion_with_mcp(
self.litellm_trace_id = litellm_trace_id
self.openai_tools = openai_tools
self.base_call_args = base_call_args
self.request_tags = request_tags
self.collected_chunks: List[ModelResponseStream] = []
self.tool_calls: Optional[List] = None
self.tool_results: Optional[List] = None
@ -303,6 +308,17 @@ async def acompletion_with_mcp(
return chunk
async def _drain_inner_stream(self):
try:
while True:
await self._stream_iterator.__anext__()
except StopAsyncIteration:
pass
except Exception:
logging.getLogger("LiteLLM").exception(
"Error draining inner MCP stream after final chunk; spend logging may be incomplete"
)
async def __anext__(self):
# Phase 1: Collect and yield initial stream chunks
if not self.stream_exhausted:
@ -332,15 +348,16 @@ async def acompletion_with_mcp(
)
if is_final:
# This is the final chunk, mark stream as exhausted
self.stream_exhausted = True
# Process tool calls after we've collected all chunks
await self._process_tool_calls()
# Apply MCP metadata (tool_calls and tool_results) to final chunk
chunk = self._add_mcp_tool_metadata_to_final_chunk(chunk)
# If we have tool results, prepare follow-up call immediately
if self.tool_results and self.complete_response:
await self._prepare_follow_up_call()
# Drain inner stream so CustomStreamWrapper fires its
# end-of-stream handler (dispatch_success_handlers →
# _ProxyDBLogger → LiteLLM_SpendLogs). The CSW may
# yield one usage chunk before raising StopAsyncIteration.
await self._drain_inner_stream()
return chunk
except StopAsyncIteration:
@ -354,6 +371,7 @@ async def acompletion_with_mcp(
# If we have tool results, prepare follow-up call
if self.tool_results and self.complete_response:
await self._prepare_follow_up_call()
await self._drain_inner_stream()
return final_chunk
# Phase 2: Yield follow-up stream chunks if available
@ -426,6 +444,7 @@ async def acompletion_with_mcp(
raw_headers=self.raw_headers,
litellm_call_id=self.litellm_call_id,
litellm_trace_id=self.litellm_trace_id,
request_tags=self.request_tags,
)
async def _prepare_follow_up_call(self):
@ -485,6 +504,7 @@ async def acompletion_with_mcp(
litellm_trace_id=kwargs.get("litellm_trace_id"),
openai_tools=openai_tools,
base_call_args=base_call_args,
request_tags=request_tags,
)
# Create a wrapper class that delegates to our custom iterator
@ -596,6 +616,7 @@ async def acompletion_with_mcp(
raw_headers=raw_headers,
litellm_call_id=kwargs.get("litellm_call_id"),
litellm_trace_id=kwargs.get("litellm_trace_id"),
request_tags=request_tags,
)
if not tool_results:

38
litellm/responses/mcp/litellm_proxy_mcp_handler.py
View file

@ -17,6 +17,7 @@ from litellm._logging import verbose_logger
from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._experimental.mcp_server.utils import split_server_prefix_from_name
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.responses.main import aresponses
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.types.llms.openai import ResponsesAPIResponse
@ -59,6 +60,20 @@ class LiteLLM_Proxy_MCP_Handler:
This handles when a user passes mcp server_url="litellm_proxy" in their tools.
"""
@staticmethod
def _get_parent_request_tags(kwargs: Optional[dict[str, Any]]) -> list[str]:
"""Tags from the parent LLM request, using the same extraction logic as standard logging (incl. User-Agent)."""
if not kwargs:
return []
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
litellm_params = kwargs.get("litellm_params") or kwargs
proxy_server_request = litellm_params.get("proxy_server_request") or kwargs.get("proxy_server_request") or {}
return StandardLoggingPayloadSetup._get_request_tags(
litellm_params=litellm_params,
proxy_server_request=proxy_server_request,
)
@staticmethod
def _should_use_litellm_mcp_gateway(tools: Optional[Iterable[ToolParam]]) -> bool:
"""
@ -162,6 +177,7 @@ class LiteLLM_Proxy_MCP_Handler:
litellm_trace_id: Optional[str] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None,
request_tags: Optional[list[str]] = None,
) -> tuple[List[MCPTool], List[str]]:
"""
Get available tools from the MCP server manager.
@ -250,6 +266,7 @@ class LiteLLM_Proxy_MCP_Handler:
log_list_tools_to_spendlogs=True,
list_tools_log_source="responses",
litellm_trace_id=litellm_trace_id,
request_tags=request_tags,
)
allowed_mcp_server_ids = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
@ -351,6 +368,7 @@ class LiteLLM_Proxy_MCP_Handler:
user_api_key_auth: Any,
mcp_tools_with_litellm_proxy: List[ToolParam],
litellm_trace_id: Optional[str] = None,
request_tags: Optional[list[str]] = None,
) -> tuple[List[Any], dict[str, str]]:
"""
Centralized method to process MCP tools through the complete pipeline.
@ -371,6 +389,7 @@ class LiteLLM_Proxy_MCP_Handler:
user_api_key_auth,
mcp_tools_with_litellm_proxy,
litellm_trace_id=litellm_trace_id,
request_tags=request_tags,
)
openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(deduplicated_mcp_tools)
@ -384,6 +403,7 @@ class LiteLLM_Proxy_MCP_Handler:
litellm_trace_id: Optional[str] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None,
request_tags: Optional[list[str]] = None,
) -> tuple[List[Any], dict[str, str]]:
"""
Process MCP tools through filtering and deduplication pipeline without OpenAI transformation.
@ -411,6 +431,7 @@ class LiteLLM_Proxy_MCP_Handler:
litellm_trace_id=litellm_trace_id,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
request_tags=request_tags,
)
# Step 2: Filter tools based on allowed_tools parameter
@ -597,6 +618,7 @@ class LiteLLM_Proxy_MCP_Handler:
raw_headers: Optional[Dict[str, str]] = None,
litellm_call_id: Optional[str] = None,
litellm_trace_id: Optional[str] = None,
request_tags: Optional[list[str]] = None,
) -> List[Dict[str, Any]]:
"""Execute tool calls and return results."""
from fastapi import HTTPException
@ -672,17 +694,19 @@ class LiteLLM_Proxy_MCP_Handler:
}
if litellm_trace_id:
logging_request_data["litellm_trace_id"] = litellm_trace_id
user_identifier = None
if request_tags:
logging_request_data["metadata"]["tags"] = request_tags
if user_api_key_auth is not None:
user_api_key = getattr(user_api_key_auth, "api_key", None)
if user_api_key:
logging_request_data["metadata"]["user_api_key"] = user_api_key
LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
data=logging_request_data,
user_api_key_dict=user_api_key_auth,
_metadata_variable_name="metadata",
)
user_identifier = getattr(user_api_key_auth, "end_user_id", None) or getattr(
user_api_key_auth, "user_id", None
)
if user_identifier:
logging_request_data["user"] = user_identifier
if user_identifier:
logging_request_data["user"] = user_identifier
litellm_logging_obj: Optional[LiteLLMLoggingObj] = None
try:

1
litellm/responses/mcp/mcp_streaming_iterator.py
View file

@ -630,6 +630,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
raw_headers=self.raw_headers,
litellm_call_id=self.litellm_call_id,
litellm_trace_id=self.litellm_trace_id,
request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(self.original_request_params),
)
# Create completion events and output_item.done events for tool execution

21
litellm/types/integrations/prometheus.py
View file

@ -188,6 +188,8 @@ class UserAPIKeyLabelNames(Enum):
STREAM = "stream"
ORG_ID = "org_id"
ORG_ALIAS = "org_alias"
MCP_TOOL_NAME = "mcp_tool_name"
MCP_SERVER_NAME = "mcp_server_name"
DEFINED_PROMETHEUS_METRICS = Literal[
@ -264,6 +266,9 @@ DEFINED_PROMETHEUS_METRICS = Literal[
"litellm_check_batch_cost_jobs_processed_total",
"litellm_check_batch_cost_errors_total",
"litellm_check_batch_cost_last_run_timestamp",
# MCP tool call metrics
"litellm_mcp_tool_calls_total",
"litellm_mcp_tool_call_spend_metric",
]
@ -737,6 +742,20 @@ class PrometheusMetricLabels:
litellm_check_batch_cost_last_run_timestamp: List[str] = []
# MCP tool call metrics
litellm_mcp_tool_calls_total: list[str] = [
UserAPIKeyLabelNames.MCP_TOOL_NAME.value,
UserAPIKeyLabelNames.MCP_SERVER_NAME.value,
UserAPIKeyLabelNames.API_KEY_HASH.value,
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.END_USER.value,
]
litellm_mcp_tool_call_spend_metric: list[str] = list(litellm_mcp_tool_calls_total)
@staticmethod
def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> List[str]:
default_labels = getattr(PrometheusMetricLabels, label_name)
@ -840,6 +859,8 @@ class UserAPIKeyLabelValues:
stream: Optional[str] = None
org_id: Optional[str] = None
org_alias: Optional[str] = None
mcp_tool_name: Optional[str] = None
mcp_server_name: Optional[str] = None
# Added for test compatibility.
def __init__(self, **kwargs: Any) -> None:

1
litellm/types/utils.py
View file

@ -152,6 +152,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_output_config: Optional[bool]
supports_image_size: Optional[bool]
bedrock_output_config_effort_ceiling: Optional[Literal["low", "medium", "high", "max", "xhigh"]]
bedrock_converse_supports_strict_tools: Optional[bool]
class SearchContextCostPerQuery(TypedDict, total=False):

1
litellm/utils.py
View file

@ -5465,6 +5465,7 @@ def _get_model_info_helper(
supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None),
supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None),
bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None),
bedrock_converse_supports_strict_tools=_model_info.get("bedrock_converse_supports_strict_tools", None),
supports_computer_use=_model_info.get("supports_computer_use", None),
search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None),
web_search_billing_unit=_model_info.get("web_search_billing_unit", None),

126
model_prices_and_context_window.json
View file

@ -1154,6 +1154,7 @@
"bedrock_output_config_effort_ceiling": "max"
},
"anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -1203,6 +1204,7 @@
"supports_output_config": true
},
"global.anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -1237,6 +1239,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"us.anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1271,6 +1274,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"eu.anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1305,6 +1309,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"au.anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1471,6 +1476,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -1505,6 +1511,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"global.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -1539,6 +1546,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"us.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1573,6 +1581,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"eu.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1607,6 +1616,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"au.anthropic.claude-opus-4-8": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -1641,6 +1651,7 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"jp.anthropic.claude-opus-4-7": {
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 6.875e-06,
"cache_read_input_token_cost": 5.5e-07,
"input_cost_per_token": 5.5e-06,
@ -1672,16 +1683,16 @@
"supports_minimal_reasoning_effort": true
},
"anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -1705,16 +1716,16 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"global.anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -1738,16 +1749,16 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"us.anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_1hr": 4.4e-06,
"cache_read_input_token_cost": 2.2e-07,
"input_cost_per_token": 2.2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -1771,16 +1782,16 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"eu.anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_1hr": 4.4e-06,
"cache_read_input_token_cost": 2.2e-07,
"input_cost_per_token": 2.2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -1804,16 +1815,16 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"au.anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_1hr": 4.4e-06,
"cache_read_input_token_cost": 2.2e-07,
"input_cost_per_token": 2.2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -1837,16 +1848,16 @@
"bedrock_output_config_effort_ceiling": "xhigh"
},
"jp.anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_1hr": 4.4e-06,
"cache_read_input_token_cost": 2.2e-07,
"input_cost_per_token": 2.2e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -2082,7 +2093,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"bedrock_converse_supports_strict_tools": false
},
"anthropic.claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 3.75e-06,
@ -2409,7 +2421,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"bedrock_converse_supports_strict_tools": false
},
"assemblyai/best": {
"input_cost_per_second": 3.333e-05,
@ -2710,16 +2723,16 @@
"supports_vision": true
},
"azure_ai/claude-sonnet-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -10474,16 +10487,16 @@
"supports_web_search": true
},
"claude-sonnet-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -14813,7 +14826,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"bedrock_converse_supports_strict_tools": false
},
"eu.anthropic.claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
@ -20232,7 +20246,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"bedrock_converse_supports_strict_tools": false
},
"global.anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.25e-06,
@ -33332,7 +33347,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"bedrock_converse_supports_strict_tools": false
},
"us.deepseek.r1-v1:0": {
"input_cost_per_token": 1.35e-06,
@ -35384,16 +35400,16 @@
"supports_vision": true
},
"vertex_ai/claude-sonnet-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
@ -42909,16 +42925,16 @@
}
},
"vertex_ai/claude-sonnet-5@default": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token": 1e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,

4
ruff-strict-budget.json
View file

@ -60,7 +60,7 @@
"limit": 4
},
"BLE001": {
"limit": 2904
"limit": 2903
},
"C401": {
"limit": 11
@ -255,7 +255,7 @@
"limit": 480
},
"S110": {
"limit": 237
"limit": 236
},
"S112": {
"limit": 24

4
ruff.toml
View file

@ -1,5 +1,5 @@
lint.ignore = ["F405", "E402", "E501", "F403"]
lint.extend-select = ["E501", "T20", "PGH004", "RUF008", "RUF009", "RUF100"]
lint.ignore = ["F405", "E402", "F403"]
lint.extend-select = ["T20", "PGH004", "RUF008", "RUF009", "RUF100"]
# RUF100 (unused-noqa) only knows the rules enabled in THIS config, so it would strip
# `# noqa` directives that protect rules enforced elsewhere. List those codes as external
# so RUF100 leaves their directives alone: the strict gate (ruff-strict.toml) and upstream

49
tests/batches_tests/test_batches_logging_unit_tests.py
View file

@ -255,6 +255,55 @@ async def test_batch_retrieve_cost_tracking_with_completed_batch_no_explicit_cos
assert mock_batch.usage == expected_usage
@pytest.mark.asyncio
async def test_handle_completed_batch_computes_real_cost_from_output_file(
sample_file_content_dict,
):
"""Integration: a completed batch's cost and usage are computed from its output
file via the real cost-calc chain (only the file download is stubbed). This is
the function the retrieve handler invokes on completion; a dropped output line, a
wrong token sum, or mispriced model fails this test.
"""
from litellm.batches.batch_utils import _handle_completed_batch
from litellm.types.utils import LiteLLMBatch
batch = LiteLLMBatch(
id="batch-real-cost-123",
object="batch",
endpoint="/v1/chat/completions",
input_file_id="file-input-123",
completion_window="24h",
status="completed",
output_file_id="file-output-123",
created_at=1234567890,
)
with patch(
"litellm.batches.batch_utils._get_batch_output_file_content_as_dictionary",
new=AsyncMock(return_value=sample_file_content_dict),
):
cost, usage, models = await _handle_completed_batch(
batch=batch, custom_llm_provider="openai"
)
pricing = litellm.model_cost["gpt-4o-mini-2024-07-18"]
expected_cost = (
42 * pricing["input_cost_per_token_batches"]
+ 20 * pricing["output_cost_per_token_batches"]
)
assert cost == pytest.approx(expected_cost)
assert cost > 0
assert (
cost
< 42 * pricing["input_cost_per_token"] + 20 * pricing["output_cost_per_token"]
)
assert usage.prompt_tokens == 42
assert usage.completion_tokens == 20
assert usage.total_tokens == 62
assert models == ["gpt-4o-mini-2024-07-18", "gpt-4o-mini-2024-07-18"]
@pytest.mark.asyncio
async def test_batch_retrieve_cost_tracking_with_explicit_cost_data():
"""

79
tests/e2e/batches/COVERAGE.md Normal file
View file

@ -0,0 +1,79 @@
# Batches Test Coverage Matrix
Live e2e coverage of the Batches API over a real proxy, real provider keys, and
real cost. Synchronous tier only: a batch's completion window is 24h, so these
tests never wait for `completed`. They assert the proxy accepts, routes, retrieves,
cancels, and lists a batch; everything created is deleted on teardown.
## Provider x operation
Only supported cells are tested. The capability table in `capabilities.py` holds one
row per supported (provider, scenario) pair, so there are no skipped cells in the
parametrized run.
| Provider | create | retrieve | cancel | list | file backing |
|-----------|--------|----------|--------|------|--------------|
| OpenAI | yes | yes | yes | yes | OpenAI Files |
| Azure | yes | yes | yes | yes | Azure Files |
| Vertex AI | yes | yes | yes | yes | GCS bucket |
| Bedrock | yes | yes | no (limited upstream) | no | S3 bucket |
| Anthropic | no | yes (env-gated) | no | no | Anthropic Files |
Bedrock cancel is unreliable upstream and list is unsupported, so both are gated off
(`can_cancel=False`, `can_list=False`). Anthropic cannot create/cancel/list through
litellm, so it has a standalone retrieve test that skips unless `ANTHROPIC_BATCH_ID`
points at a real Anthropic batch.
## Routing scenarios (per `litellm/proxy/batches_endpoints/endpoints.py`)
Each create-capable provider runs all four. The test asserts the returned file id
and batch id carry the shape that scenario must produce (`matches_id_shape`):
| Scenario | How the batch is routed | File id | Batch id |
|----------|-------------------------|---------|----------|
| `encoded` | upload with `?model=` -> model-encoded file id -> create with just that id | model-encoded | model-encoded |
| `unified` | upload with `target_model_names=` -> unified managed file id -> create with that id | managed | managed |
| `model_param` | raw file (provider-fallback upload) -> create with `model` in the body | raw | model-encoded |
| `provider_fallback` | raw file -> `POST /{provider}/v1/batches`, env creds, no model | raw | raw (native provider shape) |
"managed" ids base64-decode to a `litellm_proxy` marker; "model-encoded" ids keep the
provider prefix and base64-encode `litellm:<id>;model,<model>`; "raw" ids are the
provider's native ids. Asserting these catches a proxy that returns a raw id where it
should manage it, or vice versa. On top of the id shape, a misroute to the wrong
provider also fails create (the file id / model do not belong there), and the
`provider_fallback` raw batch id is additionally checked against the provider's native
shape (`raw_id_matches_provider`).
## Key model restriction
`test_batch_key_model_access_denied` mints a key restricted to one model
(`resources.key(models=[...])`) and proves the proxy returns 403
`key_model_access_denied` both when that key uploads a file for a disallowed model
(files endpoint) and when it creates a batch for a disallowed model (batches
endpoint).
## Per-endpoint output assertions
Each endpoint's full response is validated, not just the id. File upload asserts
`object=="file"`, `purpose=="batch"`, a positive `bytes`, a status, and a created-at.
Batch create / retrieve assert `object=="batch"`, `endpoint=="/v1/chat/completions"`,
`completion_window=="24h"`, a non-empty `input_file_id`, and a created-at; retrieve
additionally cross-checks that `id` and `input_file_id` match the created batch.
Cancel asserts the same id, `object=="batch"`, and a cancelling/cancelled status. List
asserts the `object=="list"` envelope and that the created batch is present as a batch.
File delete asserts `object=="file"` and `deleted==True`.
## This suite's files
| File | Covers |
|------|--------|
| `batch_client.py` | typed file upload/download + batch create/retrieve/cancel/list/delete over the shared Gateway; denial helpers |
| `capabilities.py` | the provider x scenario matrix + id-shape classifiers + per-provider raw-id assertion |
| `test_batches_e2e.py` | parametrized lifecycle with per-endpoint output assertions, file upload/delete outputs, key-model-access denial, anthropic retrieve |
## Out of scope (intentionally)
Driving a batch to `completed`, cost tracking on completion, and the DB write-back
are not covered here; the 24h window makes them unfit for a synchronous gate. That
logic belongs in a DI-stubbed proxy integration test under `tests/test_litellm/proxy/`
where the provider client is injected to return `completed` deterministically.

165
tests/e2e/batches/batch_client.py Normal file
View file

@ -0,0 +1,165 @@
"""Client for the batches e2e suite: file upload/download and the batch
operations (create / retrieve / cancel / list) over the shared Gateway.
`create_batch` returns the raw HTTP outcome (StreamingResponse) so a 403 model
access denial and a provider-native batch body both surface; the test parses
BatchObject from the body. A `provider` arg routes a call to /{provider}/v1/...,
which the provider-fallback scenario needs (its ids are raw, not model-encoded).
The request/response models are co-located here because only this suite uses them.
"""
from __future__ import annotations
from dataclasses import dataclass
from pydantic import BaseModel
from e2e_gateway import Gateway, build_gateway
from e2e_http import (
FileUploadForm,
NoBody,
Result,
StreamingResponse,
UnknownApiError,
)
class FileObject(BaseModel):
id: str
object: str | None = None
purpose: str | None = None
bytes: int | None = None
status: str | None = None
created_at: int | None = None
class BatchObject(BaseModel):
id: str
object: str | None = None
status: str
endpoint: str | None = None
input_file_id: str | None = None
output_file_id: str | None = None
completion_window: str | None = None
created_at: int | None = None
model: str | None = None
class BatchList(BaseModel):
object: str | None = None
data: list[BatchObject] = []
class FileDeleteResponse(BaseModel):
id: str
object: str | None = None
deleted: bool
class BatchCreateBody(BaseModel):
input_file_id: str
endpoint: str = "/v1/chat/completions"
completion_window: str = "24h"
model: str | None = None
class ModelQuery(BaseModel):
model: str | None = None
def is_model_access_denied(resp: StreamingResponse) -> bool:
"""True if the proxy rejected the call because the key may not access the model."""
return resp.status_code == 403 and "key_model_access_denied" in resp.body
def is_result_access_denied[R: BaseModel](result: Result[R]) -> bool:
match result:
case UnknownApiError(status_code=403, body=body):
return "key_model_access_denied" in body
case _:
return False
@dataclass(frozen=True, slots=True)
class BatchClient:
gateway: Gateway
def upload_file(
self,
*,
content: bytes,
form: FileUploadForm,
key: str,
model: str | None = None,
provider: str | None = None,
) -> Result[FileObject]:
return self.gateway.transport.upload(
_files_path(provider),
headers=self.gateway.transport.bearer(key),
form=form,
filename="batch_input.jsonl",
content=content,
params=ModelQuery(model=model),
response_type=FileObject,
)
def create_batch(
self, *, body: BatchCreateBody, key: str, provider: str | None = None
) -> StreamingResponse:
return self.gateway.transport.send(
_batches_path(provider),
headers=self.gateway.transport.bearer(key),
json=body,
)
def retrieve_batch(
self, batch_id: str, *, key: str, provider: str | None = None
) -> Result[BatchObject]:
return self.gateway.transport.get(
f"{_batches_path(provider)}/{batch_id}",
headers=self.gateway.transport.bearer(key),
params=NoBody(),
response_type=BatchObject,
)
def cancel_batch(
self, batch_id: str, *, key: str, provider: str | None = None
) -> Result[BatchObject]:
return self.gateway.transport.post(
f"{_batches_path(provider)}/{batch_id}/cancel",
headers=self.gateway.transport.bearer(key),
json=NoBody(),
response_type=BatchObject,
)
def list_batches(
self, *, key: str, provider: str | None = None
) -> Result[BatchList]:
return self.gateway.transport.get(
_batches_path(provider),
headers=self.gateway.transport.bearer(key),
params=NoBody(),
response_type=BatchList,
)
def delete_file(
self, file_id: str, *, key: str, provider: str | None = None
) -> Result[FileDeleteResponse]:
return self.gateway.transport.delete(
f"{_files_path(provider)}/{file_id}",
headers=self.gateway.transport.bearer(key),
json=NoBody(),
response_type=FileDeleteResponse,
)
def _files_path(provider: str | None) -> str:
return f"/{provider}/v1/files" if provider else "/v1/files"
def _batches_path(provider: str | None) -> str:
return f"/{provider}/v1/batches" if provider else "/v1/batches"
def build_client() -> BatchClient:
return BatchClient(gateway=build_gateway())

147
tests/e2e/batches/capabilities.py Normal file
View file

@ -0,0 +1,147 @@
"""The declarative provider x routing-scenario matrix the lifecycle test runs.
One Capability per supported (provider, scenario) pair, so the parametrized test
has no dead/skipped cells. `provider` is litellm's custom_llm_provider, used to
route provider-fallback calls to /{provider}/v1/... and to assert the raw batch id
shape (the only scenario whose id is not re-encoded by the proxy). Operations that
a provider does not support (Bedrock: no cancel, no list) are gated per row.
"""
from __future__ import annotations
import base64
from dataclasses import dataclass
from typing import Literal
Scenario = Literal["encoded", "unified", "model_param", "provider_fallback"]
IdShape = Literal["managed", "model_encoded", "raw"]
SCENARIOS: tuple[Scenario, ...] = (
"encoded",
"unified",
"model_param",
"provider_fallback",
)
@dataclass(frozen=True, slots=True)
class Provider:
name: str
model: str
raw_model: str
can_cancel: bool
can_list: bool
@dataclass(frozen=True, slots=True)
class Capability:
provider: str
model: str
raw_model: str
scenario: Scenario
can_cancel: bool
can_list: bool
@property
def id(self) -> str:
return f"{self.provider}-{self.scenario}"
@property
def jsonl_model(self) -> str:
"""Model name embedded in the uploaded JSONL ``body.model``.
Only the unified upload path rewrites JSONL on upload
(``target_model_names`` → ``llm_router.acreate_file`` →
``replace_model_in_jsonl``), so that scenario can use the LiteLLM alias
and rely on the proxy to swap it to the deployment model. Every other
scenario uploads raw JSONL with no rewrite, so the provider's real
deployment name is required or create fails upstream validation."""
return self.model if self.scenario == "unified" else self.raw_model
PROVIDERS: tuple[Provider, ...] = (
Provider("openai", "openai-batch", "gpt-4o-mini", can_cancel=True, can_list=True),
Provider("azure", "azure-batch", "gpt-4.1-mini-batch", can_cancel=True, can_list=True),
Provider(
"vertex_ai", "vertex-batch", "gemini-2.5-flash", can_cancel=True, can_list=True
),
# Provider(
# "bedrock",
# "bedrock-batch",
# "us.anthropic.claude-haiku-4-5-20251001-v1:0",
# can_cancel=False,
# can_list=False,
# ),
)
CAPABILITIES: tuple[Capability, ...] = tuple(
Capability(p.name, p.model, p.raw_model, scenario, p.can_cancel, p.can_list)
for p in PROVIDERS
for scenario in SCENARIOS
)
def raw_id_matches_provider(provider: str, batch_id: str) -> bool:
"""The provider-fallback path returns the provider's native batch id (unencoded),
so its shape discriminates which provider actually handled the batch."""
if provider in ("openai", "azure"):
return batch_id.startswith("batch")
if provider == "vertex_ai":
# Vertex returns the batch prediction job id, which depending on the
# routing path arrives either as the full resource name
# (projects/.../batchPredictionJobs/<id>) or as just the trailing
# numeric id, so accept either form.
return (
batch_id.startswith("projects/")
or "batchPredictionJobs" in batch_id
or batch_id.isdigit()
)
if provider == "bedrock":
return batch_id.startswith("arn:aws")
return True
FILE_ID_SHAPE: dict[Scenario, IdShape] = {
"encoded": "model_encoded",
"unified": "managed",
"model_param": "raw",
"provider_fallback": "raw",
}
BATCH_ID_SHAPE: dict[Scenario, IdShape] = {
"encoded": "model_encoded",
"unified": "managed",
"model_param": "model_encoded",
"provider_fallback": "raw",
}
def _b64_decode(value: str) -> str:
padded = value + "=" * (-len(value) % 4)
try:
return base64.urlsafe_b64decode(padded).decode()
except Exception:
return ""
def is_managed_id(id_str: str) -> bool:
"""A litellm managed unified file/batch id base64-decodes to a litellm_proxy marker."""
return _b64_decode(id_str).startswith("litellm_proxy")
def is_model_encoded_id(id_str: str) -> bool:
"""A model-encoded id keeps the provider prefix and base64-encodes litellm:<id>;model,<m>."""
for prefix in ("file-", "batch_"):
if id_str.startswith(prefix):
decoded = _b64_decode(id_str[len(prefix) :])
return decoded.startswith("litellm:") and ";model," in decoded
return False
def matches_id_shape(shape: IdShape, id_str: str) -> bool:
if shape == "managed":
return is_managed_id(id_str)
if shape == "model_encoded":
return is_model_encoded_id(id_str)
return not is_managed_id(id_str) and not is_model_encoded_id(id_str)

16
tests/e2e/batches/conftest.py Normal file
View file

@ -0,0 +1,16 @@
"""Batches suite's `client` fixture.
The shared lifecycle (resources/scoped_key), proxy liveness skip, and e2e marker
live in the parent tests/e2e/conftest.py. BatchClient holds the shared Gateway, so
the `resources` fixture cleans up keys through it; tests register file deletes and
batch cancels via `resources.defer(...)`.
"""
import pytest
from batch_client import BatchClient, build_client
@pytest.fixture(scope="session")
def client() -> BatchClient:
return build_client()

289
tests/e2e/batches/test_batches_e2e.py Normal file
View file

@ -0,0 +1,289 @@
"""Live e2e for the Batches API across every provider LiteLLM supports.
Synchronous tier only: a batch's completion window is 24h, so these never wait for
"completed". Each case uploads a tiny JSONL, creates the batch through one of the
four routing scenarios, asserts it was accepted (non-terminal status) and routed to
the right provider, then retrieves / cancels / lists where the provider supports it.
Everything created is deleted on teardown. Completion + cost tracking are out of
scope here (see COVERAGE.md).
Routing signal: for provider_fallback the raw batch id discriminates the provider;
for the encoded/unified/model_param scenarios the proxy re-encodes the id, so the
load-bearing signal is that create SUCCEEDS against that provider's own model - a
misroute to the wrong provider fails the create.
"""
from __future__ import annotations
import json
import os
import time
from typing import Callable
import pytest
from batch_client import (
BatchClient,
BatchCreateBody,
BatchObject,
FileObject,
is_model_access_denied,
is_result_access_denied,
)
from capabilities import (
BATCH_ID_SHAPE,
CAPABILITIES,
FILE_ID_SHAPE,
Capability,
matches_id_shape,
raw_id_matches_provider,
)
from e2e_http import (
FileUploadForm,
Result,
StreamingResponse,
require_successful_call,
unwrap,
)
from lifecycle import ResourceManager
pytestmark = pytest.mark.e2e
CREATED_BATCH_STATUSES = {"validating", "in_progress", "finalizing"}
BATCH_CANCEL_DELAY_SECONDS = 2
BATCH_TERMINAL_BEFORE_CANCEL = {"failed", "cancelled", "expired"}
def render_jsonl(model: str) -> bytes:
line = {
"custom_id": "req-1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": model,
"messages": [{"role": "user", "content": "ping"}],
"max_tokens": 8,
},
}
return (json.dumps(line) + "\n").encode()
def upload_for_scenario(
client: BatchClient, cap: Capability, content: bytes, key: str
) -> Result[FileObject]:
if cap.scenario == "encoded":
return client.upload_file(
content=content,
form=FileUploadForm(purpose="batch"),
model=cap.model,
key=key,
)
if cap.scenario == "unified":
return client.upload_file(
content=content,
form=FileUploadForm(purpose="batch", target_model_names=cap.model),
key=key,
)
return client.upload_file(
content=content,
form=FileUploadForm(purpose="batch"),
key=key,
provider=cap.provider,
)
def create_for_scenario(
client: BatchClient, cap: Capability, file_id: str, key: str
) -> StreamingResponse:
if cap.scenario == "model_param":
return client.create_batch(
body=BatchCreateBody(input_file_id=file_id, model=cap.model), key=key
)
if cap.scenario == "provider_fallback":
return client.create_batch(
body=BatchCreateBody(input_file_id=file_id), key=key, provider=cap.provider
)
return client.create_batch(body=BatchCreateBody(input_file_id=file_id), key=key)
def op_provider(cap: Capability) -> str | None:
"""provider_fallback ids are raw, so retrieve/cancel/list/delete need the provider
hint; the other scenarios encode it into the id and route automatically."""
return cap.provider if cap.scenario == "provider_fallback" else None
def quietly(action: Callable[[], object]) -> Callable[[], None]:
"""Adapt a value-returning call into a best-effort cleanup the teardown can run."""
def run() -> None:
action()
return run
def assert_file_object(file: FileObject) -> None:
assert file.object == "file", f"file.object={file.object!r}"
assert file.purpose == "batch", f"file.purpose={file.purpose!r}"
assert file.bytes is not None and file.bytes > 0, f"file.bytes={file.bytes!r}"
assert file.status, "file.status missing"
assert (
file.created_at is not None and file.created_at > 0
), "file.created_at missing"
def assert_batch_object(batch: BatchObject) -> None:
assert batch.object == "batch", f"batch.object={batch.object!r}"
if batch.endpoint:
assert (
batch.endpoint == "/v1/chat/completions"
), f"batch.endpoint={batch.endpoint!r}"
assert batch.completion_window == "24h", f"window={batch.completion_window!r}"
assert batch.input_file_id, "batch.input_file_id missing"
assert (
batch.created_at is not None and batch.created_at > 0
), "batch.created_at missing"
@pytest.mark.parametrize("cap", CAPABILITIES, ids=[c.id for c in CAPABILITIES])
def test_batch_lifecycle(
cap: Capability, client: BatchClient, resources: ResourceManager
) -> None:
key = resources.key()
provider = op_provider(cap)
file = unwrap(upload_for_scenario(client, cap, render_jsonl(cap.jsonl_model), key))
resources.defer(
quietly(lambda: client.delete_file(file.id, key=key, provider=provider))
)
assert_file_object(file)
assert matches_id_shape(
FILE_ID_SHAPE[cap.scenario], file.id
), f"{cap.id}: file id {file.id!r} is not a {FILE_ID_SHAPE[cap.scenario]} id"
created = create_for_scenario(client, cap, file.id, key)
require_successful_call(created)
batch = BatchObject.model_validate_json(created.body)
resources.defer(
quietly(lambda: client.cancel_batch(batch.id, key=key, provider=provider))
)
assert batch.id, f"create returned no batch id (body={created.body[:200]})"
assert (
batch.status in CREATED_BATCH_STATUSES
), f"freshly created batch has non-transitional status {batch.status!r}"
assert_batch_object(batch)
assert matches_id_shape(
BATCH_ID_SHAPE[cap.scenario], batch.id
), f"{cap.id}: batch id {batch.id!r} is not a {BATCH_ID_SHAPE[cap.scenario]} id"
if cap.scenario == "provider_fallback":
assert raw_id_matches_provider(
cap.provider, batch.id
), f"{cap.provider} batch id {batch.id!r} not in that provider's native shape; misrouted?"
fetched = unwrap(client.retrieve_batch(batch.id, key=key, provider=provider))
assert_batch_object(fetched)
assert fetched.id == batch.id
assert (
fetched.input_file_id == batch.input_file_id
), "retrieve changed input_file_id"
assert fetched.status, "retrieved batch has no status"
if cap.can_cancel:
time.sleep(BATCH_CANCEL_DELAY_SECONDS)
pre_cancel = unwrap(client.retrieve_batch(batch.id, key=key, provider=provider))
assert (
pre_cancel.status not in BATCH_TERMINAL_BEFORE_CANCEL
), (
f"batch reached {pre_cancel.status!r} before cancel; "
"provider likely rejected the input"
)
if pre_cancel.status == "completed":
return
cancelled = unwrap(client.cancel_batch(batch.id, key=key, provider=provider))
assert cancelled.id == batch.id
assert cancelled.object == "batch"
# Vertex cancel is async: the job may still show its pre-cancel status
# briefly before transitioning to cancelling/cancelled.
valid_post_cancel = {"cancelling", "cancelled"}
if cap.provider == "vertex_ai":
valid_post_cancel |= CREATED_BATCH_STATUSES
assert cancelled.status in valid_post_cancel, (
f"unexpected post-cancel status {cancelled.status!r}"
)
if cap.can_list:
listed = unwrap(client.list_batches(key=key, provider=provider))
# OpenAI includes object="list"; Azure provider list often omits the envelope field.
if listed.object is not None:
assert listed.object == "list", f"list envelope object={listed.object!r}"
match = next((b for b in listed.data if b.id == batch.id), None)
assert match is not None, "created batch absent from list"
assert match.object == "batch"
def test_batch_key_model_access_denied(
client: BatchClient, resources: ResourceManager
) -> None:
key = resources.key(models=["openai-batch"])
denied_upload = client.upload_file(
content=render_jsonl("azure-batch"),
form=FileUploadForm(purpose="batch"),
model="azure-batch",
key=key,
)
assert is_result_access_denied(
denied_upload
), f"restricted key uploaded a file for a disallowed model: {denied_upload}"
raw_file = unwrap(
client.upload_file(
content=render_jsonl("openai-batch"),
form=FileUploadForm(purpose="batch"),
key=key,
provider="openai",
)
).id
resources.defer(
quietly(lambda: client.delete_file(raw_file, key=key, provider="openai"))
)
denied_create = client.create_batch(
body=BatchCreateBody(input_file_id=raw_file, model="azure-batch"), key=key
)
assert is_model_access_denied(
denied_create
), f"restricted key created a batch for a disallowed model (status {denied_create.status_code})"
def test_file_upload_and_delete_outputs(
client: BatchClient, resources: ResourceManager
) -> None:
key = resources.key()
file = unwrap(
client.upload_file(
content=render_jsonl("openai-batch"),
form=FileUploadForm(purpose="batch"),
model="openai-batch",
key=key,
)
)
assert_file_object(file)
deleted = unwrap(client.delete_file(file.id, key=key))
assert deleted.id, "delete response has no id"
assert deleted.object == "file", f"delete object={deleted.object!r}"
assert deleted.deleted is True, "file was not reported deleted"
def test_anthropic_batch_retrieve(client: BatchClient, scoped_key: str) -> None:
batch_id = os.environ.get("ANTHROPIC_BATCH_ID")
if not batch_id:
pytest.skip(
"set ANTHROPIC_BATCH_ID to a real anthropic batch id to exercise retrieve"
)
fetched = unwrap(
client.retrieve_batch(batch_id, key=scoped_key, provider="anthropic")
)
assert fetched.id == batch_id
assert fetched.status

57
tests/e2e/e2e_http.py
View file

@ -36,6 +36,16 @@ class NoBody(BaseModel):
"""Empty body/query for routes that take none."""
class FileUploadForm(BaseModel):
"""Multipart form fields for POST /v1/files. The file bytes are passed
separately; `model` is not here because the proxy reads it from the query
(?model=) not the form."""
purpose: str = "batch"
target_model_names: str | None = None
custom_llm_provider: str | None = None
# ---------- Result types ----------
R = TypeVar("R", bound=BaseModel)
@ -304,3 +314,50 @@ def stream(
"""Streaming (SSE) call: consumes the stream counting events, and captures the
x-litellm-call-id + content-type headers. Body is elided."""
return send(url, headers=headers, json=json, stream=True, timeout=timeout)
def upload[R: BaseModel](
url: URL,
*,
headers: BaseModel,
form: FileUploadForm,
filename: str,
content: bytes,
params: BaseModel | None = None,
response_type: type[R],
timeout: float = 60.0,
) -> Result[R]:
"""Multipart POST for file uploads (/v1/files). Form fields come from `form`,
the file bytes are sent as the `file` part, and `params` carries any query
routing (e.g. ?model=). requests sets the multipart Content-Type itself."""
dumped: dict[str, object] = form.model_dump(by_alias=True, exclude_none=True)
data = {key: str(value) for key, value in dumped.items()}
try:
resp = requests.post(
str(url),
headers=_headers(headers),
params=_params(params),
data=data,
files={"file": (filename, content, "application/jsonl")},
timeout=timeout,
)
except requests.RequestException as exc:
return NetworkError(message=str(exc))
return _classify(resp, response_type)
def download(
url: URL, *, headers: BaseModel, timeout: float = 60.0
) -> StreamingResponse:
"""Raw GET for file content (/v1/files/{id}/content): provider-native bytes, no
schema. Returns the decoded body and the x-litellm-call-id header."""
try:
resp = requests.get(str(url), headers=_headers(headers), timeout=timeout)
except requests.RequestException as exc:
return StreamingResponse(status_code=-1, body=str(exc))
return StreamingResponse(
status_code=resp.status_code,
call_id=_hdr(resp, "x-litellm-call-id"),
content_type=_hdr(resp, "content-type"),
body=resp.text,
)

52
tests/e2e/gateway/litellm-config.yml
View file

@ -208,6 +208,57 @@ model_list:
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: us-central1
# batch models exercised by tests/e2e/batches/
- model_name: openai-batch
litellm_params:
model: openai/gpt-4o-mini
api_key: os.environ/OPENAI_API_KEY
model_info:
mode: batch
- model_name: azure-batch
litellm_params:
model: azure/gpt-4.1-mini-batch
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
api_version: "2024-07-01-preview"
model_info:
mode: batch
- model_name: vertex-batch
litellm_params:
model: vertex_ai/gemini-2.5-flash
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: us-central1
vertex_credentials: os.environ/VERTEXAI_CREDENTIALS
bucket_name: os.environ/GCS_BUCKET_NAME
model_info:
mode: batch
- model_name: bedrock-batch
litellm_params:
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
s3_bucket_name: os.environ/AWS_BATCH_S3_BUCKET
s3_region_name: us-west-2
s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID
s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN
model_info:
mode: batch
files_settings:
- custom_llm_provider: openai
api_key: os.environ/OPENAI_API_KEY
- custom_llm_provider: azure
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
api_version: "2024-07-01-preview"
- custom_llm_provider: vertex_ai
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: us-central1
vertex_credentials: os.environ/VERTEXAI_CREDENTIALS
bucket_name: os.environ/GCS_BUCKET_NAME
mcp_servers:
deepwiki_mcp:
@ -234,4 +285,3 @@ guardrails:
CREDIT_CARD: BLOCK
US_SSN: BLOCK
PHONE_NUMBER: BLOCK

9
tests/e2e/lifecycle.py
View file

@ -98,9 +98,12 @@ class ResourceManager:
"""Register a teardown action for any resource the test just created."""
self._cleanups.append(cleanup)
def key(self) -> str:
"""Create an all-models virtual key; delete it on teardown."""
key = self.client.generate_key(KeyGenerateBody(models=[]))
def key(self, models: list[str] | None = None, user_id: str | None = "e2e-test-user") -> str:
"""Create a virtual key; delete it on teardown. `models` restricts which
models the key may call (None/[] means all). `user_id` is required for
managed-batch ACL: the proxy stores created_by=user_id and checks it on
retrieve/cancel; None here means the 403 guard fires."""
key = self.client.generate_key(KeyGenerateBody(models=models or [], user_id=user_id))
self.defer(lambda: self.client.delete_key(key))
return key

10
tests/e2e/spend_tracking/SPEND_TRACKING_COVERAGE_MATRIX.md
View file

@ -19,7 +19,7 @@ proxy + SpendLogs rows. Status: `covered` / `partial` / `gap`.
|------|----------|-------|--------|----------|
| `_get_status_for_spend_log` | `test_spend_tracking_utils.py` | unit | covered | yes (status read off the row) |
| cache-hit `request_id` suffix | `test_spend_tracking_utils.py` | unit | covered | yes (`test_cache_hit_is_zero_cost_and_suffixed`) |
| failure status + zero spend | `test_spend_tracking_utils.py` | unit | covered | no (live failure logging is non-deterministic across providers) |
| failure status + zero spend | `test_spend_tracking_utils.py` | unit | partial | yes (`test_failure_call_writes_failure_status_row`) |
| per-model / per-provider attribution | `test_spend_tracking_utils.py` | unit | covered | yes (`test_each_model_on_a_shared_key_gets_its_own_row`) |
| field population (model/tokens/api_key/team/org) | `test_spend_tracking_utils.py` | unit | partial | yes (asserts real values) |
| `request_tags` propagation | `test_db_spend_update_writer.py` | unit | partial | yes (`test_request_tags_round_trip`) |
@ -40,9 +40,9 @@ proxy + SpendLogs rows. Status: `covered` / `partial` / `gap`.
| Entity | Existing | Status | Live e2e |
|--------|----------|--------|----------|
| API key | `test_db_spend_update_writer.py`, `test_spend_counters.py` | covered | yes (`test_key_spend_equals_sum_of_logs`) |
| Tag | `test_update_daily_tag_spend.py` | partial | yes (`test_request_tags_round_trip`, propagation only) |
| Tag | `test_update_daily_tag_spend.py` | partial | yes (`test_tag_spend_matches_sum_of_tagged_logs`) |
| End-user | `test_proxy_update_spend.py` | covered | yes |
| Spend == sum(logs) consistency | none | gap | yes (key aggregate == sum of rows) |
| Spend == sum(logs) consistency | none | gap | yes (key + tag aggregate == sum of rows) |
## Spend read endpoints (verification surface)
@ -50,7 +50,7 @@ proxy + SpendLogs rows. Status: `covered` / `partial` / `gap`.
|----------|----------|--------|----------|
| `/spend/logs` (request_id / api_key) | `test_spend_management_endpoints.py` | covered | yes (primary read path; `test_spend_logs_endpoint_returns_spend` asserts 200 + spend, never 5xx) |
| `/spend/calculate` | `local_testing/test_spend_calculate_endpoint.py` | covered | yes (`test_spend_calculate_returns_nonzero_cost`) |
| `/spend/tags` | `test_spend_management_endpoints.py` | partial | yes (`test_spend_routes.py` route probe) |
| `/spend/tags` | `test_spend_management_endpoints.py` | partial | yes (tag accuracy test) |
| whole spend GET surface (22 routes) | unit per-handler | partial | yes (`test_spend_routes.py` probes each for 404/5xx) |
## What this suite pins
@ -63,8 +63,10 @@ proxy + SpendLogs rows. Status: `covered` / `partial` / `gap`.
| `test_cache_hit_is_zero_cost_and_suffixed` | cache hits not double-charged; `_cache_hit` suffix |
| `test_key_spend_equals_sum_of_logs` | key aggregate == sum of rows |
| `test_request_tags_round_trip` | tags persist onto the row |
| `test_tag_spend_matches_sum_of_tagged_logs` | `/spend/tags` SUM/COUNT == tagged rows |
| `test_end_user_spend_attributed_on_row` | `end_user` attributed + costed |
| `test_each_model_on_a_shared_key_gets_its_own_row` | per-model/provider rows, correct model + cost, distinct request_ids matching response id |
| `test_failure_call_writes_failure_status_row` | failed call -> `status=failure`, `spend=0` |
| `test_spend_calculate_returns_nonzero_cost` | cost-map smoke (no batch wait) |
| `test_spend_logs_endpoint_returns_spend` | `/spend/logs` returns 200 + the key's spend, never a 5xx (intermittent-500 regression) |
| `test_spend_routes.py` (23) | no spend route 404s or 5xxs |

35
tests/e2e/spend_tracking/spend_e2e_client.py
View file

@ -2,8 +2,8 @@
Generic proxy operations (keys, customers, chat/embed, route probing, SpendLogs
polling) come from the shared Gateway, DI'd in (composition, not inheritance).
This client adds only the spend surface: /spend/calculate, key-spend
polling, and the route probes the breadth test uses.
This client adds only the spend surface: /spend/calculate, /spend/tags,
key-spend polling, and the route probes the breadth test uses.
Re-exports unwrap / is_ok / unique_marker / SpendLogRow so the tests import their
helpers from one place.
@ -22,6 +22,7 @@ from e2e_http import (
ProbeResult,
Result,
StreamingResponse,
Success,
is_ok,
unwrap,
)
@ -38,6 +39,8 @@ from models import (
SpendCalculateBody,
SpendCalculateResponse,
SpendLogRow,
SpendTagsResponse,
TagSpend,
)
__all__ = [
@ -139,6 +142,34 @@ class SpendClient:
)
).cost
def spend_by_tags(self) -> list[TagSpend]:
result = self.gateway.transport.get(
"/spend/tags",
headers=self.gateway.transport.master,
params=NoBody(),
response_type=SpendTagsResponse,
)
match result:
case Success(data=data):
return data.spend_per_tag or []
case _:
return []
def poll_tag_spend(self, tag: str, *, minimum: float = 0.0) -> TagSpend | None:
"""Poll /spend/tags until the tag's aggregate reaches `minimum`; last seen."""
deadline = time.monotonic() + self.gateway.poll_timeout
entry: TagSpend | None = None
while time.monotonic() < deadline:
matches = [
t for t in self.spend_by_tags() if t.individual_request_tag == tag
]
if matches:
entry = matches[0]
if (entry.total_spend or 0.0) >= minimum:
return entry
time.sleep(self.gateway.poll_interval)
return entry
def poll_key_spend(self, key: str, *, minimum: float = 0.0) -> float:
deadline = time.monotonic() + self.gateway.poll_timeout
spend = 0.0

58
tests/e2e/spend_tracking/test_spend_tracking_e2e.py
View file

@ -23,7 +23,7 @@ import pytest
from e2e_http import Success
from lifecycle import ResourceManager
from models import SpendLogs, SpendLogsParams
from spend_e2e_client import SpendClient, SpendLogRow, unique_marker, unwrap
from spend_e2e_client import SpendClient, SpendLogRow, is_ok, unique_marker, unwrap
pytestmark = pytest.mark.e2e
@ -218,6 +218,43 @@ def test_request_tags_round_trip(client: SpendClient, scoped_key: str) -> None:
)
def test_tag_spend_matches_sum_of_tagged_logs(
client: SpendClient, scoped_key: str
) -> None:
# Unique tag so /spend/tags can't be polluted by other rows; unique content
# per call so both are fresh misses (paid), not cache hits.
tag = f"e2e-tagspend-{unique_marker()}"
for _ in range(2):
_ = unwrap(
client.chat(
scoped_key,
"gemini-2.5-flash",
f"hi {unique_marker()}",
tags=[tag],
max_tokens=16,
)
)
rows = client.poll_logs_for_key(
scoped_key,
min_rows=2,
predicate=lambda rs: sum((r.spend or 0) for r in rs) > 0,
)
tagged = [r for r in rows if tag in (r.request_tags or [])]
assert len(tagged) >= 2, f"expected 2 tagged rows, saw {_summarize(rows)}"
logs_total = sum((r.spend or 0) for r in tagged)
assert logs_total > 0
entry = client.poll_tag_spend(tag, minimum=logs_total * 0.999)
assert entry is not None, f"tag {tag!r} never appeared in /spend/tags"
assert _approx_equal(entry.total_spend or 0, logs_total), (
f"/spend/tags total_spend {entry} != sum of tagged rows {logs_total}"
)
assert (entry.log_count or 0) == len(tagged), (
f"/spend/tags log_count {entry.log_count} != tagged rows {len(tagged)}"
)
def test_end_user_spend_attributed_on_row(
client: SpendClient, scoped_key: str, resources: ResourceManager
) -> None:
@ -283,6 +320,25 @@ def test_each_model_on_a_shared_key_gets_its_own_row(
), f"claude row request_id {claude_row.request_id} != response id {claude.id}"
def test_failure_call_writes_failure_status_row(
client: SpendClient, scoped_key: str
) -> None:
result = client.chat(scoped_key, "gemini-2.5-flash", "", max_tokens=1)
if is_ok(result):
pytest.skip("call unexpectedly succeeded; could not induce a failure row")
rows = client.poll_logs_for_key(
scoped_key, predicate=lambda rs: any(r.status == "failure" for r in rs)
)
failure_rows = [r for r in rows if r.status == "failure"]
if not failure_rows:
pytest.skip(
"no failure-status row was logged for the rejected call; "
"failure logging is environment-specific"
)
assert (failure_rows[0].spend or 0) == 0.0, "failed call must not be charged"
def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
cost = client.calculate_spend(
"gemini-2.5-flash", "estimate the cost of this request"

74
tests/e2e/transport.py
View file

@ -13,7 +13,14 @@ from typing import Protocol
from pydantic import BaseModel
import e2e_http
from e2e_http import URL, AuthHeaders, ProbeResult, Result, StreamingResponse
from e2e_http import (
URL,
AuthHeaders,
FileUploadForm,
ProbeResult,
Result,
StreamingResponse,
)
class Transport(Protocol):
@ -50,6 +57,20 @@ class Transport(Protocol):
def probe(self, path: str, *, params: BaseModel) -> ProbeResult: ...
def upload[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
form: FileUploadForm,
filename: str,
content: bytes,
params: BaseModel | None = None,
response_type: type[R],
) -> Result[R]: ...
def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: ...
def bearer(self, key: str) -> AuthHeaders: ...
@property
@ -143,6 +164,33 @@ class HttpTransport:
timeout=self.request_timeout,
)
def upload[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
form: FileUploadForm,
filename: str,
content: bytes,
params: BaseModel | None = None,
response_type: type[R],
) -> Result[R]:
return e2e_http.upload(
self._url(path),
headers=headers,
form=form,
filename=filename,
content=content,
params=params,
response_type=response_type,
timeout=self.request_timeout,
)
def download(self, path: str, *, headers: BaseModel) -> StreamingResponse:
return e2e_http.download(
self._url(path), headers=headers, timeout=self.request_timeout
)
# Top-level management/admin route groups. In a split deployment these are served
# by the control plane (a different service from the LLM data plane). LLM routes
@ -242,3 +290,27 @@ class SplitTransport:
def probe(self, path: str, *, params: BaseModel) -> ProbeResult:
return self._route(path).probe(path, params=params)
def upload[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
form: FileUploadForm,
filename: str,
content: bytes,
params: BaseModel | None = None,
response_type: type[R],
) -> Result[R]:
return self._route(path).upload(
path,
headers=headers,
form=form,
filename=filename,
content=content,
params=params,
response_type=response_type,
)
def download(self, path: str, *, headers: BaseModel) -> StreamingResponse:
return self._route(path).download(path, headers=headers)

74
tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py
View file

@ -1404,7 +1404,7 @@ async def test_store_unified_file_id_with_none_file_object():
from litellm.proxy._types import UserAPIKeyAuth
prisma_client = AsyncMock()
prisma_client.db.litellm_managedfiletable.create = AsyncMock(
prisma_client.db.litellm_managedfiletable.upsert = AsyncMock(
return_value=MagicMock()
)
internal_usage_cache = MagicMock()
@ -1424,11 +1424,73 @@ async def test_store_unified_file_id_with_none_file_object():
user_api_key_dict=UserAPIKeyAuth(user_id="test-user"),
)
# Verify DB create was called with expected data (without file_object)
prisma_client.db.litellm_managedfiletable.create.assert_called_once()
call_args = prisma_client.db.litellm_managedfiletable.create.call_args
assert call_args.kwargs["data"]["unified_file_id"] == "test-unified-file-id"
assert "file_object" not in call_args.kwargs["data"]
# Verify DB upsert was called idempotently with expected create data (without file_object)
prisma_client.db.litellm_managedfiletable.upsert.assert_called_once()
call_args = prisma_client.db.litellm_managedfiletable.upsert.call_args
assert call_args.kwargs["where"] == {"unified_file_id": "test-unified-file-id"}
create_data = call_args.kwargs["data"]["create"]
assert create_data["unified_file_id"] == "test-unified-file-id"
assert "file_object" not in create_data
@pytest.mark.asyncio
async def test_store_unified_file_id_updates_file_metadata_on_existing_row():
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.llms.openai import OpenAIFileObject
prisma_client = AsyncMock()
prisma_client.db.litellm_managedfiletable.upsert = AsyncMock(
return_value=MagicMock()
)
internal_usage_cache = MagicMock()
internal_usage_cache.async_set_cache = AsyncMock()
proxy_managed_files = _PROXY_LiteLLMManagedFiles(
internal_usage_cache=internal_usage_cache,
prisma_client=prisma_client,
)
user_api_key_dict = UserAPIKeyAuth(user_id="test-user")
await proxy_managed_files.store_unified_file_id(
file_id="test-unified-file-id",
file_object=None,
litellm_parent_otel_span=None,
model_mappings={"model-123": "file-provider-xyz"},
user_api_key_dict=user_api_key_dict,
)
file_object = OpenAIFileObject(
id="file-provider-xyz",
object="file",
bytes=1234,
created_at=1234567890,
filename="output.jsonl",
purpose="batch_output",
status="processed",
)
file_object._hidden_params = {
"storage_backend": "s3",
"storage_url": "s3://bucket/output.jsonl",
}
await proxy_managed_files.store_unified_file_id(
file_id="test-unified-file-id",
file_object=file_object,
litellm_parent_otel_span=None,
model_mappings={"model-123": "file-provider-xyz"},
user_api_key_dict=user_api_key_dict,
)
first_update = prisma_client.db.litellm_managedfiletable.upsert.await_args_list[
0
].kwargs["data"]["update"]
second_update = prisma_client.db.litellm_managedfiletable.upsert.await_args_list[
1
].kwargs["data"]["update"]
assert "file_object" not in first_update
assert second_update["file_object"] == file_object.model_dump_json()
assert second_update["storage_backend"] == "s3"
assert second_update["storage_url"] == "s3://bucket/output.jsonl"
@pytest.mark.asyncio

7
tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py
View file

@ -527,12 +527,11 @@ def test_backward_compatibility_regular_nova_model():
assert result["imageGenerationConfig"]["cfg_scale"] == 7
def test_amazon_titan_image_gen():
"""Test Amazon Titan image generation with cost tracking."""
def test_amazon_nova_canvas_image_gen():
"""Test Amazon Nova Canvas image generation with cost tracking."""
from litellm import image_generation
# Use v2 as v1 has reached end of life
model_id = "bedrock/amazon.titan-image-generator-v2:0"
model_id = "bedrock/amazon.nova-canvas-v1:0"
response = litellm.image_generation(
model=model_id,

4
tests/local_testing/test_embedding.py
View file

@ -765,6 +765,10 @@ def test_fireworks_embeddings():
pass
except litellm.InternalServerError as e:
pass
except litellm.APIError as e:
if "suspended" in str(e):
pytest.skip(f"Fireworks account suspended: {e}")
pytest.fail(f"Error occurred: {e}")
except Exception as e:
pytest.fail(f"Error occurred: {e}")

68
tests/logging_callback_tests/test_standard_logging_payload.py
View file

@ -335,6 +335,74 @@ def test_get_model_cost_information():
)
def test_get_model_cost_information_custom_pricing_uses_base_model():
result = StandardLoggingPayloadSetup.get_model_cost_information(
base_model="bedrock/invoke/global.anthropic.claude-opus-4-6-v1",
custom_pricing=True,
custom_llm_provider="bedrock",
init_response_obj={"model": "invoke_test_claude"},
)
assert result["model_map_value"] is not None
assert result["model_map_key"] != "invoke_test_claude"
def test_standard_logging_payload_uses_deployment_when_no_base_model():
"""metadata["deployment"] is used for cost-map lookup when base_model is not set."""
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import (
Logging,
get_standard_logging_object_payload,
)
logging_obj = Logging(
model="invoke_test_claude",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="completion",
start_time=datetime.now(),
litellm_call_id="test-deploy-fallback",
function_id="test-fn",
)
kwargs = {
"model": "invoke_test_claude",
"messages": [{"role": "user", "content": "hi"}],
"custom_llm_provider": "bedrock",
"litellm_params": {
"metadata": {
"deployment": "bedrock/invoke/global.anthropic.claude-opus-4-6-v1",
},
},
}
mock_response = {
"id": "chatcmpl-deploy-test",
"object": "chat.completion",
"model": "invoke_test_claude",
"usage": {"prompt_tokens": 5, "completion_tokens": 10, "total_tokens": 15},
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "hello"},
"finish_reason": "stop",
}
],
}
payload = get_standard_logging_object_payload(
kwargs=kwargs,
init_response_obj=mock_response,
start_time=datetime.now(),
end_time=datetime.now(),
logging_obj=logging_obj,
status="success",
)
assert payload is not None
assert payload["model_map_information"]["model_map_value"] is not None
assert payload["model_map_information"]["model_map_key"] != "invoke_test_claude"
def test_get_hidden_params():
"""Test get_hidden_params with different inputs"""
# Test with None

340
tests/proxy_unit_tests/test_check_batch_cost.py
View file

@ -1,13 +1,35 @@
"""
Unit tests for CheckBatchCost class.
Covers: stale-row cleanup (file_purpose scoping), paginated find_many,
and the batch_processed-column fallback query.
the batch_processed-column fallback query, and routing of unmanaged
Vertex batches (raw gs:// input_file_id, no managed unified id).
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
_IS_B64 = "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id"
def _unmanaged_vertex_file_object(
input_file_id="gs://bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash/abc.jsonl",
status="validating",
):
"""A LiteLLMBatch JSON blob shaped like what the managed-files hook stores for an
unmanaged Vertex batch (raw gs:// input_file_id)."""
from litellm.types.utils import LiteLLMBatch
return LiteLLMBatch(
id="8823717160934178816",
completion_window="24h",
created_at=1,
endpoint="/v1/chat/completions",
input_file_id=input_file_id,
object="batch",
status=status,
).model_dump_json()
class TestCheckBatchCost:
"""Test suite for CheckBatchCost class"""
@ -375,6 +397,76 @@ class TestCheckBatchCost:
), "update() must include batch_processed=True when column is present"
assert update_data["status"] == "complete"
@pytest.mark.asyncio
@pytest.mark.parametrize("terminal_status", ["failed", "expired", "cancelled"])
async def test_terminal_status_marks_job_processed(
self,
check_batch_cost_instance,
mock_prisma_client,
mock_llm_router,
terminal_status,
):
"""When the provider reports a terminal status (failed/expired/cancelled), the row
must be written back with that status and batch_processed=True so it stops being
polled forever.
"""
from unittest.mock import patch
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(
return_value=0
)
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(
return_value=None
)
mock_job = MagicMock()
mock_job.id = "job-terminal-1"
mock_job.unified_object_id = "dW5pZmllZF9iYXRjaF9pZA=="
mock_job.created_by = "user-1"
assert check_batch_cost_instance._has_batch_processed_column is True
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(
return_value=[mock_job]
)
mock_response = MagicMock()
mock_response.status = terminal_status
mock_response.model_dump_json.return_value = (
f'{{"id":"batch-1","status":"{terminal_status}"}}'
)
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;"
with (
patch(
"litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id",
side_effect=[decoded_id, None],
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_model_id_from_unified_batch_id",
return_value="model-123",
),
patch(
"litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id",
return_value="batch-456",
),
):
await check_batch_cost_instance.check_batch_cost()
assert (
mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1
), f"Expected update() to be called exactly once for a {terminal_status} job"
update_data = mock_prisma_client.db.litellm_managedobjecttable.update.call_args[
1
]["data"]
assert update_data["status"] == terminal_status
assert (
update_data["batch_processed"] is True
), "terminal-status update() must set batch_processed=True so polling stops"
@pytest.mark.asyncio
async def test_raw_output_file_id_converted_to_managed_id(
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
@ -512,3 +604,249 @@ class TestCheckBatchCost:
}
assert mock_response.output_file_id == fake_managed_output_id
assert mock_response.error_file_id == fake_managed_error_id
class TestUnmanagedVertexRouting:
"""Routing of unmanaged Vertex batches whose unified_object_id is a raw provider job id."""
def _instance(self, track_unmanaged, router):
from litellm_enterprise.proxy.common_utils.check_batch_cost import (
CheckBatchCost,
)
return CheckBatchCost(
proxy_logging_obj=MagicMock(),
prisma_client=MagicMock(),
llm_router=router,
track_unmanaged_vertex_batch_cost=track_unmanaged,
)
def _job(self, file_object=None):
job = MagicMock()
job.unified_object_id = "8823717160934178816"
job.file_object = (
file_object if file_object is not None else _unmanaged_vertex_file_object()
)
return job
def test_flag_off_skips_unmanaged_id_unchanged(self):
"""Default (flag off): a raw numeric unified_object_id is skipped exactly as before;
no model derivation or router lookup happens."""
router = MagicMock()
instance = self._instance(track_unmanaged=False, router=router)
prom = MagicMock()
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with("invalid_unified_id")
router.resolve_model_name_from_model_id.assert_not_called()
router.get_model_ids.assert_not_called()
def _vertex_deployment(self):
deployment = MagicMock()
deployment.litellm_params.custom_llm_provider = "vertex_ai"
deployment.litellm_params.model = "vertex_ai/gemini-2.5-flash"
return deployment
def test_flag_on_routes_to_vertex_deployment(self):
"""Flag on: derive the bare model from the gs:// path, resolve it to a deployment id,
and use the raw unified_object_id as the provider batch id."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = "gemini-2.5-flash"
router.get_model_ids.return_value = ["deploy-1"]
router.get_deployment = MagicMock(return_value=self._vertex_deployment())
instance = self._instance(track_unmanaged=True, router=router)
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), MagicMock())
assert result == ("deploy-1", "8823717160934178816")
# bare model name (trailing GCS segment), not the full publishers/.. path
router.resolve_model_name_from_model_id.assert_called_once_with(
"gemini-2.5-flash"
)
router.get_model_ids.assert_called_once_with(model_name="gemini-2.5-flash")
def test_flag_on_skips_non_vertex_deployment_sharing_model_group(self):
"""Flag on, but the only deployment for the model group is a non-vertex_ai
provider: must not be selected, even though the model group name matches."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = "gemini-2.5-flash"
router.get_model_ids.return_value = ["deploy-openai"]
non_vertex_deployment = MagicMock()
non_vertex_deployment.litellm_params.custom_llm_provider = "openai"
non_vertex_deployment.litellm_params.model = "gpt-4o"
router.get_deployment = MagicMock(return_value=non_vertex_deployment)
instance = self._instance(track_unmanaged=True, router=router)
prom = MagicMock()
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with(
"unmanaged_no_matching_deployment"
)
def test_flag_on_uses_later_vertex_deployment_with_matching_suffix(self):
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = "azure-gemini"
router.get_model_ids.return_value = ["deploy-azure"]
non_vertex_deployment = MagicMock()
non_vertex_deployment.litellm_params.custom_llm_provider = "azure"
non_vertex_deployment.litellm_params.model = "azure/gemini-2.5-flash"
router.get_deployment = MagicMock(return_value=non_vertex_deployment)
router.get_model_list.return_value = [
{
"model_name": "azure-gemini",
"litellm_params": {
"model": "azure/gemini-2.5-flash",
"custom_llm_provider": "azure",
},
"model_info": {"id": "deploy-azure"},
},
{
"model_name": "vertex-gemini",
"litellm_params": {
"model": "vertex_ai/gemini-2.5-flash",
"custom_llm_provider": "vertex_ai",
},
"model_info": {"id": "deploy-vertex"},
},
]
instance = self._instance(track_unmanaged=True, router=router)
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), MagicMock())
assert result == ("deploy-vertex", "8823717160934178816")
router.get_model_ids.assert_called_once_with(model_name="azure-gemini")
def test_flag_on_no_matching_deployment_records_metric(self):
"""Flag on but no vertex_ai deployment for the model: skip with a distinct metric."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = None
router.get_model_ids.return_value = []
instance = self._instance(track_unmanaged=True, router=router)
prom = MagicMock()
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(self._job(), prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with(
"unmanaged_no_matching_deployment"
)
def test_flag_on_non_gcs_input_is_not_unmanaged_vertex(self):
"""Flag on, but input_file_id is not a gs:// publishers path: treat as unroutable,
do not attempt model derivation."""
router = MagicMock()
instance = self._instance(track_unmanaged=True, router=router)
prom = MagicMock()
job = self._job(
file_object=_unmanaged_vertex_file_object(input_file_id="file-abc-123")
)
with patch(_IS_B64, return_value=False):
result = instance._resolve_job_routing(job, prom)
assert result is None
prom.record_check_batch_cost_error.assert_called_once_with("invalid_unified_id")
router.resolve_model_name_from_model_id.assert_not_called()
@pytest.mark.asyncio
async def test_end_to_end_costs_unmanaged_batch(self):
"""Flag on, completed unmanaged batch: the poller polls Vertex with the raw job id,
computes cost, and marks batch_processed=True. Fails before this change (the row is
skipped at the unified-id gate)."""
router = MagicMock()
router.resolve_model_name_from_model_id.return_value = "gemini-2.5-flash"
router.get_model_ids.return_value = ["deploy-1"]
mock_response = MagicMock()
mock_response.status = "completed"
mock_response.output_file_id = "gs://bucket/out/predictions.jsonl"
mock_response.error_file_id = None
mock_response.completed_at = None
mock_response.created_at = None
mock_response.model_dump_json.return_value = (
'{"id":"8823717160934178816","status":"completed"}'
)
router.aretrieve_batch = AsyncMock(return_value=mock_response)
router.get_deployment_credentials_with_provider = MagicMock(
return_value={"vertex_project": "p", "vertex_location": "us-central1"}
)
deployment = MagicMock()
deployment.litellm_params.custom_llm_provider = "vertex_ai"
deployment.litellm_params.model = "vertex_ai/gemini-2.5-flash"
deployment.model_name = "gemini-2.5-flash"
deployment.model_info.model_dump.return_value = {}
router.get_deployment = MagicMock(return_value=deployment)
instance = self._instance(track_unmanaged=True, router=router)
instance.proxy_logging_obj.get_proxy_hook.return_value = None
instance._has_batch_processed_column = True
prisma = instance.prisma_client
prisma.db = MagicMock()
prisma.db.litellm_managedobjecttable = MagicMock()
prisma.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=0)
prisma.db.litellm_managedobjecttable.update = AsyncMock()
prisma.db.litellm_managedobjecttable.find_many = AsyncMock(
return_value=[self._job()]
)
prisma.db.litellm_usertable = MagicMock()
prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
mock_file_content = MagicMock()
mock_file_content.content = b'{"id":"req-1"}'
with (
patch(_IS_B64, side_effect=[False, None]),
patch(
"litellm.files.main.afile_content",
new_callable=AsyncMock,
return_value=mock_file_content,
),
patch(
"litellm.batches.batch_utils._get_file_content_as_dictionary",
return_value=[{"id": "req-1"}],
),
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=(
0.01,
{"prompt_tokens": 10, "completion_tokens": 5},
["gemini-2.5-flash"],
),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
return_value=("gemini-2.5-flash", "vertex_ai", None, None),
),
patch(
"litellm.litellm_core_utils.litellm_logging.Logging"
) as mock_logging_cls,
):
mock_logging_obj = MagicMock()
mock_logging_obj.async_success_handler = AsyncMock()
mock_logging_cls.return_value = mock_logging_obj
await instance.check_batch_cost()
router.aretrieve_batch.assert_awaited_once()
assert router.aretrieve_batch.call_args[1]["model"] == "deploy-1"
assert router.aretrieve_batch.call_args[1]["batch_id"] == "8823717160934178816"
mock_logging_obj.async_success_handler.assert_awaited_once()
assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.01
assert prisma.db.litellm_managedobjecttable.update.call_count == 1
update_data = prisma.db.litellm_managedobjecttable.update.call_args[1]["data"]
assert update_data["batch_processed"] is True
assert update_data["status"] == "complete"

12
tests/proxy_unit_tests/test_proxy_server.py
View file

@ -2844,7 +2844,9 @@ async def test_get_config_callbacks_with_all_types(client_no_auth):
async def test_get_config_callbacks_environment_variables(client_no_auth):
"""
Test that /get/config/callbacks correctly includes environment variables
for each callback type. Values are returned as-is from the config (no decryption).
for each callback type. Under ``client_no_auth`` the resolved role is
not ``PROXY_ADMIN``, so values matched by the redaction helper come back
as ``"REDACTED"`` and other values pass through verbatim.
"""
from litellm.proxy.proxy_server import ProxyConfig
@ -2886,12 +2888,11 @@ async def test_get_config_callbacks_environment_variables(client_no_auth):
assert langfuse_callback["type"] == "success"
assert "variables" in langfuse_callback
# Verify langfuse env vars are present (values returned as-is, no decryption)
langfuse_vars = langfuse_callback["variables"]
assert "LANGFUSE_PUBLIC_KEY" in langfuse_vars
assert langfuse_vars["LANGFUSE_PUBLIC_KEY"] == "test-public-key"
assert langfuse_vars["LANGFUSE_PUBLIC_KEY"] == "REDACTED"
assert "LANGFUSE_SECRET_KEY" in langfuse_vars
assert langfuse_vars["LANGFUSE_SECRET_KEY"] == "test-secret-key"
assert langfuse_vars["LANGFUSE_SECRET_KEY"] == "REDACTED"
assert "LANGFUSE_HOST" in langfuse_vars
assert langfuse_vars["LANGFUSE_HOST"] == "https://cloud.langfuse.com"
@ -2901,14 +2902,13 @@ async def test_get_config_callbacks_environment_variables(client_no_auth):
assert otel_callback["type"] == "success_and_failure"
assert "variables" in otel_callback
# Verify otel env vars are present
otel_vars = otel_callback["variables"]
assert "OTEL_EXPORTER" in otel_vars
assert otel_vars["OTEL_EXPORTER"] == "otlp"
assert "OTEL_ENDPOINT" in otel_vars
assert otel_vars["OTEL_ENDPOINT"] == "http://localhost:4317"
assert "OTEL_HEADERS" in otel_vars
assert otel_vars["OTEL_HEADERS"] == "key=value"
assert otel_vars["OTEL_HEADERS"] == "REDACTED"
@pytest.mark.asyncio

276
tests/test_litellm/integrations/test_prometheus_mcp_tool_metrics.py Normal file
View file

@ -0,0 +1,276 @@
"""
Unit tests for MCP tool call Prometheus metrics (LIT-3765).
These metrics expose ``mcp_tool_call_metadata`` in Prometheus so Grafana
dashboards can break down MCP usage by server and tool name.
Run with:
uv run pytest tests/test_litellm/integrations/test_prometheus_mcp_tool_metrics.py -v
"""
from typing import get_args
from unittest.mock import MagicMock
import pytest
from litellm.integrations.prometheus import PrometheusLogger
from litellm.types.integrations.prometheus import (
DEFINED_PROMETHEUS_METRICS,
PrometheusMetricLabels,
UserAPIKeyLabelNames,
UserAPIKeyLabelValues,
)
MCP_METRICS = (
"litellm_mcp_tool_calls_total",
"litellm_mcp_tool_call_spend_metric",
)
def _make_mock_logger():
logger = MagicMock()
for name in MCP_METRICS:
setattr(logger, name, MagicMock())
logger.get_labels_for_metric = MagicMock(
return_value=PrometheusMetricLabels.litellm_mcp_tool_calls_total,
)
return logger
def _make_enum_values(
*,
mcp_tool_name: str = "get_weather",
mcp_server_name: str = "weather-server",
) -> UserAPIKeyLabelValues:
return UserAPIKeyLabelValues(
mcp_tool_name=mcp_tool_name,
mcp_server_name=mcp_server_name,
hashed_api_key="sk-hash-123",
api_key_alias="test-key",
team="team-1",
team_alias="Test Team",
user="user-1",
end_user="end-user-1",
)
def _make_payload(
*,
mcp_tool_name: str = "get_weather",
mcp_server_name: str = "weather-server",
response_cost: float = 0.005,
) -> dict:
return {
"model": "gpt-4o",
"model_group": "gpt-4o",
"model_id": "model-123",
"api_base": "https://api.openai.com",
"custom_llm_provider": "openai",
"response_cost": response_cost,
"completion_tokens": 50,
"prompt_tokens": 100,
"total_tokens": 150,
"request_tags": [],
"stream": False,
"metadata": {
"user_api_key_hash": "sk-hash-123",
"user_api_key_alias": "test-key",
"user_api_key_team_id": "team-1",
"user_api_key_team_alias": "Test Team",
"user_api_key_user_id": "user-1",
"user_api_key_user_email": None,
"user_api_key_org_id": None,
"user_api_key_org_alias": None,
"mcp_tool_call_metadata": {
"name": mcp_tool_name,
"mcp_server_name": mcp_server_name,
"namespaced_tool_name": f"{mcp_server_name}/{mcp_tool_name}",
"arguments": {"city": "SF"},
"result": {"temp": 72},
},
},
}
class TestMCPMetricRegistration:
def test_metrics_in_defined_prometheus_metrics(self):
defined = get_args(DEFINED_PROMETHEUS_METRICS)
for name in MCP_METRICS:
assert name in defined, f"{name} missing from DEFINED_PROMETHEUS_METRICS"
def test_metric_labels_defined(self):
for name in MCP_METRICS:
assert hasattr(PrometheusMetricLabels, name), f"{name} missing from PrometheusMetricLabels"
def test_mcp_labels_include_tool_and_server_name(self):
labels = PrometheusMetricLabels.litellm_mcp_tool_calls_total
assert UserAPIKeyLabelNames.MCP_TOOL_NAME.value in labels
assert UserAPIKeyLabelNames.MCP_SERVER_NAME.value in labels
def test_spend_metric_shares_label_set_with_calls_metric(self):
assert (
PrometheusMetricLabels.litellm_mcp_tool_call_spend_metric
== PrometheusMetricLabels.litellm_mcp_tool_calls_total
)
assert (
PrometheusMetricLabels.litellm_mcp_tool_call_spend_metric
is not PrometheusMetricLabels.litellm_mcp_tool_calls_total
)
def test_enum_values_accept_mcp_fields(self):
vals = _make_enum_values()
assert vals.mcp_tool_name == "get_weather"
assert vals.mcp_server_name == "weather-server"
def test_enum_values_default_mcp_fields_to_none(self):
vals = UserAPIKeyLabelValues(user="u1")
assert vals.mcp_tool_name is None
assert vals.mcp_server_name is None
class TestIncrementMCPToolCallMetrics:
def test_increments_calls_counter_when_mcp_metadata_present(self):
logger = _make_mock_logger()
payload = _make_payload()
enum_values = _make_enum_values()
PrometheusLogger._increment_mcp_tool_call_metrics(
logger,
standard_logging_payload=payload,
enum_values=enum_values,
response_cost=0.005,
)
logger.litellm_mcp_tool_calls_total.labels.assert_called_once()
logger.litellm_mcp_tool_calls_total.labels().inc.assert_called_once_with(1.0)
def test_increments_spend_counter_when_cost_positive(self):
logger = _make_mock_logger()
payload = _make_payload(response_cost=0.01)
enum_values = _make_enum_values()
PrometheusLogger._increment_mcp_tool_call_metrics(
logger,
standard_logging_payload=payload,
enum_values=enum_values,
response_cost=0.01,
)
logger.litellm_mcp_tool_call_spend_metric.labels.assert_called_once()
logger.litellm_mcp_tool_call_spend_metric.labels().inc.assert_called_once_with(0.01)
def test_skips_spend_counter_when_cost_zero(self):
logger = _make_mock_logger()
payload = _make_payload(response_cost=0.0)
enum_values = _make_enum_values()
PrometheusLogger._increment_mcp_tool_call_metrics(
logger,
standard_logging_payload=payload,
enum_values=enum_values,
response_cost=0.0,
)
logger.litellm_mcp_tool_calls_total.labels.assert_called_once()
logger.litellm_mcp_tool_call_spend_metric.labels.assert_not_called()
def test_noop_when_no_mcp_metadata(self):
logger = _make_mock_logger()
payload = _make_payload()
payload["metadata"]["mcp_tool_call_metadata"] = None
enum_values = _make_enum_values()
PrometheusLogger._increment_mcp_tool_call_metrics(
logger,
standard_logging_payload=payload,
enum_values=enum_values,
response_cost=0.005,
)
for name in MCP_METRICS:
getattr(logger, name).labels.assert_not_called()
def test_noop_when_metadata_missing(self):
logger = _make_mock_logger()
payload = {"metadata": None}
enum_values = _make_enum_values()
PrometheusLogger._increment_mcp_tool_call_metrics(
logger,
standard_logging_payload=payload,
enum_values=enum_values,
response_cost=0.005,
)
for name in MCP_METRICS:
getattr(logger, name).labels.assert_not_called()
def test_label_values_carry_tool_and_server_name(self):
logger = _make_mock_logger()
payload = _make_payload(
mcp_tool_name="search_docs",
mcp_server_name="docs-mcp",
)
enum_values = _make_enum_values()
PrometheusLogger._increment_mcp_tool_call_metrics(
logger,
standard_logging_payload=payload,
enum_values=enum_values,
response_cost=0.005,
)
labels_passed = logger.litellm_mcp_tool_calls_total.labels.call_args
assert labels_passed.kwargs["mcp_tool_name"] == "search_docs"
assert labels_passed.kwargs["mcp_server_name"] == "docs-mcp"
def test_label_values_carry_team_and_key_from_parent(self):
logger = _make_mock_logger()
payload = _make_payload()
enum_values = UserAPIKeyLabelValues(
hashed_api_key="sk-parent-key",
api_key_alias="parent-alias",
team="parent-team",
team_alias="Parent Team",
user="parent-user",
end_user="parent-end-user",
)
PrometheusLogger._increment_mcp_tool_call_metrics(
logger,
standard_logging_payload=payload,
enum_values=enum_values,
response_cost=0.005,
)
labels_passed = logger.litellm_mcp_tool_calls_total.labels.call_args
assert labels_passed.kwargs["hashed_api_key"] == "sk-parent-key"
assert labels_passed.kwargs["team"] == "parent-team"
assert labels_passed.kwargs["team_alias"] == "Parent Team"
assert labels_passed.kwargs["user"] == "parent-user"
def test_handles_missing_server_name_gracefully(self):
logger = _make_mock_logger()
payload = _make_payload()
payload["metadata"]["mcp_tool_call_metadata"] = {
"name": "standalone_tool",
"arguments": {},
"result": {},
}
enum_values = _make_enum_values()
PrometheusLogger._increment_mcp_tool_call_metrics(
logger,
standard_logging_payload=payload,
enum_values=enum_values,
response_cost=0.0,
)
labels_passed = logger.litellm_mcp_tool_calls_total.labels.call_args
assert labels_passed.kwargs["mcp_tool_name"] == "standalone_tool"
assert labels_passed.kwargs["mcp_server_name"] is None
if __name__ == "__main__":
pytest.main([__file__, "-v"])

154
tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py Normal file
View file

@ -0,0 +1,154 @@
"""Regression tests for Bedrock Converse ``toolSpec.strict`` forwarding.
Bedrock Converse routes Claude Opus 4.7/4.8 and Claude Sonnet 4 through an
Anthropic-compatible validator that rejects ``toolSpec.strict`` even though
Anthropic's native API accepts ``strict`` as a top-level tool field. See
BerriAI/litellm#31582.
"""
import pytest
from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt
from litellm.llms.bedrock.common_utils import bedrock_converse_supports_strict_tools
_STRICT_TOOL = [
{
"type": "function",
"function": {
"name": "get_weather",
"strict": True,
"description": "Get the weather for a city",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"unit": {"type": "string", "enum": ["celsius"]},
},
"required": ["city", "unit"],
"additionalProperties": False,
},
},
}
]
@pytest.mark.parametrize(
"model_id",
[
"bedrock/us.anthropic.claude-opus-4-7",
"bedrock/us.anthropic.claude-opus-4-8",
"anthropic.claude-opus-4-7",
"anthropic.claude-opus-4-8",
"anthropic.claude-opus-4-7-v1:0",
"bedrock/eu.anthropic.claude-opus-4-8-v1:0",
"bedrock/global.anthropic.claude-opus-4-7",
# Sonnet 4 also rejects toolSpec.strict on Bedrock Converse
"anthropic.claude-sonnet-4-20250514-v1:0",
"bedrock/global.anthropic.claude-sonnet-4-20250514-v1:0",
"bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0",
"bedrock/eu.anthropic.claude-sonnet-4-20250514-v1:0",
"bedrock/apac.anthropic.claude-sonnet-4-20250514-v1:0",
],
)
def test_bedrock_tools_pt_strict_dropped_for_strict_unsupported_models(
model_id: str,
) -> None:
"""Opus 4.7/4.8 and Sonnet 4 reject toolSpec.strict and additionalProperties."""
result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id)
tool_spec = result[0]["toolSpec"]
assert (
"strict" not in tool_spec
), f"strict leaked into toolSpec for {model_id}: {tool_spec}"
assert (
"additionalProperties" not in tool_spec["inputSchema"]["json"]
), f"additionalProperties leaked into toolSpec for {model_id}: {tool_spec}"
@pytest.mark.parametrize(
"model_id",
[
"anthropic.claude-sonnet-4-5-20250929-v1:0",
"bedrock/us.anthropic.claude-sonnet-4-6",
"bedrock/us.anthropic.claude-opus-4-6",
"bedrock/us.anthropic.claude-opus-4-5",
],
)
def test_bedrock_tools_pt_strict_kept_for_other_anthropic(model_id: str) -> None:
"""Sonnet 4.5/4.6 and Opus <=4.6 accept toolSpec.strict — keep forwarding it."""
result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id)
assert (
result[0]["toolSpec"]["strict"] is True
), f"strict missing for {model_id}: {result[0]['toolSpec']}"
@pytest.mark.parametrize(
"model_id",
[
"us.amazon.nova-micro-v1:0",
"meta.llama3-2-11b-instruct-v1:0",
],
)
def test_bedrock_tools_pt_strict_dropped_for_non_anthropic(model_id: str) -> None:
"""Non-Anthropic Bedrock families reject toolSpec.strict — must be dropped."""
result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id)
assert "strict" not in result[0]["toolSpec"]
def test_bedrock_converse_supports_strict_tools_helper() -> None:
"""Direct check for the gate helper used by factory.py."""
assert (
bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-7")
is False
)
assert (
bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-8")
is False
)
assert (
bedrock_converse_supports_strict_tools(
"anthropic.claude-sonnet-4-5-20250929-v1:0"
)
is True
)
assert (
bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-6")
is True
)
assert bedrock_converse_supports_strict_tools("us.amazon.nova-micro-v1:0") is False
assert bedrock_converse_supports_strict_tools("") is False
# Sonnet 4 also rejects strict on Bedrock Converse
assert (
bedrock_converse_supports_strict_tools(
"anthropic.claude-sonnet-4-20250514-v1:0"
)
is False
)
assert (
bedrock_converse_supports_strict_tools(
"bedrock/global.anthropic.claude-sonnet-4-20250514-v1:0"
)
is False
)
@pytest.mark.parametrize(
"cost_map_key",
[
"anthropic.claude-opus-4-7",
"us.anthropic.claude-opus-4-7",
"anthropic.claude-opus-4-8",
"us.anthropic.claude-opus-4-8",
"anthropic.claude-sonnet-4-20250514-v1:0",
"global.anthropic.claude-sonnet-4-20250514-v1:0",
"us.anthropic.claude-sonnet-4-20250514-v1:0",
"eu.anthropic.claude-sonnet-4-20250514-v1:0",
"apac.anthropic.claude-sonnet-4-20250514-v1:0",
],
)
def test_strict_tools_flag_set_in_model_cost_map(cost_map_key: str) -> None:
"""The gate is driven by ``bedrock_converse_supports_strict_tools: false`` in
``model_prices_and_context_window.json``, not hardcoded model patterns."""
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
cost_map = GetModelCostMap.load_local_model_cost_map()
assert cost_map[cost_map_key]["bedrock_converse_supports_strict_tools"] is False

36
tests/test_litellm/litellm_core_utils/test_image_handling.py
View file

@ -7,6 +7,7 @@ import litellm
from litellm import constants
from litellm.litellm_core_utils.prompt_templates import image_handling
from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
convert_url_to_base64,
)
@ -218,6 +219,41 @@ def test_streaming_download_handles_petabyte_file(monkeypatch):
assert "exceeds maximum allowed size" in str(excinfo.value)
def test_data_url_is_returned_unchanged_without_fetch(monkeypatch):
"""
A data URL is already inline base64 image data, so convert_url_to_base64
must return it as-is instead of attempting an HTTP fetch.
"""
class ExplodingClient:
def get(self, url, follow_redirects=True):
raise AssertionError("data URLs must not trigger an HTTP fetch")
monkeypatch.setattr(litellm, "module_level_client", ExplodingClient())
data_url = "data:image/png;base64,iVBORw0KGgo="
assert convert_url_to_base64(data_url) == data_url
@pytest.mark.asyncio
async def test_async_data_url_is_returned_unchanged_without_fetch(monkeypatch):
"""
The async path must short-circuit data URLs identically to the sync path,
otherwise async OCR flows would attempt an impossible HTTP fetch.
"""
class ExplodingAsyncClient:
async def get(self, url, follow_redirects=True):
raise AssertionError("data URLs must not trigger an HTTP fetch")
monkeypatch.setattr(litellm, "module_level_aclient", ExplodingAsyncClient())
data_url = "data:image/png;base64,iVBORw0KGgo="
assert await async_convert_url_to_base64(data_url) == data_url
def test_image_size_limit_disabled(monkeypatch):
"""
Test that setting MAX_IMAGE_URL_DOWNLOAD_SIZE_MB to 0 disables all image URL downloads.

174
tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py Normal file
View file

@ -0,0 +1,174 @@
import json
import os
import sys
import types
from unittest.mock import MagicMock
import pytest
sys.path.insert(0, os.path.abspath("../../../../..")) # Adds the parent directory to the system path
from litellm.llms.bedrock.realtime.handler import BedrockRealtime
from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
class FakePayloadPart:
def __init__(self, bytes_):
self.bytes_ = bytes_
class FakeInputChunk:
def __init__(self, value):
self.value = value
class FakeInputStream:
def __init__(self):
self.sent = []
self.closed = False
async def send(self, event):
self.sent.append(event)
async def close(self):
self.closed = True
class SendFailingInputStream(FakeInputStream):
async def send(self, event):
raise RuntimeError("bedrock send failed")
class FailOnPromptEndStream(FakeInputStream):
async def send(self, event):
payload = json.loads(event.value.bytes_.decode("utf-8"))
if "promptEnd" in payload.get("event", {}):
raise RuntimeError("bedrock rejected promptEnd")
self.sent.append(event)
class FakeBedrockStream:
def __init__(self, input_stream=None):
self.input_stream = input_stream if input_stream is not None else FakeInputStream()
class DisconnectingClientWS:
def __init__(self, messages):
self._messages = list(messages)
async def receive_text(self):
if self._messages:
return self._messages.pop(0)
raise RuntimeError("client disconnected")
class ClosableClientWS:
def __init__(self):
self.closed = False
async def close(self):
self.closed = True
class EndedBedrockReceiver:
async def receive(self):
return None
class EndedBedrockStream:
async def await_output(self):
return (None, EndedBedrockReceiver())
@pytest.fixture
def stub_aws_models(monkeypatch):
package = types.ModuleType("aws_sdk_bedrock_runtime")
models = types.ModuleType("aws_sdk_bedrock_runtime.models")
models.BidirectionalInputPayloadPart = FakePayloadPart
models.InvokeModelWithBidirectionalStreamInputChunk = FakeInputChunk
package.models = models
monkeypatch.setitem(sys.modules, "aws_sdk_bedrock_runtime", package)
monkeypatch.setitem(sys.modules, "aws_sdk_bedrock_runtime.models", models)
class TestBedrockRealtimeHandler:
"""Client disconnect must close the Bedrock session gracefully (LIT-2239 regression)"""
@pytest.mark.asyncio
async def test_client_disconnect_flushes_session_close_messages(self, stub_aws_models):
handler = BedrockRealtime()
config = BedrockRealtimeConfig()
stream = FakeBedrockStream()
client_ws = DisconnectingClientWS(
[json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})]
)
await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {})
sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent]
event_names = [next(iter(event["event"])) for event in sent_events]
assert event_names[0] == "sessionStart"
assert event_names[-2:] == ["promptEnd", "sessionEnd"]
assert stream.input_stream.closed
@pytest.mark.asyncio
async def test_client_disconnect_before_session_update_sends_nothing(self, stub_aws_models):
handler = BedrockRealtime()
config = BedrockRealtimeConfig()
stream = FakeBedrockStream()
await handler._forward_client_to_bedrock(
DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}
)
assert stream.input_stream.sent == []
assert stream.input_stream.closed
@pytest.mark.asyncio
async def test_input_stream_closed_even_when_close_flush_fails(self, stub_aws_models):
handler = BedrockRealtime()
config = BedrockRealtimeConfig()
stream = FakeBedrockStream(input_stream=SendFailingInputStream())
client_ws = DisconnectingClientWS(
[json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})]
)
await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {})
assert stream.input_stream.closed
@pytest.mark.asyncio
async def test_close_flush_continues_after_partial_send_failure(self, stub_aws_models):
handler = BedrockRealtime()
config = BedrockRealtimeConfig()
stream = FakeBedrockStream(input_stream=FailOnPromptEndStream())
client_ws = DisconnectingClientWS(
[json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})]
)
await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {})
sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent]
event_names = [next(iter(event["event"])) for event in sent_events]
assert "sessionEnd" in event_names
assert stream.input_stream.closed
@pytest.mark.asyncio
async def test_bedrock_stream_end_closes_client_websocket(self):
handler = BedrockRealtime()
client_ws = ClosableClientWS()
await handler._forward_bedrock_to_client(
EndedBedrockStream(),
client_ws,
BedrockRealtimeConfig(),
"amazon.nova-sonic-v1:0",
MagicMock(),
{},
)
assert client_ws.closed
if __name__ == "__main__":
pytest.main([__file__, "-v"])

295
tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
View file

@ -5,11 +5,16 @@ from unittest.mock import MagicMock
import pytest
sys.path.insert(
0, os.path.abspath("../../../../..")
) # Adds the parent directory to the system path
sys.path.insert(0, os.path.abspath("../../../../..")) # Adds the parent directory to the system path
from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
import base64
from litellm.llms.bedrock.realtime.transformation import (
TRIGGER_LEADING_SILENCE,
TRIGGER_TRAILING_SILENCE,
BedrockRealtimeConfig,
)
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
from litellm.types.llms.openai import OpenAIRealtimeEventTypes
@ -67,19 +72,14 @@ class TestBedrockRealtimeConfig:
}
]
session_config = config.session_configuration_request(
"amazon.nova-sonic-v1:0", tools=tools
)
session_config = config.session_configuration_request("amazon.nova-sonic-v1:0", tools=tools)
session_dict = json.loads(session_config)
prompt_start = session_dict["prompt_start"]["event"]["promptStart"]
assert "toolConfiguration" in prompt_start
assert "tools" in prompt_start["toolConfiguration"]
assert len(prompt_start["toolConfiguration"]["tools"]) == 1
assert (
prompt_start["toolConfiguration"]["tools"][0]["toolSpec"]["name"]
== "get_weather"
)
assert prompt_start["toolConfiguration"]["tools"][0]["toolSpec"]["name"] == "get_weather"
def test_transform_tools_to_bedrock_format(self):
"""Test OpenAI tool format to Bedrock format transformation"""
@ -93,9 +93,7 @@ class TestBedrockRealtimeConfig:
"description": "Get current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string", "description": "City name"}
},
"properties": {"location": {"type": "string", "description": "City name"}},
"required": ["location"],
},
},
@ -120,18 +118,11 @@ class TestBedrockRealtimeConfig:
# Test PCM16 format
assert config._map_audio_format_to_sample_rate("pcm16", is_output=True) == 24000
assert (
config._map_audio_format_to_sample_rate("pcm16", is_output=False) == 16000
)
assert config._map_audio_format_to_sample_rate("pcm16", is_output=False) == 16000
# Test G.711 formats
assert (
config._map_audio_format_to_sample_rate("g711_ulaw", is_output=True) == 8000
)
assert (
config._map_audio_format_to_sample_rate("g711_alaw", is_output=False)
== 8000
)
assert config._map_audio_format_to_sample_rate("g711_ulaw", is_output=True) == 8000
assert config._map_audio_format_to_sample_rate("g711_alaw", is_output=False) == 8000
def test_transform_session_update_event(self):
"""Test session.update event transformation"""
@ -158,12 +149,7 @@ class TestBedrockRealtimeConfig:
# Verify session start message
session_start = json.loads(messages[0])
assert (
session_start["event"]["sessionStart"]["inferenceConfiguration"][
"temperature"
]
== 0.9
)
assert session_start["event"]["sessionStart"]["inferenceConfiguration"]["temperature"] == 0.9
def test_transform_session_update_with_tools(self):
"""Test session.update with tools"""
@ -237,12 +223,7 @@ class TestBedrockRealtimeConfig:
content_start = json.loads(messages[0])
assert content_start["event"]["contentStart"]["type"] == "TOOL"
assert content_start["event"]["contentStart"]["role"] == "TOOL"
assert (
content_start["event"]["contentStart"]["toolResultInputConfiguration"][
"toolUseId"
]
== "call_123"
)
assert content_start["event"]["contentStart"]["toolResultInputConfiguration"]["toolUseId"] == "call_123"
def test_transform_input_audio_buffer_append(self):
"""Test input_audio_buffer.append transformation"""
@ -260,12 +241,7 @@ class TestBedrockRealtimeConfig:
content_start = json.loads(messages[0])
assert content_start["event"]["contentStart"]["type"] == "AUDIO"
assert (
content_start["event"]["contentStart"]["audioInputConfiguration"][
"sampleRateHertz"
]
== 16000
)
assert content_start["event"]["contentStart"]["audioInputConfiguration"]["sampleRateHertz"] == 16000
audio_input = json.loads(messages[1])
assert audio_input["event"]["audioInput"]["content"] == "base64_audio_data_here"
@ -286,6 +262,144 @@ class TestBedrockRealtimeConfig:
assert "contentEnd" in content_end["event"]
class TestBedrockRealtimeResponseCreate:
"""response.create must trigger Nova Sonic generation (LIT-2239 regression)"""
def _start_session(self, config):
config.transform_realtime_request(
json.dumps(
{
"type": "session.update",
"session": {"instructions": "You are a helpful assistant."},
}
),
"amazon.nova-sonic-v1:0",
)
def test_response_create_before_session_update_is_noop(self):
config = BedrockRealtimeConfig()
messages = config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0")
assert messages == []
def test_response_create_emits_spoken_trigger_audio(self):
config = BedrockRealtimeConfig()
self._start_session(config)
messages = config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0")
assert len(messages) > 1
content_start = json.loads(messages[0])["event"]["contentStart"]
assert content_start["promptName"] == config.prompt_name
assert content_start["contentName"] == config.audio_content_name
assert content_start["type"] == "AUDIO"
assert content_start["interactive"] is True
assert content_start["role"] == "USER"
assert content_start["audioInputConfiguration"]["sampleRateHertz"] == 16000
audio_events = [json.loads(message)["event"]["audioInput"] for message in messages[1:]]
assert all(event["promptName"] == config.prompt_name for event in audio_events)
assert all(event["contentName"] == config.audio_content_name for event in audio_events)
sent_pcm = b"".join(base64.b64decode(event["content"]) for event in audio_events)
assert sent_pcm == TRIGGER_LEADING_SILENCE + ready_trigger_pcm() + TRIGGER_TRAILING_SILENCE
def test_second_response_create_reuses_open_audio_content(self):
config = BedrockRealtimeConfig()
self._start_session(config)
first = config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0")
second = config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0")
assert len(second) == len(first) - 1
assert all("audioInput" in json.loads(message)["event"] for message in second)
def test_response_create_is_noop_when_client_streams_audio(self):
config = BedrockRealtimeConfig()
self._start_session(config)
config.transform_realtime_request(
json.dumps({"type": "input_audio_buffer.append", "audio": "c2lsZW5jZQ=="}),
"amazon.nova-sonic-v1:0",
)
messages = config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0")
assert messages == []
def test_client_audio_after_trigger_reopens_block_at_client_sample_rate(self):
config = BedrockRealtimeConfig()
config.transform_realtime_request(
json.dumps(
{
"type": "session.update",
"session": {
"instructions": "You are a helpful assistant.",
"input_audio_format": "g711_ulaw",
},
}
),
"amazon.nova-sonic-v1:0",
)
config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0")
trigger_content_name = config.audio_content_name
messages = config.transform_realtime_request(
json.dumps({"type": "input_audio_buffer.append", "audio": "c2lsZW5jZQ=="}),
"amazon.nova-sonic-v1:0",
)
events = [json.loads(message)["event"] for message in messages]
assert [next(iter(event)) for event in events] == [
"contentEnd",
"contentStart",
"audioInput",
]
assert events[0]["contentEnd"]["contentName"] == trigger_content_name
new_content_start = events[1]["contentStart"]
assert new_content_start["contentName"] == config.audio_content_name
assert new_content_start["contentName"] != trigger_content_name
assert new_content_start["audioInputConfiguration"]["sampleRateHertz"] == 8000
assert events[2]["audioInput"]["contentName"] == config.audio_content_name
def test_client_audio_after_trigger_reuses_block_at_matching_sample_rate(self):
config = BedrockRealtimeConfig()
self._start_session(config)
config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0")
trigger_content_name = config.audio_content_name
messages = config.transform_realtime_request(
json.dumps({"type": "input_audio_buffer.append", "audio": "c2lsZW5jZQ=="}),
"amazon.nova-sonic-v1:0",
)
assert len(messages) == 1
audio_input = json.loads(messages[0])["event"]["audioInput"]
assert audio_input["contentName"] == trigger_content_name
def test_session_close_messages_close_audio_prompt_and_session(self):
config = BedrockRealtimeConfig()
self._start_session(config)
config.transform_realtime_request(json.dumps({"type": "response.create"}), "amazon.nova-sonic-v1:0")
close_messages = [json.loads(message)["event"] for message in config.session_close_messages()]
assert [next(iter(event)) for event in close_messages] == [
"contentEnd",
"promptEnd",
"sessionEnd",
]
assert close_messages[0]["contentEnd"]["contentName"] == config.audio_content_name
assert close_messages[1]["promptEnd"]["promptName"] == config.prompt_name
assert config.session_close_messages() == []
def test_session_close_messages_before_session_update_is_empty(self):
config = BedrockRealtimeConfig()
assert config.session_close_messages() == []
class TestBedrockRealtimeResponseTransformation:
"""Test suite for response transformation"""
@ -296,11 +410,7 @@ class TestBedrockRealtimeResponseTransformation:
logging_obj.litellm_trace_id = "trace_123"
bedrock_message = {
"event": {
"sessionStart": {
"inferenceConfiguration": {"maxTokens": 1024, "temperature": 0.7}
}
}
"event": {"sessionStart": {"inferenceConfiguration": {"maxTokens": 1024, "temperature": 0.7}}}
}
result = config.transform_realtime_response(
@ -330,9 +440,7 @@ class TestBedrockRealtimeResponseTransformation:
logging_obj.litellm_trace_id = "trace_123"
# First create a content start to initialize IDs
content_start_message = {
"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}
}
content_start_message = {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}
result1 = config.transform_realtime_response(
json.dumps(content_start_message),
@ -368,9 +476,7 @@ class TestBedrockRealtimeResponseTransformation:
)
# Check for text delta
text_deltas = [
msg for msg in result2["response"] if msg["type"] == "response.text.delta"
]
text_deltas = [msg for msg in result2["response"] if msg["type"] == "response.text.delta"]
assert len(text_deltas) == 1
assert text_deltas[0]["delta"] == "Hello, world!"
@ -384,9 +490,7 @@ class TestBedrockRealtimeResponseTransformation:
logging_obj.litellm_trace_id = "trace_123"
# First create a content start for audio
content_start_message = {
"event": {"contentStart": {"role": "ASSISTANT", "type": "AUDIO"}}
}
content_start_message = {"event": {"contentStart": {"role": "ASSISTANT", "type": "AUDIO"}}}
result1 = config.transform_realtime_response(
json.dumps(content_start_message),
@ -404,9 +508,7 @@ class TestBedrockRealtimeResponseTransformation:
)
# Now send audio output
audio_output_message = {
"event": {"audioOutput": {"content": "base64_audio_content"}}
}
audio_output_message = {"event": {"audioOutput": {"content": "base64_audio_content"}}}
result2 = config.transform_realtime_response(
json.dumps(audio_output_message),
@ -424,9 +526,7 @@ class TestBedrockRealtimeResponseTransformation:
)
# Check for audio delta
audio_deltas = [
msg for msg in result2["response"] if msg["type"] == "response.audio.delta"
]
audio_deltas = [msg for msg in result2["response"] if msg["type"] == "response.audio.delta"]
assert len(audio_deltas) == 1
assert audio_deltas[0]["delta"] == "base64_audio_content"
@ -504,14 +604,67 @@ class TestBedrockRealtimeResponseTransformation:
# Should have text.done, content_part.done, and output_item.done
assert len(result["response"]) == 3
text_done = [
msg for msg in result["response"] if msg["type"] == "response.text.done"
][0]
text_done = [msg for msg in result["response"] if msg["type"] == "response.text.done"][0]
assert text_done["text"] == "Hello, world!"
# Delta chunks should be reset
assert result["current_delta_chunks"] is None
def test_content_end_end_turn_emits_response_done(self):
"""END_TURN contentEnd must produce response.done (LIT-2239 regression)"""
config = BedrockRealtimeConfig()
logging_obj = MagicMock()
logging_obj.litellm_trace_id = "trace_123"
content_end_message = {"event": {"contentEnd": {"stopReason": "END_TURN", "type": "AUDIO"}}}
result = config.transform_realtime_response(
json.dumps(content_end_message),
"amazon.nova-sonic-v1:0",
logging_obj,
realtime_response_transform_input={
"session_configuration_request": json.dumps({"configured": True}),
"current_output_item_id": "item_123",
"current_response_id": "resp_123",
"current_conversation_id": "conv_123",
"current_delta_chunks": [],
"current_item_chunks": [],
"current_delta_type": "audio",
},
)
response_done_events = [msg for msg in result["response"] if msg["type"] == "response.done"]
assert len(response_done_events) == 1
assert response_done_events[0]["response"]["status"] == "completed"
assert result["current_output_item_id"] is None
assert result["current_response_id"] is None
assert result["current_delta_type"] is None
def test_content_end_partial_turn_does_not_emit_response_done(self):
config = BedrockRealtimeConfig()
logging_obj = MagicMock()
logging_obj.litellm_trace_id = "trace_123"
content_end_message = {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN", "type": "TEXT"}}}
result = config.transform_realtime_response(
json.dumps(content_end_message),
"amazon.nova-sonic-v1:0",
logging_obj,
realtime_response_transform_input={
"session_configuration_request": json.dumps({"configured": True}),
"current_output_item_id": "item_123",
"current_response_id": "resp_123",
"current_conversation_id": "conv_123",
"current_delta_chunks": [],
"current_item_chunks": [],
"current_delta_type": "text",
},
)
assert all(msg["type"] != "response.done" for msg in result["response"])
assert result["current_response_id"] == "resp_123"
def test_transform_prompt_end_response(self):
"""Test promptEnd response transformation"""
config = BedrockRealtimeConfig()
@ -552,9 +705,7 @@ class TestBedrockRealtimeResponseTransformation:
logging_obj.litellm_trace_id = "trace_123"
# Create a sequence of messages
content_start = {
"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}
}
content_start = {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}
text_output1 = {"event": {"textOutput": {"content": "Hello"}}}
text_output2 = {"event": {"textOutput": {"content": " world"}}}
@ -600,9 +751,7 @@ class TestBedrockRealtimeResponseTransformation:
logging_obj.litellm_trace_id = "trace_123"
# Create a sequence of messages
content_start = {
"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}
}
content_start = {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}
text_output = {"event": {"textOutput": {"content": "Hello"}}}
all_events = []
@ -636,9 +785,7 @@ class TestBedrockRealtimeResponseTransformation:
)
# Check all response_ids are the same
response_ids = [
event["response_id"] for event in all_events if "response_id" in event
]
response_ids = [event["response_id"] for event in all_events if "response_id" in event]
assert len(set(response_ids)) == 1, "Response IDs should be consistent"

97
tests/test_litellm/llms/vertex_ai/batches/test_transformation.py
View file

@ -47,9 +47,7 @@ def test_transform_openai_request_builds_full_vertex_job():
"litellm.llms.vertex_ai.batches.transformation.uuid.uuid4",
return_value="fixed-uuid",
):
job = T.transform_openai_batch_request_to_vertex_ai_batch_request(
{"input_file_id": INPUT_FILE}
)
job = T.transform_openai_batch_request_to_vertex_ai_batch_request({"input_file_id": INPUT_FILE})
assert job["displayName"] == "litellm-vertex-batch-fixed-uuid"
assert job["model"] == "publishers/google/models/gemini-1.5-flash-001"
@ -91,13 +89,11 @@ def test_transform_vertex_response_full_mapping():
assert isinstance(batch, LiteLLMBatch)
assert batch.id == "3814889423749775360"
assert batch.completion_window == "24hrs"
assert batch.completion_window == "24h"
# created_at is parsed via the shared helper (uses local tz); assert the
# transform forwards createTime through that helper rather than a hardcoded
# epoch that would be tz-dependent
assert batch.created_at == _convert_vertex_datetime_to_openai_datetime(
"2024-12-04T21:53:12.120184Z"
)
assert batch.created_at == _convert_vertex_datetime_to_openai_datetime("2024-12-04T21:53:12.120184Z")
assert batch.endpoint == ""
assert batch.object == "batch"
assert batch.input_file_id == "gs://bucket/in.jsonl"
@ -140,10 +136,7 @@ def test_transform_vertex_response_error_file_id_always_none():
],
)
def test_status_mapping_every_entry(vertex_state, expected):
assert (
T._get_batch_job_status_from_vertex_ai_batch_response({"state": vertex_state})
== expected
)
assert T._get_batch_job_status_from_vertex_ai_batch_response({"state": vertex_state}) == expected
def test_status_mapping_defaults_to_unspecified_when_missing():
@ -163,9 +156,7 @@ def test_status_mapping_unknown_state_raises_keyerror():
def test_get_batch_id_splits_path():
assert (
T._get_batch_id_from_vertex_ai_batch_response(
{"name": "projects/p/locations/l/batchPredictionJobs/999"}
)
T._get_batch_id_from_vertex_ai_batch_response({"name": "projects/p/locations/l/batchPredictionJobs/999"})
== "999"
)
@ -198,18 +189,11 @@ def test_get_input_file_id_missing_input_config():
def test_get_input_file_id_missing_gcs_source():
assert (
T._get_input_file_id_from_vertex_ai_batch_response({"inputConfig": {}}) == ""
)
assert T._get_input_file_id_from_vertex_ai_batch_response({"inputConfig": {}}) == ""
def test_get_input_file_id_empty_uris():
assert (
T._get_input_file_id_from_vertex_ai_batch_response(
{"inputConfig": {"gcsSource": {"uris": []}}}
)
== ""
)
assert T._get_input_file_id_from_vertex_ai_batch_response({"inputConfig": {"gcsSource": {"uris": []}}}) == ""
# =========================================================================== #
@ -220,18 +204,14 @@ def test_get_input_file_id_empty_uris():
def test_get_output_file_id_from_output_info():
# outputInfo branch: rstrip trailing slash, append predictions.jsonl
assert (
T._get_output_file_id_from_vertex_ai_batch_response(
{"outputInfo": {"gcsOutputDirectory": "gs://bucket/out/"}}
)
T._get_output_file_id_from_vertex_ai_batch_response({"outputInfo": {"gcsOutputDirectory": "gs://bucket/out/"}})
== "gs://bucket/out/predictions.jsonl"
)
def test_get_output_file_id_output_info_no_trailing_slash():
assert (
T._get_output_file_id_from_vertex_ai_batch_response(
{"outputInfo": {"gcsOutputDirectory": "gs://bucket/out"}}
)
T._get_output_file_id_from_vertex_ai_batch_response({"outputInfo": {"gcsOutputDirectory": "gs://bucket/out"}})
== "gs://bucket/out/predictions.jsonl"
)
@ -243,10 +223,7 @@ def test_get_output_file_id_empty_output_info_falls_through_to_output_config():
"outputInfo": {},
"outputConfig": {"gcsDestination": {"outputUriPrefix": "gs://b/cfg"}},
}
assert (
T._get_output_file_id_from_vertex_ai_batch_response(resp)
== "gs://b/cfg/predictions.jsonl"
)
assert T._get_output_file_id_from_vertex_ai_batch_response(resp) == "gs://b/cfg/predictions.jsonl"
def test_get_output_file_id_no_output_info_and_no_output_config():
@ -255,32 +232,18 @@ def test_get_output_file_id_no_output_info_and_no_output_config():
def test_get_output_file_id_output_config_missing_gcs_destination():
# outputConfig present but no gcsDestination -> returns the running "" value
assert (
T._get_output_file_id_from_vertex_ai_batch_response({"outputConfig": {}}) == ""
)
assert T._get_output_file_id_from_vertex_ai_batch_response({"outputConfig": {}}) == ""
def test_get_output_file_id_output_config_already_has_suffix():
# outputUriPrefix already ends in /predictions.jsonl -> returned as-is (no double append)
resp = {
"outputConfig": {
"gcsDestination": {"outputUriPrefix": "gs://b/cfg/predictions.jsonl"}
}
}
assert (
T._get_output_file_id_from_vertex_ai_batch_response(resp)
== "gs://b/cfg/predictions.jsonl"
)
resp = {"outputConfig": {"gcsDestination": {"outputUriPrefix": "gs://b/cfg/predictions.jsonl"}}}
assert T._get_output_file_id_from_vertex_ai_batch_response(resp) == "gs://b/cfg/predictions.jsonl"
def test_get_output_file_id_output_config_strips_trailing_slash():
resp = {
"outputConfig": {"gcsDestination": {"outputUriPrefix": "gs://b/cfg/"}}
}
assert (
T._get_output_file_id_from_vertex_ai_batch_response(resp)
== "gs://b/cfg/predictions.jsonl"
)
resp = {"outputConfig": {"gcsDestination": {"outputUriPrefix": "gs://b/cfg/"}}}
assert T._get_output_file_id_from_vertex_ai_batch_response(resp) == "gs://b/cfg/predictions.jsonl"
def test_get_output_file_id_output_info_takes_precedence_over_output_config():
@ -288,10 +251,7 @@ def test_get_output_file_id_output_info_takes_precedence_over_output_config():
"outputInfo": {"gcsOutputDirectory": "gs://from-info"},
"outputConfig": {"gcsDestination": {"outputUriPrefix": "gs://from-config"}},
}
assert (
T._get_output_file_id_from_vertex_ai_batch_response(resp)
== "gs://from-info/predictions.jsonl"
)
assert T._get_output_file_id_from_vertex_ai_batch_response(resp) == "gs://from-info/predictions.jsonl"
# =========================================================================== #
@ -301,16 +261,13 @@ def test_get_output_file_id_output_info_takes_precedence_over_output_config():
def test_get_gcs_uri_prefix_root():
assert (
T._get_gcs_uri_prefix_from_file("gs://litellm-testing-bucket/vtx_batch.jsonl")
== "gs://litellm-testing-bucket"
T._get_gcs_uri_prefix_from_file("gs://litellm-testing-bucket/vtx_batch.jsonl") == "gs://litellm-testing-bucket"
)
def test_get_gcs_uri_prefix_nested():
assert (
T._get_gcs_uri_prefix_from_file(
"gs://litellm-testing-bucket/batches/vtx_batch.jsonl"
)
T._get_gcs_uri_prefix_from_file("gs://litellm-testing-bucket/batches/vtx_batch.jsonl")
== "gs://litellm-testing-bucket/batches"
)
@ -321,21 +278,13 @@ def test_get_gcs_uri_prefix_nested():
def test_get_model_from_gcs_file_plain():
assert (
T._get_model_from_gcs_file(INPUT_FILE)
== "publishers/google/models/gemini-1.5-flash-001"
)
assert T._get_model_from_gcs_file(INPUT_FILE) == "publishers/google/models/gemini-1.5-flash-001"
def test_get_model_from_gcs_file_url_encoded():
# %2F decodes to "/" via urllib.unquote before splitting
encoded = (
"gs://bucket/publishers%2Fgoogle%2Fmodels%2Fgemini-1.5-flash-001%2Fuuid"
)
assert (
T._get_model_from_gcs_file(encoded)
== "publishers/google/models/gemini-1.5-flash-001"
)
encoded = "gs://bucket/publishers%2Fgoogle%2Fmodels%2Fgemini-1.5-flash-001%2Fuuid"
assert T._get_model_from_gcs_file(encoded) == "publishers/google/models/gemini-1.5-flash-001"
def test_get_model_from_gcs_file_no_publishers_raises():
@ -389,8 +338,6 @@ def test_list_response_empty():
def test_list_response_none_jobs_treated_as_empty():
out = T.transform_vertex_ai_batch_list_response_to_openai_list_response(
{"batchPredictionJobs": None}
)
out = T.transform_vertex_ai_batch_list_response_to_openai_list_response({"batchPredictionJobs": None})
assert out["data"] == []
assert out["first_id"] is None

159
tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py
View file

@ -33,27 +33,21 @@ class TestVertexAIGeminiImageGenerationConfig:
"""Test mapping n parameter to candidate_count"""
non_default_params = {"n": 3}
optional_params = {}
result = self.config.map_openai_params(
non_default_params, optional_params, "gemini-2.5-flash-image", False
)
result = self.config.map_openai_params(non_default_params, optional_params, "gemini-2.5-flash-image", False)
assert result.get("candidate_count") == 3
def test_map_openai_params_size(self):
"""Test mapping size parameter to aspectRatio"""
non_default_params = {"size": "1024x1024"}
optional_params = {}
result = self.config.map_openai_params(
non_default_params, optional_params, "gemini-2.5-flash-image", False
)
result = self.config.map_openai_params(non_default_params, optional_params, "gemini-2.5-flash-image", False)
assert result.get("aspectRatio") == "1:1"
def test_map_openai_params_size_16_9(self):
"""Test mapping 16:9 size"""
non_default_params = {"size": "1792x1024"}
optional_params = {}
result = self.config.map_openai_params(
non_default_params, optional_params, "gemini-2.5-flash-image", False
)
result = self.config.map_openai_params(non_default_params, optional_params, "gemini-2.5-flash-image", False)
assert result.get("aspectRatio") == "16:9"
def test_map_size_to_aspect_ratio(self):
@ -67,42 +61,106 @@ class TestVertexAIGeminiImageGenerationConfig:
def test_get_supported_openai_params_includes_native_gemini_params(self):
"""Test that native Gemini imageConfig params are supported"""
supported = self.config.get_supported_openai_params(
"gemini-3-pro-image-preview"
)
supported = self.config.get_supported_openai_params("gemini-3-pro-image-preview")
assert "aspectRatio" in supported
assert "aspect_ratio" in supported
assert "imageSize" in supported
assert "image_size" in supported
assert "imageConfig" in supported
def test_map_openai_params_aspect_ratio_camel_case(self):
"""Test mapping native aspectRatio parameter"""
result = self.config.map_openai_params(
{"aspectRatio": "9:16"}, {}, "gemini-3-pro-image-preview", False
)
result = self.config.map_openai_params({"aspectRatio": "9:16"}, {}, "gemini-3-pro-image-preview", False)
assert result["aspectRatio"] == "9:16"
def test_map_openai_params_aspect_ratio_snake_case(self):
"""Test mapping native aspect_ratio parameter"""
result = self.config.map_openai_params(
{"aspect_ratio": "16:9"}, {}, "gemini-3-pro-image-preview", False
)
result = self.config.map_openai_params({"aspect_ratio": "16:9"}, {}, "gemini-3-pro-image-preview", False)
assert result["aspectRatio"] == "16:9"
def test_map_openai_params_image_size_camel_case(self):
"""Test mapping native imageSize parameter"""
result = self.config.map_openai_params(
{"imageSize": "4K"}, {}, "gemini-3-pro-image-preview", False
)
result = self.config.map_openai_params({"imageSize": "4K"}, {}, "gemini-3-pro-image-preview", False)
assert result["imageSize"] == "4K"
def test_map_openai_params_image_size_snake_case(self):
"""Test mapping native image_size parameter"""
result = self.config.map_openai_params(
{"image_size": "2K"}, {}, "gemini-3-pro-image-preview", False
)
result = self.config.map_openai_params({"image_size": "2K"}, {}, "gemini-3-pro-image-preview", False)
assert result["imageSize"] == "2K"
def test_map_openai_params_image_config_dict_stored_whole(self):
"""imageConfig dict is stored as-is so all fields survive"""
result = self.config.map_openai_params(
{"imageConfig": {"aspectRatio": "16:9", "imageSize": "2K"}},
{},
"gemini-3.1-flash-image",
False,
)
assert result["imageConfig"] == {"aspectRatio": "16:9", "imageSize": "2K"}
def test_map_openai_params_image_config_all_fields(self):
"""All ImageConfig fields (personGeneration, imageOutputOptions) pass through"""
payload = {
"imageConfig": {
"aspectRatio": "9:16",
"imageSize": "4K",
"personGeneration": "DONT_ALLOW",
"imageOutputOptions": {
"mimeType": "image/jpeg",
"compressionQuality": 80,
},
}
}
result = self.config.map_openai_params(payload, {}, "gemini-3.1-flash-image", False)
assert result["imageConfig"] == payload["imageConfig"]
def test_map_openai_params_image_config_non_dict_warns_and_drops(self):
"""Non-dict imageConfig is dropped with a warning, not silently discarded"""
with patch("litellm.llms.vertex_ai.image_generation.vertex_gemini_transformation.verbose_logger") as mock_log:
result = self.config.map_openai_params(
{"imageConfig": "bad-string-value"}, {}, "gemini-3.1-flash-image", False
)
assert "imageConfig" not in result
mock_log.warning.assert_called_once()
def test_transform_image_generation_request_from_image_config(self):
"""Full imageConfig dict is forwarded verbatim into generationConfig"""
full_config = {
"aspectRatio": "16:9",
"imageSize": "2K",
"personGeneration": "DONT_ALLOW",
"imageOutputOptions": {"mimeType": "image/jpeg", "compressionQuality": 85},
}
mapped = self.config.map_openai_params(
{"imageConfig": full_config},
{},
"gemini-3.1-flash-image",
False,
)
request = self.config.transform_image_generation_request(
model="gemini-3.1-flash-image",
prompt="A nano banana on a desk",
optional_params=mapped,
litellm_params={},
headers={},
)
assert request["generationConfig"]["imageConfig"] == full_config
def test_transform_image_generation_flat_params_override_image_config(self):
"""Explicit flat params win over the same key inside imageConfig"""
request = self.config.transform_image_generation_request(
model="gemini-3.1-flash-image",
prompt="A nano banana",
optional_params={
"imageConfig": {"aspectRatio": "1:1", "personGeneration": "DONT_ALLOW"},
"aspectRatio": "16:9", # should win
},
litellm_params={},
headers={},
)
assert request["generationConfig"]["imageConfig"]["aspectRatio"] == "16:9"
assert request["generationConfig"]["imageConfig"]["personGeneration"] == "DONT_ALLOW"
def test_transform_image_generation_request_basic(self):
"""Test basic request transformation"""
request = self.config.transform_image_generation_request(
@ -141,9 +199,7 @@ class TestVertexAIGeminiImageGenerationConfig:
def test_map_openai_params_web_search_options(self):
"""Test web_search_options maps to googleSearch tool"""
result = self.config.map_openai_params(
{"web_search_options": {}}, {}, "gemini-3.1-flash-image-preview", False
)
result = self.config.map_openai_params({"web_search_options": {}}, {}, "gemini-3.1-flash-image-preview", False)
assert result["tools"] == [{"googleSearch": {}}]
def test_transform_image_generation_request_with_web_search_tools(self):
@ -173,9 +229,7 @@ class TestVertexAIGeminiImageGenerationConfig:
headers={},
)
assert request["tools"] == [{"googleMaps": {}}]
assert request["toolConfig"] == {
"retrievalConfig": {"latLng": {"latitude": 37.7, "longitude": -122.4}}
}
assert request["toolConfig"] == {"retrievalConfig": {"latLng": {"latitude": 37.7, "longitude": -122.4}}}
def test_transform_image_generation_request_with_candidate_count(self):
"""Test request transformation with candidate_count"""
@ -344,10 +398,7 @@ class TestVertexAIGeminiImageGenerationConfig:
assert len(result.data) == 1
assert result.data[0].b64_json == "base64_encoded_image_data"
assert (
result.data[0].provider_specific_fields["thought_signature"]
== "test_signature_abc123"
)
assert result.data[0].provider_specific_fields["thought_signature"] == "test_signature_abc123"
def test_transform_image_generation_response_tracks_web_search_requests(self):
"""Grounding queries are carried onto usage so search spend can be billed"""
@ -366,9 +417,7 @@ class TestVertexAIGeminiImageGenerationConfig:
}
]
},
"groundingMetadata": {
"webSearchQueries": ["eiffel tower", "paris skyline"]
},
"groundingMetadata": {"webSearchQueries": ["eiffel tower", "paris skyline"]},
}
],
"usageMetadata": {
@ -410,18 +459,14 @@ class TestVertexAIImagenImageGenerationConfig:
"""Test mapping n parameter to sampleCount"""
non_default_params = {"n": 3}
optional_params = {}
result = self.config.map_openai_params(
non_default_params, optional_params, "imagegeneration@006", False
)
result = self.config.map_openai_params(non_default_params, optional_params, "imagegeneration@006", False)
assert result.get("sampleCount") == 3
def test_map_openai_params_size(self):
"""Test mapping size parameter to aspectRatio"""
non_default_params = {"size": "1024x1024"}
optional_params = {}
result = self.config.map_openai_params(
non_default_params, optional_params, "imagegeneration@006", False
)
result = self.config.map_openai_params(non_default_params, optional_params, "imagegeneration@006", False)
assert result.get("aspectRatio") == "1:1"
def test_map_size_to_aspect_ratio(self):
@ -462,9 +507,7 @@ class TestVertexAIImagenImageGenerationConfig:
model="imagegeneration@006",
prompt="A cat",
optional_params={},
litellm_params={
"metadata": {"requester_metadata": {"team": "platform", "env": "prod"}}
},
litellm_params={"metadata": {"requester_metadata": {"team": "platform", "env": "prod"}}},
headers={},
)
assert request["labels"] == {"team": "platform", "env": "prod"}
@ -474,9 +517,7 @@ class TestVertexAIImagenImageGenerationConfig:
"""Test response transformation"""
mock_response = MagicMock(spec=httpx.Response)
mock_response.status_code = 200
mock_response.json.return_value = {
"predictions": [{"bytesBase64Encoded": "base64_encoded_image_data"}]
}
mock_response.json.return_value = {"predictions": [{"bytesBase64Encoded": "base64_encoded_image_data"}]}
mock_response.headers = {}
from litellm.types.utils import ImageResponse
@ -539,9 +580,7 @@ class TestGetVertexAIImageGenerationConfig:
config = get_vertex_ai_image_generation_config("gemini-3-pro-image-preview")
assert isinstance(config, VertexAIGeminiImageGenerationConfig)
config = get_vertex_ai_image_generation_config(
"vertex_ai/gemini-2.5-flash-image"
)
config = get_vertex_ai_image_generation_config("vertex_ai/gemini-2.5-flash-image")
assert isinstance(config, VertexAIGeminiImageGenerationConfig)
def test_get_imagen_model_config(self):
@ -572,12 +611,8 @@ class TestVertexAIImageGenerationIntegration:
"""Test that Gemini config can validate environment"""
config = VertexAIGeminiImageGenerationConfig()
with (
patch.object(
config, "_resolve_vertex_project", return_value="test-project"
),
patch.object(
config, "_resolve_vertex_location", return_value="us-central1"
),
patch.object(config, "_resolve_vertex_project", return_value="test-project"),
patch.object(config, "_resolve_vertex_location", return_value="us-central1"),
patch.object(config, "_ensure_access_token", return_value=("token", None)),
):
headers = config.validate_environment(
@ -597,12 +632,8 @@ class TestVertexAIImageGenerationIntegration:
"""Test that Imagen config can validate environment"""
config = VertexAIImagenImageGenerationConfig()
with (
patch.object(
config, "_resolve_vertex_project", return_value="test-project"
),
patch.object(
config, "_resolve_vertex_location", return_value="us-central1"
),
patch.object(config, "_resolve_vertex_project", return_value="test-project"),
patch.object(config, "_resolve_vertex_location", return_value="us-central1"),
patch.object(config, "_ensure_access_token", return_value=("token", None)),
):
headers = config.validate_environment(

317
tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
View file

@ -3011,3 +3011,320 @@ async def test_token_endpoint_client_secret_basic_without_secret_returns_400():
code_verifier="verifier",
)
assert exc_info.value.status_code == 400
# -------------------------------------------------------------------
# Non-oauth2 (auth_type=none, access-group gated) servers must not be
# driven through the gateway OAuth authorize/token/register/discovery
# flow, and must not be advertised as OAuth-protected in discovery docs.
# -------------------------------------------------------------------
def _access_group_none_server(server_name="access_group_server"):
"""A non-oauth2, access-group gated MCP server: no client_id, no OAuth."""
from litellm.proxy._types import MCPTransport
from litellm.types.mcp import MCPAuth
from litellm.types.mcp_server.mcp_server_manager import MCPServer
return MCPServer(
server_id=server_name,
name=server_name,
server_name=server_name,
alias=server_name,
transport=MCPTransport.http,
auth_type=MCPAuth.none,
access_groups=["eng"],
)
@pytest.mark.asyncio
async def test_authorize_endpoint_rejects_non_oauth2_server():
"""authorize() against a none-auth server returns an accurate 'does not use OAuth' 400,
not the misleading 'client_id is required' that fired before the auth_type was checked."""
try:
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
authorize,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
global_mcp_server_manager.registry.clear()
server = _access_group_none_server()
global_mcp_server_manager.registry[server.server_id] = server
mock_request = MagicMock()
mock_request.base_url = "https://litellm.example.com/"
mock_request.headers = {}
try:
with pytest.raises(HTTPException) as exc_info:
await authorize(
request=mock_request,
client_id=None,
mcp_server_name="access_group_server",
redirect_uri="http://127.0.0.1:60108/callback",
state="test_state",
)
assert exc_info.value.status_code == 400
detail_text = str(exc_info.value.detail)
assert "does not use OAuth" in detail_text
assert "client_id is required" not in detail_text
finally:
global_mcp_server_manager.registry.clear()
@pytest.mark.asyncio
async def test_token_endpoint_rejects_non_oauth2_server():
"""token_endpoint() against a none-auth server returns 'does not use OAuth' 400 instead
of the misleading 'token url is not set'."""
try:
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
token_endpoint,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
global_mcp_server_manager.registry.clear()
server = _access_group_none_server()
global_mcp_server_manager.registry[server.server_id] = server
mock_request = MagicMock()
mock_request.base_url = "https://litellm.example.com/"
mock_request.headers = {}
try:
with pytest.raises(HTTPException) as exc_info:
await token_endpoint(
request=mock_request,
grant_type="authorization_code",
code="auth-code",
redirect_uri="http://localhost/callback",
client_id="some-client",
mcp_server_name="access_group_server",
client_secret=None,
code_verifier="verifier",
)
assert exc_info.value.status_code == 400
detail_text = str(exc_info.value.detail)
assert "does not use OAuth" in detail_text
assert "token url is not set" not in detail_text
finally:
global_mcp_server_manager.registry.clear()
@pytest.mark.asyncio
async def test_register_client_rejects_non_oauth2_server():
"""register_client() against a named none-auth server returns 'does not use OAuth' 400
instead of the misleading 'authorization url is not set'."""
try:
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
register_client,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
global_mcp_server_manager.registry.clear()
server = _access_group_none_server()
global_mcp_server_manager.registry[server.server_id] = server
mock_request = MagicMock()
mock_request.base_url = "https://litellm.example.com/"
mock_request.headers = {}
try:
with pytest.raises(HTTPException) as exc_info:
with patch(
"litellm.proxy._experimental.mcp_server.discoverable_endpoints._read_request_body",
new=AsyncMock(return_value={}),
):
await register_client(request=mock_request, mcp_server_name="access_group_server")
assert exc_info.value.status_code == 400
detail_text = str(exc_info.value.detail)
assert "does not use OAuth" in detail_text
assert "authorization url is not set" not in detail_text
finally:
global_mcp_server_manager.registry.clear()
@pytest.mark.asyncio
async def test_oauth_protected_resource_404_for_non_oauth2_server():
"""Discovery must not advertise a none-auth server as an OAuth-protected resource."""
try:
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
_build_oauth_protected_resource_response,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
global_mcp_server_manager.registry.clear()
server = _access_group_none_server()
global_mcp_server_manager.registry[server.server_id] = server
mock_request = MagicMock()
mock_request.base_url = "https://litellm.example.com/"
mock_request.headers = {}
try:
with pytest.raises(HTTPException) as exc_info:
await _build_oauth_protected_resource_response(
request=mock_request,
mcp_server_name="access_group_server",
use_standard_pattern=False,
)
assert exc_info.value.status_code == 404
assert "not an OAuth-protected resource" in str(exc_info.value.detail)
finally:
global_mcp_server_manager.registry.clear()
@pytest.mark.asyncio
async def test_oauth_authorization_server_404_for_non_oauth2_server():
"""Discovery must not advertise a none-auth server as an OAuth authorization server."""
try:
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
_build_oauth_authorization_server_response,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
global_mcp_server_manager.registry.clear()
server = _access_group_none_server()
global_mcp_server_manager.registry[server.server_id] = server
mock_request = MagicMock()
mock_request.base_url = "https://litellm.example.com/"
mock_request.headers = {}
try:
with pytest.raises(HTTPException) as exc_info:
_build_oauth_authorization_server_response(
request=mock_request,
mcp_server_name="access_group_server",
)
assert exc_info.value.status_code == 404
assert "not an OAuth authorization server" in str(exc_info.value.detail)
finally:
global_mcp_server_manager.registry.clear()
@pytest.mark.asyncio
async def test_oauth_protected_resource_passthrough_none_auth_not_404():
"""Regression guard for the protected-resource auth_type gate placement: a none-auth
server that opted into OAuth pass-through must still proxy upstream metadata, it must
NOT be 404'd. The gate has to sit after the pass-through branch."""
try:
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
_build_oauth_protected_resource_response,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
from litellm.proxy._types import MCPTransport
from litellm.types.mcp import MCPAuth
from litellm.types.mcp_server.mcp_server_manager import MCPServer
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
global_mcp_server_manager.registry.clear()
passthrough_server = MCPServer(
server_id="passthrough_server",
name="passthrough_server",
server_name="passthrough_server",
alias="passthrough_server",
url="https://upstream.example.com/mcp",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
oauth_passthrough=True,
extra_headers=["Authorization"],
)
global_mcp_server_manager.registry[passthrough_server.server_id] = passthrough_server
mock_request = MagicMock()
mock_request.base_url = "https://litellm.example.com/"
mock_request.headers = {}
try:
with patch(
"litellm.proxy._experimental.mcp_server.discoverable_endpoints.fetch_upstream_oauth_protected_resource",
new=AsyncMock(return_value={"authorization_servers": ["https://upstream-idp.example.com"]}),
):
response = await _build_oauth_protected_resource_response(
request=mock_request,
mcp_server_name="passthrough_server",
use_standard_pattern=False,
)
assert response["authorization_servers"] == ["https://upstream-idp.example.com"]
assert response["resource"].endswith("/passthrough_server/mcp")
finally:
global_mcp_server_manager.registry.clear()
@pytest.mark.asyncio
async def test_oauth_protected_resource_404_for_unknown_server_name():
"""A discovery request for an unknown server name returns the same 404 as a non-oauth2
server (not a 200 metadata doc with broken URLs), so the well-known paths cannot be used
to enumerate non-OAuth server names."""
try:
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
_build_oauth_protected_resource_response,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
global_mcp_server_manager.registry.clear()
mock_request = MagicMock()
mock_request.base_url = "https://litellm.example.com/"
mock_request.headers = {}
with pytest.raises(HTTPException) as exc_info:
await _build_oauth_protected_resource_response(
request=mock_request,
mcp_server_name="does_not_exist",
use_standard_pattern=True,
)
assert exc_info.value.status_code == 404
@pytest.mark.asyncio
async def test_oauth_authorization_server_404_for_unknown_server_name():
"""A named authorization-server discovery request for an unknown server returns 404, not a
200 metadata document pointing at non-existent /{name}/authorize and /{name}/token."""
try:
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
_build_oauth_authorization_server_response,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
except ImportError:
pytest.skip("MCP discoverable endpoints not available")
global_mcp_server_manager.registry.clear()
mock_request = MagicMock()
mock_request.base_url = "https://litellm.example.com/"
mock_request.headers = {}
with pytest.raises(HTTPException) as exc_info:
_build_oauth_authorization_server_response(
request=mock_request,
mcp_server_name="does_not_exist",
)
assert exc_info.value.status_code == 404

102
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
View file

@ -4164,6 +4164,7 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab
_get_tools_from_mcp_servers,
)
from litellm.proxy._types import UserAPIKeyAuth
from mcp.types import Tool as MCPTool
except ImportError:
pytest.skip("MCP server not available")
@ -4177,12 +4178,20 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab
server_a.auth_type = None
server_a.extra_headers = None
tool_1 = MagicMock()
tool_1.name = "server_a-tool_1"
tool_1 = MCPTool(
name="server_a-tool_1",
description="test tool",
inputSchema={"type": "object"},
)
dummy_logging_obj = MagicMock()
dummy_logging_obj.model_call_details = {"metadata": {"spend_logs_metadata": {}}}
dummy_logging_obj.async_success_handler = AsyncMock()
function_setup_kwargs = {}
def _capture_function_setup(*_args, **kwargs):
function_setup_kwargs.update(kwargs)
return dummy_logging_obj, None
with (
patch(
@ -4206,7 +4215,7 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab
),
patch(
"litellm.proxy._experimental.mcp_server.server.function_setup",
return_value=(dummy_logging_obj, None),
side_effect=_capture_function_setup,
),
):
mock_manager._get_tools_from_server = AsyncMock(return_value=[tool_1])
@ -4218,13 +4227,15 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab
mcp_server_auth_headers=None,
log_list_tools_to_spendlogs=True,
list_tools_log_source="mcp_protocol",
request_tags=["team-a"],
)
assert tools == [tool_1]
dummy_logging_obj.async_success_handler.assert_awaited_once()
assert dummy_logging_obj.async_success_handler.await_args.kwargs["result"] == [
tool_1
tool_1.model_dump(mode="json")
]
assert function_setup_kwargs["metadata"]["tags"] == ["team-a"]
spend_meta = dummy_logging_obj.model_call_details["metadata"]["spend_logs_metadata"]
assert spend_meta["tool_count_total"] == 1
@ -6498,3 +6509,86 @@ class TestMCPMetaTraceCarrier:
assert _mcp_meta_trace_carrier(SimpleNamespace(meta=None)) is None
only_progress = RequestParams.Meta.model_validate({"progressToken": "p1"})
assert _mcp_meta_trace_carrier(SimpleNamespace(meta=only_progress)) is None
@pytest.mark.asyncio
async def test_get_allowed_mcp_servers_includes_active_servers_submitted_by_user():
"""BYOM submitters can see approved servers they submitted without allow_all_keys."""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
submitted_server = _make_mcp_server_for_scope_filter("submitted-1", "user_mcp")
submitter = UserAPIKeyAuth(
user_id="submitter-user",
user_role=LitellmUserRoles.INTERNAL_USER,
api_key="sk-submitter",
)
other_user = UserAPIKeyAuth(
user_id="other-user",
user_role=LitellmUserRoles.INTERNAL_USER,
api_key="sk-other",
)
async def _submitted_ids(prisma_client, user_id):
return ["submitted-1"] if user_id == "submitter-user" else []
with (
patch.object(
global_mcp_server_manager,
"get_registry",
return_value={"submitted-1": submitted_server},
),
patch(
"litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp."
"MCPRequestHandler.get_allowed_mcp_servers",
AsyncMock(return_value=[]),
),
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
patch(
"litellm.proxy._experimental.mcp_server.db.get_active_submitted_mcp_server_ids_for_user",
side_effect=_submitted_ids,
),
):
submitter_allowed = await global_mcp_server_manager.get_allowed_mcp_servers(submitter)
other_allowed = await global_mcp_server_manager.get_allowed_mcp_servers(other_user)
assert "submitted-1" in submitter_allowed
assert "submitted-1" not in other_allowed
@pytest.mark.asyncio
async def test_get_active_submitted_mcp_server_ids_for_user_queries_active_rows():
from litellm.proxy._experimental.mcp_server.db import (
get_active_submitted_mcp_server_ids_for_user,
)
from litellm.proxy._types import MCPApprovalStatus
row = MagicMock()
row.server_id = "submitted-1"
prisma_client = MagicMock()
prisma_client.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[row])
result = await get_active_submitted_mcp_server_ids_for_user(prisma_client, "submitter-user")
assert result == ["submitted-1"]
prisma_client.db.litellm_mcpservertable.find_many.assert_awaited_once_with(
where={
"submitted_by": "submitter-user",
"approval_status": MCPApprovalStatus.active,
},
)
@pytest.mark.asyncio
async def test_get_active_submitted_mcp_server_ids_for_user_empty_user_id_skips_db():
from litellm.proxy._experimental.mcp_server.db import (
get_active_submitted_mcp_server_ids_for_user,
)
prisma_client = MagicMock()
prisma_client.db.litellm_mcpservertable.find_many = AsyncMock()
assert await get_active_submitted_mcp_server_ids_for_user(prisma_client, "") == []
prisma_client.db.litellm_mcpservertable.find_many.assert_not_awaited()

244
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
View file

@ -3198,6 +3198,250 @@ class TestMCPServerManager:
assert result == []
mock_inner.assert_not_called()
@pytest.mark.asyncio
async def test_no_mcp_servers_sentinel_excludes_submitted_byom_servers(self):
from litellm.proxy import proxy_server as proxy_server_module
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth
class _Cache:
async def async_get_cache(self, key: str):
return ["submitted-server"]
manager = MCPServerManager()
manager.registry = {
"submitted-server": MCPServer(
server_id="submitted-server",
name="submitted",
transport=MCPTransport.http,
)
}
object_permission = LiteLLM_ObjectPermissionTable(
object_permission_id="perm_no_mcp",
mcp_servers=["no-mcp-servers"],
mcp_access_groups=[],
)
user_api_key_auth = UserAPIKeyAuth(
api_key="sk-test",
user_id="user-123",
object_permission=object_permission,
object_permission_id="perm_no_mcp",
)
with (
patch.object(proxy_server_module, "user_api_key_cache", _Cache()),
patch.object(proxy_server_module, "prisma_client", None),
patch.object(
manager, "get_allow_all_keys_server_ids", return_value=["global-server"]
),
patch.object(
MCPRequestHandler,
"get_allowed_mcp_servers",
new_callable=AsyncMock,
return_value=["leaked-server"],
) as mock_inner,
):
result = await manager.get_allowed_mcp_servers(user_api_key_auth)
assert result == []
mock_inner.assert_not_called()
@pytest.mark.asyncio
async def test_explicitly_scoped_key_excludes_submitted_byom_servers(self):
from litellm.proxy import proxy_server as proxy_server_module
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth
cache = MagicMock()
cache.async_get_cache = AsyncMock(return_value=["submitted-server"])
manager = MCPServerManager()
manager.registry = {
"submitted-server": MCPServer(
server_id="submitted-server",
name="submitted",
transport=MCPTransport.http,
),
"scoped-server": MCPServer(
server_id="scoped-server",
name="scoped",
transport=MCPTransport.http,
),
}
object_permission = LiteLLM_ObjectPermissionTable(
object_permission_id="perm_scoped",
mcp_servers=["scoped-server"],
mcp_access_groups=[],
)
user_api_key_auth = UserAPIKeyAuth(
api_key="sk-test",
user_id="user-123",
object_permission=object_permission,
object_permission_id="perm_scoped",
)
with (
patch.object(proxy_server_module, "user_api_key_cache", cache),
patch.object(proxy_server_module, "prisma_client", None),
patch.object(manager, "get_allow_all_keys_server_ids", return_value=[]),
patch.object(
MCPRequestHandler,
"get_allowed_mcp_servers",
new_callable=AsyncMock,
return_value=["scoped-server"],
),
):
result = await manager.get_allowed_mcp_servers(user_api_key_auth)
assert result == ["scoped-server"]
cache.async_get_cache.assert_not_awaited()
@pytest.mark.asyncio
async def test_toolset_scope_excludes_submitted_byom_servers(self):
from litellm.proxy import proxy_server as proxy_server_module
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy._experimental.mcp_server.mcp_context import (
_mcp_active_toolset_id,
)
from litellm.proxy._types import UserAPIKeyAuth
cache = MagicMock()
cache.async_get_cache = AsyncMock(return_value=["submitted-server"])
manager = MCPServerManager()
manager.registry = {
"submitted-server": MCPServer(
server_id="submitted-server",
name="submitted",
transport=MCPTransport.http,
),
"toolset-server": MCPServer(
server_id="toolset-server",
name="toolset",
transport=MCPTransport.http,
),
}
user_api_key_auth = UserAPIKeyAuth(api_key="sk-test", user_id="user-123")
token = _mcp_active_toolset_id.set("toolset-abc")
try:
with (
patch.object(proxy_server_module, "user_api_key_cache", cache),
patch.object(proxy_server_module, "prisma_client", None),
patch.object(
manager, "get_allow_all_keys_server_ids", return_value=["global-server"]
),
patch.object(
MCPRequestHandler,
"get_allowed_mcp_servers",
new_callable=AsyncMock,
return_value=["toolset-server"],
),
):
result = await manager.get_allowed_mcp_servers(user_api_key_auth)
finally:
_mcp_active_toolset_id.reset(token)
assert result == ["toolset-server"]
@pytest.mark.asyncio
async def test_invalidate_byom_submitted_servers_cache_deletes_key(self):
from litellm.proxy import proxy_server as proxy_server_module
cache = MagicMock()
cache.async_delete_cache = AsyncMock()
manager = MCPServerManager()
with patch.object(proxy_server_module, "user_api_key_cache", cache):
await manager.invalidate_byom_submitted_servers_cache("user-123")
await manager.invalidate_byom_submitted_servers_cache(None)
cache.async_delete_cache.assert_awaited_once_with(key="byom_submitted_servers:user-123")
@pytest.mark.asyncio
async def test_get_active_submitted_ids_cache_miss_queries_db_and_caches(self):
from litellm.proxy import proxy_server as proxy_server_module
from litellm.proxy._types import UserAPIKeyAuth
cache = MagicMock()
cache.async_get_cache = AsyncMock(return_value=None)
cache.async_set_cache = AsyncMock()
manager = MCPServerManager()
manager.registry = {
"submitted-server": MCPServer(
server_id="submitted-server",
name="submitted",
transport=MCPTransport.http,
)
}
user_api_key_auth = UserAPIKeyAuth(api_key="sk-test", user_id="user-123")
with (
patch.object(proxy_server_module, "user_api_key_cache", cache),
patch.object(proxy_server_module, "prisma_client", MagicMock()),
patch(
"litellm.proxy._experimental.mcp_server.db.get_active_submitted_mcp_server_ids_for_user",
AsyncMock(return_value=["submitted-server", "unknown-server"]),
),
):
result = await manager._get_active_submitted_mcp_server_ids_for_user(user_api_key_auth)
assert result == ["submitted-server"]
cache.async_set_cache.assert_awaited_once_with(
key="byom_submitted_servers:user-123",
value=["submitted-server", "unknown-server"],
ttl=60,
)
@pytest.mark.asyncio
async def test_get_allowed_mcp_servers_fallback_keeps_submitted_byom_servers(self):
from litellm.proxy import proxy_server as proxy_server_module
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy._types import UserAPIKeyAuth
class _Cache:
async def async_get_cache(self, key: str):
assert key == "byom_submitted_servers:user-123"
return ["submitted-server"]
manager = MCPServerManager()
manager.registry = {
"submitted-server": MCPServer(
server_id="submitted-server",
name="submitted",
transport=MCPTransport.http,
)
}
user_api_key_auth = UserAPIKeyAuth(
api_key="sk-test",
user_id="user-123",
)
with (
patch.object(proxy_server_module, "user_api_key_cache", _Cache()),
patch.object(proxy_server_module, "prisma_client", None),
patch.object(
manager, "get_allow_all_keys_server_ids", return_value=["global-server"]
),
patch.object(
MCPRequestHandler,
"get_allowed_mcp_servers",
new_callable=AsyncMock,
side_effect=RuntimeError("permission resolver failed"),
),
):
result = await manager.get_allowed_mcp_servers(user_api_key_auth)
assert set(result) == {"global-server", "submitted-server"}
@pytest.mark.asyncio
async def test_get_allowed_mcp_servers_anonymous_delegate_requires_oauth2(self):
"""Anonymous delegated auth listing should only include oauth2 servers."""

6
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py
View file

@ -432,6 +432,11 @@ class TestCallToolRestApiVirtualTools:
new_callable=AsyncMock,
return_value=fake_result,
) as mock_execute,
patch(
"litellm.proxy._experimental.mcp_server.rest_endpoints._fire_mcp_success_logging",
new_callable=AsyncMock,
side_effect=RuntimeError("logging failed"),
) as mock_fire_logging,
):
result = await self._get_call_fn()(
request=request,
@ -439,6 +444,7 @@ class TestCallToolRestApiVirtualTools:
)
mock_execute.assert_awaited_once()
mock_fire_logging.assert_awaited_once()
assert mock_execute.await_args.kwargs["name"] == "github-create_issue"
assert result.isError is False

48
tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py
View file

@ -1,6 +1,8 @@
import asyncio
import json
from datetime import datetime
from typing import Any, Dict, Optional
from unittest.mock import MagicMock
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
@ -330,7 +332,8 @@ class TestExecuteWithMcpClient:
@pytest.mark.asyncio
async def test_m2m_does_not_build_presented_store(self, monkeypatch):
"""M2M (client_credentials): to_server_spec returns None, so no presented provider is built;
the auto-fetch path is unchanged (no cred_provider, the incoming header dropped as before)."""
the auto-fetch path is unchanged (no cred_provider, the incoming header dropped as before).
"""
captured: dict = {}
def fake_build_stdio_env(server, raw_headers):
@ -377,7 +380,8 @@ class TestExecuteWithMcpClient:
@pytest.mark.asyncio
async def test_token_exchange_does_not_build_presented_store(self, monkeypatch):
"""OBO / token-exchange (auth_type oauth2_token_exchange, not oauth2): excluded by the
auth_type == oauth2 guard, so no presented provider is built and the v1 exchange path runs."""
auth_type == oauth2 guard, so no presented provider is built and the v1 exchange path runs.
"""
captured: dict = {}
def fake_build_stdio_env(server, raw_headers):
@ -1196,6 +1200,13 @@ class TestCallToolRestAPI:
fake_execute_mcp_tool,
raising=False,
)
fire_logging = AsyncMock(side_effect=RuntimeError("logging failed"))
monkeypatch.setattr(
rest_endpoints,
"_fire_mcp_success_logging",
fire_logging,
raising=False,
)
request_payload = {
"server_id": "server-1",
@ -1217,6 +1228,23 @@ class TestCallToolRestAPI:
assert captured["name"] == "demo-tool"
assert captured["arguments"] == {"foo": "bar"}
assert captured["allowed_mcp_servers"] == [stub_server]
fire_logging.assert_awaited_once()
async def test_success_logging_cancellation_propagates(self, monkeypatch):
fire_logging = AsyncMock(side_effect=asyncio.CancelledError())
monkeypatch.setattr(
rest_endpoints,
"_fire_mcp_success_logging",
fire_logging,
raising=False,
)
with pytest.raises(asyncio.CancelledError):
await rest_endpoints._safe_fire_mcp_success_logging(
object(), {"result": "ok"}, datetime.now(), datetime.now()
)
fire_logging.assert_awaited_once()
class TestGetToolsForSingleServer:
@ -1809,9 +1837,9 @@ class TestPreviewOpenAPITools:
names = [t["name"] for t in result["tools"]]
anthropic_re = re.compile(r"^[a-zA-Z0-9_-]{1,128}$")
for name in names:
assert anthropic_re.match(name), (
f"preview tool name {name!r} violates ^[a-zA-Z0-9_-]+$"
)
assert anthropic_re.match(
name
), f"preview tool name {name!r} violates ^[a-zA-Z0-9_-]+$"
assert "actions_download-job-logs-for-workflow-run" in names
assert "pulls_list-files" in names
@ -1868,7 +1896,9 @@ class TestPreviewOpenAPITools:
registered_summary_to_name: dict = {}
def fake_create_tool_function(path, method, operation, base_url): # noqa: ANN001
def fake_create_tool_function(
path, method, operation, base_url
): # noqa: ANN001
def _f():
return None
@ -1881,7 +1911,9 @@ class TestPreviewOpenAPITools:
)
class _StubRegistry:
def register_tool(self, name, description, input_schema, handler): # noqa: ANN001
def register_tool(
self, name, description, input_schema, handler
): # noqa: ANN001
registered_summary_to_name[description] = name
monkeypatch.setattr(

73
tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py
View file

@ -1104,3 +1104,76 @@ async def test_async_post_call_failure_hook_records_recovered_partial_spend():
mock_update_database.assert_called_once()
assert mock_update_database.call_args[1]["response_cost"] == 3.5e-05
@pytest.mark.asyncio
async def test_track_cost_callback_enriches_user_id_for_mcp_style_metadata():
"""MCP tool calls may only carry user_api_key; user/team rollups still need user_id."""
from litellm.proxy._types import UserAPIKeyAuth
logger = _ProxyDBLogger()
key_obj = UserAPIKeyAuth(
api_key="hashed-key",
user_id="mcp-user@example.com",
team_id="team-123",
org_id="org-456",
key_alias="mcp-key",
)
kwargs = {
"call_type": "call_mcp_tool",
"model": "MCP: echo",
"litellm_params": {
"metadata": {
"user_api_key": "hashed-key",
}
},
"standard_logging_object": {
"response_cost": 10.0,
"request_tags": [],
"metadata": {},
},
}
with (
patch(
"litellm.proxy.hooks.proxy_track_cost_callback.get_key_object",
new_callable=AsyncMock,
return_value=key_obj,
),
patch(
"litellm.proxy.proxy_server.increment_spend_counters",
new_callable=AsyncMock,
) as mock_increment,
patch(
"litellm.proxy.proxy_server.update_cache",
new_callable=AsyncMock,
),
patch(
"litellm.proxy.proxy_server.proxy_logging_obj",
) as mock_proxy_logging,
):
mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock()
mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock()
await logger._PROXY_track_cost_callback(
kwargs=kwargs,
completion_response={"id": "mcp-call-1"},
start_time=datetime.now(),
end_time=datetime.now(),
)
mock_increment.assert_awaited_once()
assert mock_increment.call_args.kwargs["user_id"] == "mcp-user@example.com"
assert mock_increment.call_args.kwargs["team_id"] == "team-123"
assert mock_increment.call_args.kwargs["org_id"] == "org-456"
update_kwargs = (
mock_proxy_logging.db_spend_update_writer.update_database.await_args.kwargs
)
assert update_kwargs["user_id"] == "mcp-user@example.com"
assert update_kwargs["team_id"] == "team-123"
assert (
kwargs["litellm_params"]["metadata"]["user_api_key_user_id"]
== "mcp-user@example.com"
)

197
tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
View file

@ -2068,6 +2068,7 @@ class TestTemporaryMCPSessionEndpoints:
request = MagicMock()
server = generate_mock_mcp_server_config_record(server_id="server-1")
server.auth_type = MCPAuth.oauth2
authorize_response = MagicMock()
admin_auth = generate_mock_user_api_key_auth(
user_role=LitellmUserRoles.PROXY_ADMIN,
@ -2110,6 +2111,91 @@ class TestTemporaryMCPSessionEndpoints:
scope="scope1",
)
@pytest.mark.asyncio
async def test_mcp_authorize_rejects_non_oauth2_server(self):
"""mcp_authorize must reject a none-auth server with an accurate 'does not use OAuth'
400 before the client_id check, never delegating to authorize_with_server."""
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
mcp_authorize,
)
server = generate_mock_mcp_server_config_record(server_id="none-server")
server.auth_type = MCPAuth.none
admin_auth = generate_mock_user_api_key_auth(
user_role=LitellmUserRoles.PROXY_ADMIN,
)
with (
patch(
"litellm.proxy.management_endpoints.mcp_management_endpoints._get_cached_temporary_mcp_server_or_404",
return_value=server,
),
patch(
"litellm.proxy.management_endpoints.mcp_management_endpoints.authorize_with_server",
AsyncMock(),
) as authorize_mock,
):
with pytest.raises(HTTPException) as exc_info:
await mcp_authorize(
request=MagicMock(),
server_id="none-server",
user_api_key_dict=admin_auth,
client_id=None,
redirect_uri="https://example.com/callback",
state="state123",
)
assert exc_info.value.status_code == 400
detail_text = str(exc_info.value.detail)
assert "does not use OAuth" in detail_text
assert "missing_client_id" not in detail_text
authorize_mock.assert_not_awaited()
@pytest.mark.asyncio
async def test_mcp_token_rejects_non_oauth2_server(self):
"""mcp_token must reject a none-auth server with 'does not use OAuth' 400 before the
client_id check, never delegating to exchange_token_with_server."""
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
mcp_token,
)
server = generate_mock_mcp_server_config_record(server_id="none-server")
server.auth_type = MCPAuth.none
admin_auth = generate_mock_user_api_key_auth(
user_role=LitellmUserRoles.PROXY_ADMIN,
)
with (
patch(
"litellm.proxy.management_endpoints.mcp_management_endpoints._get_cached_temporary_mcp_server_or_404",
return_value=server,
),
patch(
"litellm.proxy.management_endpoints.mcp_management_endpoints.exchange_token_with_server",
AsyncMock(),
) as exchange_mock,
):
with pytest.raises(HTTPException) as exc_info:
await mcp_token(
request=MagicMock(),
server_id="none-server",
user_api_key_dict=admin_auth,
grant_type="authorization_code",
code="code-123",
redirect_uri="https://example.com/callback",
client_id=None,
client_secret=None,
code_verifier="verifier",
refresh_token=None,
scope=None,
)
assert exc_info.value.status_code == 400
detail_text = str(exc_info.value.detail)
assert "does not use OAuth" in detail_text
assert "missing_client_id" not in detail_text
exchange_mock.assert_not_awaited()
@pytest.mark.asyncio
async def test_mcp_token_proxies_to_exchange_endpoint(self):
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
@ -2118,6 +2204,7 @@ class TestTemporaryMCPSessionEndpoints:
request = MagicMock()
server = generate_mock_mcp_server_config_record(server_id="server-1")
server.auth_type = MCPAuth.oauth2
exchange_response = {"access_token": "token"}
admin_auth = generate_mock_user_api_key_auth(
user_role=LitellmUserRoles.PROXY_ADMIN,
@ -2170,6 +2257,7 @@ class TestTemporaryMCPSessionEndpoints:
request = MagicMock()
server = generate_mock_mcp_server_config_record(server_id="server-1")
server.auth_type = MCPAuth.oauth2
exchange_response = {"access_token": "new-token", "refresh_token": "new-rt"}
admin_auth = generate_mock_user_api_key_auth(
user_role=LitellmUserRoles.PROXY_ADMIN,
@ -2222,6 +2310,7 @@ class TestTemporaryMCPSessionEndpoints:
request = MagicMock()
server = generate_mock_mcp_server_config_record(server_id="server-1")
server.auth_type = MCPAuth.oauth2
register_response = {"client_id": "generated"}
request_body = {
"client_name": "LiteLLM",
@ -3126,41 +3215,18 @@ class TestMCPApprovalWorkflow:
assert result.pending_review == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
"user_role, expected_global_value",
[
(LitellmUserRoles.PROXY_ADMIN, "super-secret"),
(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ""),
],
)
async def test_get_submissions_redacts_global_env_for_view_only_admin(
self, user_role, expected_global_value
):
"""Read-only admins reviewing the submission queue must not receive the
submitter's global env var secrets; full admins still see them."""
async def test_get_submissions_sanitizes_for_view_only_admin(self):
"""PROXY_ADMIN_VIEW_ONLY reviewing the submission queue must go through
the non-admin sanitizer that fetch/list endpoints use: url,
static_headers, env, env_vars, and credentials are all dropped. A
mutation swapping the gate back to the old partial-blank pattern (which
left url/static_headers/env and env-var names intact) would fail this."""
from litellm.proxy._types import MCPSubmissionsSummary
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
get_mcp_server_submissions,
)
base = generate_mock_mcp_server_db_record(alias="Pending")
item = LiteLLM_MCPServerTable(
**{
**base.model_dump(),
"env_vars": [
{
"name": "ADMIN_API_KEY",
"value": "super-secret",
"scope": "global",
},
{
"name": "USER_TOKEN",
"value": "placeholder-hint",
"scope": "user",
},
],
}
)
item = _leaky_list_server()
item.approval_status = "pending_review"
summary = MCPSubmissionsSummary(
total=1, pending_review=1, active=0, rejected=0, items=[item]
@ -3177,12 +3243,70 @@ class TestMCPApprovalWorkflow:
),
):
result = await get_mcp_server_submissions(
user_api_key_dict=generate_mock_user_api_key_auth(user_role=user_role),
user_api_key_dict=generate_mock_user_api_key_auth(
user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY
),
)
by_name = {ev.name: ev for ev in result.items[0].env_vars}
assert by_name["ADMIN_API_KEY"].value == expected_global_value
assert by_name["USER_TOKEN"].value == "placeholder-hint"
assert len(result.items) == 1
sanitized = result.items[0]
assert sanitized.url is None
assert sanitized.static_headers is None
assert sanitized.env == {}
assert sanitized.env_vars is None
assert sanitized.credentials is None
# The source record must not be mutated by sanitization.
assert item.url == "https://leaky.example.com/mcp?api_key=sk-embedded-in-url"
assert item.static_headers == {"Authorization": "Bearer sk-secret-header"}
@pytest.mark.asyncio
async def test_get_submissions_full_admin_still_sees_secrets(self):
"""The view-only redaction must not over-redact for a full PROXY_ADMIN,
who needs url/static_headers/env/env_vars to review the pending
submission. Only the explicit credentials field is cleared."""
from litellm.proxy._types import MCPSubmissionsSummary
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
get_mcp_server_submissions,
)
item = _leaky_list_server()
item.approval_status = "pending_review"
summary = MCPSubmissionsSummary(
total=1, pending_review=1, active=0, rejected=0, items=[item]
)
with (
patch(
"litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw",
return_value=MagicMock(),
),
patch(
"litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_submissions",
AsyncMock(return_value=summary),
),
):
result = await get_mcp_server_submissions(
user_api_key_dict=generate_mock_user_api_key_auth(
user_role=LitellmUserRoles.PROXY_ADMIN
),
)
assert len(result.items) == 1
raw = result.items[0]
assert raw.url == "https://leaky.example.com/mcp?api_key=sk-embedded-in-url"
assert raw.static_headers == {"Authorization": "Bearer sk-secret-header"}
assert raw.env == {"UPSTREAM_TOKEN": "sk-secret-env"}
assert raw.credentials is None
assert raw.env_vars is not None
assert len(raw.env_vars) == 1
# ``model_construct`` in ``_leaky_list_server`` skips validation, so
# env_vars stays as raw dicts; mirror the fixture shape here.
entry = raw.env_vars[0]
name = entry["name"] if isinstance(entry, dict) else entry.name
value = entry["value"] if isinstance(entry, dict) else entry.value
assert name == "GLOBAL_KEY"
assert value == "super-secret"
@pytest.mark.asyncio
async def test_approve_non_pending_server_raises_400(self):
@ -3223,8 +3347,10 @@ class TestMCPApprovalWorkflow:
pending_server.approval_status = MCPApprovalStatus.pending_review
approved_server = generate_mock_mcp_server_db_record()
approved_server.approval_status = MCPApprovalStatus.active
approved_server.submitted_by = "submitter-user"
mock_manager = MagicMock()
mock_manager.invalidate_byom_submitted_servers_cache = AsyncMock()
mock_manager.reload_servers_from_database = AsyncMock()
with (
@ -3250,6 +3376,9 @@ class TestMCPApprovalWorkflow:
)
mock_manager.reload_servers_from_database.assert_awaited_once()
mock_manager.invalidate_byom_submitted_servers_cache.assert_awaited_once_with(
"submitter-user"
)
assert result is not None
@pytest.mark.asyncio

4
tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py
View file

@ -84,7 +84,7 @@ class TestVertexAIBatchPassthroughHandler:
"input_file_id": "file-123",
"output_file_id": "file-456",
"error_file_id": None,
"completion_window": "24hrs",
"completion_window": "24h",
}
mock_transformation._get_batch_id_from_vertex_ai_batch_response.return_value = (
"123456789"
@ -451,7 +451,7 @@ class TestVertexAIBatchPassthroughHandler:
"input_file_id": "file-123",
"output_file_id": "file-456",
"error_file_id": None,
"completion_window": "24hrs",
"completion_window": "24h",
}
mock_transformation._get_batch_id_from_vertex_ai_batch_response.return_value = (
"123456789"

216
tests/test_litellm/proxy/proxy_server/test_routes_config.py
View file

@ -741,6 +741,222 @@ def test_get_config_callbacks_internal_error(client, auth_as, mock_prisma, monke
)
_CALLBACK_ENV_FIXTURE = {
"LANGFUSE_PUBLIC_KEY": "pk-public-1234567890",
"LANGFUSE_SECRET_KEY": "sk-langfuse-super-secret",
"LANGFUSE_HOST": "https://cloud.langfuse.com",
"DD_API_KEY": "dd-super-secret-api-key",
"DD_SITE": "datadoghq.com",
"OTEL_HEADERS": "Authorization=Bearer otel-super-secret",
"OTEL_ENDPOINT": "https://otlp.example.com",
"SLACK_WEBHOOK_URL": "https://hooks.slack.com/services/T000/B000/SLACK-WEBHOOK-FIXTURE-SECRET",
}
def _install_callbacks_config(monkeypatch, mock_prisma):
from litellm.proxy import proxy_server as ps
_install_litellm_config(mock_prisma)
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
monkeypatch.setattr(ps, "llm_router", None)
fake_proxy_config = MagicMock()
fake_proxy_config.get_config = AsyncMock(
return_value={
"litellm_settings": {"success_callback": ["langfuse", "datadog", "otel"]},
"general_settings": {"alerting": ["slack"]},
"environment_variables": dict(_CALLBACK_ENV_FIXTURE),
}
)
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
def _callback_variables(body: dict, name: str) -> dict:
return next(
cb["variables"] for cb in body["callbacks"] if cb["name"] == name
)
def test_get_config_callbacks_redacts_secret_env_vars_for_view_only_admin(
client, auth_as, mock_prisma, monkeypatch
):
from litellm.proxy._types import LitellmUserRoles
_install_callbacks_config(monkeypatch, mock_prisma)
with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY):
response = client.get("/get/config/callbacks")
assert response.status_code == 200
body = response.json()
for secret in (
_CALLBACK_ENV_FIXTURE["LANGFUSE_SECRET_KEY"],
_CALLBACK_ENV_FIXTURE["DD_API_KEY"],
_CALLBACK_ENV_FIXTURE["OTEL_HEADERS"],
_CALLBACK_ENV_FIXTURE["LANGFUSE_PUBLIC_KEY"],
):
assert secret not in response.text
langfuse_vars = _callback_variables(body, "langfuse")
assert langfuse_vars["LANGFUSE_PUBLIC_KEY"] == "REDACTED"
assert langfuse_vars["LANGFUSE_SECRET_KEY"] == "REDACTED"
assert langfuse_vars["LANGFUSE_HOST"] == _CALLBACK_ENV_FIXTURE["LANGFUSE_HOST"]
datadog_vars = _callback_variables(body, "datadog")
assert datadog_vars["DD_API_KEY"] == "REDACTED"
assert datadog_vars["DD_SITE"] == _CALLBACK_ENV_FIXTURE["DD_SITE"]
otel_vars = _callback_variables(body, "otel")
assert otel_vars["OTEL_HEADERS"] == "REDACTED"
assert otel_vars["OTEL_ENDPOINT"] == _CALLBACK_ENV_FIXTURE["OTEL_ENDPOINT"]
def test_get_config_callbacks_full_admin_still_sees_secret_env_vars(
client, auth_as, mock_prisma, monkeypatch
):
from litellm.proxy._types import LitellmUserRoles
_install_callbacks_config(monkeypatch, mock_prisma)
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.get("/get/config/callbacks")
assert response.status_code == 200
body = response.json()
langfuse_vars = _callback_variables(body, "langfuse")
assert langfuse_vars["LANGFUSE_SECRET_KEY"] == _CALLBACK_ENV_FIXTURE["LANGFUSE_SECRET_KEY"]
assert langfuse_vars["LANGFUSE_PUBLIC_KEY"] == _CALLBACK_ENV_FIXTURE["LANGFUSE_PUBLIC_KEY"]
datadog_vars = _callback_variables(body, "datadog")
assert datadog_vars["DD_API_KEY"] == _CALLBACK_ENV_FIXTURE["DD_API_KEY"]
otel_vars = _callback_variables(body, "otel")
assert otel_vars["OTEL_HEADERS"] == _CALLBACK_ENV_FIXTURE["OTEL_HEADERS"]
def test_get_config_callbacks_redacts_slack_webhook_urls_for_view_only_admin(
client, auth_as, mock_prisma, monkeypatch
):
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
_install_callbacks_config(monkeypatch, mock_prisma)
webhooks = {
"spend_reports": "https://hooks.slack.com/services/T000/B000/SPEND-WEBHOOK-SECRET",
"budget_alerts": "https://hooks.slack.com/services/T000/B111/BUDGET-WEBHOOK-SECRET",
}
monkeypatch.setattr(
ps.proxy_logging_obj.slack_alerting_instance,
"alert_to_webhook_url",
webhooks,
raising=False,
)
def _slack_block(body):
return next(a for a in body["alerts"] if a["name"] == "slack")
with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY):
view_resp = client.get("/get/config/callbacks")
assert view_resp.status_code == 200
for url in webhooks.values():
assert url not in view_resp.text
assert _CALLBACK_ENV_FIXTURE["SLACK_WEBHOOK_URL"] not in view_resp.text
view_slack = _slack_block(view_resp.json())
assert view_slack["alerts_to_webhook"] == {
"spend_reports": "REDACTED",
"budget_alerts": "REDACTED",
}
assert view_slack["variables"]["SLACK_WEBHOOK_URL"] == "REDACTED"
with auth_as(LitellmUserRoles.PROXY_ADMIN):
admin_resp = client.get("/get/config/callbacks")
assert admin_resp.status_code == 200
admin_slack = _slack_block(admin_resp.json())
assert admin_slack["alerts_to_webhook"] == webhooks
assert admin_slack["variables"]["SLACK_WEBHOOK_URL"] != "REDACTED"
def test_redact_callback_env_vars_helper_handles_none_and_non_secret_keys():
from litellm.proxy import proxy_server as ps
out = ps._redact_callback_env_vars(
{
"LANGFUSE_SECRET_KEY": "sk-leak",
"LANGFUSE_HOST": "https://cloud.langfuse.com",
"DD_API_KEY": None,
"GALILEO_USERNAME": "galileo-user-1234",
"GENERIC_LOGGER_HEADERS": "Authorization=Bearer x",
"GCS_PATH_SERVICE_ACCOUNT": "/etc/secrets/gcs.json",
"SLACK_WEBHOOK_URL": "https://hooks.slack.com/services/T/B/token",
"SMTP_USERNAME": "smtp-user-1234",
}
)
assert out == {
"LANGFUSE_SECRET_KEY": "REDACTED",
"LANGFUSE_HOST": "https://cloud.langfuse.com",
"DD_API_KEY": None,
"GALILEO_USERNAME": "REDACTED",
"GENERIC_LOGGER_HEADERS": "REDACTED",
"GCS_PATH_SERVICE_ACCOUNT": "REDACTED",
"SLACK_WEBHOOK_URL": "REDACTED",
"SMTP_USERNAME": "REDACTED",
}
def test_get_config_callbacks_redacts_email_alerting_vars_for_view_only_admin(
client, auth_as, mock_prisma, monkeypatch
):
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
_install_litellm_config(mock_prisma)
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
monkeypatch.setattr(ps, "llm_router", None)
fake_proxy_config = MagicMock()
fake_proxy_config.get_config = AsyncMock(
return_value={
"litellm_settings": {"success_callback": []},
"general_settings": {"alerting": ["email"]},
"environment_variables": {
"SMTP_HOST": "smtp.resend.com",
"SMTP_PORT": "587",
"SMTP_USERNAME": "smtp-user-fixture-1234",
"SMTP_PASSWORD": "smtp-password-fixture-1234",
"SMTP_SENDER_EMAIL": "alerts@example.com",
"TEST_EMAIL_ADDRESS": "admin@example.com",
"EMAIL_LOGO_URL": "https://example.com/logo.png",
"EMAIL_SUPPORT_CONTACT": "support@example.com",
},
}
)
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
def _email_block(body):
return next(a for a in body["alerts"] if a["name"] == "email")
with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY):
view_resp = client.get("/get/config/callbacks")
assert view_resp.status_code == 200
for secret in ("smtp-user-fixture-1234", "smtp-password-fixture-1234"):
assert secret not in view_resp.text
view_email = _email_block(view_resp.json())["variables"]
assert view_email["SMTP_PASSWORD"] == "REDACTED"
assert view_email["SMTP_USERNAME"] == "REDACTED"
assert view_email["SMTP_HOST"] == "smtp.resend.com"
assert view_email["SMTP_PORT"] == "587"
assert view_email["SMTP_SENDER_EMAIL"] == "alerts@example.com"
with auth_as(LitellmUserRoles.PROXY_ADMIN):
admin_resp = client.get("/get/config/callbacks")
assert admin_resp.status_code == 200
admin_email = _email_block(admin_resp.json())["variables"]
assert admin_email["SMTP_USERNAME"] == "smtp-user-fixture-1234"
assert admin_email["SMTP_PASSWORD"] != "REDACTED"
assert admin_email["SMTP_HOST"] == "smtp.resend.com"
# ---------------------------------------------------------------------------
# GET /config/yaml
# ---------------------------------------------------------------------------

29
tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -1051,7 +1051,10 @@ async def test_ui_view_spend_logs_sort_by_ttft_ms(client, monkeypatch):
page_size = params[-2] if len(params) >= 2 else 50
skip = params[-1] if len(params) >= 1 else 0
return [
{**{k: v for k, v in row.items() if k != "_ttft_ms"}, "total_count": len(base_logs)}
{
**{k: v for k, v in row.items() if k != "_ttft_ms"},
"total_count": len(base_logs),
}
for row in sorted_logs[skip : skip + page_size]
]
@ -2917,10 +2920,11 @@ async def test_build_ui_spend_logs_response_dict_rows_session_counts():
)
session_id = "sess-abc-123"
api_key = "hashed-key-xyz"
dict_rows = [
{"request_id": "req-1", "session_id": session_id, "call_type": "completion"},
{"request_id": "req-2", "session_id": session_id, "call_type": "mcp_tool_call"},
{"request_id": "req-3", "session_id": None, "call_type": "completion"},
{"request_id": "req-1", "session_id": session_id, "call_type": "completion", "api_key": api_key},
{"request_id": "req-2", "session_id": session_id, "call_type": "mcp_tool_call", "api_key": api_key},
{"request_id": "req-3", "session_id": None, "call_type": "completion", "api_key": api_key},
]
mock_prisma = MagicMock()
@ -2929,6 +2933,15 @@ async def test_build_ui_spend_logs_response_dict_rows_session_counts():
{"session_id": session_id, "_count": {"session_id": 2}},
]
)
mock_prisma.db.query_raw = AsyncMock(
return_value=[
{
"session_id": session_id,
"mcp_tool_call_count": 1,
"mcp_tool_call_spend": 10.0,
}
]
)
result = await _build_ui_spend_logs_response(
prisma_client=mock_prisma,
@ -2946,6 +2959,10 @@ async def test_build_ui_spend_logs_response_dict_rows_session_counts():
# Rows with the shared session_id should have session_total_count=2
assert rows[0]["session_total_count"] == 2
assert rows[1]["session_total_count"] == 2
assert rows[0]["mcp_tool_call_count"] == 1
assert rows[0]["mcp_tool_call_spend"] == 10.0
assert rows[1]["mcp_tool_call_count"] == 1
assert rows[1]["mcp_tool_call_spend"] == 10.0
# Row without a session_id defaults to 1
assert rows[2]["session_total_count"] == 1
@ -4104,7 +4121,9 @@ async def test_cold_storage_handler_returns_none_when_no_logger_configured(monke
@pytest.mark.asyncio
async def test_cold_storage_handler_resolves_configured_logger_from_registry(monkeypatch):
async def test_cold_storage_handler_resolves_configured_logger_from_registry(
monkeypatch,
):
from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler
logger = _FakeColdStorageLogger({"messages": "from-registry"})

20
tests/test_litellm/proxy/test_proxy_server.py
View file

@ -896,7 +896,9 @@ def test_get_config_custom_callback_api_env_vars(monkeypatch):
# Bypass auth dependency
original_overrides = app.dependency_overrides.copy()
app.dependency_overrides[user_api_key_auth] = lambda: MagicMock()
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234"
)
client = TestClient(app)
try:
@ -950,7 +952,9 @@ def test_get_config_returns_email_settings(monkeypatch):
monkeypatch.setattr(proxy_config, "get_config", AsyncMock(return_value=config_data))
original_overrides = app.dependency_overrides.copy()
app.dependency_overrides[user_api_key_auth] = lambda: MagicMock()
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234"
)
client = TestClient(app)
try:
@ -1007,7 +1011,9 @@ def test_get_config_returns_slack_webhook(monkeypatch):
monkeypatch.setattr(proxy_config, "get_config", AsyncMock(return_value=config_data))
original_overrides = app.dependency_overrides.copy()
app.dependency_overrides[user_api_key_auth] = lambda: MagicMock()
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234"
)
client = TestClient(app)
try:
@ -1061,7 +1067,9 @@ def test_get_config_cleared_slack_webhook_not_overridden_by_os_env(monkeypatch):
monkeypatch.setattr(proxy_config, "get_config", AsyncMock(return_value=config_data))
original_overrides = app.dependency_overrides.copy()
app.dependency_overrides[user_api_key_auth] = lambda: MagicMock()
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234"
)
client = TestClient(app)
try:
@ -5205,7 +5213,9 @@ def test_get_config_normalizes_string_callbacks(monkeypatch):
monkeypatch.setattr(proxy_config, "get_config", AsyncMock(return_value=config_data))
original_overrides = app.dependency_overrides.copy()
app.dependency_overrides[user_api_key_auth] = lambda: MagicMock()
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234"
)
client = TestClient(app)
try:

26
tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
View file

@ -2367,3 +2367,29 @@ def test_update_ui_settings_writes_audit_log(monkeypatch):
assert after["disable_custom_api_keys"] is True
finally:
app.dependency_overrides.pop(user_api_key_auth, None)
def test_update_mcp_semantic_filter_settings_requires_proxy_admin(monkeypatch):
"""Non-admin callers must not mutate global MCP semantic filter settings."""
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
async def _internal_user_auth():
return UserAPIKeyAuth(
user_id="internal-user-1",
api_key="hashed-internal-key",
user_role=LitellmUserRoles.INTERNAL_USER,
)
app.dependency_overrides[user_api_key_auth] = _internal_user_auth
try:
resp = client.patch(
"/update/mcp_semantic_filter_settings",
json={"enabled": True, "top_k": 99, "similarity_threshold": 0.01},
)
assert resp.status_code == 403
assert "proxy admin" in resp.json()["detail"].lower()
finally:
app.dependency_overrides.pop(user_api_key_auth, None)

239
tests/test_litellm/responses/mcp/test_chat_completions_handler.py
View file

@ -1105,3 +1105,242 @@ async def test_execute_tool_calls_sets_proxy_server_request_arguments(monkeypatc
"param1": "value1",
"param2": 123,
}, "arguments should be parsed correctly"
@pytest.mark.asyncio
async def test_acompletion_with_mcp_streaming_drain_error_does_not_drop_final_chunk(monkeypatch):
"""
Regression test: after yielding the final chunk, MCPStreamingIterator drains
the inner CustomStreamWrapper to fire end-of-stream spend logging. If the
inner stream raises a non-StopAsyncIteration error during that drain (e.g.
a transient APIError on the trailing usage chunk), the error must not
escape __anext__ and drop the already-assembled final chunk.
"""
from unittest.mock import MagicMock
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
from litellm.utils import CustomStreamWrapper
tools = [{"type": "mcp", "server_url": "litellm_proxy/mcp/local"}]
openai_tools = [{"type": "function", "function": {"name": "local_search"}}]
def create_chunk(content, finish_reason=None):
return ModelResponseStream(
id="test-stream",
model="test-model",
created=1234567890,
object="chat.completion.chunk",
choices=[
StreamingChoices(
index=0,
delta=Delta(content=content, role="assistant"),
finish_reason=finish_reason,
)
],
)
chunks = [
create_chunk("Hello"),
create_chunk(" world", finish_reason="stop"),
]
logging_obj = MagicMock()
logging_obj.model_call_details = {}
class DrainErrorStreamingResponse(CustomStreamWrapper):
def __init__(self):
super().__init__(
completion_stream=None,
model="test-model",
logging_obj=logging_obj,
)
self.chunks = chunks
self._index = 0
def __aiter__(self):
return self
async def __anext__(self):
if self._index < len(self.chunks):
chunk = self.chunks[self._index]
self._index += 1
return chunk
if self._index == len(self.chunks):
self._index += 1
raise RuntimeError("connection dropped on trailing usage chunk")
raise StopAsyncIteration
mock_acompletion = AsyncMock(return_value=DrainErrorStreamingResponse())
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_should_use_litellm_mcp_gateway",
staticmethod(lambda tools: True),
)
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_parse_mcp_tools",
staticmethod(lambda tools: (tools, [])),
)
async def mock_process(**_):
return (tools, {"local_search": "local"})
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_process_mcp_tools_without_openai_transform",
mock_process,
)
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_transform_mcp_tools_to_openai",
staticmethod(lambda *_, **__: openai_tools),
)
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_should_auto_execute_tools",
staticmethod(lambda **_: True),
)
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_extract_tool_calls_from_chat_response",
staticmethod(lambda **_: []),
)
monkeypatch.setattr(
ResponsesAPIRequestUtils,
"extract_mcp_headers_from_request",
staticmethod(lambda **_: (None, None, None, None)),
)
with patch("litellm.acompletion", mock_acompletion):
result = await acompletion_with_mcp(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hello"}],
tools=tools,
stream=True,
)
all_chunks = []
async for chunk in result:
all_chunks.append(chunk)
final_chunks = [
chunk
for chunk in all_chunks
if chunk.choices and chunk.choices[0].finish_reason == "stop"
]
assert len(final_chunks) == 1, f"Final chunk must survive a drain error. Got chunks: {all_chunks}"
assert all_chunks[-1].choices[0].finish_reason == "stop"
@pytest.mark.asyncio
async def test_acompletion_with_mcp_streaming_drains_inner_stream_after_exhaustion(monkeypatch):
from unittest.mock import MagicMock
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
from litellm.utils import CustomStreamWrapper
tools = [{"type": "mcp", "server_url": "litellm_proxy/mcp/local"}]
openai_tools = [{"type": "function", "function": {"name": "local_search"}}]
def create_chunk(content):
return ModelResponseStream(
id="test-stream",
model="test-model",
created=1234567890,
object="chat.completion.chunk",
choices=[
StreamingChoices(
index=0,
delta=Delta(content=content, role="assistant"),
finish_reason=None,
)
],
)
chunks = [create_chunk("Hello"), create_chunk(" world")]
logging_obj = MagicMock()
logging_obj.model_call_details = {}
class ExhaustingStreamingResponse(CustomStreamWrapper):
def __init__(self):
super().__init__(
completion_stream=None,
model="test-model",
logging_obj=logging_obj,
)
self.chunks = chunks
self._index = 0
self.drained_after_exhaustion = False
def __aiter__(self):
return self
async def __anext__(self):
if self._index < len(self.chunks):
chunk = self.chunks[self._index]
self._index += 1
return chunk
if self._index == len(self.chunks):
self._index += 1
raise StopAsyncIteration
self.drained_after_exhaustion = True
raise StopAsyncIteration
initial_stream = ExhaustingStreamingResponse()
mock_acompletion = AsyncMock(return_value=initial_stream)
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_should_use_litellm_mcp_gateway",
staticmethod(lambda tools: True),
)
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_parse_mcp_tools",
staticmethod(lambda tools: (tools, [])),
)
async def mock_process(**_):
return (tools, {"local_search": "local"})
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_process_mcp_tools_without_openai_transform",
mock_process,
)
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_transform_mcp_tools_to_openai",
staticmethod(lambda *_, **__: openai_tools),
)
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_should_auto_execute_tools",
staticmethod(lambda **_: True),
)
monkeypatch.setattr(
LiteLLM_Proxy_MCP_Handler,
"_extract_tool_calls_from_chat_response",
staticmethod(lambda **_: []),
)
monkeypatch.setattr(
ResponsesAPIRequestUtils,
"extract_mcp_headers_from_request",
staticmethod(lambda **_: (None, None, None, None)),
)
with patch("litellm.acompletion", mock_acompletion):
result = await acompletion_with_mcp(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hello"}],
tools=tools,
stream=True,
)
all_chunks = []
async for chunk in result:
all_chunks.append(chunk)
assert len(all_chunks) == 3
assert initial_stream.drained_after_exhaustion is True

74
tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py
View file

@ -401,3 +401,77 @@ async def test_get_mcp_tools_from_manager_enables_list_tools_logging(monkeypatch
assert mock_get_tools.await_args is not None
assert mock_get_tools.await_args.kwargs["log_list_tools_to_spendlogs"] is True
assert mock_get_tools.await_args.kwargs["list_tools_log_source"] == "responses"
def test_get_parent_request_tags_from_metadata():
tags = LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(
{"metadata": {"tags": ["team-a", "prod"]}}
)
assert tags == ["team-a", "prod"]
def test_get_parent_request_tags_from_nested_litellm_params():
tags = LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(
{
"metadata": {"tags": ["top-level"]},
"litellm_params": {
"metadata": {"tags": ["nested"]},
"proxy_server_request": {"headers": {"user-agent": "client/1.0"}},
},
}
)
assert tags == ["nested", "User-Agent: client", "User-Agent: client/1.0"]
@pytest.mark.asyncio
async def test_get_mcp_tools_from_manager_forwards_request_tags(monkeypatch):
mock_get_tools = AsyncMock(return_value=[])
monkeypatch.setattr(
"litellm.proxy._experimental.mcp_server.server._get_tools_from_mcp_servers",
mock_get_tools,
)
fake_manager = types.SimpleNamespace(
get_allowed_mcp_servers=AsyncMock(return_value=[]),
get_mcp_servers_from_ids=MagicMock(return_value=[]),
get_mcp_server_by_name=MagicMock(return_value=None),
)
monkeypatch.setattr(
"litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager",
fake_manager,
)
await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager(
user_api_key_auth=types.SimpleNamespace(api_key="k", user_id="u"),
mcp_tools_with_litellm_proxy=[
{"type": "mcp", "server_url": "litellm_proxy/mcp/deepwiki"}
],
request_tags=["team-a"],
)
assert mock_get_tools.await_args.kwargs["request_tags"] == ["team-a"]
@pytest.mark.asyncio
async def test_execute_tool_calls_propagates_request_tags_to_function_setup(monkeypatch):
_setup_proxy_logging(monkeypatch)
_setup_mcp_call_environment(monkeypatch)
captured = {}
def fake_function_setup(*_args, **kwargs):
captured.update(kwargs)
return None, None
handler_module = importlib.import_module(
"litellm.responses.mcp.litellm_proxy_mcp_handler"
)
monkeypatch.setattr(handler_module, "function_setup", fake_function_setup)
tool_name = "deepwiki-read_wiki_structure"
await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
tool_server_map={tool_name: "deepwiki"},
tool_calls=[{"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}],
user_api_key_auth=None,
request_tags=["team-a", "prod"],
)
assert captured["metadata"]["tags"] == ["team-a", "prod"]

41
tests/test_litellm/test_claude_sonnet_5_config.py
View file

@ -76,12 +76,23 @@ def test_sonnet_5_pricing_and_capabilities():
assert info["max_output_tokens"] == 128000
assert info["max_tokens"] == 128000
# Standard Sonnet pricing: $3 / $15 per MTok, with the 1.25x cache-write
# and 0.1x cache-read multipliers.
assert info["input_cost_per_token"] == 3e-06
assert info["output_cost_per_token"] == 1.5e-05
assert info["cache_creation_input_token_cost"] == 3.75e-06
assert info["cache_read_input_token_cost"] == 3e-07
# Introductory Sonnet 5 pricing through 2026-08-31: $2 / $10 per MTok,
# with the 1.25x cache-write and 0.1x cache-read multipliers. On
# 2026-09-01 flip these five fields back to the sticker rate, here and
# in both cost-map JSON files (all ten claude-sonnet-5 entries):
# input_cost_per_token: 3e-06
# output_cost_per_token: 1.5e-05
# cache_creation_input_token_cost: 3.75e-06
# cache_creation_input_token_cost_above_1hr: 6e-06
# cache_read_input_token_cost: 3e-07
# Regional Bedrock profiles (us./eu./au./jp.) stay at 1.1x those values:
# 3.3e-06 / 1.65e-05 / 4.125e-06 / 6.6e-06 / 3.3e-07 (see
# test_sonnet_5_bedrock_regional_pricing below).
assert info["input_cost_per_token"] == 2e-06
assert info["output_cost_per_token"] == 1e-05
assert info["cache_creation_input_token_cost"] == 2.5e-06
assert info["cache_creation_input_token_cost_above_1hr"] == 4e-06
assert info["cache_read_input_token_cost"] == 2e-07
# gen-5 adaptive-thinking profile: effort-driven, no sampling params, no
# assistant prefill.
@ -102,16 +113,18 @@ def test_sonnet_5_bedrock_regional_pricing():
model_data = _load_root_cost_map()
base_pricing = {
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 2e-06,
"output_cost_per_token": 1e-05,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
}
regional_pricing = {
"input_cost_per_token": 3.3e-06,
"output_cost_per_token": 1.65e-05,
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 2.2e-06,
"output_cost_per_token": 1.1e-05,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_1hr": 4.4e-06,
"cache_read_input_token_cost": 2.2e-07,
}
expected = {

8
tests/test_litellm/test_router_retry_policy_update.py
View file

@ -273,7 +273,13 @@ async def test_config_update_persists_and_reads_back_retry_policy(monkeypatch):
assert isinstance(router.retry_policy, RetryPolicy)
assert router.retry_policy.RateLimitErrorRetries == 7
read_back = (await proxy_server.get_config())["router_settings"]["retry_policy"]
read_back = (
await proxy_server.get_config(
user_api_key_dict=UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234"
)
)
)["router_settings"]["retry_policy"]
assert read_back.BadRequestErrorRetries == 5
assert read_back.TimeoutErrorRetries == 3
assert read_back.RateLimitErrorRetries == 7

1
tests/test_litellm/test_utils.py
View file

@ -858,6 +858,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"type": "string",
"enum": ["low", "medium", "high", "max", "xhigh"],
},
"bedrock_converse_supports_strict_tools": {"type": "boolean"},
"tpm": {"type": "number"},
"provider_specific_entry": {"type": "object"},
"supported_endpoints": {

10
ui/litellm-dashboard/e2e_tests/migration.serverRootPath.config.ts
View file

@ -6,14 +6,6 @@ import { defineConfig, devices } from "@playwright/test";
* running. globalSetup logs in at `${SERVER_ROOT_PATH}/ui/login` so the admin
* storage state is valid under the prefix.
*/
if (!process.env.SERVER_ROOT_PATH) {
throw new Error(
"migration.serverRootPath.config.ts requires SERVER_ROOT_PATH to be set (e.g. SERVER_ROOT_PATH=/litellm). " +
"Without it this config silently re-runs the default mount and never exercises the prefix. " +
"For the root-less run use the default playwright.config.ts (npm run e2e:migration).",
);
}
export default defineConfig({
testDir: "./tests/migration",
testMatch: ["migratedPages.spec.ts"],
@ -34,5 +26,5 @@ export default defineConfig({
projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }],
timeout: 3 * 60 * 1000,
expect: { timeout: 10 * 1000 },
globalSetup: require.resolve("./globalSetup"),
globalSetup: require.resolve("./migration.serverRootPath.globalSetup"),
});

12
ui/litellm-dashboard/e2e_tests/migration.serverRootPath.globalSetup.ts Normal file
View file

@ -0,0 +1,12 @@
import globalSetup from "./globalSetup";
export default async function migrationServerRootPathGlobalSetup() {
if (!process.env.SERVER_ROOT_PATH) {
throw new Error(
"migration.serverRootPath.config.ts requires SERVER_ROOT_PATH to be set (e.g. SERVER_ROOT_PATH=/litellm). " +
"Without it this config silently re-runs the default mount and never exercises the prefix. " +
"For the root-less run use the default playwright.config.ts (npm run e2e:migration).",
);
}
await globalSetup();
}

16
ui/litellm-dashboard/eslint-suppressions.json
View file

@ -376,7 +376,7 @@
"count": 1
}
},
"src/components/DefaultUserSettings.tsx": {
"src/app/(dashboard)/users/_components/DefaultUserSettings.tsx": {
"no-restricted-imports": {
"count": 1
}
@ -1001,7 +1001,7 @@
"count": 1
}
},
"src/components/edit_user.tsx": {
"src/app/(dashboard)/users/_components/edit_user.tsx": {
"no-restricted-imports": {
"count": 1
}
@ -1957,12 +1957,12 @@
"count": 2
}
},
"src/components/user_edit_view.test.tsx": {
"src/app/(dashboard)/users/_components/user_edit_view.test.tsx": {
"react/display-name": {
"count": 1
}
},
"src/components/user_edit_view.tsx": {
"src/app/(dashboard)/users/_components/user_edit_view.tsx": {
"no-restricted-imports": {
"count": 1
},
@ -2047,7 +2047,7 @@
"count": 2
}
},
"src/components/view_users.tsx": {
"src/app/(dashboard)/users/_components/view_users.tsx": {
"no-restricted-imports": {
"count": 1
},
@ -2055,7 +2055,7 @@
"count": 1
}
},
"src/components/view_users/columns.tsx": {
"src/app/(dashboard)/users/_components/view_users/columns.tsx": {
"max-params": {
"count": 1
},
@ -2063,12 +2063,12 @@
"count": 1
}
},
"src/components/view_users/table.tsx": {
"src/app/(dashboard)/users/_components/view_users/table.tsx": {
"no-restricted-imports": {
"count": 1
}
},
"src/components/view_users/user_info_view.tsx": {
"src/app/(dashboard)/users/_components/view_users/user_info_view.tsx": {
"no-restricted-imports": {
"count": 1
},

295
ui/litellm-dashboard/package-lock.json generated
View file

@ -18,6 +18,7 @@
"@types/papaparse": "5.5.2",
"antd": "5.29.3",
"cva": "1.0.0-beta.4",
"date-fns": "3.6.0",
"dayjs": "1.11.19",
"jwt-decode": "4.0.0",
"lucide-react": "0.513.0",
@ -31,7 +32,6 @@
"react-json-view-lite": "2.5.0",
"react-markdown": "9.1.0",
"react-syntax-highlighter": "15.6.6",
"remark-gfm": "4.0.1",
"tailwind-merge": "3.4.0",
"uuid": "14.0.0"
},
@ -8601,16 +8601,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/markdown-table": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz",
"integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
@ -8620,34 +8610,6 @@
"node": ">= 0.4"
}
},
"node_modules/mdast-util-find-and-replace": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.2.tgz",
"integrity": "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==",
"license": "MIT",
"dependencies": {
"@types/mdast": "^4.0.0",
"escape-string-regexp": "^5.0.0",
"unist-util-is": "^6.0.0",
"unist-util-visit-parents": "^6.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-find-and-replace/node_modules/escape-string-regexp": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz",
"integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==",
"license": "MIT",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/mdast-util-from-markdown": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz",
@ -8672,107 +8634,6 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz",
"integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==",
"license": "MIT",
"dependencies": {
"mdast-util-from-markdown": "^2.0.0",
"mdast-util-gfm-autolink-literal": "^2.0.0",
"mdast-util-gfm-footnote": "^2.0.0",
"mdast-util-gfm-strikethrough": "^2.0.0",
"mdast-util-gfm-table": "^2.0.0",
"mdast-util-gfm-task-list-item": "^2.0.0",
"mdast-util-to-markdown": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm-autolink-literal": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz",
"integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==",
"license": "MIT",
"dependencies": {
"@types/mdast": "^4.0.0",
"ccount": "^2.0.0",
"devlop": "^1.0.0",
"mdast-util-find-and-replace": "^3.0.0",
"micromark-util-character": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm-footnote": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz",
"integrity": "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==",
"license": "MIT",
"dependencies": {
"@types/mdast": "^4.0.0",
"devlop": "^1.1.0",
"mdast-util-from-markdown": "^2.0.0",
"mdast-util-to-markdown": "^2.0.0",
"micromark-util-normalize-identifier": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm-strikethrough": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz",
"integrity": "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==",
"license": "MIT",
"dependencies": {
"@types/mdast": "^4.0.0",
"mdast-util-from-markdown": "^2.0.0",
"mdast-util-to-markdown": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm-table": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz",
"integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==",
"license": "MIT",
"dependencies": {
"@types/mdast": "^4.0.0",
"devlop": "^1.0.0",
"markdown-table": "^3.0.0",
"mdast-util-from-markdown": "^2.0.0",
"mdast-util-to-markdown": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm-task-list-item": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz",
"integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==",
"license": "MIT",
"dependencies": {
"@types/mdast": "^4.0.0",
"devlop": "^1.0.0",
"mdast-util-from-markdown": "^2.0.0",
"mdast-util-to-markdown": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-mdx-expression": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz",
@ -8987,127 +8848,6 @@
"micromark-util-types": "^2.0.0"
}
},
"node_modules/micromark-extension-gfm": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz",
"integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==",
"license": "MIT",
"dependencies": {
"micromark-extension-gfm-autolink-literal": "^2.0.0",
"micromark-extension-gfm-footnote": "^2.0.0",
"micromark-extension-gfm-strikethrough": "^2.0.0",
"micromark-extension-gfm-table": "^2.0.0",
"micromark-extension-gfm-tagfilter": "^2.0.0",
"micromark-extension-gfm-task-list-item": "^2.0.0",
"micromark-util-combine-extensions": "^2.0.0",
"micromark-util-types": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-autolink-literal": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz",
"integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==",
"license": "MIT",
"dependencies": {
"micromark-util-character": "^2.0.0",
"micromark-util-sanitize-uri": "^2.0.0",
"micromark-util-symbol": "^2.0.0",
"micromark-util-types": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-footnote": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz",
"integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==",
"license": "MIT",
"dependencies": {
"devlop": "^1.0.0",
"micromark-core-commonmark": "^2.0.0",
"micromark-factory-space": "^2.0.0",
"micromark-util-character": "^2.0.0",
"micromark-util-normalize-identifier": "^2.0.0",
"micromark-util-sanitize-uri": "^2.0.0",
"micromark-util-symbol": "^2.0.0",
"micromark-util-types": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-strikethrough": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz",
"integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==",
"license": "MIT",
"dependencies": {
"devlop": "^1.0.0",
"micromark-util-chunked": "^2.0.0",
"micromark-util-classify-character": "^2.0.0",
"micromark-util-resolve-all": "^2.0.0",
"micromark-util-symbol": "^2.0.0",
"micromark-util-types": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-table": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.1.tgz",
"integrity": "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==",
"license": "MIT",
"dependencies": {
"devlop": "^1.0.0",
"micromark-factory-space": "^2.0.0",
"micromark-util-character": "^2.0.0",
"micromark-util-symbol": "^2.0.0",
"micromark-util-types": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-tagfilter": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz",
"integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==",
"license": "MIT",
"dependencies": {
"micromark-util-types": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-task-list-item": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz",
"integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==",
"license": "MIT",
"dependencies": {
"devlop": "^1.0.0",
"micromark-factory-space": "^2.0.0",
"micromark-util-character": "^2.0.0",
"micromark-util-symbol": "^2.0.0",
"micromark-util-types": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-factory-destination": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz",
@ -11701,24 +11441,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/remark-gfm": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz",
"integrity": "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==",
"license": "MIT",
"dependencies": {
"@types/mdast": "^4.0.0",
"mdast-util-gfm": "^3.0.0",
"micromark-extension-gfm": "^3.0.0",
"remark-parse": "^11.0.0",
"remark-stringify": "^11.0.0",
"unified": "^11.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/remark-parse": {
"version": "11.0.0",
"resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz",
@ -11752,21 +11474,6 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/remark-stringify": {
"version": "11.0.0",
"resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz",
"integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==",
"license": "MIT",
"dependencies": {
"@types/mdast": "^4.0.0",
"mdast-util-to-markdown": "^2.0.0",
"unified": "^11.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/require-from-string": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",

2
ui/litellm-dashboard/package.json
View file

@ -34,6 +34,7 @@
"@types/papaparse": "5.5.2",
"antd": "5.29.3",
"cva": "1.0.0-beta.4",
"date-fns": "3.6.0",
"dayjs": "1.11.19",
"jwt-decode": "4.0.0",
"lucide-react": "0.513.0",
@ -47,7 +48,6 @@
"react-json-view-lite": "2.5.0",
"react-markdown": "9.1.0",
"react-syntax-highlighter": "15.6.6",
"remark-gfm": "4.0.1",
"tailwind-merge": "3.4.0",
"uuid": "14.0.0"
},

1
ui/litellm-dashboard/src/app/(dashboard)/memory/components/index.tsx
View file

@ -1 +0,0 @@
export { MemoryView, default } from "./MemoryView";

8
ui/litellm-dashboard/src/components/BulkEditUsers.test.tsx → ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx
View file

@ -1,11 +1,11 @@
import userEvent from "@testing-library/user-event";
import { describe, expect, it, vi, beforeEach } from "vitest";
import { renderWithProviders, screen, waitFor } from "../../tests/test-utils";
import { renderWithProviders, screen, waitFor } from "../../../../../tests/test-utils";
import BulkEditUserModal from "./BulkEditUsers";
import { userBulkUpdateUserCall, teamBulkMemberAddCall } from "./networking";
import NotificationsManager from "./molecules/notifications_manager";
import { userBulkUpdateUserCall, teamBulkMemberAddCall } from "@/components/networking";
import NotificationsManager from "@/components/molecules/notifications_manager";
vi.mock("./networking", () => ({
vi.mock("@/components/networking", () => ({
userBulkUpdateUserCall: vi.fn(),
teamBulkMemberAddCall: vi.fn(),
}));

4
ui/litellm-dashboard/src/components/BulkEditUsers.tsx → ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx
View file

@ -1,8 +1,8 @@
import React, { useState } from "react";
import { Modal, Typography, Divider, Table, Select, InputNumber, Card, Space, Checkbox } from "antd";
import { userBulkUpdateUserCall, teamBulkMemberAddCall, Member } from "./networking";
import { userBulkUpdateUserCall, teamBulkMemberAddCall, Member } from "@/components/networking";
import { UserEditView } from "./user_edit_view";
import NotificationsManager from "./molecules/notifications_manager";
import NotificationsManager from "@/components/molecules/notifications_manager";
import MessageManager from "@/components/molecules/message_manager";
const { Text, Title } = Typography;

8
ui/litellm-dashboard/src/components/DefaultUserSettings.test.tsx → ui/litellm-dashboard/src/app/(dashboard)/users/_components/DefaultUserSettings.test.tsx
View file

@ -1,15 +1,15 @@
import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { beforeEach, describe, expect, it, vi } from "vitest";
import DefaultUserSettings from "./DefaultUserSettings";
import * as networking from "./networking";
import * as networking from "@/components/networking";
vi.mock("./networking", () => ({
vi.mock("@/components/networking", () => ({
getInternalUserSettings: vi.fn(),
updateInternalUserSettings: vi.fn(),
modelAvailableCall: vi.fn(),
}));
vi.mock("./common_components/budget_duration_dropdown", () => ({
vi.mock("@/components/common_components/budget_duration_dropdown", () => ({
default: ({ value, onChange }: { value: string | null; onChange: (value: string | null) => void }) => (
<select data-testid="budget-duration" value={value || ""} onChange={(e) => onChange(e.target.value || null)}>
<option value="">Select duration</option>
@ -20,7 +20,7 @@ vi.mock("./common_components/budget_duration_dropdown", () => ({
getBudgetDurationLabel: (value: string) => value,
}));
vi.mock("./key_team_helpers/fetch_available_models_team_key", () => ({
vi.mock("@/components/key_team_helpers/fetch_available_models_team_key", () => ({
getModelDisplayName: (model: string) => model,
}));

10
ui/litellm-dashboard/src/components/DefaultUserSettings.tsx → ui/litellm-dashboard/src/app/(dashboard)/users/_components/DefaultUserSettings.tsx
View file

@ -2,11 +2,13 @@ import React, { useState, useEffect } from "react";
import { Card, Title, Text, Divider, TextInput } from "@tremor/react";
import { Button, Typography, Spin, Switch, Select, InputNumber } from "antd";
import { PlusOutlined, DeleteOutlined } from "@ant-design/icons";
import { getInternalUserSettings, updateInternalUserSettings, modelAvailableCall } from "./networking";
import BudgetDurationDropdown, { getBudgetDurationLabel } from "./common_components/budget_duration_dropdown";
import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key";
import { getInternalUserSettings, updateInternalUserSettings, modelAvailableCall } from "@/components/networking";
import BudgetDurationDropdown, {
getBudgetDurationLabel,
} from "@/components/common_components/budget_duration_dropdown";
import { getModelDisplayName } from "@/components/key_team_helpers/fetch_available_models_team_key";
import { formatNumberWithCommas } from "@/utils/dataUtils";
import NotificationManager from "./molecules/notifications_manager";
import NotificationManager from "@/components/molecules/notifications_manager";
interface DefaultUserSettingsProps {
accessToken: string | null;

4
ui/litellm-dashboard/src/components/edit_user.tsx → ui/litellm-dashboard/src/app/(dashboard)/users/_components/edit_user.tsx
View file

@ -3,8 +3,8 @@ import { TextInput, SelectItem } from "@tremor/react";
import { Button as Button2, Modal, Form, Select as Select2, InputNumber } from "antd";
import NumericalInput from "./shared/numerical_input";
import BudgetDurationDropdown from "./common_components/budget_duration_dropdown";
import NumericalInput from "@/components/shared/numerical_input";
import BudgetDurationDropdown from "@/components/common_components/budget_duration_dropdown";
interface EditUserModalProps {
visible: boolean;

1
ui/litellm-dashboard/src/app/(dashboard)/users/_components/index.tsx Normal file
View file

@ -0,0 +1 @@
export { default as ViewUserDashboard } from "./view_users";

6
ui/litellm-dashboard/src/components/user_edit_view.test.tsx → ui/litellm-dashboard/src/app/(dashboard)/users/_components/user_edit_view.test.tsx
View file

@ -1,14 +1,14 @@
import { cleanup, screen, waitFor } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { renderWithProviders } from "../../tests/test-utils";
import { renderWithProviders } from "../../../../../tests/test-utils";
import { UserEditView } from "./user_edit_view";
vi.mock("./key_team_helpers/fetch_available_models_team_key", () => ({
vi.mock("@/components/key_team_helpers/fetch_available_models_team_key", () => ({
getModelDisplayName: vi.fn((model: string) => model),
}));
vi.mock("../utils/roles", () => ({
vi.mock("@/utils/roles", () => ({
all_admin_roles: ["Admin", "Admin Viewer", "proxy_admin", "proxy_admin_viewer", "org_admin"],
}));

8
ui/litellm-dashboard/src/components/user_edit_view.tsx → ui/litellm-dashboard/src/app/(dashboard)/users/_components/user_edit_view.tsx
View file

@ -2,10 +2,10 @@ import { InfoCircleOutlined } from "@ant-design/icons";
import { Button, SelectItem, TextInput, Textarea } from "@tremor/react";
import { Checkbox, Form, Select, Tooltip } from "antd";
import React, { useState } from "react";
import { all_admin_roles } from "../utils/roles";
import BudgetDurationDropdown from "./common_components/budget_duration_dropdown";
import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key";
import NumericalInput from "./shared/numerical_input";
import { all_admin_roles } from "@/utils/roles";
import BudgetDurationDropdown from "@/components/common_components/budget_duration_dropdown";
import { getModelDisplayName } from "@/components/key_team_helpers/fetch_available_models_team_key";
import NumericalInput from "@/components/shared/numerical_input";
interface UserEditViewProps {
userData: any;

4
ui/litellm-dashboard/src/components/view_users.test.tsx → ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx
View file

@ -5,7 +5,7 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import ViewUserDashboard from "./view_users";
// Mock the networking module
vi.mock("./networking", () => ({
vi.mock("@/components/networking", () => ({
userListCall: vi.fn().mockResolvedValue({
users: [
{
@ -45,7 +45,7 @@ vi.mock("./networking", () => ({
}));
// Mock NotificationsManager
vi.mock("./molecules/notifications_manager", () => ({
vi.mock("@/components/molecules/notifications_manager", () => ({
default: {
success: vi.fn(),
fromBackend: vi.fn(),

14
ui/litellm-dashboard/src/components/view_users.tsx → ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.tsx
View file

@ -3,7 +3,7 @@ import React, { useEffect, useState } from "react";
import { Button } from "antd";
import BulkEditUserModal from "./BulkEditUsers";
import { CreateUserButton } from "./CreateUserButton";
import { CreateUserButton } from "@/components/CreateUserButton";
import EditUserModal from "./edit_user";
import {
getPossibleUserRoles,
@ -12,21 +12,21 @@ import {
userListCall,
UserListResponse,
userUpdateUserCall,
} from "./networking";
import OnboardingModal, { InvitationLink } from "./onboarding_link";
} from "@/components/networking";
import OnboardingModal, { InvitationLink } from "@/components/onboarding_link";
import { updateExistingKeys } from "@/utils/dataUtils";
import { isAdminRole, isProxyAdminRole } from "@/utils/roles";
import { useDebouncedState } from "@tanstack/react-pacer/debouncer";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { Typography } from "antd";
import DeleteResourceModal from "./common_components/DeleteResourceModal";
import NotificationsManager from "./molecules/notifications_manager";
import { modelAvailableCall, userDeleteCall } from "./networking";
import DeleteResourceModal from "@/components/common_components/DeleteResourceModal";
import NotificationsManager from "@/components/molecules/notifications_manager";
import { modelAvailableCall, userDeleteCall } from "@/components/networking";
import DefaultUserSettings from "./DefaultUserSettings";
import { columns } from "./view_users/columns";
import { UserDataTable } from "./view_users/table";
import { UserInfo } from "./view_users/types";
import { UserInfo } from "@/components/networking";
import { Skeleton } from "antd";
const { Text, Title } = Typography;

2
ui/litellm-dashboard/src/components/view_users/columns.tsx → ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/columns.tsx
View file

@ -1,7 +1,7 @@
import { ColumnDef } from "@tanstack/react-table";
import { Badge, Grid, Icon } from "@tremor/react";
import { Tooltip, Checkbox, Tag } from "antd";
import { UserInfo } from "./types";
import { UserInfo } from "@/components/networking";
import { PencilAltIcon, TrashIcon, InformationCircleIcon, RefreshIcon } from "@heroicons/react/outline";
import { CopyOutlined } from "@ant-design/icons";
import { formatNumberWithCommas, copyToClipboard } from "@/utils/dataUtils";

2
ui/litellm-dashboard/src/components/view_users/table.test.tsx → ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/table.test.tsx
View file

@ -2,7 +2,7 @@ import { act, fireEvent, render, screen } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";
import { columns } from "./columns";
import { UserDataTable } from "./table";
import { UserInfo } from "./types";
import { UserInfo } from "@/components/networking";
const defaultFilters = {
email: "",

Some files were not shown because too many files have changed in this diff Show more