scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

Merge pull request #23882 from Harshit28j/litellm_silent_metrics_v2_patched

Browse source 
patch dev
This commit is contained in:
yuneng-jiang 2026-03-17 12:47:06 -07:00 • committed by GitHub
commit 27f78084dc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 661 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/integrations/prometheus.py
View file

@ -1657,6 +1657,9 @@ class PrometheusLogger(CustomLogger):
return
_metadata = data.get("metadata", {}) or {}
model_id = _metadata.get("model_info", {}).get("id") or data.get(
"model_info", {}
).get("id")
enum_values = UserAPIKeyLabelValues(
end_user=user_api_key_dict.end_user_id,
hashed_api_key=user_api_key_dict.api_key,
@ -1674,6 +1677,7 @@ class PrometheusLogger(CustomLogger):
),
client_ip=_metadata.get("requester_ip_address"),
user_agent=_metadata.get("user_agent"),
model_id=model_id,
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(

483
litellm/model_prices_and_context_window_backup.json
View file

@ -8535,6 +8535,227 @@
}
]
},
"dashscope/qwen3-max": {
"litellm_provider": "dashscope",
"max_input_tokens": 258048,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"tiered_pricing": [
{
"input_cost_per_token": 1.2e-06,
"output_cost_per_token": 6e-06,
"range": [
0,
32000.0
]
},
{
"input_cost_per_token": 2.4e-06,
"output_cost_per_token": 1.2e-05,
"range": [
32000.0,
128000.0
]
},
{
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"range": [
128000.0,
252000.0
]
}
]
},
"dashscope/qwen3-max-2026-01-23": {
"litellm_provider": "dashscope",
"max_input_tokens": 258048,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"tiered_pricing": [
{
"input_cost_per_token": 1.2e-06,
"output_cost_per_token": 6e-06,
"range": [
0,
32000.0
]
},
{
"input_cost_per_token": 2.4e-06,
"output_cost_per_token": 1.2e-05,
"range": [
32000.0,
128000.0
]
},
{
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"range": [
128000.0,
252000.0
]
}
]
},
"dashscope/qwen3-next-80b-a3b-instruct": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_tool_choice": true
},
"dashscope/qwen3-next-80b-a3b-thinking": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"dashscope/qwen3-vl-235b-a22b-instruct": {
"input_cost_per_token": 4e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 1.6e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-235b-a22b-thinking": {
"input_cost_per_token": 4e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-32b-instruct": {
"input_cost_per_token": 1.6e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6.4e-07,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-32b-thinking": {
"input_cost_per_token": 1.6e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 2.87e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-plus": {
"litellm_provider": "dashscope",
"max_input_tokens": 260096,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"tiered_pricing": [
{
"input_cost_per_token": 2e-07,
"output_cost_per_token": 1.6e-06,
"range": [
0,
32000.0
]
},
{
"input_cost_per_token": 3e-07,
"output_cost_per_token": 2.4e-06,
"range": [
32000.0,
128000.0
]
},
{
"input_cost_per_token": 6e-07,
"output_cost_per_token": 4.8e-06,
"range": [
128000.0,
256000.0
]
}
]
},
"dashscope/qwen3.5-plus": {
"litellm_provider": "dashscope",
"max_input_tokens": 991808,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"tiered_pricing": [
{
"input_cost_per_token": 4e-07,
"output_cost_per_token": 2.4e-06,
"range": [
0,
256000.0
]
},
{
"input_cost_per_token": 5e-07,
"output_cost_per_token": 3e-06,
"range": [
256000.0,
1000000.0
]
}
]
},
"dashscope/qwq-plus": {
"input_cost_per_token": 8e-07,
"litellm_provider": "dashscope",
@ -23151,14 +23372,59 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"openrouter/anthropic/claude-sonnet-4.6": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost": 3e-07,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"source": "https://openrouter.ai/anthropic/claude-sonnet-4.6",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"openrouter/anthropic/claude-opus-4.5": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 5e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"max_tokens": 32000,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"source": "https://openrouter.ai/anthropic/claude-sonnet-4.6",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"openrouter/anthropic/claude-opus-4.6": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 5e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"supports_assistant_prefill": true,
@ -23168,7 +23434,7 @@
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
"tool_use_system_prompt_tokens": 346
},
"openrouter/anthropic/claude-sonnet-4.5": {
"input_cost_per_image": 0.0048,
@ -23525,46 +23791,39 @@
"supports_web_search": true,
"tpm": 800000
},
"openrouter/google/gemini-pro-1.5": {
"input_cost_per_image": 0.00265,
"input_cost_per_token": 2.5e-06,
"openrouter/google/gemini-3.1-pro-preview": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_200k_tokens": 4e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_above_200k_tokens": 1.8e-05,
"source": "https://openrouter.ai/google/gemini-3.1-pro-preview",
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"openrouter/google/gemini-pro-vision": {
"input_cost_per_image": 0.0025,
"input_cost_per_token": 1.25e-07,
"litellm_provider": "openrouter",
"max_tokens": 45875,
"mode": "chat",
"output_cost_per_token": 3.75e-07,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"openrouter/google/palm-2-chat-bison": {
"input_cost_per_token": 5e-07,
"litellm_provider": "openrouter",
"max_tokens": 25804,
"mode": "chat",
"output_cost_per_token": 5e-07,
"supports_tool_choice": true
},
"openrouter/google/palm-2-codechat-bison": {
"input_cost_per_token": 5e-07,
"litellm_provider": "openrouter",
"max_tokens": 20070,
"mode": "chat",
"output_cost_per_token": 5e-07,
"supports_tool_choice": true
},
"openrouter/gryphe/mythomax-l2-13b": {
"input_cost_per_token": 1.875e-06,
"litellm_provider": "openrouter",
@ -24100,6 +24359,29 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-5.1-codex-max": {
"cache_read_input_token_cost": 1.25e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1e-05,
"source": "https://openrouter.ai/openai/gpt-5.1-codex-max",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"openrouter/openai/gpt-5.2": {
"input_cost_per_image": 0,
"cache_read_input_token_cost": 1.75e-07,
@ -24314,6 +24596,44 @@
"supports_tool_choice": true,
"supports_function_calling": true
},
"openrouter/qwen/qwen3-coder-plus": {
"input_cost_per_token": 1e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 997952,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 5e-06,
"source": "https://openrouter.ai/qwen/qwen3-coder-plus",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/qwen/qwen3-235b-a22b-2507": {
"input_cost_per_token": 7.1e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 1e-07,
"source": "https://openrouter.ai/qwen/qwen3-235b-a22b-2507",
"supports_function_calling": true,
"supports_tool_choice": true
},
"openrouter/qwen/qwen3-235b-a22b-thinking-2507": {
"input_cost_per_token": 1.1e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://openrouter.ai/qwen/qwen3-235b-a22b-thinking-2507",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/switchpoint/router": {
"input_cost_per_token": 8.5e-07,
"litellm_provider": "openrouter",
@ -24390,21 +24710,21 @@
"supports_tool_choice": true
},
"openrouter/xiaomi/mimo-v2-flash": {
"input_cost_per_token": 9e-08,
"output_cost_per_token": 2.9e-07,
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 0.0,
"litellm_provider": "openrouter",
"max_input_tokens": 262144,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": false,
"supports_prompt_caching": false
},
"input_cost_per_token": 9e-08,
"output_cost_per_token": 2.9e-07,
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 0.0,
"litellm_provider": "openrouter",
"max_input_tokens": 262144,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": false,
"supports_prompt_caching": false
},
"openrouter/z-ai/glm-4.7": {
"input_cost_per_token": 4e-07,
"output_cost_per_token": 1.5e-06,
@ -24438,6 +24758,19 @@
"supports_vision": true,
"supports_prompt_caching": false
},
"openrouter/z-ai/glm-5": {
"input_cost_per_token": 8e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 202752,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.56e-06,
"source": "https://openrouter.ai/z-ai/glm-5",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/minimax/minimax-m2.1": {
"input_cost_per_token": 2.7e-07,
"output_cost_per_token": 1.2e-06,
@ -31080,6 +31413,50 @@
"supports_vision": true,
"supports_web_search": true
},
"zai.glm-4.7": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"zai/glm-5": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1e-06,
"output_cost_per_token": 3.2e-06,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-5-code": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1.2e-06,
"output_cost_per_token": 5e-06,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-4.7": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 1.1e-07,

45
litellm/proxy/common_request_processing.py
View file

@ -26,7 +26,7 @@ from litellm.constants import (
DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE,
STREAM_SSE_DATA_PREFIX,
)
from litellm.litellm_core_utils.dd_tracing import set_active_span_tag, tracer
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
get_response_headers,
@ -38,6 +38,7 @@ from litellm.proxy.common_utils.callback_utils import (
get_logging_caching_headers,
get_remaining_tokens_and_requests_from_request_data,
)
from litellm.proxy.dd_span_tagger import DDSpanTagger
from litellm.proxy.route_llm_request import route_request
from litellm.proxy.utils import ProxyLogging
from litellm.router import Router
@ -237,26 +238,6 @@ async def create_response(
)
def _add_dd_apm_tags_for_litellm_call_id(litellm_call_id: Optional[str]) -> None:
"""
Attach LiteLLM call id to the active Datadog APM span.
This enables searching APM traces by LiteLLM call id returned in
`x-litellm-call-id`.
"""
if not litellm_call_id:
return
try:
set_active_span_tag("litellm.call_id", str(litellm_call_id))
except Exception:
# Tagging is best-effort and should never impact request processing.
verbose_proxy_logger.debug(
"Failed to tag active ddtrace span with litellm.call_id",
exc_info=True,
)
def _override_openai_response_model(
*,
response_obj: Any,
@ -619,7 +600,27 @@ class ProxyBaseLLMRequestProcessing:
self.data["litellm_call_id"] = request.headers.get(
"x-litellm-call-id", str(uuid.uuid4())
)
_add_dd_apm_tags_for_litellm_call_id(self.data.get("litellm_call_id"))
DDSpanTagger.tag_call_id(self.data.get("litellm_call_id"))
DDSpanTagger.tag_request(
user_api_key_dict=user_api_key_dict,
requested_model=self.data.get("model"),
)
### AUTO STREAM USAGE TRACKING ###
# If always_include_stream_usage is enabled and this is a streaming request
# automatically add stream_options={'include_usage': True} if not already set
if (
general_settings.get("always_include_stream_usage", False) is True
and self.data.get("stream", False) is True
):
# Only set if stream_options is not already provided by the client
if "stream_options" not in self.data:
self.data["stream_options"] = {"include_usage": True}
elif (
isinstance(self.data["stream_options"], dict)
and "include_usage" not in self.data["stream_options"]
):
self.data["stream_options"]["include_usage"] = True
### CALL HOOKS ### - modify/reject incoming data before calling the model
## LOGGING OBJECT ## - initialize logging object for logging success/failure events for call

60
litellm/proxy/dd_span_tagger.py Normal file
View file

@ -0,0 +1,60 @@
from typing import Optional
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.dd_tracing import set_active_span_tag
from litellm.proxy._types import UserAPIKeyAuth
class DDSpanTagger:
"""Best-effort helpers for tagging the active Datadog APM span with LiteLLM request metadata."""
@staticmethod
def tag_call_id(litellm_call_id: Optional[str]) -> None:
"""
Attach LiteLLM call id to the active Datadog APM span.
This enables searching APM traces by LiteLLM call id returned in
`x-litellm-call-id`.
"""
if not litellm_call_id:
return
try:
set_active_span_tag("litellm.call_id", str(litellm_call_id))
except Exception:
verbose_proxy_logger.debug(
"Failed to tag active ddtrace span with litellm.call_id",
exc_info=True,
)
@staticmethod
def tag_request(
user_api_key_dict: UserAPIKeyAuth,
requested_model: Optional[str],
) -> None:
"""
Attach key and model tags to the active Datadog APM span.
Tags set (all best-effort, skipped when value is absent):
- ``litellm.key_alias`` — human-readable alias for the API key
- ``litellm.key_hash`` — hashed API key (safe to log; never the raw secret)
- ``litellm.requested_model``— model name as sent by the client
Use cases:
- Trace all requests from a specific user/key: filter by ``litellm.key_alias`` or
``litellm.key_hash``.
- Trace all requests for a specific model: filter by ``litellm.requested_model``.
Note: key_alias / key_hash are not available for unauthenticated (e.g. 401) requests.
"""
try:
if user_api_key_dict.key_alias:
set_active_span_tag("litellm.key_alias", str(user_api_key_dict.key_alias))
if user_api_key_dict.token:
set_active_span_tag("litellm.key_hash", str(user_api_key_dict.token))
if requested_model:
set_active_span_tag("litellm.requested_model", str(requested_model))
except Exception:
verbose_proxy_logger.debug(
"Failed to tag active ddtrace span with key/model tags",
exc_info=True,
)

25
litellm/router.py
View file

@ -1331,14 +1331,37 @@ class Router:
def _get_silent_experiment_kwargs(self, **kwargs) -> dict:
"""
Prepare kwargs for a silent experiment by ensuring isolation from the primary call.
Guarantee metadata isolation: safe_deep_copy falls back to the original
reference when deepcopy fails (e.g. metadata contains UserAPIKeyAuth with
parent_otel_span — an OTel Span that is not deepcopy-able). Force a shallow
copy of the metadata dict so mutations (model_group, is_silent_experiment)
never corrupt the main call's metadata.
"""
# Copy kwargs to ensure isolation (use safe_deep_copy to handle non-serializable objects like OTEL spans)
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
silent_kwargs = safe_deep_copy(kwargs)
# safe_deep_copy may fall back to the original metadata reference when
# deepcopy fails (UserAPIKeyAuth.parent_otel_span is not deepcopy-able).
# Detect this via identity check and force a shallow copy so that setting
# model_group / is_silent_experiment on the silent dict doesn't corrupt
# the primary call's metadata.
original_metadata = kwargs.get("metadata")
if (
original_metadata is not None
and silent_kwargs.get("metadata") is original_metadata
):
silent_kwargs["metadata"] = dict(original_metadata)
if "metadata" not in silent_kwargs:
silent_kwargs["metadata"] = {}
# OTel spans are not safe to use across event loops. The silent
# experiment runs in a new event loop, so strip the span to prevent
# cross-loop tracing races or span corruption.
silent_kwargs["metadata"].pop("litellm_parent_otel_span", None)
silent_kwargs["metadata"]["is_silent_experiment"] = True
# Pop logging objects and call IDs to ensure a fresh logging context

66
tests/test_litellm/proxy/test_common_request_processing.py
View file

@ -1,7 +1,7 @@
import copy
import datetime
from typing import AsyncGenerator
from unittest.mock import AsyncMock, MagicMock
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import Request, status
@ -13,13 +13,13 @@ from litellm.integrations.opentelemetry import UserAPIKeyAuth
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
ProxyConfig,
_add_dd_apm_tags_for_litellm_call_id,
_extract_error_from_sse_chunk,
_get_cost_breakdown_from_logging_obj,
_override_openai_response_model,
_parse_event_data_for_error,
create_response,
)
from litellm.proxy.dd_span_tagger import DDSpanTagger
from litellm.proxy.utils import ProxyLogging
@ -82,13 +82,15 @@ class TestProxyBaseLLMRequestProcessing:
def test_add_dd_apm_tags_for_litellm_call_id_uses_dd_tracing_helper(self, monkeypatch):
mock_set_active_span_tag = MagicMock(return_value=True)
import litellm.proxy.dd_span_tagger
monkeypatch.setattr(
litellm.proxy.common_request_processing,
litellm.proxy.dd_span_tagger,
"set_active_span_tag",
mock_set_active_span_tag,
)
_add_dd_apm_tags_for_litellm_call_id("test-call-id")
DDSpanTagger.tag_call_id("test-call-id")
mock_set_active_span_tag.assert_called_once_with(
"litellm.call_id", "test-call-id"
@ -1387,3 +1389,59 @@ class TestStreamingOverheadHeader:
"It was missing — this is the streaming overhead header regression."
)
assert custom_headers["x-litellm-overhead-duration-ms"] == "55.3"
class TestDDSpanTaggerTagRequest:
"""Tests for DDSpanTagger.tag_request - key/model DD span tagging."""
def _make_user_api_key_dict(self, key_alias=None, token=None):
from litellm.proxy._types import UserAPIKeyAuth
d = UserAPIKeyAuth()
d.key_alias = key_alias
d.token = token
return d
def test_tags_key_alias_and_model(self):
"""key_alias and requested_model are set on the span when present."""
user_key = self._make_user_api_key_dict(key_alias="my-prod-key", token="hashed123")
with patch(
"litellm.proxy.dd_span_tagger.set_active_span_tag"
) as mock_set_tag:
DDSpanTagger.tag_request(
user_api_key_dict=user_key,
requested_model="gpt-4o",
)
mock_set_tag.assert_any_call("litellm.key_alias", "my-prod-key")
mock_set_tag.assert_any_call("litellm.key_hash", "hashed123")
mock_set_tag.assert_any_call("litellm.requested_model", "gpt-4o")
def test_no_tags_when_key_absent(self):
"""No key tags are set when key_alias and token are None (e.g. 401 path)."""
user_key = self._make_user_api_key_dict(key_alias=None, token=None)
with patch(
"litellm.proxy.dd_span_tagger.set_active_span_tag"
) as mock_set_tag:
DDSpanTagger.tag_request(
user_api_key_dict=user_key,
requested_model=None,
)
mock_set_tag.assert_not_called()
def test_only_model_tagged_when_no_key_info(self):
"""requested_model is tagged even when there's no key info."""
user_key = self._make_user_api_key_dict(key_alias=None, token=None)
with patch(
"litellm.proxy.dd_span_tagger.set_active_span_tag"
) as mock_set_tag:
DDSpanTagger.tag_request(
user_api_key_dict=user_key,
requested_model="claude-3-5-sonnet",
)
mock_set_tag.assert_called_once_with("litellm.requested_model", "claude-3-5-sonnet")

60
tests/test_litellm/test_router_silent_experiment.py
View file

@ -7,10 +7,36 @@ import litellm
from litellm.router import Router
class _NonCopyableSpan:
"""Mimics an OTel Span which raises on deepcopy, forcing safe_deep_copy
to fall back to the original reference."""
def __deepcopy__(self, memo):
raise TypeError("OTel spans cannot be deepcopied")
class _FakeUserAPIKeyAuth:
"""Mimics UserAPIKeyAuth which contains a parent_otel_span that is not
deepcopy-able. This is what actually causes safe_deep_copy to fail for
the metadata dict in production — safe_deep_copy handles the top-level
litellm_parent_otel_span specially (pops it before copying), but does
NOT handle user_api_key_auth.parent_otel_span inside it."""
def __init__(self, key_alias, parent_otel_span):
self.key_alias = key_alias
self.parent_otel_span = parent_otel_span
def __deepcopy__(self, memo):
raise TypeError("Contains OTel span that cannot be deepcopied")
def test_get_silent_experiment_kwargs():
"""
Test _get_silent_experiment_kwargs returns isolated kwargs with silent experiment metadata.
Direct call for router code coverage.
Uses a non-copyable user_api_key_auth (mimicking the real proxy scenario)
so that safe_deep_copy falls back to the original metadata reference —
exercising the identity-check fix path.
"""
model_list = [
{
@ -19,11 +45,41 @@ def test_get_silent_experiment_kwargs():
},
]
router = Router(model_list=model_list)
kwargs = {"metadata": {"foo": "bar"}, "litellm_call_id": "call-123"}
mock_span = _NonCopyableSpan()
mock_auth = _FakeUserAPIKeyAuth(
key_alias="HaneefKeyNonTeamProd",
parent_otel_span=mock_span,
)
kwargs = {
"metadata": {
"foo": "bar",
"litellm_parent_otel_span": mock_span,
"user_api_key_auth": mock_auth,
},
"litellm_call_id": "call-123",
"stream": True,
"proxy_server_request": {"body": {"model": "test"}},
}
result = router._get_silent_experiment_kwargs(**kwargs)
assert result["metadata"]["is_silent_experiment"] is True
assert result["metadata"]["foo"] == "bar"
assert "litellm_call_id" not in result
# stream must be forced to False so callbacks fire in background
assert result["stream"] is False
# proxy_server_request must be preserved for spend log metadata
assert "proxy_server_request" in result
# CRITICAL: metadata must be a DIFFERENT dict object than the original,
# so that setting model_group / is_silent_experiment on the silent dict
# doesn't corrupt the primary call's metadata.
assert result["metadata"] is not kwargs["metadata"]
# OTel span must be stripped from the silent copy — it's not safe to use
# across event loops (silent experiment runs in a new event loop).
assert "litellm_parent_otel_span" not in result["metadata"]
# Original metadata must NOT be mutated — must carry the real span,
# not safe_deep_copy's temporary "placeholder" string.
assert "is_silent_experiment" not in kwargs["metadata"]
assert kwargs["metadata"]["litellm_parent_otel_span"] is mock_span
assert kwargs["metadata"]["user_api_key_auth"] is mock_auth
def test_silent_experiment_completion_direct():