From a112e78795cfb00bf32fc7288d52b1bc185d9080 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Tue, 3 Mar 2026 13:52:51 -0800 Subject: [PATCH] feat(proxy): add key_alias, key_hash, requested_model tags to DD APM spans --- ...odel_prices_and_context_window_backup.json | 252 ++++++++++++++++++ litellm/proxy/common_request_processing.py | 37 +++ .../proxy/test_common_request_processing.py | 59 +++- 3 files changed, 347 insertions(+), 1 deletion(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5de764c5cec..412f99791a0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9779,6 +9779,122 @@ } ] }, + "dashscope/qwen3-max-2026-01-23": { + "litellm_provider": "dashscope", + "max_input_tokens": 258048, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "tiered_pricing": [ + { + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 6e-06, + "range": [ + 0, + 32000.0 + ] + }, + { + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 1.2e-05, + "range": [ + 32000.0, + 128000.0 + ] + }, + { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "range": [ + 128000.0, + 252000.0 + ] + } + ] + }, + "dashscope/qwen3-next-80b-a3b-instruct": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "dashscope", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supports_function_calling": true, + "supports_tool_choice": true + }, + "dashscope/qwen3-next-80b-a3b-thinking": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "dashscope", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "dashscope/qwen3-vl-235b-a22b-instruct": { + "input_cost_per_token": 4e-07, + "litellm_provider": "dashscope", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "dashscope/qwen3-vl-235b-a22b-thinking": { + "input_cost_per_token": 4e-07, + "litellm_provider": "dashscope", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "dashscope/qwen3-vl-32b-instruct": { + "input_cost_per_token": 1.6e-07, + "litellm_provider": "dashscope", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6.4e-07, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "dashscope/qwen3-vl-32b-thinking": { + "input_cost_per_token": 1.6e-07, + "litellm_provider": "dashscope", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 2.87e-06, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "dashscope/qwen3-vl-plus": { "litellm_provider": "dashscope", "max_input_tokens": 260096, @@ -25806,6 +25922,30 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, + "openrouter/anthropic/claude-sonnet-4.6": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_token": 3e-06, + "input_cost_per_token_above_200k_tokens": 6e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_above_200k_tokens": 2.25e-05, + "source": "https://openrouter.ai/anthropic/claude-sonnet-4.6", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "openrouter/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, @@ -26156,6 +26296,39 @@ "supports_web_search": true, "tpm": 800000 }, + "openrouter/google/gemini-3.1-pro-preview": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "source": "https://openrouter.ai/google/gemini-3.1-pro-preview", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/gryphe/mythomax-l2-13b": { "input_cost_per_token": 1.875e-06, "litellm_provider": "openrouter", @@ -26533,6 +26706,29 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/openai/gpt-5.1-codex-max": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://openrouter.ai/openai/gpt-5.1-codex-max", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/openai/gpt-5.2": { "input_cost_per_image": 0, "cache_read_input_token_cost": 1.75e-07, @@ -26687,6 +26883,19 @@ "supports_tool_choice": true, "supports_function_calling": true }, + "openrouter/qwen/qwen3-coder-plus": { + "input_cost_per_token": 1e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 997952, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 5e-06, + "source": "https://openrouter.ai/qwen/qwen3-coder-plus", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/qwen/qwen3-235b-a22b-2507": { "input_cost_per_token": 7.1e-08, "litellm_provider": "openrouter", @@ -26822,6 +27031,19 @@ "supports_vision": true, "supports_prompt_caching": false }, + "openrouter/z-ai/glm-5": { + "input_cost_per_token": 8e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 202752, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.56e-06, + "source": "https://openrouter.ai/z-ai/glm-5", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/minimax/minimax-m2.1": { "input_cost_per_token": 2.7e-07, "output_cost_per_token": 1.2e-06, @@ -34315,6 +34537,36 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "zai/glm-5": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3.2e-06, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://docs.z.ai/guides/overview/pricing" + }, + "zai/glm-5-code": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 5e-06, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://docs.z.ai/guides/overview/pricing" + }, "zai/glm-4.7": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 1.1e-07, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 1269f58213a..ac8839f3d18 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -265,6 +265,39 @@ def _add_dd_apm_tags_for_litellm_call_id(litellm_call_id: Optional[str]) -> None ) +def _add_dd_apm_tags_for_request( + user_api_key_dict: UserAPIKeyAuth, + requested_model: Optional[str], +) -> None: + """ + Attach key and model tags to the active Datadog APM span. + + Tags set (all best-effort, skipped when value is absent): + - ``litellm.key_alias`` — human-readable alias for the API key + - ``litellm.key_hash`` — hashed API key (safe to log; never the raw secret) + - ``litellm.requested_model``— model name as sent by the client + + Use cases: + - Trace all requests from a specific user/key: filter by ``litellm.key_alias`` or + ``litellm.key_hash``. + - Trace all requests for a specific model: filter by ``litellm.requested_model``. + + Note: key_alias / key_hash are not available for unauthenticated (e.g. 401) requests. + """ + try: + if user_api_key_dict.key_alias: + set_active_span_tag("litellm.key_alias", str(user_api_key_dict.key_alias)) + if user_api_key_dict.token: + set_active_span_tag("litellm.key_hash", str(user_api_key_dict.token)) + if requested_model: + set_active_span_tag("litellm.requested_model", str(requested_model)) + except Exception: + verbose_proxy_logger.debug( + "Failed to tag active ddtrace span with key/model tags", + exc_info=True, + ) + + def _override_openai_response_model( *, response_obj: Any, @@ -663,6 +696,10 @@ class ProxyBaseLLMRequestProcessing: "x-litellm-call-id", str(uuid.uuid4()) ) _add_dd_apm_tags_for_litellm_call_id(self.data.get("litellm_call_id")) + _add_dd_apm_tags_for_request( + user_api_key_dict=user_api_key_dict, + requested_model=self.data.get("model"), + ) ### AUTO STREAM USAGE TRACKING ### # If always_include_stream_usage is enabled and this is a streaming request diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 9b905d24fd1..323aa718dfd 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -1,7 +1,7 @@ import copy import datetime from typing import AsyncGenerator -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import Request, status @@ -14,6 +14,7 @@ from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, ProxyConfig, _add_dd_apm_tags_for_litellm_call_id, + _add_dd_apm_tags_for_request, _extract_error_from_sse_chunk, _get_cost_breakdown_from_logging_obj, _override_openai_response_model, @@ -1564,3 +1565,59 @@ class TestStreamingOverheadHeader: "It was missing — this is the streaming overhead header regression." ) assert custom_headers["x-litellm-overhead-duration-ms"] == "55.3" + + +class TestAddDdApmTagsForRequest: + """Tests for _add_dd_apm_tags_for_request - key/model DD span tagging.""" + + def _make_user_api_key_dict(self, key_alias=None, token=None): + from litellm.proxy._types import UserAPIKeyAuth + + d = UserAPIKeyAuth() + d.key_alias = key_alias + d.token = token + return d + + def test_tags_key_alias_and_model(self): + """key_alias and requested_model are set on the span when present.""" + user_key = self._make_user_api_key_dict(key_alias="my-prod-key", token="hashed123") + + with patch( + "litellm.proxy.common_request_processing.set_active_span_tag" + ) as mock_set_tag: + _add_dd_apm_tags_for_request( + user_api_key_dict=user_key, + requested_model="gpt-4o", + ) + + mock_set_tag.assert_any_call("litellm.key_alias", "my-prod-key") + mock_set_tag.assert_any_call("litellm.key_hash", "hashed123") + mock_set_tag.assert_any_call("litellm.requested_model", "gpt-4o") + + def test_no_tags_when_key_absent(self): + """No key tags are set when key_alias and token are None (e.g. 401 path).""" + user_key = self._make_user_api_key_dict(key_alias=None, token=None) + + with patch( + "litellm.proxy.common_request_processing.set_active_span_tag" + ) as mock_set_tag: + _add_dd_apm_tags_for_request( + user_api_key_dict=user_key, + requested_model=None, + ) + + mock_set_tag.assert_not_called() + + def test_only_model_tagged_when_no_key_info(self): + """requested_model is tagged even when there's no key info.""" + user_key = self._make_user_api_key_dict(key_alias=None, token=None) + + with patch( + "litellm.proxy.common_request_processing.set_active_span_tag" + ) as mock_set_tag: + _add_dd_apm_tags_for_request( + user_api_key_dict=user_key, + requested_model="claude-3-5-sonnet", + ) + + mock_set_tag.assert_called_once_with("litellm.requested_model", "claude-3-5-sonnet")