From 4032af34c955047937e1acd0e4456236243265a9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Wed, 18 Mar 2026 10:50:51 -0700 Subject: [PATCH] add DD Tracing --- ...odel_prices_and_context_window_backup.json | 483 ++---------------- litellm/proxy/auth/auth_checks.py | 106 ++-- litellm/proxy/auth/user_api_key_auth.py | 229 +++++---- 3 files changed, 228 insertions(+), 590 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 87f27b4847d..0f84bba941d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -8535,227 +8535,6 @@ } ] }, - "dashscope/qwen3-max": { - "litellm_provider": "dashscope", - "max_input_tokens": 258048, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "tiered_pricing": [ - { - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 6e-06, - "range": [ - 0, - 32000.0 - ] - }, - { - "input_cost_per_token": 2.4e-06, - "output_cost_per_token": 1.2e-05, - "range": [ - 32000.0, - 128000.0 - ] - }, - { - "input_cost_per_token": 3e-06, - "output_cost_per_token": 1.5e-05, - "range": [ - 128000.0, - 252000.0 - ] - } - ] - }, - "dashscope/qwen3-max-2026-01-23": { - "litellm_provider": "dashscope", - "max_input_tokens": 258048, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "tiered_pricing": [ - { - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 6e-06, - "range": [ - 0, - 32000.0 - ] - }, - { - "input_cost_per_token": 2.4e-06, - "output_cost_per_token": 1.2e-05, - "range": [ - 32000.0, - 128000.0 - ] - }, - { - "input_cost_per_token": 3e-06, - "output_cost_per_token": 1.5e-05, - "range": [ - 128000.0, - 252000.0 - ] - } - ] - }, - "dashscope/qwen3-next-80b-a3b-instruct": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "dashscope", - "max_input_tokens": 262144, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, - "dashscope/qwen3-next-80b-a3b-thinking": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "dashscope", - "max_input_tokens": 262144, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "dashscope/qwen3-vl-235b-a22b-instruct": { - "input_cost_per_token": 4e-07, - "litellm_provider": "dashscope", - "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1.6e-06, - "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "dashscope/qwen3-vl-235b-a22b-thinking": { - "input_cost_per_token": 4e-07, - "litellm_provider": "dashscope", - "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 4e-06, - "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "dashscope/qwen3-vl-32b-instruct": { - "input_cost_per_token": 1.6e-07, - "litellm_provider": "dashscope", - "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.4e-07, - "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "dashscope/qwen3-vl-32b-thinking": { - "input_cost_per_token": 1.6e-07, - "litellm_provider": "dashscope", - "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 2.87e-06, - "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "dashscope/qwen3-vl-plus": { - "litellm_provider": "dashscope", - "max_input_tokens": 260096, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "tiered_pricing": [ - { - "input_cost_per_token": 2e-07, - "output_cost_per_token": 1.6e-06, - "range": [ - 0, - 32000.0 - ] - }, - { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 2.4e-06, - "range": [ - 32000.0, - 128000.0 - ] - }, - { - "input_cost_per_token": 6e-07, - "output_cost_per_token": 4.8e-06, - "range": [ - 128000.0, - 256000.0 - ] - } - ] - }, - "dashscope/qwen3.5-plus": { - "litellm_provider": "dashscope", - "max_input_tokens": 991808, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "tiered_pricing": [ - { - "input_cost_per_token": 4e-07, - "output_cost_per_token": 2.4e-06, - "range": [ - 0, - 256000.0 - ] - }, - { - "input_cost_per_token": 5e-07, - "output_cost_per_token": 3e-06, - "range": [ - 256000.0, - 1000000.0 - ] - } - ] - }, "dashscope/qwq-plus": { "input_cost_per_token": 8e-07, "litellm_provider": "dashscope", @@ -23372,59 +23151,14 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-sonnet-4.6": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost": 3e-07, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "source": "https://openrouter.ai/anthropic/claude-sonnet-4.6", - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 - }, "openrouter/anthropic/claude-opus-4.5": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "openrouter", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "source": "https://openrouter.ai/anthropic/claude-sonnet-4.6", - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 - }, - "openrouter/anthropic/claude-opus-4.6": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 5e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, @@ -23434,7 +23168,7 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "tool_use_system_prompt_tokens": 159 }, "openrouter/anthropic/claude-sonnet-4.5": { "input_cost_per_image": 0.0048, @@ -23791,39 +23525,46 @@ "supports_web_search": true, "tpm": 800000 }, - "openrouter/google/gemini-3.1-pro-preview": { - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, - "input_cost_per_token": 2e-06, - "input_cost_per_token_above_200k_tokens": 4e-06, + "openrouter/google/gemini-pro-1.5": { + "input_cost_per_image": 0.00265, + "input_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.2e-05, - "output_cost_per_token_above_200k_tokens": 1.8e-05, - "source": "https://openrouter.ai/google/gemini-3.1-pro-preview", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, + "output_cost_per_token": 7.5e-06, "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true }, + "openrouter/google/gemini-pro-vision": { + "input_cost_per_image": 0.0025, + "input_cost_per_token": 1.25e-07, + "litellm_provider": "openrouter", + "max_tokens": 45875, + "mode": "chat", + "output_cost_per_token": 3.75e-07, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "openrouter/google/palm-2-chat-bison": { + "input_cost_per_token": 5e-07, + "litellm_provider": "openrouter", + "max_tokens": 25804, + "mode": "chat", + "output_cost_per_token": 5e-07, + "supports_tool_choice": true + }, + "openrouter/google/palm-2-codechat-bison": { + "input_cost_per_token": 5e-07, + "litellm_provider": "openrouter", + "max_tokens": 20070, + "mode": "chat", + "output_cost_per_token": 5e-07, + "supports_tool_choice": true + }, "openrouter/gryphe/mythomax-l2-13b": { "input_cost_per_token": 1.875e-06, "litellm_provider": "openrouter", @@ -24359,29 +24100,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "openrouter/openai/gpt-5.1-codex-max": { - "cache_read_input_token_cost": 1.25e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://openrouter.ai/openai/gpt-5.1-codex-max", - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-5.2": { "input_cost_per_image": 0, "cache_read_input_token_cost": 1.75e-07, @@ -24596,44 +24314,6 @@ "supports_tool_choice": true, "supports_function_calling": true }, - "openrouter/qwen/qwen3-coder-plus": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 997952, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://openrouter.ai/qwen/qwen3-coder-plus", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "openrouter/qwen/qwen3-235b-a22b-2507": { - "input_cost_per_token": 7.1e-08, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 1e-07, - "source": "https://openrouter.ai/qwen/qwen3-235b-a22b-2507", - "supports_function_calling": true, - "supports_tool_choice": true - }, - "openrouter/qwen/qwen3-235b-a22b-thinking-2507": { - "input_cost_per_token": 1.1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://openrouter.ai/qwen/qwen3-235b-a22b-thinking-2507", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "openrouter/switchpoint/router": { "input_cost_per_token": 8.5e-07, "litellm_provider": "openrouter", @@ -24710,21 +24390,21 @@ "supports_tool_choice": true }, "openrouter/xiaomi/mimo-v2-flash": { - "input_cost_per_token": 9e-08, - "output_cost_per_token": 2.9e-07, - "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 0.0, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_reasoning": true, - "supports_vision": false, - "supports_prompt_caching": false - }, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 2.9e-07, + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_vision": false, + "supports_prompt_caching": false + }, "openrouter/z-ai/glm-4.7": { "input_cost_per_token": 4e-07, "output_cost_per_token": 1.5e-06, @@ -24758,19 +24438,6 @@ "supports_vision": true, "supports_prompt_caching": false }, - "openrouter/z-ai/glm-5": { - "input_cost_per_token": 8e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 202752, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.56e-06, - "source": "https://openrouter.ai/z-ai/glm-5", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "openrouter/minimax/minimax-m2.1": { "input_cost_per_token": 2.7e-07, "output_cost_per_token": 1.2e-06, @@ -31413,50 +31080,6 @@ "supports_vision": true, "supports_web_search": true }, - "zai.glm-4.7": { - "input_cost_per_token": 6e-07, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.2e-06, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/" - }, - "zai/glm-5": { - "cache_creation_input_token_cost": 0, - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3.2e-06, - "litellm_provider": "zai", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "mode": "chat", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "source": "https://docs.z.ai/guides/overview/pricing" - }, - "zai/glm-5-code": { - "cache_creation_input_token_cost": 0, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 5e-06, - "litellm_provider": "zai", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "mode": "chat", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "source": "https://docs.z.ai/guides/overview/pricing" - }, "zai/glm-4.7": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 1.1e-07, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index e0b056d450f..76ed284c62c 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -28,6 +28,7 @@ from litellm.constants import ( DEFAULT_MAX_RECURSE_DEPTH, EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE, ) +from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.proxy._types import ( RBAC_ROLES, @@ -118,50 +119,55 @@ async def common_checks( # 2. If team can call model if _model and team_object: - if not can_team_access_model( - model=_model, - team_object=team_object, - llm_router=llm_router, - team_model_aliases=valid_token.team_model_aliases if valid_token else None, - ): - raise ProxyException( - message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", - type=ProxyErrorTypes.team_model_access_denied, - param="model", - code=status.HTTP_401_UNAUTHORIZED, - ) + with tracer.trace("litellm.proxy.auth.common_checks.can_team_access_model"): + if not can_team_access_model( + model=_model, + team_object=team_object, + llm_router=llm_router, + team_model_aliases=valid_token.team_model_aliases if valid_token else None, + ): + raise ProxyException( + message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", + type=ProxyErrorTypes.team_model_access_denied, + param="model", + code=status.HTTP_401_UNAUTHORIZED, + ) ## 2.1 If user can call model (if personal key) if _model and team_object is None and user_object is not None: - await can_user_call_model( - model=_model, - llm_router=llm_router, - user_object=user_object, - ) + with tracer.trace("litellm.proxy.auth.common_checks.can_user_call_model"): + await can_user_call_model( + model=_model, + llm_router=llm_router, + user_object=user_object, + ) # 3. If team is in budget - await _team_max_budget_check( - team_object=team_object, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks.team_max_budget_check"): + await _team_max_budget_check( + team_object=team_object, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # 3.1. If organization is in budget - await _organization_max_budget_check( - valid_token=valid_token, - team_object=team_object, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace("litellm.proxy.auth.common_checks.organization_max_budget_check"): + await _organization_max_budget_check( + valid_token=valid_token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) - await _tag_max_budget_check( - request_body=request_body, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks.tag_max_budget_check"): + await _tag_max_budget_check( + request_body=request_body, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # 4. If user is in budget ## 4.1 check personal budget, if personal key @@ -179,14 +185,15 @@ async def common_checks( ) ## 4.2 check team member budget, if team key - await _check_team_member_budget( - team_object=team_object, - user_object=user_object, - valid_token=valid_token, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace("litellm.proxy.auth.common_checks.check_team_member_budget"): + await _check_team_member_budget( + team_object=team_object, + user_object=user_object, + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) # 5. If end_user ('user' passed to /chat/completions, /embeddings endpoint) is in budget if end_user_object is not None and end_user_object.litellm_budget_table is not None: @@ -294,11 +301,12 @@ async def common_checks( ) # 11. [OPTIONAL] Vector store checks - is the object allowed to access the vector store - await vector_store_access_check( - request_body=request_body, - team_object=team_object, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks.vector_store_access_check"): + await vector_store_access_check( + request_body=request_body, + team_object=team_object, + valid_token=valid_token, + ) return True diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 7290528cb5a..ffc75a5429f 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -419,13 +419,12 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 custom_auth_api_key: bool = False try: - # get the request body - - await pre_db_read_auth_checks( - request_data=request_data, - request=request, - route=route, - ) + with tracer.trace("litellm.proxy.auth.pre_db_read_auth_checks"): + await pre_db_read_auth_checks( + request_data=request_data, + request=request, + route=route, + ) pass_through_endpoints: Optional[List[dict]] = general_settings.get( "pass_through_endpoints", None ) @@ -459,9 +458,10 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ### USER-DEFINED AUTH FUNCTION ### if enterprise_custom_auth is not None: - response = await enterprise_custom_auth( - request=request, api_key=api_key, user_custom_auth=user_custom_auth - ) + with tracer.trace("litellm.proxy.auth.enterprise_custom_auth"): + response = await enterprise_custom_auth( + request=request, api_key=api_key, user_custom_auth=user_custom_auth + ) if response is not None and isinstance(response, UserAPIKeyAuth): return UserAPIKeyAuth.model_validate(response) elif response is not None and isinstance(response, str): @@ -522,18 +522,19 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 is_jwt = jwt_handler.is_jwt(token=api_key) verbose_proxy_logger.debug("is_jwt: %s", is_jwt) if is_jwt: - result = await JWTAuthManager.auth_builder( - request_data=request_data, - general_settings=general_settings, - api_key=api_key, - jwt_handler=jwt_handler, - route=route, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - parent_otel_span=parent_otel_span, - request_headers=dict(request.headers), - ) + with tracer.trace("litellm.proxy.auth.jwt_auth_builder"): + result = await JWTAuthManager.auth_builder( + request_data=request_data, + general_settings=general_settings, + api_key=api_key, + jwt_handler=jwt_handler, + route=route, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + parent_otel_span=parent_otel_span, + request_headers=dict(request.headers), + ) is_proxy_admin = result["is_proxy_admin"] team_id = result["team_id"] @@ -605,19 +606,20 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 else None, ) # run through common checks - _ = await common_checks( - request=request, - request_body=request_data, - team_object=team_object, - user_object=user_object, - end_user_object=end_user_object, - general_settings=general_settings, - global_proxy_spend=global_proxy_spend, - route=route, - llm_router=llm_router, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks"): + _ = await common_checks( + request=request, + request_body=request_data, + team_object=team_object, + user_object=user_object, + end_user_object=end_user_object, + general_settings=general_settings, + global_proxy_spend=global_proxy_spend, + route=route, + llm_router=llm_router, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # return UserAPIKeyAuth object return cast(UserAPIKeyAuth, valid_token) @@ -675,15 +677,15 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 try: end_user_params["end_user_id"] = end_user_id - # get end-user object - _end_user_object = await get_end_user_object( - end_user_id=end_user_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - route=route, - ) + with tracer.trace("litellm.proxy.auth.get_end_user_object"): + _end_user_object = await get_end_user_object( + end_user_id=end_user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + route=route, + ) if _end_user_object is not None: end_user_params[ "allowed_model_region" @@ -725,14 +727,15 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 # note: never string compare api keys, this is vulenerable to a time attack. Use secrets.compare_digest instead ## Check CACHE try: - valid_token = await get_key_object( - hashed_token=hash_token(api_key), - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - check_cache_only=True, - ) + with tracer.trace("litellm.proxy.auth.get_key_object_check_cache"): + valid_token = await get_key_object( + hashed_token=hash_token(api_key), + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + check_cache_only=True, + ) except Exception: verbose_logger.debug("api key not found in cache.") valid_token = None @@ -992,14 +995,15 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 # Check 2. If user_id for this token is in budget - done in common_checks() if valid_token.user_id is not None: try: - user_obj = await get_user_object( - user_id=valid_token.user_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - user_id_upsert=False, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace("litellm.proxy.auth.get_user_object"): + user_obj = await get_user_object( + user_id=valid_token.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) except Exception as e: verbose_logger.debug( "litellm.proxy.auth.user_api_key_auth.py::user_api_key_auth() - Unable to get user from db/cache. Setting user_obj to None. Exception received - {}".format( @@ -1073,46 +1077,47 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 param=abbreviate_api_key(api_key=api_key), ) - # Check 4. Token Spend is under budget - if RouteChecks.is_llm_api_route(route=route): - await _virtual_key_max_budget_check( + with tracer.trace("litellm.proxy.auth.budget_checks"): + # Check 4. Token Spend is under budget + if RouteChecks.is_llm_api_route(route=route): + await _virtual_key_max_budget_check( + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + user_obj=user_obj, + ) + + # Check 5. Max Budget Alert Check + await _virtual_key_max_budget_alert_check( valid_token=valid_token, proxy_logging_obj=proxy_logging_obj, user_obj=user_obj, ) - # Check 5. Max Budget Alert Check - await _virtual_key_max_budget_alert_check( - valid_token=valid_token, - proxy_logging_obj=proxy_logging_obj, - user_obj=user_obj, - ) - - # Check 6. Soft Budget Check - await _virtual_key_soft_budget_check( - valid_token=valid_token, - proxy_logging_obj=proxy_logging_obj, - user_obj=user_obj, - ) - - # Check 5. Token Model Spend is under Model budget - max_budget_per_model = valid_token.model_max_budget - current_model = request_data.get("model", None) - - if ( - max_budget_per_model is not None - and isinstance(max_budget_per_model, dict) - and len(max_budget_per_model) > 0 - and prisma_client is not None - and current_model is not None - and valid_token.token is not None - ): - ## GET THE SPEND FOR THIS MODEL - await model_max_budget_limiter.is_key_within_model_budget( - user_api_key_dict=valid_token, - model=current_model, + # Check 6. Soft Budget Check + await _virtual_key_soft_budget_check( + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + user_obj=user_obj, ) + # Check 5. Token Model Spend is under Model budget + max_budget_per_model = valid_token.model_max_budget + current_model = request_data.get("model", None) + + if ( + max_budget_per_model is not None + and isinstance(max_budget_per_model, dict) + and len(max_budget_per_model) > 0 + and prisma_client is not None + and current_model is not None + and valid_token.token is not None + ): + ## GET THE SPEND FOR THIS MODEL + await model_max_budget_limiter.is_key_within_model_budget( + user_api_key_dict=valid_token, + model=current_model, + ) + # Check 6: Additional Common Checks across jwt + key auth if valid_token.team_id is not None: _team_obj: Optional[LiteLLM_TeamTable] = LiteLLM_TeamTable( @@ -1138,11 +1143,12 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 litellm.max_budget > 0 and prisma_client is not None ): # user set proxy max budget cache_key = "{}:spend".format(litellm_proxy_admin_name) - global_proxy_spend = await _fetch_global_spend_with_event_coordination( - cache_key=cache_key, - user_api_key_cache=user_api_key_cache, - prisma_client=prisma_client, - ) + with tracer.trace("litellm.proxy.auth.get_global_proxy_spend"): + global_proxy_spend = await _fetch_global_spend_with_event_coordination( + cache_key=cache_key, + user_api_key_cache=user_api_key_cache, + prisma_client=prisma_client, + ) if global_proxy_spend is not None: call_info = CallInfo( @@ -1159,19 +1165,20 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 user_info=call_info, ) ) - _ = await common_checks( - request=request, - request_body=request_data, - team_object=_team_obj, - user_object=user_obj, - end_user_object=_end_user_object, - general_settings=general_settings, - global_proxy_spend=global_proxy_spend, - route=route, - llm_router=llm_router, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks"): + _ = await common_checks( + request=request, + request_body=request_data, + team_object=_team_obj, + user_object=user_obj, + end_user_object=_end_user_object, + general_settings=general_settings, + global_proxy_spend=global_proxy_spend, + route=route, + llm_router=llm_router, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # Token passed all checks if valid_token is None: raise HTTPException(401, detail="Invalid API key")