From 620370e40c1f23a6006adb7d144af2778d39bdbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luiz=20Renn=C3=B3=20Costa?= Date: Tue, 23 Sep 2025 18:24:10 -0300 Subject: [PATCH] fix: get metadata info from both metadata and litellm_metadata fields (#14783) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: get metadata info from both metadata and litellm_metadata fields * fix: implemented requested changes (metadata field doesnt always exist) * chore: moved get_metadata_variable_name_from_kwargs to callback_utils so it can be used on get_model_group_from_litellm_kwargs --------- Co-authored-by: Luiz Rennó Costa --- litellm/proxy/common_utils/callback_utils.py | 21 +++++++++++++++++-- .../hooks/parallel_request_limiter_v3.py | 7 +++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index d52592952bc..fb7ada8ab10 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional import litellm from litellm import get_secret @@ -289,7 +289,7 @@ def initialize_callbacks_on_proxy( # noqa: PLR0915 def get_model_group_from_litellm_kwargs(kwargs: dict) -> Optional[str]: _litellm_params = kwargs.get("litellm_params", None) or {} - _metadata = _litellm_params.get("metadata", None) or {} + _metadata = _litellm_params.get(get_metadata_variable_name_from_kwargs(kwargs)) or {} _model_group = _metadata.get("model_group", None) if _model_group is not None: return _model_group @@ -365,3 +365,20 @@ def add_guardrail_to_applied_guardrails_header( _metadata["applied_guardrails"].append(guardrail_name) else: _metadata["applied_guardrails"] = [guardrail_name] + + +def get_metadata_variable_name_from_kwargs( + kwargs: dict + ) -> Literal["metadata", "litellm_metadata"]: + """ + Helper to return what the "metadata" field should be called in the request data + + - New endpoints return `litellm_metadata` + - Old endpoints return `metadata` + + Context: + - LiteLLM used `metadata` as an internal field for storing metadata + - OpenAI then started using this field for their metadata + - LiteLLM is now moving to using `litellm_metadata` for our metadata + """ + return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 8ef0a662ffe..b6b82e4b376 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -25,6 +25,7 @@ from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -708,6 +709,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) from litellm.proxy.common_utils.callback_utils import ( get_model_group_from_litellm_kwargs, + get_metadata_variable_name_from_kwargs ) from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.utils import ModelResponse, Usage @@ -723,7 +725,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) # Get metadata from kwargs - litellm_metadata = kwargs["litellm_params"]["metadata"] + litellm_metadata = kwargs["litellm_params"].get(get_metadata_variable_name_from_kwargs(kwargs), {}) if litellm_metadata is None: return user_api_key = litellm_metadata.get("user_api_key") @@ -736,7 +738,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # Get total tokens from response total_tokens = 0 - if isinstance(response_obj, ModelResponse): + # spot fix for /responses api + if (isinstance(response_obj, ModelResponse) or isinstance(response_obj, BaseLiteLLMOpenAIResponseObject)): _usage = getattr(response_obj, "usage", None) if _usage and isinstance(_usage, Usage): if rate_limit_type == "output":