scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

fix: get metadata info from both metadata and litellm_metadata fields (#14783)

Browse source 
* fix: get metadata info from both metadata and litellm_metadata fields

* fix: implemented requested changes (metadata field doesnt always exist)

* chore: moved get_metadata_variable_name_from_kwargs to callback_utils so it can be used on get_model_group_from_litellm_kwargs

---------

Co-authored-by: Luiz Rennó Costa <luiz.renno@ifood.com.br>
This commit is contained in:
Luiz Rennó Costa 2025-09-23 18:24:10 -03:00 • committed by GitHub
parent c87e6a849b
commit 620370e40c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 24 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
litellm/proxy/common_utils/callback_utils.py
View file

@ -1,4 +1,4 @@
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Literal, Optional
import litellm
from litellm import get_secret
@ -289,7 +289,7 @@ def initialize_callbacks_on_proxy( # noqa: PLR0915
def get_model_group_from_litellm_kwargs(kwargs: dict) -> Optional[str]:
_litellm_params = kwargs.get("litellm_params", None) or {}
_metadata = _litellm_params.get("metadata", None) or {}
_metadata = _litellm_params.get(get_metadata_variable_name_from_kwargs(kwargs)) or {}
_model_group = _metadata.get("model_group", None)
if _model_group is not None:
return _model_group
@ -365,3 +365,20 @@ def add_guardrail_to_applied_guardrails_header(
_metadata["applied_guardrails"].append(guardrail_name)
else:
_metadata["applied_guardrails"] = [guardrail_name]
def get_metadata_variable_name_from_kwargs(
kwargs: dict
) -> Literal["metadata", "litellm_metadata"]:
"""
Helper to return what the "metadata" field should be called in the request data
- New endpoints return `litellm_metadata`
- Old endpoints return `metadata`
Context:
- LiteLLM used `metadata` as an internal field for storing metadata
- OpenAI then started using this field for their metadata
- LiteLLM is now moving to using `litellm_metadata` for our metadata
"""
return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"

7
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -25,6 +25,7 @@ from litellm import DualCache
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@ -708,6 +709,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
from litellm.proxy.common_utils.callback_utils import (
get_model_group_from_litellm_kwargs,
get_metadata_variable_name_from_kwargs
)
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import ModelResponse, Usage
@ -723,7 +725,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
# Get metadata from kwargs
litellm_metadata = kwargs["litellm_params"]["metadata"]
litellm_metadata = kwargs["litellm_params"].get(get_metadata_variable_name_from_kwargs(kwargs), {})
if litellm_metadata is None:
return
user_api_key = litellm_metadata.get("user_api_key")
@ -736,7 +738,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# Get total tokens from response
total_tokens = 0
if isinstance(response_obj, ModelResponse):
# spot fix for /responses api
if (isinstance(response_obj, ModelResponse) or isinstance(response_obj, BaseLiteLLMOpenAIResponseObject)):
_usage = getattr(response_obj, "usage", None)
if _usage and isinstance(_usage, Usage):
if rate_limit_type == "output":