From e08b4767e925c5c55c62a7881440da4bd6641a7c Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 18 Dec 2025 17:25:43 -0300 Subject: [PATCH 1/5] fix: case-insensitive model cost map lookup Users were getting "does not support parameters: ['tools']" errors when using lowercase model names (e.g., "qwen/qwen3-next-80b-a3b-thinking") because the model cost map has mixed-case keys and the lookup was case-sensitive. Added _get_model_cost_key() helper that tries exact match first (O(1)), then falls back to case-insensitive search if not found. --- litellm/utils.py | 96 ++++++++++++++-------- tests/local_testing/test_get_model_info.py | 83 +++++++++++++++++++ 2 files changed, 144 insertions(+), 35 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index dfc0df5c9a2..8eb71817111 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4697,6 +4697,25 @@ def _strip_model_name(model: str, custom_llm_provider: Optional[str]) -> str: return model +def _get_model_cost_key(potential_key: str) -> Optional[str]: + """ + Get the actual key from model_cost, with case-insensitive fallback. + + Returns the key if found (exact match preferred, then case-insensitive), or None if not found. + """ + # Try exact match first (most common case, O(1)) + if potential_key in litellm.model_cost: + return potential_key + + # Fallback to case-insensitive match + potential_key_lower = potential_key.lower() + for key in litellm.model_cost: + if key.lower() == potential_key_lower: + return key + + return None + + def _get_model_info_from_model_cost(key: str) -> dict: return litellm.model_cost[key] @@ -4845,10 +4864,10 @@ def _is_potential_model_name_in_model_cost( potential_model_names: PotentialModelNamesAndCustomLLMProvider, ) -> bool: """ - Check if the potential model name is in the model cost. + Check if the potential model name is in the model cost (case-insensitive). """ return any( - potential_model_name in litellm.model_cost + _get_model_cost_key(potential_model_name) is not None for potential_model_name in potential_model_names.values() ) @@ -4926,44 +4945,51 @@ def _get_model_info_helper( # noqa: PLR0915 _model_info: Optional[Dict[str, Any]] = None key: Optional[str] = None - if combined_model_name in litellm.model_cost: - key = combined_model_name - _model_info = _get_model_info_from_model_cost(key=cast(str, key)) - if not _check_provider_match( - model_info=_model_info, custom_llm_provider=custom_llm_provider - ): - _model_info = None - if _model_info is None and model in litellm.model_cost: - key = model - _model_info = _get_model_info_from_model_cost(key=cast(str, key)) - if not _check_provider_match( - model_info=_model_info, custom_llm_provider=custom_llm_provider - ): - _model_info = None - if ( - _model_info is None - and combined_stripped_model_name in litellm.model_cost - ): - key = combined_stripped_model_name - _model_info = _get_model_info_from_model_cost(key=cast(str, key)) - if not _check_provider_match( - model_info=_model_info, custom_llm_provider=custom_llm_provider - ): - _model_info = None - if _model_info is None and stripped_model_name in litellm.model_cost: - key = stripped_model_name - _model_info = _get_model_info_from_model_cost(key=cast(str, key)) - if not _check_provider_match( - model_info=_model_info, custom_llm_provider=custom_llm_provider - ): - _model_info = None - if _model_info is None and split_model in litellm.model_cost: - key = split_model + # Use case-insensitive lookup for all model name checks + _matched_key = _get_model_cost_key(combined_model_name) + if _matched_key is not None: + key = _matched_key _model_info = _get_model_info_from_model_cost(key=cast(str, key)) if not _check_provider_match( model_info=_model_info, custom_llm_provider=custom_llm_provider ): _model_info = None + if _model_info is None: + _matched_key = _get_model_cost_key(model) + if _matched_key is not None: + key = _matched_key + _model_info = _get_model_info_from_model_cost(key=cast(str, key)) + if not _check_provider_match( + model_info=_model_info, custom_llm_provider=custom_llm_provider + ): + _model_info = None + if _model_info is None: + _matched_key = _get_model_cost_key(combined_stripped_model_name) + if _matched_key is not None: + key = _matched_key + _model_info = _get_model_info_from_model_cost(key=cast(str, key)) + if not _check_provider_match( + model_info=_model_info, custom_llm_provider=custom_llm_provider + ): + _model_info = None + if _model_info is None: + _matched_key = _get_model_cost_key(stripped_model_name) + if _matched_key is not None: + key = _matched_key + _model_info = _get_model_info_from_model_cost(key=cast(str, key)) + if not _check_provider_match( + model_info=_model_info, custom_llm_provider=custom_llm_provider + ): + _model_info = None + if _model_info is None: + _matched_key = _get_model_cost_key(split_model) + if _matched_key is not None: + key = _matched_key + _model_info = _get_model_info_from_model_cost(key=cast(str, key)) + if not _check_provider_match( + model_info=_model_info, custom_llm_provider=custom_llm_provider + ): + _model_info = None if _model_info is None or key is None: raise ValueError( diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 823f03185d5..b84fc22af01 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -372,3 +372,86 @@ def test_get_model_info_cost_calculator_bedrock_region_cris_stripped(model, prov print("info", info) assert info["key"] == "us.anthropic.claude-3-haiku-20240307-v1:0" assert info["litellm_provider"] == "bedrock" + + +def test_get_model_info_case_insensitive_lookup(monkeypatch): + """ + Test that model info lookup is case-insensitive. + + This ensures that users can use lowercase model names even when the model cost + map has mixed-case keys (e.g., "Qwen/Qwen3-Next-80B-A3B-Thinking"). + + Related Slack discussion: Users were getting "does not support parameters: ['tools']" + errors when using lowercase model names like "qwen/qwen3-next-80b-a3b-thinking" + because the lookup was case-sensitive. + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + + # Register a test model with mixed-case name + litellm.register_model( + { + "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": { + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0002, + "litellm_provider": "together_ai", + "supports_function_calling": True, + } + } + ) + + # Test 1: Exact case should work + info = litellm.get_model_info( + model="Qwen/Qwen3-Next-80B-A3B-Thinking", custom_llm_provider="together_ai" + ) + assert info is not None + assert info["supports_function_calling"] is True + + # Test 2: Lowercase should also work (case-insensitive lookup) + info_lower = litellm.get_model_info( + model="qwen/qwen3-next-80b-a3b-thinking", custom_llm_provider="together_ai" + ) + assert info_lower is not None + assert info_lower["supports_function_calling"] is True + + # Test 3: Mixed case should also work + info_mixed = litellm.get_model_info( + model="QWEN/qwen3-NEXT-80b-a3b-thinking", custom_llm_provider="together_ai" + ) + assert info_mixed is not None + assert info_mixed["supports_function_calling"] is True + + +def test_get_model_info_case_insensitive_supports_function_calling(monkeypatch): + """ + Test that supports_function_calling check works with case-insensitive model lookup. + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + + # Register a model with mixed-case name that supports function calling + litellm.register_model( + { + "test_provider/TestModel-ABC": { + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0002, + "litellm_provider": "test_provider", + "supports_function_calling": True, + } + } + ) + + # Test that supports_function_calling works with lowercase model name + from litellm.utils import supports_function_calling + + # Exact case + assert ( + supports_function_calling("TestModel-ABC", custom_llm_provider="test_provider") + is True + ) + + # Lowercase (should now work with case-insensitive lookup) + assert ( + supports_function_calling("testmodel-abc", custom_llm_provider="test_provider") + is True + ) From 1203e8416279a58c8404e9a1af87e4cfda48b4be Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Fri, 9 Jan 2026 23:43:44 +0530 Subject: [PATCH 2/5] fix(otel): update semantic conventions to 1.38 (gen_ai attributes) (#18793) --- litellm/integrations/opentelemetry.py | 174 +++++++++---- litellm/proxy/_types.py | 144 +++++++---- .../integrations/test_opentelemetry.py | 234 ++++++++++++++++++ 3 files changed, 452 insertions(+), 100 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 7e0cfab617b..a223925d59a 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -594,9 +594,9 @@ class OpenTelemetry(CustomLogger): def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]: """Extract dynamic headers from kwargs if available.""" - standard_callback_dynamic_params: Optional[ - StandardCallbackDynamicParams - ] = kwargs.get("standard_callback_dynamic_params") + standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = ( + kwargs.get("standard_callback_dynamic_params") + ) if not standard_callback_dynamic_params: return None @@ -797,7 +797,7 @@ class OpenTelemetry(CustomLogger): and self._token_usage_histogram ): in_attrs = {**common_attrs, "gen_ai.token.type": "input"} - out_attrs = {**common_attrs, "gen_ai.token.type": "completion"} + out_attrs = {**common_attrs, "gen_ai.token.type": "output"} self._token_usage_histogram.record( usage.get("prompt_tokens", 0), attributes=in_attrs ) @@ -1488,21 +1488,21 @@ class OpenTelemetry(CustomLogger): if usage: self.safe_set_attribute( span=span, - key=SpanAttributes.LLM_USAGE_TOTAL_TOKENS.value, + key=SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS.value, value=usage.get("total_tokens"), ) # The number of tokens used in the LLM response (completion). self.safe_set_attribute( span=span, - key=SpanAttributes.LLM_USAGE_COMPLETION_TOKENS.value, + key=SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS.value, value=usage.get("completion_tokens"), ) # The number of tokens used in the LLM prompt. self.safe_set_attribute( span=span, - key=SpanAttributes.LLM_USAGE_PROMPT_TOKENS.value, + key=SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS.value, value=usage.get("prompt_tokens"), ) @@ -1520,53 +1520,75 @@ class OpenTelemetry(CustomLogger): self.set_tools_attributes(span, tools) if kwargs.get("messages"): - for idx, prompt in enumerate(kwargs.get("messages")): - if prompt.get("role"): - self.safe_set_attribute( - span=span, - key=f"{SpanAttributes.LLM_PROMPTS.value}.{idx}.role", - value=prompt.get("role"), - ) + transformed_messages = ( + self._transform_messages_to_otel_semantic_conventions( + kwargs.get("messages") + ) + ) + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_INPUT_MESSAGES.value, + value=safe_dumps(transformed_messages), + ) - if prompt.get("content"): - if not isinstance(prompt.get("content"), str): - prompt["content"] = str(prompt.get("content")) - self.safe_set_attribute( - span=span, - key=f"{SpanAttributes.LLM_PROMPTS.value}.{idx}.content", - value=prompt.get("content"), - ) + if kwargs.get("system_instructions"): + transformed_system_instructions = ( + self._transform_messages_to_otel_semantic_conventions( + kwargs.get("system_instructions") + ) + ) + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value, + value=safe_dumps(transformed_system_instructions), + ) + + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_OPERATION_NAME.value, + value=( + "chat" + if standard_logging_payload.get("call_type") == "completion" + else standard_logging_payload.get("call_type") or "chat" + ), + ) + + if standard_logging_payload.get("request_id"): + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_REQUEST_ID.value, + value=standard_logging_payload.get("request_id"), + ) ############################################# ########## LLM Response Attributes ########## ############################################# if response_obj is not None: if response_obj.get("choices"): + transformed_choices = ( + self._transform_choices_to_otel_semantic_conventions( + response_obj.get("choices") + ) + ) + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_OUTPUT_MESSAGES.value, + value=safe_dumps(transformed_choices), + ) + + finish_reasons = [] + for idx, choice in enumerate(response_obj.get("choices")): + if choice.get("finish_reason"): + finish_reasons.append(choice.get("finish_reason")) + + if finish_reasons: + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS.value, + value=safe_dumps(finish_reasons), + ) + for idx, choice in enumerate(response_obj.get("choices")): if choice.get("finish_reason"): - self.safe_set_attribute( - span=span, - key=f"{SpanAttributes.LLM_COMPLETIONS.value}.{idx}.finish_reason", - value=choice.get("finish_reason"), - ) - if choice.get("message"): - if choice.get("message").get("role"): - self.safe_set_attribute( - span=span, - key=f"{SpanAttributes.LLM_COMPLETIONS.value}.{idx}.role", - value=choice.get("message").get("role"), - ) - if choice.get("message").get("content"): - if not isinstance( - choice.get("message").get("content"), str - ): - choice["message"]["content"] = str( - choice.get("message").get("content") - ) - self.safe_set_attribute( - span=span, - key=f"{SpanAttributes.LLM_COMPLETIONS.value}.{idx}.content", - value=choice.get("message").get("content"), - ) message = choice.get("message") tool_calls = message.get("tool_calls") @@ -1608,6 +1630,66 @@ class OpenTelemetry(CustomLogger): primitive_value = self._cast_as_primitive_value_type(value) span.set_attribute(key, primitive_value) + def _transform_messages_to_otel_semantic_conventions( + self, messages: Union[List[dict], str] + ) -> List[dict]: + """ + Transforms LiteLLM/OpenAI style messages into OTEL GenAI 1.38 compliant format. + OTEL expects a 'parts' array instead of a single 'content' string. + """ + if isinstance(messages, str): + # Handle system_instructions passed as a string + return [ + {"role": "system", "parts": [{"type": "text", "content": messages}]} + ] + + transformed = [] + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + parts = [] + + if isinstance(content, str): + parts.append({"type": "text", "content": content}) + elif isinstance(content, list): + # Handle multi-modal content if necessary + for part in content: + if isinstance(part, dict): + parts.append(part) + else: + parts.append({"type": "text", "content": str(part)}) + + transformed_msg = {"role": role, "parts": parts} + if "id" in msg: + transformed_msg["id"] = msg["id"] + if "tool_calls" in msg: + transformed_msg["tool_calls"] = msg["tool_calls"] + if "tool_call_id" in msg: + transformed_msg["tool_call_id"] = msg["tool_call_id"] + transformed.append(transformed_msg) + + return transformed + + def _transform_choices_to_otel_semantic_conventions( + self, choices: List[dict] + ) -> List[dict]: + """ + Transforms choices into OTEL GenAI 1.38 compliant format for output.messages. + """ + transformed = [] + for choice in choices: + message = choice.get("message") or {} + finish_reason = choice.get("finish_reason") + + transformed_msg = self._transform_messages_to_otel_semantic_conventions( + [message] + )[0] + if finish_reason: + transformed_msg["finish_reason"] = finish_reason + + transformed.append(transformed_msg) + return transformed + def set_raw_request_attributes(self, span: Span, kwargs, response_obj): try: kwargs.get("optional_params", {}) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index be1e4dbcdc9..fc69836cf75 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -830,9 +830,9 @@ class GenerateRequestBase(LiteLLMPydanticObjectBase): allowed_cache_controls: Optional[list] = [] config: Optional[dict] = {} permissions: Optional[dict] = {} - model_max_budget: Optional[ - dict - ] = {} # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {} + model_max_budget: Optional[dict] = ( + {} + ) # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {} model_config = ConfigDict(protected_namespaces=()) model_rpm_limit: Optional[dict] = None @@ -1364,12 +1364,12 @@ class NewCustomerRequest(BudgetNewRequest): blocked: bool = False # allow/disallow requests for this end-user budget_id: Optional[str] = None # give either a budget_id or max_budget spend: Optional[float] = None - allowed_model_region: Optional[ - AllowedModelRegion - ] = None # require all user requests to use models in this specific region - default_model: Optional[ - str - ] = None # if no equivalent model in allowed region - default all requests to this model + allowed_model_region: Optional[AllowedModelRegion] = ( + None # require all user requests to use models in this specific region + ) + default_model: Optional[str] = ( + None # if no equivalent model in allowed region - default all requests to this model + ) @model_validator(mode="before") @classmethod @@ -1391,12 +1391,12 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase): blocked: bool = False # allow/disallow requests for this end-user max_budget: Optional[float] = None budget_id: Optional[str] = None # give either a budget_id or max_budget - allowed_model_region: Optional[ - AllowedModelRegion - ] = None # require all user requests to use models in this specific region - default_model: Optional[ - str - ] = None # if no equivalent model in allowed region - default all requests to this model + allowed_model_region: Optional[AllowedModelRegion] = ( + None # require all user requests to use models in this specific region + ) + default_model: Optional[str] = ( + None # if no equivalent model in allowed region - default all requests to this model + ) class DeleteCustomerRequest(LiteLLMPydanticObjectBase): @@ -1482,15 +1482,15 @@ class NewTeamRequest(TeamBase): ] = None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm model_tpm_limit: Optional[Dict[str, int]] = None - team_member_budget: Optional[ - float - ] = None # allow user to set a budget for all team members - team_member_rpm_limit: Optional[ - int - ] = None # allow user to set RPM limit for all team members - team_member_tpm_limit: Optional[ - int - ] = None # allow user to set TPM limit for all team members + team_member_budget: Optional[float] = ( + None # allow user to set a budget for all team members + ) + team_member_rpm_limit: Optional[int] = ( + None # allow user to set RPM limit for all team members + ) + team_member_tpm_limit: Optional[int] = ( + None # allow user to set TPM limit for all team members + ) team_member_key_duration: Optional[str] = None # e.g. "1d", "1w", "1m" allowed_vector_store_indexes: Optional[List[AllowedVectorStoreIndexItem]] = None @@ -1578,9 +1578,9 @@ class BlockKeyRequest(LiteLLMPydanticObjectBase): class AddTeamCallback(LiteLLMPydanticObjectBase): callback_name: str - callback_type: Optional[ - Literal["success", "failure", "success_and_failure"] - ] = "success_and_failure" + callback_type: Optional[Literal["success", "failure", "success_and_failure"]] = ( + "success_and_failure" + ) callback_vars: Dict[str, str] @model_validator(mode="before") @@ -1893,9 +1893,9 @@ class ConfigList(LiteLLMPydanticObjectBase): stored_in_db: Optional[bool] field_default_value: Any premium_field: bool = False - nested_fields: Optional[ - List[FieldDetail] - ] = None # For nested dictionary or Pydantic fields + nested_fields: Optional[List[FieldDetail]] = ( + None # For nested dictionary or Pydantic fields + ) class UserHeaderMapping(LiteLLMPydanticObjectBase): @@ -2289,9 +2289,9 @@ class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase): budget_id: Optional[str] = None created_at: datetime updated_at: datetime - user: Optional[ - Any - ] = None # You might want to replace 'Any' with a more specific type if available + user: Optional[Any] = ( + None # You might want to replace 'Any' with a more specific type if available + ) litellm_budget_table: Optional[LiteLLM_BudgetTable] = None model_config = ConfigDict(protected_namespaces=()) @@ -2826,6 +2826,18 @@ class SpanAttributes(str, enum.Enum): LLM_RESPONSE_MODEL = "gen_ai.response.model" LLM_USAGE_COMPLETION_TOKENS = "gen_ai.usage.completion_tokens" LLM_USAGE_PROMPT_TOKENS = "gen_ai.usage.prompt_tokens" + + # OTEL 1.38 attributes + GEN_AI_INPUT_MESSAGES = "gen_ai.input.messages" + GEN_AI_OUTPUT_MESSAGES = "gen_ai.output.messages" + GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens" + GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens" + GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens" + GEN_AI_OPERATION_NAME = "gen_ai.operation.name" + GEN_AI_REQUEST_ID = "gen_ai.request.id" + GEN_AI_SYSTEM_INSTRUCTIONS = "gen_ai.system_instructions" + GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons" + LLM_TOKEN_TYPE = "gen_ai.token.type" # To be added # LLM_RESPONSE_FINISH_REASON = "gen_ai.response.finish_reasons" @@ -3251,9 +3263,9 @@ class TeamModelDeleteRequest(BaseModel): # Organization Member Requests class OrganizationMemberAddRequest(OrgMemberAddRequest): organization_id: str - max_budget_in_organization: Optional[ - float - ] = None # Users max budget within the organization + max_budget_in_organization: Optional[float] = ( + None # Users max budget within the organization + ) class OrganizationMemberDeleteRequest(MemberDeleteRequest): @@ -3468,9 +3480,9 @@ class ProviderBudgetResponse(LiteLLMPydanticObjectBase): Maps provider names to their budget configs. """ - providers: Dict[ - str, ProviderBudgetResponseObject - ] = {} # Dictionary mapping provider names to their budget configurations + providers: Dict[str, ProviderBudgetResponseObject] = ( + {} + ) # Dictionary mapping provider names to their budget configurations class ProxyStateVariables(TypedDict): @@ -3613,9 +3625,9 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase): enforce_rbac: bool = False roles_jwt_field: Optional[str] = None # v2 on role mappings role_mappings: Optional[List[RoleMapping]] = None - object_id_jwt_field: Optional[ - str - ] = None # can be either user / team, inferred from the role mapping + object_id_jwt_field: Optional[str] = ( + None # can be either user / team, inferred from the role mapping + ) scope_mappings: Optional[List[ScopeMapping]] = None enforce_scope_based_access: bool = False enforce_team_based_model_access: bool = False @@ -3866,20 +3878,44 @@ class CostEstimateResponse(LiteLLMPydanticObjectBase): num_requests_per_day: Optional[int] = None num_requests_per_month: Optional[int] = None # Per-request costs - cost_per_request: float = Field(description="Total cost per request (includes margin)") - input_cost_per_request: float = Field(description="Input token cost per request (before margin)") - output_cost_per_request: float = Field(description="Output token cost per request (before margin)") - margin_cost_per_request: float = Field(default=0.0, description="Margin/fee added per request") + cost_per_request: float = Field( + description="Total cost per request (includes margin)" + ) + input_cost_per_request: float = Field( + description="Input token cost per request (before margin)" + ) + output_cost_per_request: float = Field( + description="Output token cost per request (before margin)" + ) + margin_cost_per_request: float = Field( + default=0.0, description="Margin/fee added per request" + ) # Daily costs (if num_requests_per_day provided) - daily_cost: Optional[float] = Field(default=None, description="Total daily cost (includes margin)") - daily_input_cost: Optional[float] = Field(default=None, description="Daily input token cost") - daily_output_cost: Optional[float] = Field(default=None, description="Daily output token cost") - daily_margin_cost: Optional[float] = Field(default=None, description="Daily margin/fee") + daily_cost: Optional[float] = Field( + default=None, description="Total daily cost (includes margin)" + ) + daily_input_cost: Optional[float] = Field( + default=None, description="Daily input token cost" + ) + daily_output_cost: Optional[float] = Field( + default=None, description="Daily output token cost" + ) + daily_margin_cost: Optional[float] = Field( + default=None, description="Daily margin/fee" + ) # Monthly costs (if num_requests_per_month provided) - monthly_cost: Optional[float] = Field(default=None, description="Total monthly cost (includes margin)") - monthly_input_cost: Optional[float] = Field(default=None, description="Monthly input token cost") - monthly_output_cost: Optional[float] = Field(default=None, description="Monthly output token cost") - monthly_margin_cost: Optional[float] = Field(default=None, description="Monthly margin/fee") + monthly_cost: Optional[float] = Field( + default=None, description="Total monthly cost (includes margin)" + ) + monthly_input_cost: Optional[float] = Field( + default=None, description="Monthly input token cost" + ) + monthly_output_cost: Optional[float] = Field( + default=None, description="Monthly output token cost" + ) + monthly_margin_cost: Optional[float] = Field( + default=None, description="Monthly margin/fee" + ) # Pricing info input_cost_per_token: Optional[float] = None output_cost_per_token: Optional[float] = None diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 55b65fbb92a..95fa6ed8f60 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -1869,3 +1869,237 @@ class TestOpenTelemetryExternalSpan(unittest.TestCase): parent_span_finished.attributes, "Parent span should have model attribute from LiteLLM even on failure" ) + + +class TestOpenTelemetrySemanticConventions138(unittest.TestCase): + """ + Test suite for OpenTelemetry 1.38 Semantic Conventions compliance. + + These tests verify that LiteLLM emits span attributes following the + OpenTelemetry GenAI semantic conventions v1.38, including: + - gen_ai.input.messages (JSON string with parts array) + - gen_ai.output.messages (JSON string with parts array) + - gen_ai.usage.input_tokens / output_tokens (new naming) + - gen_ai.response.finish_reasons (JSON array) + + See: https://github.com/BerriAI/litellm/issues/17794 + """ + + def test_input_messages_uses_parts_structure(self): + """ + Test that gen_ai.input.messages uses the OTEL 1.38 parts array structure. + + Expected format: + [{"role": "user", "parts": [{"type": "text", "content": "Hello"}]}] + """ + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello world"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + }, + } + + response_obj = { + "id": "test-response-id", + "model": "gpt-4", + "choices": [ + { + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Hi there!"}, + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + # Find the call that set gen_ai.input.messages + input_messages_calls = [ + call for call in mock_span.set_attribute.call_args_list + if call[0][0] == "gen_ai.input.messages" + ] + self.assertEqual(len(input_messages_calls), 1, "Should have exactly one gen_ai.input.messages attribute") + + input_messages_value = input_messages_calls[0][0][1] + parsed = json.loads(input_messages_value) + + # Verify structure + self.assertIsInstance(parsed, list) + self.assertEqual(len(parsed), 1) + self.assertEqual(parsed[0]["role"], "user") + self.assertIn("parts", parsed[0]) + self.assertEqual(parsed[0]["parts"][0]["type"], "text") + self.assertEqual(parsed[0]["parts"][0]["content"], "Hello world") + + def test_output_messages_uses_parts_structure(self): + """ + Test that gen_ai.output.messages uses the OTEL 1.38 parts array structure. + + Expected format: + [{"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}], "finish_reason": "stop"}] + """ + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + }, + } + + response_obj = { + "id": "test-response-id", + "model": "gpt-4", + "choices": [ + { + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Hello back!"}, + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + # Find the call that set gen_ai.output.messages + output_messages_calls = [ + call for call in mock_span.set_attribute.call_args_list + if call[0][0] == "gen_ai.output.messages" + ] + self.assertEqual(len(output_messages_calls), 1, "Should have exactly one gen_ai.output.messages attribute") + + output_messages_value = output_messages_calls[0][0][1] + parsed = json.loads(output_messages_value) + + # Verify structure + self.assertIsInstance(parsed, list) + self.assertEqual(len(parsed), 1) + self.assertEqual(parsed[0]["role"], "assistant") + self.assertIn("parts", parsed[0]) + self.assertEqual(parsed[0]["parts"][0]["type"], "text") + self.assertEqual(parsed[0]["parts"][0]["content"], "Hello back!") + self.assertEqual(parsed[0]["finish_reason"], "stop") + + def test_usage_tokens_use_new_naming_convention(self): + """ + Test that token usage uses the OTEL 1.38 naming convention: + - gen_ai.usage.input_tokens (not prompt_tokens) + - gen_ai.usage.output_tokens (not completion_tokens) + """ + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + }, + } + + response_obj = { + "id": "test-response-id", + "model": "gpt-4", + "choices": [], + "usage": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + # Verify new naming convention is used + mock_span.set_attribute.assert_any_call("gen_ai.usage.input_tokens", 100) + mock_span.set_attribute.assert_any_call("gen_ai.usage.output_tokens", 50) + mock_span.set_attribute.assert_any_call("gen_ai.usage.total_tokens", 150) + + def test_finish_reasons_is_json_array(self): + """ + Test that gen_ai.response.finish_reasons is a proper JSON array. + + Expected: '["stop"]' (not "['stop']") + """ + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + }, + } + + response_obj = { + "id": "test-response-id", + "model": "gpt-4", + "choices": [ + {"finish_reason": "stop", "message": {"role": "assistant", "content": "Hi"}}, + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + # Find the call that set gen_ai.response.finish_reasons + finish_reasons_calls = [ + call for call in mock_span.set_attribute.call_args_list + if call[0][0] == "gen_ai.response.finish_reasons" + ] + self.assertEqual(len(finish_reasons_calls), 1, "Should have exactly one gen_ai.response.finish_reasons attribute") + + finish_reasons_value = finish_reasons_calls[0][0][1] + + # Verify it's valid JSON (not Python repr) + parsed = json.loads(finish_reasons_value) + self.assertEqual(parsed, ["stop"]) + + def test_operation_name_is_chat_for_completion(self): + """ + Test that gen_ai.operation.name is 'chat' for completion calls. + """ + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + }, + } + + response_obj = { + "id": "test-response-id", + "model": "gpt-4", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + mock_span.set_attribute.assert_any_call("gen_ai.operation.name", "chat") From 1aa63842495eca7fc65e146a8140e0e2c5fc60ea Mon Sep 17 00:00:00 2001 From: Chesars Date: Mon, 12 Jan 2026 09:46:06 -0300 Subject: [PATCH 3/5] fix: resolve mypy type error in _is_potential_model_name_in_model_cost Add explicit str() conversion when calling _get_model_cost_key to satisfy mypy type checking. TypedDict.values() returns object type, not str. --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 8eb71817111..a75130a31eb 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4867,7 +4867,7 @@ def _is_potential_model_name_in_model_cost( Check if the potential model name is in the model cost (case-insensitive). """ return any( - _get_model_cost_key(potential_model_name) is not None + _get_model_cost_key(str(potential_model_name)) is not None for potential_model_name in potential_model_names.values() ) From 7c61933bc52dabf9e0e7be293551d856b71b894b Mon Sep 17 00:00:00 2001 From: Dima-Mediator Date: Mon, 12 Jan 2026 22:54:24 -0500 Subject: [PATCH 4/5] Fix image tokens spend logging for /images/generations --- litellm/responses/utils.py | 4 +- .../test_standard_logging_payload.py | 47 ++++++++++ .../responses/test_responses_utils.py | 87 +++++++++++++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 49b8e123fe2..469eeb3da06 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -472,7 +472,9 @@ class ResponseAPILoggingUtils: output_tokens_details = getattr(response_api_usage, "output_tokens_details", None) if output_tokens_details: completion_tokens_details = CompletionTokensDetailsWrapper( - reasoning_tokens=getattr(output_tokens_details, "reasoning_tokens", None) + reasoning_tokens=getattr(output_tokens_details, "reasoning_tokens", None), + image_tokens=getattr(output_tokens_details, "image_tokens", None), + text_tokens=getattr(output_tokens_details, "text_tokens", None), ) chat_usage = Usage( diff --git a/tests/logging_callback_tests/test_standard_logging_payload.py b/tests/logging_callback_tests/test_standard_logging_payload.py index 1c01ab91938..4ead642c462 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload.py +++ b/tests/logging_callback_tests/test_standard_logging_payload.py @@ -80,6 +80,53 @@ def test_get_usage(response_obj, expected_values): assert usage.total_tokens == expected_values[2] +def test_get_usage_from_image_generation_response(): + """ + Test that image generation usage (with input_tokens/output_tokens format) + is correctly transformed to standard usage format with image_tokens preserved. + + Note: get_usage_from_response_obj() is used by multiple endpoints including + /images/generations and Response API (/responses), both of which use the + input_tokens/output_tokens format instead of prompt_tokens/completion_tokens. + + This tests the fix for the bug where image_tokens were being lost during + spend log creation for /images/generations endpoint. + """ + # Simulating image generation response usage from OpenAI + response_obj = { + "usage": { + "input_tokens": 13, + "output_tokens": 372, + "total_tokens": 385, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 13, + }, + "output_tokens_details": { + "image_tokens": 272, + "text_tokens": 100, + }, + } + } + + usage = StandardLoggingPayloadSetup.get_usage_from_response_obj(response_obj) + + # Check basic token counts are mapped correctly + assert usage.prompt_tokens == 13 + assert usage.completion_tokens == 372 + assert usage.total_tokens == 385 + + # Check that prompt_tokens_details contains image_tokens and text_tokens + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.image_tokens == 0 + assert usage.prompt_tokens_details.text_tokens == 13 + + # Check that completion_tokens_details contains image_tokens and text_tokens + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.image_tokens == 272 + assert usage.completion_tokens_details.text_tokens == 100 + + def test_get_additional_headers(): additional_headers = { "x-ratelimit-limit-requests": "2000", diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 09628cd4a76..3eb9c63e1be 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -222,3 +222,90 @@ class TestResponseAPILoggingUtils: assert result.prompt_tokens == 15 assert result.completion_tokens == 25 assert result.total_tokens == 40 # 15 + 25 + + def test_transform_response_api_usage_with_image_tokens(self): + """Test transformation handles image_tokens from image generation responses. + + Note: _transform_response_api_usage_to_chat_usage() is used by multiple + endpoints including /images/generations and Response API (/responses), + both of which use the input_tokens/output_tokens format. + + This tests the fix for image generation responses that include image_tokens + in both input_tokens_details and output_tokens_details. + + Example from gpt-image-1.5: + - input: text prompt with 13 tokens + - output: generated image with 272 image tokens + 100 text tokens + """ + # Setup - simulating image generation usage from OpenAI + usage = { + "input_tokens": 13, + "output_tokens": 372, + "total_tokens": 385, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 13, + }, + "output_tokens_details": { + "image_tokens": 272, + "text_tokens": 100, + }, + } + + # Execute + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + + # Assert - verify basic token counts + assert isinstance(result, Usage) + assert result.prompt_tokens == 13 + assert result.completion_tokens == 372 + assert result.total_tokens == 385 + + # Assert - verify prompt_tokens_details includes image_tokens and text_tokens + assert result.prompt_tokens_details is not None + assert result.prompt_tokens_details.image_tokens == 0 + assert result.prompt_tokens_details.text_tokens == 13 + + # Assert - verify completion_tokens_details includes image_tokens and text_tokens + assert result.completion_tokens_details is not None + assert result.completion_tokens_details.image_tokens == 272 + assert result.completion_tokens_details.text_tokens == 100 + + def test_transform_response_api_usage_mixed_details(self): + """Test transformation handles mixed token details (cached + image + audio).""" + # Setup - hypothetical usage with mixed token types + usage = { + "input_tokens": 100, + "output_tokens": 200, + "total_tokens": 300, + "input_tokens_details": { + "cached_tokens": 50, + "audio_tokens": 10, + "image_tokens": 20, + "text_tokens": 20, + }, + "output_tokens_details": { + "reasoning_tokens": 30, + "image_tokens": 100, + "text_tokens": 70, + }, + } + + # Execute + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + + # Assert - all token detail types should be preserved + assert result.prompt_tokens_details is not None + assert result.prompt_tokens_details.cached_tokens == 50 + assert result.prompt_tokens_details.audio_tokens == 10 + assert result.prompt_tokens_details.image_tokens == 20 + assert result.prompt_tokens_details.text_tokens == 20 + + assert result.completion_tokens_details is not None + assert result.completion_tokens_details.reasoning_tokens == 30 + assert result.completion_tokens_details.image_tokens == 100 + assert result.completion_tokens_details.text_tokens == 70 From de6330b6b6b892cccc45a288f1fdfce0761a3976 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 13 Jan 2026 16:59:17 +0530 Subject: [PATCH 5/5] Fix test_async_otel_callback[False] --- .../test_otel_logging.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/logging_callback_tests/test_otel_logging.py b/tests/logging_callback_tests/test_otel_logging.py index 3350c6c2dbd..b53be44fe0f 100644 --- a/tests/logging_callback_tests/test_otel_logging.py +++ b/tests/logging_callback_tests/test_otel_logging.py @@ -27,6 +27,9 @@ exporter = InMemorySpanExporter() @pytest.mark.parametrize("streaming", [True, False]) async def test_async_otel_callback(streaming): litellm.set_verbose = True + + # Clear exporter at the start to ensure clean state + exporter.clear() litellm.callbacks = [OpenTelemetry(config=OpenTelemetryConfig(exporter=exporter))] @@ -83,9 +86,9 @@ def validate_litellm_request(span): "llm.user", "gen_ai.response.id", "gen_ai.response.model", - "llm.usage.total_tokens", - "gen_ai.usage.completion_tokens", - "gen_ai.usage.prompt_tokens", + "gen_ai.usage.total_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.input_tokens", ] # get the str of all the span attributes @@ -149,6 +152,10 @@ async def test_awesome_otel_with_message_logging_off(streaming, global_redact): tests when OpenTelemetry(message_logging=False) is set """ litellm.set_verbose = True + + # Clear exporter at the start to ensure clean state + exporter.clear() + litellm.callbacks = [OpenTelemetry(config=OpenTelemetryConfig(exporter=exporter))] if global_redact is False: otel_logger = OpenTelemetry( @@ -201,9 +208,9 @@ def validate_redacted_message_span_attributes(span): "llm.request.type", "gen_ai.response.id", "gen_ai.response.model", - "llm.usage.total_tokens", - "gen_ai.usage.completion_tokens", - "gen_ai.usage.prompt_tokens", + "gen_ai.usage.total_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.input_tokens", ] _all_attributes = set( @@ -230,6 +237,8 @@ def validate_redacted_message_span_attributes(span): attr.startswith("metadata.") or attr.startswith("hidden_params") or attr.startswith("gen_ai.cost.") + or attr.startswith("gen_ai.operation.") + or attr.startswith("gen_ai.request.") ), f"Non-metadata attribute found: {attr}" pass