diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 94902460c81..e6147409ed8 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -24,6 +24,10 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import ( from litellm.integrations.otel.model.semconv import Metric from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string +from litellm.litellm_core_utils.service_tier_utils import ( + get_requested_service_tier, + get_served_service_tier, +) from litellm.secret_managers.main import get_secret_bool, str_to_bool from litellm.types.services import ServiceLoggerPayload from litellm.types.utils import ( @@ -74,6 +78,11 @@ PREPROCESSING_DURATION_MS_ATTRIBUTE = "litellm.preprocessing.duration_ms" TEAM_METADATA_ATTRIBUTE = "litellm.team.metadata" MODEL_GROUP_ATTRIBUTE = "litellm.model_group" PROVIDER_MODEL_ATTRIBUTE = "litellm.provider.model" +# semconv names the service tier attributes under the openai namespace, but every +# provider that reports a tier (OpenAI, Anthropic, Bedrock, Groq, Vertex) uses the +# same request param and response field, so both keys carry all of them. +REQUEST_SERVICE_TIER_ATTRIBUTE = "gen_ai.openai.request.service_tier" +RESPONSE_SERVICE_TIER_ATTRIBUTE = "gen_ai.openai.response.service_tier" # Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" @@ -1411,6 +1420,23 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if provider_model: self.safe_set_attribute(span=span, key=PROVIDER_MODEL_ATTRIBUTE, value=provider_model) + def _set_service_tier_attributes( + self, + span: Span, + standard_logging_payload: StandardLoggingPayload, + ) -> None: + """Stamp the tier the caller asked for and the tier the provider reports it + served, so tier usage is segmentable in traces. Both are optional: a caller + may not name a tier, and streaming responses carry no served tier. + """ + requested_tier = get_requested_service_tier(standard_logging_payload) + if requested_tier is not None: + self.safe_set_attribute(span=span, key=REQUEST_SERVICE_TIER_ATTRIBUTE, value=requested_tier) + + served_tier = get_served_service_tier(standard_logging_payload) + if served_tier is not None: + self.safe_set_attribute(span=span, key=RESPONSE_SERVICE_TIER_ATTRIBUTE, value=served_tier) + @staticmethod def _team_metadata_json(value: Any, allowed_keys: list[str]) -> str | None: """JSON-serialize only the allowlisted sub-keys of a team's metadata. @@ -2310,6 +2336,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): value=response_obj.get("model"), ) + self._set_service_tier_attributes(span=span, standard_logging_payload=standard_logging_payload) + usage = response_obj and response_obj.get("usage") if usage: self.safe_set_attribute( diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 78f4e213f50..b92e4f7e723 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -34,6 +34,9 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, ) +from litellm.litellm_core_utils.service_tier_utils import ( + get_service_tier_from_standard_logging_payload, +) from litellm.proxy._types import ( LiteLLM_DeletedVerificationToken, LiteLLM_TeamTable, @@ -98,16 +101,6 @@ class _ExcludedLabelMetric: return self._metric.labels(*kept_values) if kept_values else self._metric -# Tiers a caller may name in a request, across the providers that accept the -# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and -# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which -# maps "default" to "standard". Used to bound the caller-controlled fallback in -# ``get_service_tier_from_standard_logging_payload``. -KNOWN_REQUEST_SERVICE_TIERS = frozenset( - {"auto", "batch", "default", "flex", "priority", "scale", "standard", "standard_only"} -) - - def _get_budget_metrics_per_request_timeout() -> float: raw = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT") if raw is None: @@ -4181,44 +4174,6 @@ def get_custom_labels_from_metadata(metadata: dict) -> dict[str, str]: return result -def get_service_tier_from_standard_logging_payload( - standard_logging_payload: StandardLoggingPayload, -) -> str | None: - """ - Resolve the service tier a request ran on, for the ``service_tier`` label. - - The tier the provider actually served wins over the tier the caller asked for, - so latency and spend stay segmentable when the request said ``auto`` and the - provider picked the concrete tier. Providers report the served tier either at - the top level of the response (OpenAI, Bedrock, Groq) or on the usage object - (Anthropic). - - Streaming responses carry no served tier, so the requested tier is the - fallback. That value is caller-controlled and survives param mapping even - where the provider then ignores it (Bedrock and Groq accept the request and - drop an unrecognized tier), so it is only labelled when it names a known - tier; otherwise one caller could mint a Prometheus series per string. Values - the provider itself reports are not caller-controlled and stay unrestricted, - so a tier a provider adds later is still labelled correctly. - """ - response = standard_logging_payload.get("response") - usage_object = standard_logging_payload.get("metadata", {}).get("usage_object") - - served_candidates: tuple[object, ...] = ( - response.get("service_tier") if isinstance(response, dict) else None, - usage_object.get("service_tier") if isinstance(usage_object, dict) else None, - ) - served_tier = next((tier for tier in served_candidates if isinstance(tier, str) and tier), None) - if served_tier is not None: - return served_tier - - model_parameters = standard_logging_payload.get("model_parameters") - requested_tier = model_parameters.get("service_tier") if isinstance(model_parameters, dict) else None - if isinstance(requested_tier, str) and requested_tier in KNOWN_REQUEST_SERVICE_TIERS: - return requested_tier - return None - - def _get_combined_custom_metadata_from_standard_logging_payload( standard_logging_payload: dict | None, ) -> dict[str, Any]: diff --git a/litellm/litellm_core_utils/service_tier_utils.py b/litellm/litellm_core_utils/service_tier_utils.py new file mode 100644 index 00000000000..9e3f5da42c4 --- /dev/null +++ b/litellm/litellm_core_utils/service_tier_utils.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from litellm.types.utils import ServiceTier, StandardLoggingPayload + +# Tiers a caller may name in a request, across the providers that accept the +# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and +# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which +# maps "default" to "standard". Bounds the caller-controlled requested tier +# wherever it is recorded. Derived from ``ServiceTier`` so a tier added there for +# cost calculation cannot go missing here. +KNOWN_REQUEST_SERVICE_TIERS = frozenset( + tuple(tier.value for tier in ServiceTier) + ("batch", "default", "scale", "standard", "standard_only") +) + + +def get_served_service_tier(standard_logging_payload: StandardLoggingPayload) -> str | None: + """ + The tier the provider reports it actually served the request on. + + Providers report it either at the top level of the response (OpenAI, Bedrock, + Groq) or on the usage object (Anthropic). Streaming responses carry no served + tier. + """ + response = standard_logging_payload.get("response") + usage_object = standard_logging_payload.get("metadata", {}).get("usage_object") + + served_candidates: tuple[object, ...] = ( + response.get("service_tier") if isinstance(response, dict) else None, + usage_object.get("service_tier") if isinstance(usage_object, dict) else None, + ) + return next((tier for tier in served_candidates if isinstance(tier, str) and tier), None) + + +def get_requested_service_tier(standard_logging_payload: StandardLoggingPayload) -> str | None: + """ + The tier the caller asked for, as sent to the provider. + + The value is caller-controlled and survives param mapping even where the + provider then ignores it (Bedrock and Groq accept the request and drop an + unrecognized tier), so it is only reported when it names a known tier. + """ + model_parameters = standard_logging_payload.get("model_parameters") + requested_tier = model_parameters.get("service_tier") if isinstance(model_parameters, dict) else None + if isinstance(requested_tier, str) and requested_tier in KNOWN_REQUEST_SERVICE_TIERS: + return requested_tier + return None + + +def get_service_tier_from_standard_logging_payload( + standard_logging_payload: StandardLoggingPayload, +) -> str | None: + """ + Resolve the service tier a request ran on, for the Prometheus ``service_tier`` label. + + The tier the provider actually served wins over the tier the caller asked for, + so latency and spend stay segmentable when the request said ``auto`` and the + provider picked the concrete tier. + + Streaming responses carry no served tier, so the requested tier is the + fallback. Values the provider itself reports are not caller-controlled and + stay unrestricted, so a tier a provider adds later is still labelled + correctly. + """ + served_tier = get_served_service_tier(standard_logging_payload) + if served_tier is not None: + return served_tier + + return get_requested_service_tier(standard_logging_payload) diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 05205cb76f2..b300c386326 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -5886,3 +5886,124 @@ class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase): exporter="console", attributes=attributes ) ) + + +class TestOTELServiceTierAttributes(unittest.TestCase): + """The tier a request asked for and the tier the provider served must land on + the litellm_request span, so tier usage is segmentable in traces.""" + + REQUEST_KEY = "gen_ai.openai.request.service_tier" + RESPONSE_KEY = "gen_ai.openai.response.service_tier" + + def _span_attributes(self, standard_logging_object, response_obj): + otel = OpenTelemetry() + mock_span = MagicMock() + kwargs = { + "model": "gpt-5-mini", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": standard_logging_object.get("model_parameters") or {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": standard_logging_object, + } + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + return {call[0][0]: call[0][1] for call in mock_span.set_attribute.call_args_list} + + def test_served_tier_from_response_and_requested_tier_are_stamped(self): + response_obj = { + "id": "chatcmpl-1", + "model": "gpt-5-mini", + "service_tier": "scale", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {"service_tier": "auto"}, + "response": response_obj, + }, + response_obj, + ) + self.assertEqual(attributes[self.RESPONSE_KEY], "scale") + self.assertEqual(attributes[self.REQUEST_KEY], "auto") + + def test_served_tier_read_from_usage_object(self): + """Anthropic reports the served tier on the usage object, not the top level.""" + response_obj = { + "id": "chatcmpl-2", + "model": "claude-sonnet-4-5", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {"usage_object": {"service_tier": "priority"}}, + "model_parameters": {}, + "response": response_obj, + }, + response_obj, + ) + self.assertEqual(attributes[self.RESPONSE_KEY], "priority") + self.assertNotIn(self.REQUEST_KEY, attributes) + + def test_no_tier_anywhere_stamps_nothing(self): + response_obj = { + "id": "chatcmpl-3", + "model": "gpt-5-mini", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {}, + "response": response_obj, + }, + response_obj, + ) + self.assertNotIn(self.RESPONSE_KEY, attributes) + self.assertNotIn(self.REQUEST_KEY, attributes) + + def test_unknown_requested_tier_is_not_stamped(self): + """The requested tier is caller-controlled, so an unrecognized value is + dropped rather than written verbatim onto the span.""" + response_obj = { + "id": "chatcmpl-4", + "model": "gpt-5-mini", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {"service_tier": "Z" * 5000}, + "response": response_obj, + }, + response_obj, + ) + self.assertNotIn(self.REQUEST_KEY, attributes) + + def test_served_tier_is_stamped_even_when_unrecognized(self): + """The served tier comes from the provider, not the caller, so a tier a + provider adds later is still stamped.""" + response_obj = { + "id": "chatcmpl-5", + "model": "gpt-5-mini", + "service_tier": "tier-added-by-provider-later", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + attributes = self._span_attributes( + { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "model_parameters": {}, + "response": response_obj, + }, + response_obj, + ) + self.assertEqual(attributes[self.RESPONSE_KEY], "tier-added-by-provider-later") diff --git a/tests/test_litellm/integrations/test_prometheus_service_tier_label.py b/tests/test_litellm/integrations/test_prometheus_service_tier_label.py index 9d702b19c6c..b2212c4ff41 100644 --- a/tests/test_litellm/integrations/test_prometheus_service_tier_label.py +++ b/tests/test_litellm/integrations/test_prometheus_service_tier_label.py @@ -13,9 +13,9 @@ import datetime import pytest -from litellm.integrations.prometheus import ( +from litellm.integrations.prometheus import PrometheusLogger +from litellm.litellm_core_utils.service_tier_utils import ( KNOWN_REQUEST_SERVICE_TIERS, - PrometheusLogger, get_service_tier_from_standard_logging_payload, ) from litellm.types.integrations.prometheus import ( @@ -230,3 +230,12 @@ async def test_success_event_emits_service_tier_on_latency_and_spend_metrics(): ) finally: _clear_prometheus_registry() + + +def test_allowlist_covers_every_modeled_service_tier(): + """A tier modeled for cost calculation is real traffic, so it must resolve + rather than being dropped as an unknown caller value.""" + from litellm.types.utils import ServiceTier + + missing = {tier.value for tier in ServiceTier} - KNOWN_REQUEST_SERVICE_TIERS + assert not missing, f"ServiceTier values missing from the allowlist: {sorted(missing)}"