scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

feat(otel): stamp service tier attributes on inference spans (#35679)

Browse source 
* feat(otel): stamp service tier attributes on inference spans

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(otel): bound requested service tier to known values

The requested tier is caller-controlled and reaches the span verbatim, so an
arbitrary string lands on every litellm_request span on success and on failure.
A 100k character value was stamped uncapped; safe_set_attribute does not
truncate and no span limits are configured.

Apply KNOWN_REQUEST_SERVICE_TIERS in get_requested_service_tier so both the
span attribute and the Prometheus label bound the value the same way. The
served tier stays unrestricted since it comes from the provider, so a tier a
provider adds later is still reported.

Prometheus label behavior is unchanged.

* fix: derive known service tiers from the ServiceTier enum

The allowlist omitted "fast", which litellm models as a real tier and prices
through the priority cost key, so a request naming it resolved to no tier on
the span and no Prometheus label.

Deriving the set from ServiceTier keeps the two in sync, so a tier added there
for cost calculation cannot go missing here.

Behavior change: a request with service_tier "fast" now carries the tier on the
span and on the Prometheus service_tier label, where it previously resolved to
none. Every other value resolves as before.

* refactor: build the known service tiers without a mutable intermediate

The set comprehension and set literal tripped LIT002, which bounds mutable
collections. Concatenating tuples keeps the derivation from ServiceTier while
every intermediate stays immutable; the resulting frozenset is unchanged.

---------

Co-authored-by: milan <milan@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Yucheng Zhu <yucheng@berri.ai>
This commit is contained in:
devin-ai-integration[bot] 2026-08-03 23:10:01 -07:00 • committed by GitHub
parent abe3289398
commit a625d1e1ca
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 231 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
litellm/integrations/opentelemetry.py
View file

@ -24,6 +24,10 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
from litellm.integrations.otel.model.semconv import Metric
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.service_tier_utils import (
get_requested_service_tier,
get_served_service_tier,
)
from litellm.secret_managers.main import get_secret_bool, str_to_bool
from litellm.types.services import ServiceLoggerPayload
from litellm.types.utils import (
@ -74,6 +78,11 @@ PREPROCESSING_DURATION_MS_ATTRIBUTE = "litellm.preprocessing.duration_ms"
TEAM_METADATA_ATTRIBUTE = "litellm.team.metadata"
MODEL_GROUP_ATTRIBUTE = "litellm.model_group"
PROVIDER_MODEL_ATTRIBUTE = "litellm.provider.model"
# semconv names the service tier attributes under the openai namespace, but every
# provider that reports a tier (OpenAI, Anthropic, Bedrock, Groq, Vertex) uses the
# same request param and response field, so both keys carry all of them.
REQUEST_SERVICE_TIER_ATTRIBUTE = "gen_ai.openai.request.service_tier"
RESPONSE_SERVICE_TIER_ATTRIBUTE = "gen_ai.openai.response.service_tier"
# Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later
RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
@ -1411,6 +1420,23 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if provider_model:
self.safe_set_attribute(span=span, key=PROVIDER_MODEL_ATTRIBUTE, value=provider_model)
def _set_service_tier_attributes(
self,
span: Span,
standard_logging_payload: StandardLoggingPayload,
) -> None:
"""Stamp the tier the caller asked for and the tier the provider reports it
served, so tier usage is segmentable in traces. Both are optional: a caller
may not name a tier, and streaming responses carry no served tier.
"""
requested_tier = get_requested_service_tier(standard_logging_payload)
if requested_tier is not None:
self.safe_set_attribute(span=span, key=REQUEST_SERVICE_TIER_ATTRIBUTE, value=requested_tier)
served_tier = get_served_service_tier(standard_logging_payload)
if served_tier is not None:
self.safe_set_attribute(span=span, key=RESPONSE_SERVICE_TIER_ATTRIBUTE, value=served_tier)
@staticmethod
def _team_metadata_json(value: Any, allowed_keys: list[str]) -> str | None:
"""JSON-serialize only the allowlisted sub-keys of a team's metadata.
@ -2310,6 +2336,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=response_obj.get("model"),
)
self._set_service_tier_attributes(span=span, standard_logging_payload=standard_logging_payload)
usage = response_obj and response_obj.get("usage")
if usage:
self.safe_set_attribute(

51
litellm/integrations/prometheus.py
View file

@ -34,6 +34,9 @@ from litellm.litellm_core_utils.core_helpers import (
get_litellm_metadata_from_kwargs,
get_metadata_variable_name_from_kwargs,
)
from litellm.litellm_core_utils.service_tier_utils import (
get_service_tier_from_standard_logging_payload,
)
from litellm.proxy._types import (
LiteLLM_DeletedVerificationToken,
LiteLLM_TeamTable,
@ -98,16 +101,6 @@ class _ExcludedLabelMetric:
return self._metric.labels(*kept_values) if kept_values else self._metric
# Tiers a caller may name in a request, across the providers that accept the
# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and
# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which
# maps "default" to "standard". Used to bound the caller-controlled fallback in
# ``get_service_tier_from_standard_logging_payload``.
KNOWN_REQUEST_SERVICE_TIERS = frozenset(
{"auto", "batch", "default", "flex", "priority", "scale", "standard", "standard_only"}
)
def _get_budget_metrics_per_request_timeout() -> float:
raw = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT")
if raw is None:
@ -4181,44 +4174,6 @@ def get_custom_labels_from_metadata(metadata: dict) -> dict[str, str]:
return result
def get_service_tier_from_standard_logging_payload(
standard_logging_payload: StandardLoggingPayload,
) -> str | None:
"""
Resolve the service tier a request ran on, for the ``service_tier`` label.
The tier the provider actually served wins over the tier the caller asked for,
so latency and spend stay segmentable when the request said ``auto`` and the
provider picked the concrete tier. Providers report the served tier either at
the top level of the response (OpenAI, Bedrock, Groq) or on the usage object
(Anthropic).
Streaming responses carry no served tier, so the requested tier is the
fallback. That value is caller-controlled and survives param mapping even
where the provider then ignores it (Bedrock and Groq accept the request and
drop an unrecognized tier), so it is only labelled when it names a known
tier; otherwise one caller could mint a Prometheus series per string. Values
the provider itself reports are not caller-controlled and stay unrestricted,
so a tier a provider adds later is still labelled correctly.
"""
response = standard_logging_payload.get("response")
usage_object = standard_logging_payload.get("metadata", {}).get("usage_object")
served_candidates: tuple[object, ...] = (
response.get("service_tier") if isinstance(response, dict) else None,
usage_object.get("service_tier") if isinstance(usage_object, dict) else None,
)
served_tier = next((tier for tier in served_candidates if isinstance(tier, str) and tier), None)
if served_tier is not None:
return served_tier
model_parameters = standard_logging_payload.get("model_parameters")
requested_tier = model_parameters.get("service_tier") if isinstance(model_parameters, dict) else None
if isinstance(requested_tier, str) and requested_tier in KNOWN_REQUEST_SERVICE_TIERS:
return requested_tier
return None
def _get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload: dict | None,
) -> dict[str, Any]:

68
litellm/litellm_core_utils/service_tier_utils.py Normal file
View file

@ -0,0 +1,68 @@
from __future__ import annotations
from litellm.types.utils import ServiceTier, StandardLoggingPayload
# Tiers a caller may name in a request, across the providers that accept the
# parameter: OpenAI ("auto", "default", "flex", "priority", "scale"), Bedrock and
# Groq (subsets of those), Anthropic ("auto", "standard_only") and Vertex, which
# maps "default" to "standard". Bounds the caller-controlled requested tier
# wherever it is recorded. Derived from ``ServiceTier`` so a tier added there for
# cost calculation cannot go missing here.
KNOWN_REQUEST_SERVICE_TIERS = frozenset(
tuple(tier.value for tier in ServiceTier) + ("batch", "default", "scale", "standard", "standard_only")
)
def get_served_service_tier(standard_logging_payload: StandardLoggingPayload) -> str | None:
"""
The tier the provider reports it actually served the request on.
Providers report it either at the top level of the response (OpenAI, Bedrock,
Groq) or on the usage object (Anthropic). Streaming responses carry no served
tier.
"""
response = standard_logging_payload.get("response")
usage_object = standard_logging_payload.get("metadata", {}).get("usage_object")
served_candidates: tuple[object, ...] = (
response.get("service_tier") if isinstance(response, dict) else None,
usage_object.get("service_tier") if isinstance(usage_object, dict) else None,
)
return next((tier for tier in served_candidates if isinstance(tier, str) and tier), None)
def get_requested_service_tier(standard_logging_payload: StandardLoggingPayload) -> str | None:
"""
The tier the caller asked for, as sent to the provider.
The value is caller-controlled and survives param mapping even where the
provider then ignores it (Bedrock and Groq accept the request and drop an
unrecognized tier), so it is only reported when it names a known tier.
"""
model_parameters = standard_logging_payload.get("model_parameters")
requested_tier = model_parameters.get("service_tier") if isinstance(model_parameters, dict) else None
if isinstance(requested_tier, str) and requested_tier in KNOWN_REQUEST_SERVICE_TIERS:
return requested_tier
return None
def get_service_tier_from_standard_logging_payload(
standard_logging_payload: StandardLoggingPayload,
) -> str | None:
"""
Resolve the service tier a request ran on, for the Prometheus ``service_tier`` label.
The tier the provider actually served wins over the tier the caller asked for,
so latency and spend stay segmentable when the request said ``auto`` and the
provider picked the concrete tier.
Streaming responses carry no served tier, so the requested tier is the
fallback. Values the provider itself reports are not caller-controlled and
stay unrestricted, so a tier a provider adds later is still labelled
correctly.
"""
served_tier = get_served_service_tier(standard_logging_payload)
if served_tier is not None:
return served_tier
return get_requested_service_tier(standard_logging_payload)

121
tests/test_litellm/integrations/test_opentelemetry.py
View file

@ -5886,3 +5886,124 @@ class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase):
exporter="console", attributes=attributes
)
)
class TestOTELServiceTierAttributes(unittest.TestCase):
"""The tier a request asked for and the tier the provider served must land on
the litellm_request span, so tier usage is segmentable in traces."""
REQUEST_KEY = "gen_ai.openai.request.service_tier"
RESPONSE_KEY = "gen_ai.openai.response.service_tier"
def _span_attributes(self, standard_logging_object, response_obj):
otel = OpenTelemetry()
mock_span = MagicMock()
kwargs = {
"model": "gpt-5-mini",
"messages": [{"role": "user", "content": "Hello"}],
"optional_params": standard_logging_object.get("model_parameters") or {},
"litellm_params": {"custom_llm_provider": "openai"},
"standard_logging_object": standard_logging_object,
}
otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj)
return {call[0][0]: call[0][1] for call in mock_span.set_attribute.call_args_list}
def test_served_tier_from_response_and_requested_tier_are_stamped(self):
response_obj = {
"id": "chatcmpl-1",
"model": "gpt-5-mini",
"service_tier": "scale",
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
}
attributes = self._span_attributes(
{
"id": "test-id",
"call_type": "completion",
"metadata": {},
"model_parameters": {"service_tier": "auto"},
"response": response_obj,
},
response_obj,
)
self.assertEqual(attributes[self.RESPONSE_KEY], "scale")
self.assertEqual(attributes[self.REQUEST_KEY], "auto")
def test_served_tier_read_from_usage_object(self):
"""Anthropic reports the served tier on the usage object, not the top level."""
response_obj = {
"id": "chatcmpl-2",
"model": "claude-sonnet-4-5",
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
}
attributes = self._span_attributes(
{
"id": "test-id",
"call_type": "completion",
"metadata": {"usage_object": {"service_tier": "priority"}},
"model_parameters": {},
"response": response_obj,
},
response_obj,
)
self.assertEqual(attributes[self.RESPONSE_KEY], "priority")
self.assertNotIn(self.REQUEST_KEY, attributes)
def test_no_tier_anywhere_stamps_nothing(self):
response_obj = {
"id": "chatcmpl-3",
"model": "gpt-5-mini",
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
}
attributes = self._span_attributes(
{
"id": "test-id",
"call_type": "completion",
"metadata": {},
"model_parameters": {},
"response": response_obj,
},
response_obj,
)
self.assertNotIn(self.RESPONSE_KEY, attributes)
self.assertNotIn(self.REQUEST_KEY, attributes)
def test_unknown_requested_tier_is_not_stamped(self):
"""The requested tier is caller-controlled, so an unrecognized value is
dropped rather than written verbatim onto the span."""
response_obj = {
"id": "chatcmpl-4",
"model": "gpt-5-mini",
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
}
attributes = self._span_attributes(
{
"id": "test-id",
"call_type": "completion",
"metadata": {},
"model_parameters": {"service_tier": "Z" * 5000},
"response": response_obj,
},
response_obj,
)
self.assertNotIn(self.REQUEST_KEY, attributes)
def test_served_tier_is_stamped_even_when_unrecognized(self):
"""The served tier comes from the provider, not the caller, so a tier a
provider adds later is still stamped."""
response_obj = {
"id": "chatcmpl-5",
"model": "gpt-5-mini",
"service_tier": "tier-added-by-provider-later",
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
}
attributes = self._span_attributes(
{
"id": "test-id",
"call_type": "completion",
"metadata": {},
"model_parameters": {},
"response": response_obj,
},
response_obj,
)
self.assertEqual(attributes[self.RESPONSE_KEY], "tier-added-by-provider-later")

13
tests/test_litellm/integrations/test_prometheus_service_tier_label.py
View file

@ -13,9 +13,9 @@ import datetime
import pytest
from litellm.integrations.prometheus import (
from litellm.integrations.prometheus import PrometheusLogger
from litellm.litellm_core_utils.service_tier_utils import (
KNOWN_REQUEST_SERVICE_TIERS,
PrometheusLogger,
get_service_tier_from_standard_logging_payload,
)
from litellm.types.integrations.prometheus import (
@ -230,3 +230,12 @@ async def test_success_event_emits_service_tier_on_latency_and_spend_metrics():
)
finally:
_clear_prometheus_registry()
def test_allowlist_covers_every_modeled_service_tier():
"""A tier modeled for cost calculation is real traffic, so it must resolve
rather than being dropped as an unknown caller value."""
from litellm.types.utils import ServiceTier
missing = {tier.value for tier in ServiceTier} - KNOWN_REQUEST_SERVICE_TIERS
assert not missing, f"ServiceTier values missing from the allowlist: {sorted(missing)}"