scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

Merge 25c9f18285 into 5724117116

Browse source
This commit is contained in:
devin-ai-integration[bot] 2026-10-03 03:27:45 -04:00 • committed by GitHub
commit b87eb1a13e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 1418 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -29,6 +29,14 @@ LANGFUSE_CLOUD_US_ENDPOINT: Final = "https://us.cloud.langfuse.com/api/public/ot
LANGFUSE_INGESTION_VERSION_HEADER: Final = "x-langfuse-ingestion-version"
LANGFUSE_INGESTION_VERSION: Final = "4"
_TRACE_IDENTITY_FIELDS: Final = (
"user_api_key_alias",
"user_api_key_user_id",
"user_api_key_end_user_id",
"user_api_key_team_id",
"user_api_key_team_alias",
)
class LangfuseOtelLogger(OpenTelemetry):
def __init__(self, config=None, *args, **kwargs):
@ -125,6 +133,30 @@ class LangfuseOtelLogger(OpenTelemetry):
value = str(value)
safe_set_attribute(span, enum_attr.value, value)
@staticmethod
def _set_request_metadata_attributes(span: Span, kwargs: dict[str, object]) -> None:
from litellm.integrations.arize._utils import safe_set_attribute
from litellm.integrations.langfuse.langfuse import log_requester_metadata
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
standard_logging_object: Final = kwargs.get("standard_logging_object")
request_metadata: Final = (
standard_logging_object.get("metadata") if isinstance(standard_logging_object, dict) else None
)
if not isinstance(request_metadata, dict):
return
observation_metadata: Final = { # mutable-ok: stays a real dict for safe_dumps and .get reads below
key: value
for key, value in log_requester_metadata(redact_user_api_key_info(metadata=request_metadata)).items()
if value is not None
}
safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_METADATA.value, safe_dumps(observation_metadata))
trace_prefix: Final = LangfuseSpanAttributes.TRACE_METADATA.value
for field in _TRACE_IDENTITY_FIELDS:
if value := observation_metadata.get(field):
safe_set_attribute(span, f"{trace_prefix}.{field}", value)
@staticmethod
def _set_observation_output(span: Span, response_obj):
"""Helper to set observation output attributes."""
@ -244,6 +276,7 @@ class LangfuseOtelLogger(OpenTelemetry):
metadata: Final = LangfuseOtelLogger._extract_langfuse_metadata(kwargs)
LangfuseOtelLogger._set_metadata_attributes(span=span, metadata=metadata)
LangfuseOtelLogger._set_request_metadata_attributes(span=span, kwargs=kwargs)
messages: Final = kwargs.get("messages")
if messages:

24
litellm/integrations/otel/mappers/langfuse.py
View file

@ -12,6 +12,7 @@ the JSON-serialized payloads. ``trace_attributes`` maps the caller's trace contr
import json
from collections.abc import Callable
from types import MappingProxyType
from typing import Final
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
@ -28,6 +29,8 @@ from litellm.integrations.otel.model.payloads import (
LLMUsage,
)
from litellm.integrations.otel.model.trace_controls import TraceControls
from litellm.integrations.otel.model.utils import as_str
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
LANGFUSE_OBSERVATION_INPUT: Final = "langfuse.observation.input"
LANGFUSE_OBSERVATION_OUTPUT: Final = "langfuse.observation.output"
@ -35,6 +38,19 @@ LANGFUSE_TRACE_NAME: Final = "langfuse.trace.name"
LANGFUSE_TRACE_USER_ID: Final = "user.id"
LANGFUSE_TRACE_SESSION_ID: Final = "session.id"
LANGFUSE_TRACE_TAGS: Final = "langfuse.trace.tags"
LANGFUSE_OBSERVATION_METADATA: Final = "langfuse.observation.metadata"
LANGFUSE_TRACE_METADATA_PREFIX: Final = "langfuse.trace.metadata."
TRACE_IDENTITY_FIELDS: Final = (
"user_api_key_alias",
"user_api_key_user_id",
"user_api_key_end_user_id",
"user_api_key_team_id",
"user_api_key_team_alias",
)
def _identity_field(name: str) -> Callable[[LLMCallSpanData], AttrValue | None]:
return lambda d: as_str(d.request_metadata.get(name)) or None
class LangfuseMapper:
@ -45,6 +61,9 @@ class LangfuseMapper:
"langfuse.observation.id": lambda d: d.identity.call_id or None,
"langfuse.trace.metadata.team_id": lambda d: d.identity.team_id or None,
"langfuse.trace.metadata.team_alias": lambda d: d.identity.team_alias or None,
**MappingProxyType(
{f"{LANGFUSE_TRACE_METADATA_PREFIX}{name}": _identity_field(name) for name in TRACE_IDENTITY_FIELDS}
),
}
# Sub-tables folded into their respective JSON blobs.
@ -68,6 +87,11 @@ class LangfuseMapper:
# JSON-payload attributes: each builder returns the serialized blob or None.
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
LANGFUSE_OBSERVATION_METADATA: lambda d: (
safe_dumps(dict(d.request_metadata)) # mutable-ok: safe_dumps only serializes real dicts
if d.request_metadata
else None
),
"langfuse.observation.model.parameters": lambda d: json_if(
collect(LangfuseMapper._MODEL_PARAMS, d.request_params)
),

19
litellm/integrations/otel/model/metadata.py
View file

@ -42,9 +42,11 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, SESSION_ID_GENERATED_METADATA_KEY
from litellm.integrations.langfuse.langfuse import log_requester_metadata
from litellm.integrations.otel.model.semconv import resolve_operation
from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls
from litellm.integrations.otel.model.utils import as_str, as_str_mapping, to_seconds
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
if TYPE_CHECKING:
from litellm.types.utils import StandardLoggingPayload
@ -417,6 +419,23 @@ def _model_info_id(model_info: object) -> str | None:
return None
def exported_request_metadata(payload: StandardLoggingPayload) -> Mapping[str, object]:
"""The request metadata as the logging callbacks export it: ``user_api_key_*``
dropped when ``litellm.redact_user_api_key_info`` is on, header keys nested
under ``requester_metadata``, ``None`` values dropped."""
raw_meta: Final = cast( # cast-ok: StandardLoggingPayload.get returns Any | None
Mapping[str, object], payload.get("metadata") or MappingProxyType({})
)
redacted: Final = cast( # cast-ok: redact_user_api_key_info is untyped
Mapping[str, object],
redact_user_api_key_info(metadata=dict(raw_meta)), # mutable-ok: the redactor requires a real dict
)
exported: Final = cast( # cast-ok: log_requester_metadata is untyped
Mapping[str, object], log_requester_metadata(redacted)
)
return MappingProxyType({key: value for key, value in exported.items() if value is not None})
def _team_metadata_dict(value: object) -> Mapping[str, object] | None:
"""The team's free-form metadata as a raw mapping, or ``None`` when missing
or empty.

4
litellm/integrations/otel/model/payloads.py
View file

@ -12,7 +12,7 @@ from urllib.parse import urlsplit
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.otel.model.metadata import RequestContext, RequestIdentity
from litellm.integrations.otel.model.metadata import RequestContext, RequestIdentity, exported_request_metadata
from litellm.integrations.otel.model.semconv import (
GenAIOperation,
GenAIOutputType,
@ -409,6 +409,7 @@ class LLMCallSpanData:
response_cost: float | None
server: ServerInfo | None
identity: RequestIdentity
request_metadata: Mapping[str, object] = field(default_factory=dict)
is_streaming: bool | None = None
cost: LLMCost = field(default_factory=LLMCost)
tools: tuple[ToolDefinition, ...] = ()
@ -476,6 +477,7 @@ class LLMCallSpanData:
cost=LLMCost.from_breakdown(cast("Mapping[str, object] | None", payload.get("cost_breakdown"))),
server=ServerInfo.from_api_base(context.api_base),
identity=context.identity,
request_metadata=exported_request_metadata(payload),
is_streaming=as_bool(payload.get("stream")),
tools=_extract_tools(params),
messages_in=_dicts(payload.get("messages")) if capture_content else (),

1
litellm/types/integrations/langfuse_otel.py
View file

@ -29,6 +29,7 @@ class LangfuseSpanAttributes(str, Enum):
# ---- Observation input/output ----
OBSERVATION_INPUT = "langfuse.observation.input"
OBSERVATION_OUTPUT = "langfuse.observation.output"
OBSERVATION_METADATA = "langfuse.observation.metadata"
# ---- Trace-level metadata ----
TRACE_USER_ID = "user.id"

1129
tests/integration/observability/test_langfuse_otel_metadata.py Normal file
View file

File diff suppressed because it is too large Load diff

98
tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py
View file

@ -400,3 +400,101 @@ def test_resolve_mappers_composition_layers_vocabularies():
def test_resolve_mappers_rejects_unknown_name():
with pytest.raises(ValueError, match="unknown mapper name 'nope'"):
resolve_mappers(["genai", "nope"])
def test_langfuse_mapper_emits_request_metadata_and_identity():
payload: Final[dict[str, object]] = {
"call_type": "acompletion",
"custom_llm_provider": "openai",
"model": "gpt-4o",
"metadata": {
"user_api_key_alias": "prod-key",
"user_api_key_user_id": "user-1",
"user_api_key_end_user_id": "end-1",
"user_api_key_team_id": "team-9",
"user_api_key_team_alias": "team nine",
"user_api_key_max_budget": None,
"spend_logs_metadata": {"ticket": "LIT-8283"},
"requester_metadata": {"headers": {"x-tenant": "acme"}},
},
"response": {},
}
attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload))
assert json.loads(attrs["langfuse.observation.metadata"]) == {
"user_api_key_alias": "prod-key",
"user_api_key_user_id": "user-1",
"user_api_key_end_user_id": "end-1",
"user_api_key_team_id": "team-9",
"user_api_key_team_alias": "team nine",
"spend_logs_metadata": {"ticket": "LIT-8283"},
"requester_metadata": {"headers": {"x-tenant": "acme"}},
}
assert attrs["langfuse.trace.metadata.user_api_key_alias"] == "prod-key"
assert attrs["langfuse.trace.metadata.user_api_key_user_id"] == "user-1"
assert attrs["langfuse.trace.metadata.user_api_key_end_user_id"] == "end-1"
assert attrs["langfuse.trace.metadata.user_api_key_team_id"] == "team-9"
assert attrs["langfuse.trace.metadata.user_api_key_team_alias"] == "team nine"
assert attrs["langfuse.trace.metadata.team_id"] == "team-9"
assert attrs["langfuse.trace.metadata.team_alias"] == "team nine"
assert attrs["langfuse.observation.metadata.provider"] == "openai"
def test_langfuse_mapper_skips_empty_identity_values():
payload: Final[dict[str, object]] = {
"call_type": "acompletion",
"custom_llm_provider": "openai",
"model": "gpt-4o",
"metadata": {
"user_api_key_alias": None,
"user_api_key_user_id": "",
"user_api_key_end_user_id": "end-1",
},
"response": {},
}
attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload))
assert "langfuse.trace.metadata.user_api_key_alias" not in attrs
assert "langfuse.trace.metadata.user_api_key_user_id" not in attrs
assert attrs["langfuse.trace.metadata.user_api_key_end_user_id"] == "end-1"
def test_langfuse_mapper_redacts_user_api_key_fields():
import litellm
saved: Final = litellm.redact_user_api_key_info
try:
litellm.redact_user_api_key_info = True
payload: Final[dict[str, object]] = {
"call_type": "acompletion",
"custom_llm_provider": "openai",
"model": "gpt-4o",
"metadata": {
"user_api_key_alias": "prod-key",
"user_api_key_team_id": "team-9",
"spend_logs_metadata": {"ticket": "LIT-8283"},
},
"response": {},
}
attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload))
finally:
litellm.redact_user_api_key_info = saved
exported: Final = json.loads(attrs["langfuse.observation.metadata"])
assert not [key for key in exported if key.startswith("user_api_key")]
assert exported["spend_logs_metadata"] == {"ticket": "LIT-8283"}
assert not [key for key in attrs if key.startswith("langfuse.trace.metadata.user_api_key_")]
assert attrs["langfuse.trace.metadata.team_id"] == "team-9"
def test_langfuse_mapper_omits_observation_metadata_without_payload_metadata():
payload: Final[dict[str, object]] = {
"call_type": "acompletion",
"custom_llm_provider": "openai",
"model": "gpt-4o",
"metadata": {},
"response": {},
}
attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload))
assert json.loads(attrs["langfuse.observation.metadata"]) == {"requester_metadata": {}}

111
tests/unit/integrations/test_langfuse_otel.py
View file

@ -1,9 +1,11 @@
import json
import os
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
import litellm
from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
from litellm.integrations.opentelemetry import OpenTelemetryConfig
from litellm.types.llms.openai import ResponsesAPIResponse
@ -482,6 +484,115 @@ class TestLangfuseOtelIntegration:
assert isinstance(config, OpenTelemetryConfig)
# Endpoint assertion removed as side effect is gone
def test_request_metadata_is_emitted_under_langfuse_observation_and_trace_metadata_keys(self) -> None:
request_metadata: Final = {
"user_api_key_hash": "hash123",
"user_api_key_alias": "prod-key",
"user_api_key_user_id": "user-1",
"user_api_key_end_user_id": "end-user-1",
"user_api_key_team_id": "team-1",
"user_api_key_team_alias": "team-a",
"spend_logs_metadata": {"env": "prod"},
"requester_ip_address": "10.0.0.1",
"requester_metadata": {"ticket": "LIT-8283"},
}
kwargs: Final = {
"litellm_params": {"metadata": {}},
"standard_logging_object": {"metadata": request_metadata},
}
with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute:
LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None)
actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list}
assert json.loads(actual["langfuse.observation.metadata"]) == {**request_metadata}
assert {key: value for key, value in actual.items() if key.startswith("langfuse.trace.metadata.")} == {
"langfuse.trace.metadata.user_api_key_alias": "prod-key",
"langfuse.trace.metadata.user_api_key_user_id": "user-1",
"langfuse.trace.metadata.user_api_key_end_user_id": "end-user-1",
"langfuse.trace.metadata.user_api_key_team_id": "team-1",
"langfuse.trace.metadata.user_api_key_team_alias": "team-a",
}
def test_request_metadata_redaction_matches_vanilla_langfuse(self) -> None:
previous_flag: Final = litellm.redact_user_api_key_info
litellm.redact_user_api_key_info = True
request_metadata: Final = {
"user_api_key_hash": "hash123",
"user_api_key_alias": "prod-key",
"user_api_key_user_id": "user-1",
"user_api_key_end_user_id": "end-user-1",
"user_api_key_team_id": "team-1",
"user_api_key_team_alias": "team-a",
"spend_logs_metadata": {"env": "prod"},
"requester_ip_address": "10.0.0.1",
"requester_metadata": {"ticket": "LIT-8283"},
}
kwargs: Final = {
"litellm_params": {"metadata": {}},
"standard_logging_object": {"metadata": request_metadata},
}
try:
with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute:
LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None)
actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list}
assert json.loads(actual["langfuse.observation.metadata"]) == {
"spend_logs_metadata": {"env": "prod"},
"requester_ip_address": "10.0.0.1",
"requester_metadata": {"ticket": "LIT-8283"},
}
assert not [key for key in actual if key.startswith("langfuse.trace.metadata.")]
finally:
litellm.redact_user_api_key_info = previous_flag
def test_request_metadata_drops_null_fields_and_skips_empty_trace_identities(self) -> None:
request_metadata: Final = {
"user_api_key_alias": "prod-key",
"user_api_key_user_id": "user-1",
"user_api_key_end_user_id": None,
"user_api_key_team_id": "",
"user_api_key_team_alias": None,
"team_id": None,
"team_alias": None,
"spend_logs_metadata": {"env": "prod"},
}
kwargs: Final = {
"litellm_params": {"metadata": {}},
"standard_logging_object": {"metadata": request_metadata},
}
with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute:
LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None)
actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list}
assert json.loads(actual["langfuse.observation.metadata"]) == {
"user_api_key_alias": "prod-key",
"user_api_key_user_id": "user-1",
"user_api_key_team_id": "",
"spend_logs_metadata": {"env": "prod"},
"requester_metadata": {},
}
assert {key: value for key, value in actual.items() if key.startswith("langfuse.trace.metadata.")} == {
"langfuse.trace.metadata.user_api_key_alias": "prod-key",
"langfuse.trace.metadata.user_api_key_user_id": "user-1",
}
def test_request_metadata_keys_are_absent_without_standard_logging_object(self) -> None:
kwargs: Final = {"litellm_params": {"metadata": {"trace_metadata": {"k": "v"}}}}
with patch("litellm.integrations.arize._utils.safe_set_attribute") as mock_safe_set_attribute:
LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None)
actual: Final = {call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list}
assert [key for key in actual if key.startswith("langfuse.trace.metadata")] == ["langfuse.trace.metadata"]
assert "langfuse.observation.metadata" not in actual
class TestLangfuseOtelKeyDynamicConfig:
"""Key/team-scoped Langfuse credentials must define the full export target