From 75ae43e6679c9cbf7ef32c62056119cfe4a37440 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Tue, 29 Jul 2025 01:53:36 +0200 Subject: [PATCH] feat(langfuse-otel): Add comprehensive metadata support to Langfuse OpenTelemetry integration (#12956) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(langfuse-otel): Add comprehensive metadata support to Langfuse OpenTelemetry integration This commit brings the langfuse_otel integration to feature parity with the vanilla Langfuse integration by adding support for all metadata fields. Changes: - Extended LangfuseSpanAttributes enum with all supported metadata fields: - Generation-level: generation_name, generation_id, parent_observation_id, version, mask_input/output - Trace-level: trace_user_id, session_id, tags, trace_name, trace_id, trace_metadata, trace_version, trace_release, existing_trace_id, update_trace_keys - Debug: debug_langfuse - Implemented metadata extraction and mapping in langfuse_otel.py: - Added _extract_langfuse_metadata() helper to extract metadata from kwargs - Support for header-based metadata (langfuse_* headers) via proxy - Enhanced _set_langfuse_specific_attributes() to map all metadata to OTEL attributes - JSON serialization for complex types (lists, dicts) for OTEL compatibility - Updated documentation: - Added 'Metadata Support' section explaining all fields are now supported - Provided usage example showing how to pass metadata - Clarified that traces are viewed in Langfuse UI (not generic OTEL backends) - Added opentelemetry-exporter-otlp to required dependencies This allows users to pass metadata like: metadata={ 'generation_name': 'my-generation', 'trace_id': 'trace-123', 'session_id': 'session-456', 'tags': ['prod', 'v1'], 'trace_metadata': {'user_type': 'premium'} } All metadata is exported as OpenTelemetry span attributes with 'langfuse.*' prefix for easy filtering and analysis in the Langfuse UI. * Fix ruff linting error * test(langfuse-otel): Fix failing test and add comprehensive metadata tests - Fix test_set_langfuse_environment_attribute to use positional arguments instead of keyword arguments when asserting safe_set_attribute calls - Add test_extract_langfuse_metadata_basic to verify metadata extraction from litellm_params - Add test_extract_langfuse_metadata_with_header_enrichment to test integration with header-based metadata using a stubbed LangFuseLogger - Add test_set_langfuse_specific_attributes_full_mapping to comprehensively test all metadata field mappings and JSON serialization of complex types These tests ensure full coverage of the langfuse_otel metadata features added in commit ab1dbe355 and fix the CI test failure. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --------- Co-authored-by: Claude --- .../langfuse_otel_integration.md | 37 +++++++- .../integrations/langfuse/langfuse_otel.py | 84 ++++++++++++++-- litellm/types/integrations/langfuse_otel.py | 27 +++++- .../integrations/test_langfuse_otel.py | 95 ++++++++++++++++++- 4 files changed, 231 insertions(+), 12 deletions(-) diff --git a/docs/my-website/docs/observability/langfuse_otel_integration.md b/docs/my-website/docs/observability/langfuse_otel_integration.md index c45c33f0f25..4801fa8e1b0 100644 --- a/docs/my-website/docs/observability/langfuse_otel_integration.md +++ b/docs/my-website/docs/observability/langfuse_otel_integration.md @@ -24,7 +24,7 @@ The Langfuse OpenTelemetry integration allows you to send LiteLLM traces and obs 2. **API Keys**: Get your public and secret keys from your Langfuse project settings 3. **Dependencies**: Install required packages: ```bash - pip install litellm opentelemetry-api opentelemetry-sdk + pip install litellm opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp ``` ## Configuration @@ -147,6 +147,41 @@ The integration automatically collects the following data: - **Metadata**: User ID, session ID, custom tags (if provided) - **Error Information**: Exception details and stack traces (if errors occur) +## Metadata Support + +All metadata fields available in the vanilla Langfuse integration are now **fully supported** when you use the OTEL integration. + +- Any key you pass in the `metadata` dictionary (`generation_name`, `trace_id`, `session_id`, `tags`, and the rest) is exported as an OpenTelemetry span attribute. +- Attribute names are prefixed with `langfuse.` so you can filter or search for them easily in your observability backend. + Examples: `langfuse.generation.name`, `langfuse.trace.id`, `langfuse.trace.session_id`. + +### Passing Metadata – Example + +```python +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello!"}], + metadata={ + "generation_name": "welcome-message", + "trace_id": "trace-123", + "session_id": "sess-42", + "tags": ["prod", "beta-user"] + } +) +``` + +The resulting span will contain attributes similar to: + +``` +langfuse.generation.name = "welcome-message" +langfuse.trace.id = "trace-123" +langfuse.trace.session_id = "sess-42" +langfuse.trace.tags = ["prod", "beta-user"] +``` + +Use the **Langfuse UI** (Traces tab) to search, filter and analyse spans that contain the `langfuse.*` attributes. +The OTEL exporter in this integration sends data directly to Langfuse’s OTLP HTTP endpoint; it is **not** intended for Grafana, Honeycomb, Datadog, or other generic OTEL back-ends. + ## Authentication The integration uses HTTP Basic Authentication with your Langfuse public and secret keys: diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index f257da6d95e..4072be2a256 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -1,5 +1,6 @@ import base64 import os +import json # <--- NEW from typing import TYPE_CHECKING, Any, Union from urllib.parse import quote @@ -49,21 +50,92 @@ class LangfuseOtelLogger: kwargs=kwargs ) return - + + @staticmethod + def _extract_langfuse_metadata(kwargs: dict) -> dict: + """ + Extracts Langfuse metadata from the standard LiteLLM kwargs structure. + + 1. Reads kwargs["litellm_params"]["metadata"] if present and is a dict. + 2. Enriches it with any `langfuse_*` request-header params via the + existing LangFuseLogger.add_metadata_from_header helper so that proxy + users get identical behaviour across vanilla and OTEL integrations. + """ + litellm_params = kwargs.get("litellm_params", {}) or {} + metadata = litellm_params.get("metadata") or {} + # Ensure we only work with dicts + if metadata is None or not isinstance(metadata, dict): + metadata = {} + + # Re-use header extraction logic from the vanilla logger if available + try: + from litellm.integrations.langfuse.langfuse import ( + LangFuseLogger as _LFLogger, + ) + + metadata = _LFLogger.add_metadata_from_header(litellm_params, metadata) # type: ignore + except Exception: + # Fallback silently if import fails; header enrichment just won't happen + pass + + return metadata + @staticmethod def _set_langfuse_specific_attributes(span: Span, kwargs): """ - Sets Langfuse specific attributes to the span. + Sets Langfuse specific metadata attributes onto the OTEL span. + + All keys supported by the vanilla Langfuse integration are mapped to + OTEL-safe attribute names defined in LangfuseSpanAttributes. Complex + values (lists/dicts) are serialised to JSON strings for OTEL + compatibility. """ from litellm.integrations.arize._utils import safe_set_attribute - langfuse_environment = os.environ.get("LANGFUSE_TRACING_ENVIRONMENT", None) + + # 1) Environment variable override + langfuse_environment = os.environ.get("LANGFUSE_TRACING_ENVIRONMENT") if langfuse_environment: safe_set_attribute( - span=span, - key=LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value, - value=langfuse_environment + span, + LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value, + langfuse_environment, ) + # 2) Dynamic metadata from kwargs / headers + metadata = LangfuseOtelLogger._extract_langfuse_metadata(kwargs) + + # Mapping from metadata key -> OTEL attribute enum + mapping = { + "generation_name": LangfuseSpanAttributes.GENERATION_NAME, + "generation_id": LangfuseSpanAttributes.GENERATION_ID, + "parent_observation_id": LangfuseSpanAttributes.PARENT_OBSERVATION_ID, + "version": LangfuseSpanAttributes.GENERATION_VERSION, + "mask_input": LangfuseSpanAttributes.MASK_INPUT, + "mask_output": LangfuseSpanAttributes.MASK_OUTPUT, + "trace_user_id": LangfuseSpanAttributes.TRACE_USER_ID, + "session_id": LangfuseSpanAttributes.SESSION_ID, + "tags": LangfuseSpanAttributes.TAGS, + "trace_name": LangfuseSpanAttributes.TRACE_NAME, + "trace_id": LangfuseSpanAttributes.TRACE_ID, + "trace_metadata": LangfuseSpanAttributes.TRACE_METADATA, + "trace_version": LangfuseSpanAttributes.TRACE_VERSION, + "trace_release": LangfuseSpanAttributes.TRACE_RELEASE, + "existing_trace_id": LangfuseSpanAttributes.EXISTING_TRACE_ID, + "update_trace_keys": LangfuseSpanAttributes.UPDATE_TRACE_KEYS, + "debug_langfuse": LangfuseSpanAttributes.DEBUG_LANGFUSE, + } + + for key, enum_attr in mapping.items(): + if key in metadata and metadata[key] is not None: + value = metadata[key] + # Lists / dicts must be stringified for OTEL + if isinstance(value, (list, dict)): + try: + value = json.dumps(value) + except Exception: + value = str(value) + safe_set_attribute(span, enum_attr.value, value) + @staticmethod def get_langfuse_otel_config() -> LangfuseOtelConfig: """ diff --git a/litellm/types/integrations/langfuse_otel.py b/litellm/types/integrations/langfuse_otel.py index d4b61304546..37eac1f54a9 100644 --- a/litellm/types/integrations/langfuse_otel.py +++ b/litellm/types/integrations/langfuse_otel.py @@ -13,5 +13,28 @@ class LangfuseOtelConfig(BaseModel): otlp_auth_headers: Optional[str] = None protocol: Protocol = "otlp_http" -class LangfuseSpanAttributes(Enum): - LANGFUSE_ENVIRONMENT = "langfuse.environment" \ No newline at end of file +class LangfuseSpanAttributes(str, Enum): + LANGFUSE_ENVIRONMENT = "langfuse.environment" + + # ---- Generation-level metadata ---- + GENERATION_NAME = "langfuse.generation.name" + GENERATION_ID = "langfuse.generation.id" + PARENT_OBSERVATION_ID = "langfuse.generation.parent_observation_id" + GENERATION_VERSION = "langfuse.generation.version" + MASK_INPUT = "langfuse.generation.mask_input" + MASK_OUTPUT = "langfuse.generation.mask_output" + + # ---- Trace-level metadata ---- + TRACE_USER_ID = "langfuse.trace.user_id" + SESSION_ID = "langfuse.trace.session_id" + TAGS = "langfuse.trace.tags" + TRACE_NAME = "langfuse.trace.name" + TRACE_ID = "langfuse.trace.id" + TRACE_METADATA = "langfuse.trace.metadata" + TRACE_VERSION = "langfuse.trace.version" + TRACE_RELEASE = "langfuse.trace.release" + EXISTING_TRACE_ID = "langfuse.trace.existing_id" + UPDATE_TRACE_KEYS = "langfuse.trace.update_keys" + + # ---- Misc / flags ---- + DEBUG_LANGFUSE = "langfuse.debug" \ No newline at end of file diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index cf33299f62e..20486a44928 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -1,5 +1,6 @@ import os from unittest.mock import MagicMock, patch +import json import pytest @@ -91,12 +92,100 @@ class TestLangfuseOtelIntegration: with patch('litellm.integrations.arize._utils.safe_set_attribute') as mock_safe_set_attribute: LangfuseOtelLogger._set_langfuse_specific_attributes(mock_span, mock_kwargs) + # safe_set_attribute(span, key, value) → positional args mock_safe_set_attribute.assert_called_once_with( - span=mock_span, - key="langfuse.environment", - value=test_env + mock_span, + "langfuse.environment", + test_env ) + def test_extract_langfuse_metadata_basic(self): + """Ensure metadata is correctly pulled from litellm_params.""" + metadata_in = {"generation_name": "my-gen", "custom": "data"} + kwargs = {"litellm_params": {"metadata": metadata_in}} + extracted = LangfuseOtelLogger._extract_langfuse_metadata(kwargs) + assert extracted == metadata_in + + def test_extract_langfuse_metadata_with_header_enrichment(self, monkeypatch): + """_extract_langfuse_metadata should call LangFuseLogger.add_metadata_from_header when available.""" + import sys, types + + # Build a stub module + class on-the-fly + stub_module = types.ModuleType("litellm.integrations.langfuse.langfuse") + class StubLFLogger: + @staticmethod + def add_metadata_from_header(litellm_params, metadata): + # Echo back existing metadata plus a marker + return {**metadata, "enriched": True} + stub_module.LangFuseLogger = StubLFLogger # type: ignore + + # Register stub in sys.modules so import inside method succeeds + sys.modules["litellm.integrations.langfuse.langfuse"] = stub_module # type: ignore + + kwargs = {"litellm_params": {"metadata": {"foo": "bar"}}} + extracted = LangfuseOtelLogger._extract_langfuse_metadata(kwargs) + assert extracted.get("foo") == "bar" + assert extracted.get("enriched") is True + + def test_set_langfuse_specific_attributes_full_mapping(self): + """Verify every supported metadata key maps to the correct OTEL attribute and complex types are JSON-serialised.""" + # Build a sample metadata payload covering all mappings + metadata = { + "generation_name": "gen-name", + "generation_id": "gen-id", + "parent_observation_id": "parent-id", + "version": "v1", + "mask_input": True, + "mask_output": False, + "trace_user_id": "user-123", + "session_id": "sess-456", + "tags": ["tagA", "tagB"], + "trace_name": "trace-name", + "trace_id": "trace-id", + "trace_metadata": {"k": "v"}, + "trace_version": "t-ver", + "trace_release": "rel-1", + "existing_trace_id": "existing-id", + "update_trace_keys": ["key1", "key2"], + "debug_langfuse": True, + } + kwargs = {"litellm_params": {"metadata": metadata}} + + # Capture calls to safe_set_attribute + with patch('litellm.integrations.arize._utils.safe_set_attribute') as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs) + + # Build expected calls manually for clarity + from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes + expected = { + LangfuseSpanAttributes.GENERATION_NAME.value: "gen-name", + LangfuseSpanAttributes.GENERATION_ID.value: "gen-id", + LangfuseSpanAttributes.PARENT_OBSERVATION_ID.value: "parent-id", + LangfuseSpanAttributes.GENERATION_VERSION.value: "v1", + LangfuseSpanAttributes.MASK_INPUT.value: True, + LangfuseSpanAttributes.MASK_OUTPUT.value: False, + LangfuseSpanAttributes.TRACE_USER_ID.value: "user-123", + LangfuseSpanAttributes.SESSION_ID.value: "sess-456", + # Lists / dicts should be JSON strings + LangfuseSpanAttributes.TAGS.value: json.dumps(["tagA", "tagB"]), + LangfuseSpanAttributes.TRACE_NAME.value: "trace-name", + LangfuseSpanAttributes.TRACE_ID.value: "trace-id", + LangfuseSpanAttributes.TRACE_METADATA.value: json.dumps({"k": "v"}), + LangfuseSpanAttributes.TRACE_VERSION.value: "t-ver", + LangfuseSpanAttributes.TRACE_RELEASE.value: "rel-1", + LangfuseSpanAttributes.EXISTING_TRACE_ID.value: "existing-id", + LangfuseSpanAttributes.UPDATE_TRACE_KEYS.value: json.dumps(["key1", "key2"]), + LangfuseSpanAttributes.DEBUG_LANGFUSE.value: True, + } + + # Flatten the actual calls into {key: value} + actual = { + call.args[1]: call.args[2] # (span, key, value) + for call in mock_safe_set_attribute.call_args_list + } + + assert actual == expected, "Mismatch between expected and actual OTEL attribute mapping." + if __name__ == "__main__": pytest.main([__file__]) \ No newline at end of file