{t("teams.loggingSettings.langfuseSpanScopeHint")}
+ )} {renderParamControl(config, configIndex, paramName, { type: paramType, options: paramType === "select" ? paramOptions[paramName] || [] : [], diff --git a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json index b52a8f6e4a8..5f9d04881e8 100644 --- a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json @@ -17339,6 +17339,7 @@ "number": "Number" }, "numberHint": "Value must be between 0 and 1", + "langfuseSpanScopeHint": "Leave empty to use the proxy default (LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE, full if unset).", "disabledCallbacks": { "title": "Disabled Callbacks", "tooltip": "Select callbacks to disable for this key. Disabled callbacks will not receive any logging data.", diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts index 311b7297f4d..855aac16f03 100644 --- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -18844,7 +18844,10 @@ export interface paths { * - langfuse_secret: The secret for the Langfuse callback * - langfuse_host: The host for the Langfuse callback * - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT) - * - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans + * - langfuse_span_scope: For langfuse_otel, "full" sends the whole request trace, + * "llm_only" sends only model-call spans. + * Omitted follows LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE (default "full"), also configurable as + * callback_settings.otel.langfuse_destination_span_scope. * - capture_message_content: For OTel v2 callbacks, "no_content" or "span_only". * An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting. * "span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out diff --git a/tests/integration/observability/test_langfuse_destination_span_scope.py b/tests/integration/observability/test_langfuse_destination_span_scope.py new file mode 100644 index 00000000000..e393828fd19 --- /dev/null +++ b/tests/integration/observability/test_langfuse_destination_span_scope.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from typing import Final, Literal + +import pytest +from integration._support.client import Gateway, gateway_from_environment, held +from integration._support.otlp_sink import ( + AuditConfigWriter, + SpanSinks, + langfuse_callback_variables, + recorded_spans, + spans_for_trace, +) +from integration._support.process import owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + + +def _chat(request: Request) -> Reply: + assert request.target.endswith("/chat/completions"), request.target + assert b'"messages"' in request.body, request.body + return Reply( + body=( + b'{"id":"chatcmpl-scope","object":"chat.completion","created":1,"model":"gpt-4o-mini",' + b'"choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],' + b'"usage":{"prompt_tokens":7,"completion_tokens":2,"total_tokens":9}}' + ) + ) + + +@pytest.fixture(scope="module") +def provider() -> Iterator[Wire]: + with wire_server(_chat) as wire: + yield wire + + +@pytest.fixture(scope="module", params=["llm_only", None, "full"], ids=["proxy-llm-only", "proxy-unset", "proxy-full"]) +def proxy( + request: pytest.FixtureRequest, + audit_sinks: SpanSinks, + otel_audit_config: AuditConfigWriter, + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[tuple[Gateway, bool]]: + directory: Final = tmp_path_factory.mktemp("langfuse-destination-scope") + narrowed: Final = request.param == "llm_only" + default: Final = "llm_only" if narrowed else "full" + with ( + gateway_from_environment() as gateway, + owned_proxy_process( + gateway, + directory, + { + "LITELLM_OTEL_V2": "true", + "ARIZE_HTTP_ENDPOINT": audit_sinks.arize + "/v1/traces", + **({"LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE": default} if request.param is not None else {}), + }, + config=otel_audit_config(directory, {}), + remove_environment=( + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LANGFUSE_HOST", + "LITELLM_OTEL_LANGFUSE_SPAN_SCOPE", + *(("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE",) if request.param is None else ()), + ), + workers=2, + ) as owned, + ): + yield owned.gateway, narrowed + + +@pytest.mark.parametrize( + ("level", "scope", "backend"), + [ + ("key", None, "langfuse_otel"), + ("key", "full", "langfuse_otel"), + ("team", None, "langfuse_otel"), + ("key", "llm_only", "langfuse_otel"), + ("key", None, "arize"), + ], + ids=["key-default", "key-full", "team-default", "key-llm-only", "arize-default"], +) +def test_proxy_default_limits_only_destinations_without_an_explicit_scope( + level: Literal["key", "team"], + scope: str | None, + backend: Literal["langfuse_otel", "arize"], + proxy: tuple[Gateway, bool], + provider: Wire, + audit_sinks: SpanSinks, +) -> None: + gateway, narrowed = proxy + logging: Final[list[JsonValue]] = [ + { + "callback_name": backend, + "callback_type": "success", + "callback_vars": { + **( + langfuse_callback_variables(audit_sinks) + if backend == "langfuse_otel" + else {"arize_api_key": "arize-test-key", "arize_space_id": "arize-test-space"} + ), + **({"langfuse_span_scope": scope} if scope else {}), + }, + } + ] + sink: Final = audit_sinks.tenant if backend == "langfuse_otel" else audit_sinks.arize + with gateway.scenario() as scenario: + model: Final = scenario.model(api_base=provider.url + "/v1") + key: Final = ( + scenario.key(metadata={"logging": logging}) + if level == "key" + else scenario.key(team_id=scenario.team(metadata={"logging": logging})) + ) + since: Final = recorded_spans(sink)[0] + trace_id: Final = uuid.uuid4().hex + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "destination span scope"}]}, + key=key, + headers={"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"}, + ) + assert response.status_code == 200, response.text + spans: Final = held( + lambda: spans_for_trace(recorded_spans(sink, since)[1], trace_id), + lambda found: any("gen_ai.operation.name" in span["attributes"] for span in found), + holding=3, + seconds=40, + ) + if backend == "langfuse_otel" and (scope == "llm_only" or (narrowed and scope is None)): + assert len(spans) == 1, [(span["name"], span["kind"]) for span in spans] + assert "gen_ai.operation.name" in spans[0]["attributes"], spans + assert not any(span["kind"] == 2 or span["attributes"].get("db.system") == "postgresql" for span in spans) + return + assert any(span["kind"] == 2 for span in spans), [(span["name"], span["kind"]) for span in spans] + assert any("gen_ai.operation.name" in span["attributes"] for span in spans), spans + assert any(span["attributes"].get("db.system") == "postgresql" for span in spans), spans diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 60635c5ce45..5dff8f6a048 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -41,7 +41,9 @@ from litellm.integrations.otel.model.config import ( ExporterOwner, ExporterSpec, OpenTelemetryV2Config, + _cached_langfuse_destination_span_scope, is_otel_v2_enabled, + langfuse_destination_span_scope, parse_excluded_services, ) from litellm.integrations.otel.model.destination import OtelDestination @@ -115,6 +117,7 @@ def isolate_published_provider(monkeypatch): def forget_otel_v2_flag_after_each_test(): yield is_otel_v2_enabled.cache_clear() + _cached_langfuse_destination_span_scope.cache_clear() def in_fresh_context(fn, *args): @@ -3035,6 +3038,144 @@ class TestSpanScope: assert destination_for("langfuse_otel", creds).span_scope == "full" + @pytest.mark.parametrize("level", ["key", "team"]) + @pytest.mark.parametrize("source", ["env", "callback_settings"]) + @pytest.mark.parametrize( + ("default", "explicit", "expected"), + [("llm_only", None, "llm_only"), ("llm_only", "full", "full"), (None, None, "full")], + ) + def test_proxy_destination_scope_falls_back_without_overriding_tenant_choices( + self, + monkeypatch: pytest.MonkeyPatch, + allow_test_hosts: None, + level: str, + default: str | None, + explicit: str | None, + expected: str, + source: str, + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_SPAN_SCOPE", "llm_only") + monkeypatch.setattr(litellm, "callback_settings", {}) + monkeypatch.delenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", raising=False) + if default is not None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", default if source == "env" else "full") + if source == "callback_settings": + monkeypatch.setattr( + litellm, "callback_settings", {"otel": {"langfuse_destination_span_scope": default}} + ) + is_otel_v2_enabled.cache_clear() + metadata: Final = { + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "langfuse_host": "http://x", + **({"langfuse_span_scope": explicit} if explicit is not None else {}), + }, + } + ] + } + auth: Final = UserAPIKeyAuth(**({"metadata": metadata} if level == "key" else {"team_metadata": metadata})) + destinations: Final = resolve_tenant_otel_destinations(auth) + assert [destination.span_scope for destination in destinations] == [expected] + + @pytest.mark.parametrize("otel_settings", [None, "invalid", [], 42]) + @pytest.mark.parametrize("default", [None, "llm_only"]) + def test_non_mapping_otel_settings_preserve_key_destination( + self, + monkeypatch: pytest.MonkeyPatch, + allow_test_hosts: None, + otel_settings: object, + default: str | None, + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setattr(litellm, "callback_settings", {"otel": otel_settings}) + monkeypatch.delenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", raising=False) + if default is not None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", default) + is_otel_v2_enabled.cache_clear() + auth: Final = UserAPIKeyAuth( + metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "langfuse_host": "http://x", + }, + } + ] + } + ) + + assert [destination.span_scope for destination in resolve_tenant_otel_destinations(auth)] == [default or "full"] + + def test_callback_settings_destination_scope_wins_over_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", "full") + assert langfuse_destination_span_scope(" LLM_ONLY\n") == "llm_only" + + @pytest.mark.parametrize( + ("bare", "prefixed", "configured", "expected"), + [ + ("llm_only", None, None, "full"), + ("full", "llm_only", None, "llm_only"), + ("full", "full", "llm_only", "llm_only"), + ("llm_only", "llm_only", "full", "full"), + ], + ) + def test_only_prefixed_environment_sets_destination_scope( + self, + monkeypatch: pytest.MonkeyPatch, + bare: str, + prefixed: str | None, + configured: str | None, + expected: str, + ) -> None: + monkeypatch.setenv("langfuse_destination_span_scope", bare) + if prefixed is None: + monkeypatch.delenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", raising=False) + else: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", prefixed) + + assert langfuse_destination_span_scope(configured) == expected + + @pytest.mark.parametrize("spelling", ["LLM_ONLY", " llm_only\n", "full"]) + def test_destination_scope_config_normalizes_like_operator_scope( + self, + monkeypatch: pytest.MonkeyPatch, + spelling: str, + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", spelling) + assert langfuse_destination_span_scope() == spelling.strip().lower() + + @pytest.mark.parametrize("invalid", ["", "everything"]) + @pytest.mark.parametrize("source", ["env", "callback_settings"]) + def test_invalid_destination_scope_falls_back_and_warns_once( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, invalid: str, source: str + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", invalid if source == "env" else "full") + with caplog.at_level("WARNING", logger="LiteLLM"): + assert langfuse_destination_span_scope(invalid if source == "callback_settings" else None) == "full" + assert langfuse_destination_span_scope(invalid if source == "callback_settings" else None) == "full" + warnings: Final = [record.message for record in caplog.records if "DESTINATION_SPAN_SCOPE" in record.message] + assert len(warnings) == 1 + assert repr(invalid) in warnings[0] + assert "'full'" in warnings[0] and "'llm_only'" in warnings[0] + + def test_destination_scope_environment_is_a_process_snapshot(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", "llm_only") + assert langfuse_destination_span_scope() == "llm_only" + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", "full") + assert langfuse_destination_span_scope() == "llm_only" + assert langfuse_destination_span_scope("full") == "full" + assert langfuse_destination_span_scope("llm_only") == "llm_only" + def test_only_langfuse_honours_the_scope_var(self): arize = destination_for( "arize", {"arize_api_key": "k", "arize_space_id": "s", "langfuse_span_scope": "llm_only"}