mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
chore(langfuse_otel): drop the docstrings and comments the house rule disallows and type the new tests' parameters
This commit is contained in:
parent
3207920c46
commit
caf2ae25a2
6 changed files with 11 additions and 51 deletions
|
|
@ -44,7 +44,6 @@ LANGFUSE_CLOUD_EU_ENDPOINT: Final = "https://cloud.langfuse.com/api/public/otel"
|
|||
LANGFUSE_CLOUD_US_ENDPOINT: Final = "https://us.cloud.langfuse.com/api/public/otel"
|
||||
LANGFUSE_INGESTION_VERSION_HEADER: Final = "x-langfuse-ingestion-version"
|
||||
LANGFUSE_INGESTION_VERSION: Final = "4"
|
||||
# The attributes ``OpenTelemetry._record_exception_on_span`` adds to a failed call's span after these.
|
||||
_ERROR_ATTRIBUTES: Final = frozenset(
|
||||
{
|
||||
ErrorAttributes.ERROR_CODE,
|
||||
|
|
@ -77,13 +76,11 @@ def _preset_cache_key(kwargs: Mapping[str, object]) -> object:
|
|||
|
||||
|
||||
def _mask_flag(value: object) -> bool:
|
||||
"""A mask flag as the classic langfuse callback reads it: "true" / "false" strings parse, anything else by truthiness."""
|
||||
parsed: Final = str_to_bool(value) if isinstance(value, str) else None
|
||||
return bool(value) if parsed is None else parsed
|
||||
|
||||
|
||||
def _caller_tag_list(value: object) -> tuple[str, ...]:
|
||||
"""Caller tags as Langfuse ingestion reads a raw tags value: a list, a JSON array string, a comma list or one tag."""
|
||||
parsed: Final = (
|
||||
safe_json_loads(value, default=value) if isinstance(value, str) and value.strip().startswith("[") else value
|
||||
)
|
||||
|
|
@ -100,7 +97,6 @@ def _parent_of(span: object) -> trace.SpanContext | None:
|
|||
|
||||
|
||||
def _litellm_proxy_span(kwargs: Mapping[str, object]) -> trace.Span | None:
|
||||
"""The proxy request span another OTel callback opened for this request, when the request carries one."""
|
||||
litellm_params: Final = _as_gettable(kwargs.get("litellm_params")) or {}
|
||||
carried: Final = (
|
||||
(_as_gettable(litellm_params.get(key)) or {}).get("litellm_parent_otel_span")
|
||||
|
|
@ -117,12 +113,6 @@ def _litellm_proxy_span(kwargs: Mapping[str, object]) -> trace.Span | None:
|
|||
|
||||
|
||||
def _starts_its_own_trace(span: object, kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether the trace `span` belongs to was started by litellm rather than by the app calling it.
|
||||
|
||||
litellm writes onto the app's own span when one is active, and otherwise parents its span to an inbound
|
||||
traceparent, a caller-provided span or its proxy request span. The trace is litellm's only when its span has no
|
||||
parent, or hangs off a proxy request span that has none.
|
||||
"""
|
||||
if span is trace.get_current_span():
|
||||
return False
|
||||
parent: Final = _parent_of(span)
|
||||
|
|
@ -233,14 +223,12 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
|
||||
@staticmethod
|
||||
def _observation_output(response_obj: _Gettable | None) -> str | None:
|
||||
"""Serialized observation output, or None when the response yields nothing."""
|
||||
if not response_obj or not hasattr(response_obj, "get"):
|
||||
return None
|
||||
return _extract_output_items(response_obj) or _extract_choices_output(response_obj)
|
||||
|
||||
@staticmethod
|
||||
def _trace_tags(kwargs: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[str, ...]:
|
||||
"""Order-preserving dedupe of caller tags, request tags and langfuse_default_tags expansions."""
|
||||
caller_tags: Final = metadata.get("tags")
|
||||
request_tags: Final = (_as_gettable(kwargs.get("standard_logging_object")) or {}).get("request_tags")
|
||||
default_tags: Final = litellm.langfuse_default_tags
|
||||
|
|
@ -330,8 +318,6 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
input_json: str | None,
|
||||
output_json: str | None,
|
||||
) -> None:
|
||||
"""Set the derived trace name, tags, input and output into the room ``span``'s attribute limit leaves next
|
||||
to the error attributes a failed call adds after them, so they never evict or drop another attribute."""
|
||||
from litellm.integrations.otel.emitter import carried_keys, span_attribute_limit
|
||||
|
||||
tags: Final = LangfuseOtelLogger._trace_tags(kwargs, metadata)
|
||||
|
|
|
|||
|
|
@ -40,9 +40,6 @@ class LangfuseOpenTelemetryV2(OpenTelemetryV2):
|
|||
super().log_pre_api_call(model, messages, kwargs)
|
||||
|
||||
def _derives_trace_name(self, root: "Span", kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether this call may name the request root after its call type: the call is not one litellm made for
|
||||
its own features, this logger's provider started the root with no parent, and neither the caller nor an
|
||||
earlier call in the request has named it."""
|
||||
return (
|
||||
internal_call_origin(None, kwargs) is None
|
||||
and isinstance(root, ReadableSpan)
|
||||
|
|
|
|||
|
|
@ -41,7 +41,6 @@ class SpanRecord:
|
|||
dropped_attributes: int = 0
|
||||
|
||||
def tags(self) -> tuple[object, ...]:
|
||||
"""The trace tags Langfuse ingestion reads from the raw attribute."""
|
||||
raw: Final = self.attributes.get("langfuse.trace.tags")
|
||||
if isinstance(raw, tuple):
|
||||
return raw
|
||||
|
|
@ -79,9 +78,6 @@ def _span_records(body: bytes) -> tuple[SpanRecord, ...]:
|
|||
|
||||
|
||||
class LangfuseSink:
|
||||
"""The Langfuse OTLP endpoint: keeps every accepted generation and root span, or answers 503 during a scripted
|
||||
outage."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.lock: Final = threading.Lock()
|
||||
self.outage: Final = threading.Event()
|
||||
|
|
@ -128,8 +124,6 @@ class LangfuseSink:
|
|||
|
||||
|
||||
class OtlpSink:
|
||||
"""A generic OTLP endpoint: keeps every span it receives."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.lock: Final = threading.Lock()
|
||||
self.spans: Final[list[SpanRecord]] = [] # mutable-ok: spans accumulate as the exporter delivers them
|
||||
|
|
@ -142,7 +136,6 @@ class OtlpSink:
|
|||
return Reply(body=b"")
|
||||
|
||||
def root_of_request(self, marker: str) -> SpanRecord:
|
||||
"""The root span of the one trace whose spans carry the marker."""
|
||||
|
||||
def roots() -> tuple[SpanRecord, ...]:
|
||||
with self.lock:
|
||||
|
|
@ -167,7 +160,6 @@ def _chat_completion(marker: str, text: str = "langfuse-otel scripted reply") ->
|
|||
|
||||
|
||||
def _provider(request: Request) -> Reply:
|
||||
"""Scripted provider: the three surfaces echo the caller marker, a `fail-` marker answers 500, embeddings succeed."""
|
||||
if request.method != "POST" or not request.body:
|
||||
return Reply(body=b'{"object": "list", "data": []}')
|
||||
body: Final = json.loads(request.body)
|
||||
|
|
@ -192,8 +184,6 @@ def _provider(request: Request) -> Reply:
|
|||
|
||||
|
||||
def _langfuse_config(directory: Path, name: str, settings: Mapping[str, object], provider_url: str) -> Path:
|
||||
"""The integration config with `langfuse_otel`, plus an opt-in post_call `llm_as_a_judge` guardrail whose judge
|
||||
model is served by the scripted provider."""
|
||||
base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
judge: Final = {
|
||||
"model_name": JUDGE_MODEL,
|
||||
|
|
@ -300,8 +290,6 @@ class SharedRig:
|
|||
|
||||
@pytest.fixture(scope="module")
|
||||
def v2_after_otel(tmp_path_factory: pytest.TempPathFactory) -> Iterator[SharedRig]:
|
||||
"""OTel v2 with the generic `otel` callback listed before `langfuse_otel`, so the request root belongs to the
|
||||
generic exporter."""
|
||||
otel: Final = OtlpSink()
|
||||
with wire_server(otel.respond) as destination:
|
||||
environment: Final = {
|
||||
|
|
@ -339,7 +327,6 @@ def _chat(
|
|||
|
||||
|
||||
def _trace_fields(spans: tuple[SpanRecord, ...]) -> Iterator[str]:
|
||||
"""Every trace-level key any of the spans carries."""
|
||||
for record in spans:
|
||||
yield from (key for key in record.attributes if key in TRACE_FIELDS)
|
||||
|
||||
|
|
@ -654,7 +641,6 @@ def _chat_with_tools(
|
|||
@pytest.mark.parametrize(("tools", "prefix"), product((30, 33, 34, 35, 36, 40), ("", "fail-")))
|
||||
def test_trace_fields_never_evict_or_drop_another_attribute_of_a_full_span(v1: Rig, tools: int, prefix: str) -> None:
|
||||
run: Final = uuid.uuid4().hex
|
||||
# Both requests carry a caller trace id, so the joined one's traceparent adds no key the derived one lacks.
|
||||
derived: Final = _chat_with_tools(v1, f"{prefix}lit8281-full-{tools}-{run}", tools, {"trace_id": uuid.uuid4().hex})
|
||||
joined: Final = _chat_with_tools(
|
||||
v1,
|
||||
|
|
|
|||
|
|
@ -165,8 +165,6 @@ class TraceSpan:
|
|||
|
||||
|
||||
class TraceSink:
|
||||
"""The Langfuse OTLP endpoint for both the app's own exporter and litellm's: keeps every span it receives."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.lock: Final = threading.Lock()
|
||||
self.spans: Final[list[TraceSpan]] = [] # mutable-ok: spans accumulate as both exporters deliver
|
||||
|
|
@ -183,7 +181,6 @@ class TraceSink:
|
|||
return Reply(body=b"")
|
||||
|
||||
def traces(self, marker: str, calls: tuple[str, ...]) -> Mapping[str, tuple[TraceSpan, ...]]:
|
||||
"""Every span of each app trace, keyed by call, once the app root and a litellm generation both landed."""
|
||||
with self.lock:
|
||||
snapshot: Final = tuple(self.spans)
|
||||
roots: Final = {
|
||||
|
|
@ -205,7 +202,6 @@ class TraceSink:
|
|||
|
||||
|
||||
def _trace_values(spans: tuple[TraceSpan, ...], keys: tuple[str, ...]) -> Iterator[tuple[str, str]]:
|
||||
"""Every (key, value) pair of the given trace-level keys that any span of the trace carries."""
|
||||
for span in spans:
|
||||
yield from ((key, span.attributes[key]) for key in keys if key in span.attributes)
|
||||
|
||||
|
|
@ -351,12 +347,12 @@ def test_sdk_call_inside_an_app_trace_leaves_the_app_trace_name_and_io_alone(
|
|||
output: Final = child.communicate(input=b"", timeout=60)[0].decode()
|
||||
assert sorted(call for call, spans in traces.items() if _logged(spans)) == sorted(calls), output
|
||||
for call, spans in traces.items():
|
||||
app: Final = {
|
||||
app = {
|
||||
"langfuse.trace.name": f"checkout-{marker}-{call}",
|
||||
"langfuse.trace.input": json.dumps({"cart": f"{marker}-{call}"}),
|
||||
"langfuse.trace.output": json.dumps({"order": f"{marker}-{call}"}),
|
||||
}
|
||||
root: Final = next(span for span in spans if not span.parent_span_id)
|
||||
root = next(span for span in spans if not span.parent_span_id)
|
||||
assert {key: root.attributes.get(key) for key in app} == app, (call, root.attributes)
|
||||
written: Final = set(_trace_values(spans, (*app, "langfuse.trace.tags")))
|
||||
written = set(_trace_values(spans, (*app, "langfuse.trace.tags")))
|
||||
assert written == set(app.items()), (call, written)
|
||||
|
|
|
|||
|
|
@ -66,8 +66,8 @@ def _logger(*, capture: str = "span_only", mappers: Sequence[str] = ("genai", "l
|
|||
return build_otel_v2_logger(config=cfg, tracer_provider=tracer_provider), exporter
|
||||
|
||||
|
||||
def _start_root(logger, parent_context: Context | None = None):
|
||||
root = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME, parent_context)
|
||||
def _start_root(logger: OpenTelemetryV2, parent_context: Context | None = None):
|
||||
root: Final = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME, parent_context)
|
||||
set_request_root_span(root)
|
||||
return root
|
||||
|
||||
|
|
@ -466,14 +466,16 @@ def _root_trace_name_after(
|
|||
|
||||
@pytest.mark.parametrize("later", [JUDGE_PARAMS, UNNAMED], ids=["litellm-sub-call", "untagged-call"])
|
||||
@pytest.mark.parametrize("caller", CALLER_NAMES, ids=["body", "header"])
|
||||
def test_a_later_call_in_the_request_keeps_the_caller_trace_name_on_the_root(caller, later):
|
||||
def test_a_later_call_in_the_request_keeps_the_caller_trace_name_on_the_root(
|
||||
caller: Mapping[str, object], later: Mapping[str, object]
|
||||
) -> None:
|
||||
logger, _ = _logger()
|
||||
|
||||
assert _root_trace_name_after(logger, (("acompletion", caller), ("acompletion", later))) == "from-caller"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("caller", CALLER_NAMES, ids=["body", "header"])
|
||||
def test_a_caller_trace_name_replaces_the_name_an_earlier_call_derived(caller):
|
||||
def test_a_caller_trace_name_replaces_the_name_an_earlier_call_derived(caller: Mapping[str, object]) -> None:
|
||||
logger, _ = _logger()
|
||||
|
||||
assert _root_trace_name_after(logger, (("acompletion", JUDGE_PARAMS), ("acompletion", caller))) == "from-caller"
|
||||
|
|
@ -505,7 +507,7 @@ def test_a_root_another_vendor_started_gets_no_derived_trace_name():
|
|||
|
||||
|
||||
@pytest.mark.parametrize("caller", CALLER_NAMES, ids=["body", "header"])
|
||||
def test_a_root_another_vendor_started_still_gets_the_caller_trace_name(caller):
|
||||
def test_a_root_another_vendor_started_still_gets_the_caller_trace_name(caller: Mapping[str, object]) -> None:
|
||||
logger, _ = _logger()
|
||||
other_vendor, _ = _logger(mappers=("genai",))
|
||||
|
||||
|
|
|
|||
|
|
@ -941,7 +941,7 @@ class TestLangfuseOtelResponsesAPI:
|
|||
output_json = output_calls[0].args[2]
|
||||
output_data = json.loads(output_json)
|
||||
|
||||
# Verify output contains both reasoning summaries and the message
|
||||
# Verify output contains reasoning and message
|
||||
assert isinstance(output_data, list)
|
||||
assert len(output_data) == 3
|
||||
|
||||
|
|
@ -1324,8 +1324,6 @@ def _carrying(parent: Span, key: str = "metadata") -> Mapping[str, object]:
|
|||
|
||||
|
||||
class TestDerivedTraceFieldsOnlyOnTracesLitellmStarts:
|
||||
"""litellm derives a trace's name, io and tags only when its own span opened the trace."""
|
||||
|
||||
def test_a_span_under_an_inbound_traceparent_joins_the_caller_trace(self) -> None:
|
||||
span: Final = _TRACER.start_span(
|
||||
"litellm_request", context=trace.set_span_in_context(trace.NonRecordingSpan(CALLER))
|
||||
|
|
@ -1422,8 +1420,6 @@ class ExportedSpan:
|
|||
|
||||
|
||||
def _exported_full_span(filler: int, *, joins_caller: bool, failed: bool) -> ExportedSpan:
|
||||
"""A span already holding ``filler`` attributes, after langfuse_otel stamps a chat call and, when ``failed``, the
|
||||
error attributes a failed call adds."""
|
||||
span: Final = _TRACER.start_span(
|
||||
"litellm_request",
|
||||
context=trace.set_span_in_context(trace.NonRecordingSpan(CALLER)) if joins_caller else Context(),
|
||||
|
|
@ -1438,9 +1434,6 @@ def _exported_full_span(filler: int, *, joins_caller: bool, failed: bool) -> Exp
|
|||
|
||||
|
||||
class TestDerivedTraceFieldsAtTheSpanAttributeLimit:
|
||||
"""The derived trace fields take only room a span has to spare, so a full span exports what it would without
|
||||
them."""
|
||||
|
||||
@pytest.mark.parametrize("failed", [False, True])
|
||||
@pytest.mark.parametrize("filler", range(SPAN_ATTRIBUTE_LIMIT - 30, SPAN_ATTRIBUTE_LIMIT + 1))
|
||||
def test_trace_fields_never_evict_or_drop_another_attribute(self, filler: int, failed: bool) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue