From ae5f4ce2e49cf5030f7e4b789d4f27c19e6e744c Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 1 Oct 2026 23:58:20 +0000 Subject: [PATCH] test(otel): add skipped BUG cells for pre-existing Arize OTel v2 gaps Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...test_arize_otel_v2_openinference_family.py | 32 +++++++++++ .../test_arize_otel_v2_openinference_modes.py | 53 +++++++++++++++++++ ...st_arize_otel_v2_openinference_sad_edge.py | 35 ++++++++++++ .../test_arize_otel_v2_openinference_spans.py | 40 ++++++++++++++ 4 files changed, 160 insertions(+) diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_family.py b/tests/integration/observability/test_arize_otel_v2_openinference_family.py index 5d51db0b5b0..242b7bfe8c1 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_family.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_family.py @@ -307,6 +307,38 @@ def test_arize_otel_v2_b3_non_openinference_callback_family(callback: str, gatew assert baggage == expected_baggage, attributes +def test_arize_otel_v2_b3_langfuse_carries_metadata_baggage(gateway: Gateway, tmp_path: Path) -> None: + pytest.skip("BUG: LIT-9128 Langfuse OTel v2 preset omits request-metadata baggage") + marker: Final = "b3-langfuse-" + uuid.uuid4().hex + + def upstream(request: Request) -> Reply: + body: Final = _json_object(request.body) + assert body == { + "messages": [{"role": "user", "content": "weather in Paris?"}], + "model": "gpt-4o-mini", + "tool_choice": {"type": "function", "function": {"name": "lookup_weather"}}, + "tools": CHAT_TOOLS, + }, body + return _chat_response(marker) + + callback_settings: Final[dict[str, JsonValue]] = { + "otel": {"exporter": "http/protobuf", "endpoint": "unused", "mapper_names": ["genai"]} + } + with _rig( + gateway, + tmp_path, + upstream, + callbacks=("langfuse_otel",), + callback_settings=callback_settings, + environment={"LANGFUSE_HOST": "http://127.0.0.1"}, + ) as rig: + response: Final = _request(rig.proxy, rig.model, marker) + assert response.status_code == 200, response.text + assert _json_object(response.content) == _chat_caller_response(_chat_response(marker), rig.model), response.text + attributes: Final = _matching_genai_marker_span(rig.destination, marker) + assert attributes["litellm.metadata.trace_marker"] == marker, attributes + + def test_arize_otel_v2_b4_legacy_otel_is_unchanged(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "b4-" + uuid.uuid4().hex diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_modes.py b/tests/integration/observability/test_arize_otel_v2_openinference_modes.py index d7753b80637..7d0f3f31787 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_modes.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_modes.py @@ -16,6 +16,7 @@ from _openinference_support import ( _chat_response, _json_messages, _json_object, + _llm_spans_through_markers, _matching_marker_span, _matching_output_value_span, _matching_span, @@ -216,6 +217,24 @@ def test_arize_otel_v2_c5_yaml_allowlist_does_not_reach_preset_so_default_applie _assert_default_allowlist_attributes(attributes, marker) +def test_arize_otel_v2_c5_yaml_allowlist_reaches_preset(gateway: Gateway, tmp_path: Path) -> None: + pytest.skip("BUG: LIT-9124 arize preset ignores callback_settings.otel.baggage_metadata_keys from config.yaml") + marker: Final = "c5-allowlist-" + uuid.uuid4().hex + with _rig( + gateway, + tmp_path, + _upstream(marker), + callback_settings={"otel": {"baggage_metadata_keys": ["requester_metadata.trace_marker"]}}, + remove_environment=("LITELLM_OTEL_BAGGAGE_METADATA_KEYS",), + disabled_environment=("LITELLM_OTEL_BAGGAGE_METADATA_KEYS",), + ) as rig: + response: Final = _request(rig.proxy, rig.model, marker) + _assert_success_body(response, marker, rig.model) + attributes: Final = _matching_marker_span(rig.destination, marker) + assert _json_object(attributes["metadata"].encode()) == {"trace_marker": marker}, attributes + assert attributes["litellm.metadata.trace_marker"] == marker, attributes + + def test_arize_otel_v2_c6_content_capture_disabled(gateway: Gateway, tmp_path: Path) -> None: marker: Final = "c6-" + uuid.uuid4().hex with _rig( @@ -302,6 +321,40 @@ def test_arize_otel_v2_c8_request_callback_disable(gateway: Gateway, tmp_path: P _assert_success_body(disabled_response, disabled_marker, rig.model) +def test_arize_otel_v2_c8_disabled_callback_exports_no_span(gateway: Gateway, tmp_path: Path) -> None: + pytest.skip("BUG: LIT-9049 x-litellm-disable-callbacks: arize still exports the OTel v2 span") + disabled_marker: Final = "c8-disabled-" + uuid.uuid4().hex + sentinel: Final = "c8-sentinel-" + uuid.uuid4().hex + + def upstream(request: Request) -> Reply: + marker: Final = _chat_request_marker(request) + assert marker in (disabled_marker, sentinel), request + _assert_chat_request(request, messages=[{"role": "user", "content": marker}]) + return _chat_response(marker) + + with _rig( + gateway, + tmp_path, + upstream, + litellm_settings={"allow_dynamic_callback_disabling": True}, + workers=1, + ) as rig: + disabled_response: Final = _request( + rig.proxy, + rig.model, + disabled_marker, + prompt=disabled_marker, + headers={"x-litellm-disable-callbacks": "arize"}, + ) + _assert_success_body(disabled_response, disabled_marker, rig.model) + sentinel_response: Final = _request(rig.proxy, rig.model, sentinel, prompt=sentinel) + _assert_success_body(sentinel_response, sentinel, rig.model) + spans: Final = _llm_spans_through_markers(rig.destination, (sentinel,)) + assert not any(attributes.get("litellm.metadata.trace_marker") == disabled_marker for attributes in spans), ( + spans + ) + + @pytest.mark.parametrize("failure_status", (401, 500)) def test_arize_otel_v2_c9_upstream_failures_are_recorded(failure_status: int, gateway: Gateway, tmp_path: Path) -> None: marker: Final = "c9-" + uuid.uuid4().hex diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py b/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py index beaba0857a1..a05b3f0f045 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py @@ -475,6 +475,41 @@ def test_arize_otel_v2_d3_malformed_tool_calls_are_normalized(shape: str, gatewa assert "Exception while exporting Span batch" not in proxy_log.read_text(), proxy_log.read_text() +def test_arize_otel_v2_d3_non_dict_tool_call_is_not_a_caller_error(gateway: Gateway, tmp_path: Path) -> None: + pytest.skip("BUG: LIT-9125 non-dict tool_calls entry returns HTTP 400 with a server traceback") + marker: Final = "d3-non-dict-call-" + uuid.uuid4().hex + + def upstream(request: Request) -> Reply: + _assert_chat_request(request, messages=[{"role": "user", "content": "weather in Paris?"}]) + return Reply( + body=json.dumps( + { + "id": marker, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": {"role": "assistant", "content": None, "tool_calls": ["not-a-call"]}, + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}, + } + ).encode() + ) + + with _rig(gateway, tmp_path, upstream) as rig: + response: Final = _call(rig.proxy, rig.model, marker) + assert response.status_code not in range(400, 500), response.text + assert "Traceback" not in response.text, response.text + assert "AttributeError" not in response.text, response.text + readiness: Final = rig.proxy.client.get("/health/readiness") + assert readiness.status_code == 200, readiness.text + assert _json_object(readiness.content) == {"status": "healthy", "db": "connected"}, readiness.text + + @pytest.mark.parametrize("shape", ("empty", "null", "missing")) def test_arize_otel_v2_e3_empty_or_missing_tool_calls_never_indexed( shape: str, gateway: Gateway, tmp_path: Path diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_spans.py b/tests/integration/observability/test_arize_otel_v2_openinference_spans.py index a7a19333d88..d5b4dbeacea 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_spans.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_spans.py @@ -924,3 +924,43 @@ def test_arize_otel_v2_a_cache(surface: str, gateway: Gateway, tmp_path: Path) - assert len(sentinel_spans) == 1, spans _assert_tool_span_for_marker(sentinel_spans[0], sentinel) assert not any(attributes.get("litellm.metadata.trace_marker") == marker for attributes in spans), spans + + +def test_arize_otel_v2_cache_hit_exports_llm_span(gateway: Gateway, tmp_path: Path) -> None: + pytest.skip("BUG: LIT-9127 response-cache hits emit no OTel v2 LLM span") + marker: Final = "a-cache-hit-" + uuid.uuid4().hex + sentinel: Final = f"{marker}-sentinel" + + def upstream(request: Request) -> Reply: + request_marker: Final = _chat_request_marker(request) + assert request_marker in (marker, sentinel), request + _assert_chat_request(request, messages=[{"role": "user", "content": request_marker}]) + return _chat_response(request_marker) + + with _rig(gateway, tmp_path, upstream, workers=1) as rig: + first: Final = _cache_call(rig, "chat", marker) + assert first[0] == (f"call_{marker}", "lookup_weather", '{"city": "Paris"}'), first + assert first[1] == marker, first + assert not first[2].get("x-litellm-cache-key"), first[2] + second: Final = _cache_call(rig, "chat", marker, cache_hit=True) + assert second[0] == first[0], second + assert second[1] == first[1], second + assert second[2].get("x-litellm-cache-key"), second[2] + forwarded: Final = tuple( + request for request in rig.provider.drain() if request.method == "POST" and marker.encode() in request.body + ) + assert len(forwarded) == 1, forwarded + sentinel_response: Final = _cache_call(rig, "chat", sentinel) + assert not sentinel_response[2].get("x-litellm-cache-key"), sentinel_response[2] + sentinel_forwarded: Final = tuple( + request + for request in rig.provider.drain() + if request.method == "POST" and sentinel.encode() in request.body + ) + assert len(sentinel_forwarded) == 1, sentinel_forwarded + spans: Final = _llm_spans_through_markers(rig.destination, (sentinel,)) + marker_spans: Final = tuple( + attributes for attributes in spans if attributes.get("litellm.metadata.trace_marker") == marker + ) + assert len(marker_spans) == 2, spans + _assert_tool_span_for_marker(marker_spans[1], marker)