test(otel): add skipped BUG cells for pre-existing Arize OTel v2 gaps

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

tests/integration/observability/test_arize_otel_v2_openinference_family.py

@@ -307,6 +307,38 @@ def test_arize_otel_v2_b3_non_openinference_callback_family(callback: str, gatew
         assert baggage == expected_baggage, attributes
 
 
+def test_arize_otel_v2_b3_langfuse_carries_metadata_baggage(gateway: Gateway, tmp_path: Path) -> None:
+    pytest.skip("BUG: LIT-9128 Langfuse OTel v2 preset omits request-metadata baggage")
+    marker: Final = "b3-langfuse-" + uuid.uuid4().hex
+
+    def upstream(request: Request) -> Reply:
+        body: Final = _json_object(request.body)
+        assert body == {
+            "messages": [{"role": "user", "content": "weather in Paris?"}],
+            "model": "gpt-4o-mini",
+            "tool_choice": {"type": "function", "function": {"name": "lookup_weather"}},
+            "tools": CHAT_TOOLS,
+        }, body
+        return _chat_response(marker)
+
+    callback_settings: Final[dict[str, JsonValue]] = {
+        "otel": {"exporter": "http/protobuf", "endpoint": "unused", "mapper_names": ["genai"]}
+    }
+    with _rig(
+        gateway,
+        tmp_path,
+        upstream,
+        callbacks=("langfuse_otel",),
+        callback_settings=callback_settings,
+        environment={"LANGFUSE_HOST": "http://127.0.0.1"},
+    ) as rig:
+        response: Final = _request(rig.proxy, rig.model, marker)
+        assert response.status_code == 200, response.text
+        assert _json_object(response.content) == _chat_caller_response(_chat_response(marker), rig.model), response.text
+        attributes: Final = _matching_genai_marker_span(rig.destination, marker)
+        assert attributes["litellm.metadata.trace_marker"] == marker, attributes
+
+
 def test_arize_otel_v2_b4_legacy_otel_is_unchanged(gateway: Gateway, tmp_path: Path) -> None:
     marker: Final = "b4-" + uuid.uuid4().hex
 

tests/integration/observability/test_arize_otel_v2_openinference_modes.py

@@ -16,6 +16,7 @@ from _openinference_support import (
     _chat_response,
     _json_messages,
     _json_object,
+    _llm_spans_through_markers,
     _matching_marker_span,
     _matching_output_value_span,
     _matching_span,
@@ -216,6 +217,24 @@ def test_arize_otel_v2_c5_yaml_allowlist_does_not_reach_preset_so_default_applie
         _assert_default_allowlist_attributes(attributes, marker)
 
 
+def test_arize_otel_v2_c5_yaml_allowlist_reaches_preset(gateway: Gateway, tmp_path: Path) -> None:
+    pytest.skip("BUG: LIT-9124 arize preset ignores callback_settings.otel.baggage_metadata_keys from config.yaml")
+    marker: Final = "c5-allowlist-" + uuid.uuid4().hex
+    with _rig(
+        gateway,
+        tmp_path,
+        _upstream(marker),
+        callback_settings={"otel": {"baggage_metadata_keys": ["requester_metadata.trace_marker"]}},
+        remove_environment=("LITELLM_OTEL_BAGGAGE_METADATA_KEYS",),
+        disabled_environment=("LITELLM_OTEL_BAGGAGE_METADATA_KEYS",),
+    ) as rig:
+        response: Final = _request(rig.proxy, rig.model, marker)
+        _assert_success_body(response, marker, rig.model)
+        attributes: Final = _matching_marker_span(rig.destination, marker)
+        assert _json_object(attributes["metadata"].encode()) == {"trace_marker": marker}, attributes
+        assert attributes["litellm.metadata.trace_marker"] == marker, attributes
+
+
 def test_arize_otel_v2_c6_content_capture_disabled(gateway: Gateway, tmp_path: Path) -> None:
     marker: Final = "c6-" + uuid.uuid4().hex
     with _rig(
@@ -302,6 +321,40 @@ def test_arize_otel_v2_c8_request_callback_disable(gateway: Gateway, tmp_path: P
         _assert_success_body(disabled_response, disabled_marker, rig.model)
 
 
+def test_arize_otel_v2_c8_disabled_callback_exports_no_span(gateway: Gateway, tmp_path: Path) -> None:
+    pytest.skip("BUG: LIT-9049 x-litellm-disable-callbacks: arize still exports the OTel v2 span")
+    disabled_marker: Final = "c8-disabled-" + uuid.uuid4().hex
+    sentinel: Final = "c8-sentinel-" + uuid.uuid4().hex
+
+    def upstream(request: Request) -> Reply:
+        marker: Final = _chat_request_marker(request)
+        assert marker in (disabled_marker, sentinel), request
+        _assert_chat_request(request, messages=[{"role": "user", "content": marker}])
+        return _chat_response(marker)
+
+    with _rig(
+        gateway,
+        tmp_path,
+        upstream,
+        litellm_settings={"allow_dynamic_callback_disabling": True},
+        workers=1,
+    ) as rig:
+        disabled_response: Final = _request(
+            rig.proxy,
+            rig.model,
+            disabled_marker,
+            prompt=disabled_marker,
+            headers={"x-litellm-disable-callbacks": "arize"},
+        )
+        _assert_success_body(disabled_response, disabled_marker, rig.model)
+        sentinel_response: Final = _request(rig.proxy, rig.model, sentinel, prompt=sentinel)
+        _assert_success_body(sentinel_response, sentinel, rig.model)
+        spans: Final = _llm_spans_through_markers(rig.destination, (sentinel,))
+        assert not any(attributes.get("litellm.metadata.trace_marker") == disabled_marker for attributes in spans), (
+            spans
+        )
+
+
 @pytest.mark.parametrize("failure_status", (401, 500))
 def test_arize_otel_v2_c9_upstream_failures_are_recorded(failure_status: int, gateway: Gateway, tmp_path: Path) -> None:
     marker: Final = "c9-" + uuid.uuid4().hex

tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py

@@ -475,6 +475,41 @@ def test_arize_otel_v2_d3_malformed_tool_calls_are_normalized(shape: str, gatewa
     assert "Exception while exporting Span batch" not in proxy_log.read_text(), proxy_log.read_text()
 
 
+def test_arize_otel_v2_d3_non_dict_tool_call_is_not_a_caller_error(gateway: Gateway, tmp_path: Path) -> None:
+    pytest.skip("BUG: LIT-9125 non-dict tool_calls entry returns HTTP 400 with a server traceback")
+    marker: Final = "d3-non-dict-call-" + uuid.uuid4().hex
+
+    def upstream(request: Request) -> Reply:
+        _assert_chat_request(request, messages=[{"role": "user", "content": "weather in Paris?"}])
+        return Reply(
+            body=json.dumps(
+                {
+                    "id": marker,
+                    "object": "chat.completion",
+                    "created": 1,
+                    "model": "gpt-4o-mini",
+                    "choices": [
+                        {
+                            "index": 0,
+                            "finish_reason": "tool_calls",
+                            "message": {"role": "assistant", "content": None, "tool_calls": ["not-a-call"]},
+                        }
+                    ],
+                    "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12},
+                }
+            ).encode()
+        )
+
+    with _rig(gateway, tmp_path, upstream) as rig:
+        response: Final = _call(rig.proxy, rig.model, marker)
+        assert response.status_code not in range(400, 500), response.text
+        assert "Traceback" not in response.text, response.text
+        assert "AttributeError" not in response.text, response.text
+        readiness: Final = rig.proxy.client.get("/health/readiness")
+        assert readiness.status_code == 200, readiness.text
+        assert _json_object(readiness.content) == {"status": "healthy", "db": "connected"}, readiness.text
+
+
 @pytest.mark.parametrize("shape", ("empty", "null", "missing"))
 def test_arize_otel_v2_e3_empty_or_missing_tool_calls_never_indexed(
     shape: str, gateway: Gateway, tmp_path: Path

tests/integration/observability/test_arize_otel_v2_openinference_spans.py

@@ -924,3 +924,43 @@ def test_arize_otel_v2_a_cache(surface: str, gateway: Gateway, tmp_path: Path) -
         assert len(sentinel_spans) == 1, spans
         _assert_tool_span_for_marker(sentinel_spans[0], sentinel)
         assert not any(attributes.get("litellm.metadata.trace_marker") == marker for attributes in spans), spans
+
+
+def test_arize_otel_v2_cache_hit_exports_llm_span(gateway: Gateway, tmp_path: Path) -> None:
+    pytest.skip("BUG: LIT-9127 response-cache hits emit no OTel v2 LLM span")
+    marker: Final = "a-cache-hit-" + uuid.uuid4().hex
+    sentinel: Final = f"{marker}-sentinel"
+
+    def upstream(request: Request) -> Reply:
+        request_marker: Final = _chat_request_marker(request)
+        assert request_marker in (marker, sentinel), request
+        _assert_chat_request(request, messages=[{"role": "user", "content": request_marker}])
+        return _chat_response(request_marker)
+
+    with _rig(gateway, tmp_path, upstream, workers=1) as rig:
+        first: Final = _cache_call(rig, "chat", marker)
+        assert first[0] == (f"call_{marker}", "lookup_weather", '{"city": "Paris"}'), first
+        assert first[1] == marker, first
+        assert not first[2].get("x-litellm-cache-key"), first[2]
+        second: Final = _cache_call(rig, "chat", marker, cache_hit=True)
+        assert second[0] == first[0], second
+        assert second[1] == first[1], second
+        assert second[2].get("x-litellm-cache-key"), second[2]
+        forwarded: Final = tuple(
+            request for request in rig.provider.drain() if request.method == "POST" and marker.encode() in request.body
+        )
+        assert len(forwarded) == 1, forwarded
+        sentinel_response: Final = _cache_call(rig, "chat", sentinel)
+        assert not sentinel_response[2].get("x-litellm-cache-key"), sentinel_response[2]
+        sentinel_forwarded: Final = tuple(
+            request
+            for request in rig.provider.drain()
+            if request.method == "POST" and sentinel.encode() in request.body
+        )
+        assert len(sentinel_forwarded) == 1, sentinel_forwarded
+        spans: Final = _llm_spans_through_markers(rig.destination, (sentinel,))
+        marker_spans: Final = tuple(
+            attributes for attributes in spans if attributes.get("litellm.metadata.trace_marker") == marker
+        )
+        assert len(marker_spans) == 2, spans
+        _assert_tool_span_for_marker(marker_spans[1], marker)