test(otel): type the pass-through operation tests and drive them with a real logging object at fixed times

This commit is contained in:
yucheng 2026-10-27 17:17:10 -07:00
parent c7b3d77056
commit ccf84e6006
6 changed files with 36 additions and 27 deletions

View file

@ -24,6 +24,7 @@ dashboard already querying that histogram.
import asyncio import asyncio
import json import json
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import Final
import pytest import pytest
@ -504,15 +505,12 @@ def test_agent_message_is_not_labelled_as_chat(call_type):
assert dp.attributes[OPERATION_KEY] == "invoke_agent" assert dp.attributes[OPERATION_KEY] == "invoke_agent"
def test_passthrough_operation_labels_the_metrics(): def test_passthrough_operation_labels_the_metrics() -> None:
reader = InMemoryMetricReader() reader: Final = InMemoryMetricReader()
kwargs, response_obj, start, end = _build_call(call_type="pass_through_endpoint") kwargs, response_obj, start, end = _build_call(call_type="pass_through_endpoint")
labelled: Final = {**kwargs, PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}
asyncio.run( asyncio.run(_logger(reader, enable_metrics=True).async_log_success_event(labelled, response_obj, start, end))
_logger(reader, enable_metrics=True).async_log_success_event(
{**kwargs, PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}, response_obj, start, end
)
)
for dp in _metrics_by_name(reader)[OPERATION_DURATION]: for dp in _metrics_by_name(reader)[OPERATION_DURATION]:
assert dp.attributes[OPERATION_KEY] == "retrieval" assert dp.attributes[OPERATION_KEY] == "retrieval"

View file

@ -32,13 +32,13 @@ from litellm.integrations.otel import (
from litellm.integrations.otel.mappers.genai import GenAIMapper from litellm.integrations.otel.mappers.genai import GenAIMapper
from litellm.integrations.otel.model import spans as spans_mod from litellm.integrations.otel.model import spans as spans_mod
from litellm.integrations.otel.model.metadata import LLMCallEvent from litellm.integrations.otel.model.metadata import LLMCallEvent
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY
from litellm.integrations.otel.model.payloads import ( from litellm.integrations.otel.model.payloads import (
EmbeddingOutput, EmbeddingOutput,
LLMCallSpanData, LLMCallSpanData,
RequestIdentity, RequestIdentity,
_upstream_address_port, _upstream_address_port,
) )
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY
from litellm.integrations.otel.model.spans import ( from litellm.integrations.otel.model.spans import (
SPAN_REGISTRY, SPAN_REGISTRY,
LiteLLMSpanKind, LiteLLMSpanKind,
@ -224,10 +224,10 @@ def test_operation_resolution():
assert resolve_operation("call_mcp_tool") is GenAIOperation.EXECUTE_TOOL assert resolve_operation("call_mcp_tool") is GenAIOperation.EXECUTE_TOOL
def test_passthrough_operation_wins_over_the_call_type(): def test_passthrough_operation_wins_over_the_call_type() -> None:
assert resolve_operation("pass_through_endpoint", GenAIOperation.RETRIEVAL) is GenAIOperation.RETRIEVAL assert resolve_operation("pass_through_endpoint", GenAIOperation.RETRIEVAL) is GenAIOperation.RETRIEVAL
assert resolve_operation("pass_through_endpoint", "retrieval") is GenAIOperation.CHAT assert resolve_operation("pass_through_endpoint", "retrieval") is GenAIOperation.CHAT
event = LLMCallEvent.from_dict( event: Final = LLMCallEvent.from_dict(
{"call_type": "pass_through_endpoint", "model": "unknown", PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL} {"call_type": "pass_through_endpoint", "model": "unknown", PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}
) )
assert event.operation is GenAIOperation.RETRIEVAL assert event.operation is GenAIOperation.RETRIEVAL

View file

@ -1049,9 +1049,9 @@ class TestOpenTelemetrySemconvStability(unittest.TestCase):
kwargs = {"model": "claude-sonnet-4-5", "call_type": "unknown"} kwargs = {"model": "claude-sonnet-4-5", "call_type": "unknown"}
self.assertEqual(h._get_span_name(kwargs), "chat claude-sonnet-4-5") self.assertEqual(h._get_span_name(kwargs), "chat claude-sonnet-4-5")
def test_passthrough_operation_names_the_span(self): def test_passthrough_operation_names_the_span(self) -> None:
h = self._make(env="gen_ai_latest_experimental") h: Final = self._make(env="gen_ai_latest_experimental")
kwargs = { kwargs: Final = {
"model": "unknown", "model": "unknown",
"call_type": "pass_through_endpoint", "call_type": "pass_through_endpoint",
PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL, PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL,

View file

@ -1,7 +1,7 @@
import json import json
import logging import logging
from collections.abc import Iterator from collections.abc import Iterator
from datetime import datetime from datetime import datetime, timezone
from typing import Final from typing import Final
from unittest.mock import MagicMock from unittest.mock import MagicMock
@ -122,9 +122,18 @@ def test_streaming_generate_content_bills_against_the_requested_provider(
) )
def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call( def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call(
call_type: str, expected: GenAIOperation | None call_type: str, expected: GenAIOperation | None
): ) -> None:
logging_obj = _logging_obj() start: Final = datetime(2025, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
logging_obj.model_call_details["call_type"] = call_type logging_obj: Final = LiteLLMLoggingObj(
model="unknown",
messages=[],
stream=True,
call_type=call_type,
start_time=start,
litellm_call_id="search-1",
function_id="search",
)
logging_obj.update_environment_variables(litellm_params={}, optional_params={})
PassThroughStreamingHandler._build_passthrough_logging_result( PassThroughStreamingHandler._build_passthrough_logging_result(
litellm_logging_obj=logging_obj, litellm_logging_obj=logging_obj,
@ -132,9 +141,9 @@ def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call(
url_route="https://doc-app.internal/search", url_route="https://doc-app.internal/search",
request_body={}, request_body={},
endpoint_type=EndpointType.GENERIC, endpoint_type=EndpointType.GENERIC,
start_time=datetime.now(), start_time=start,
raw_bytes=[b'data: {"chunk": 1}\n\n'], raw_bytes=[b'data: {"chunk": 1}\n\n'],
end_time=datetime.now(), end_time=datetime(2025, 1, 2, 3, 4, 9, tzinfo=timezone.utc),
model=None, model=None,
) )

View file

@ -659,14 +659,14 @@ def test_vertex_ai_live_route_without_usage_frames_yields_no_logging_response(
) )
async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_named_it( async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_named_it(
reported: GenAIOperation | None, expected: GenAIOperation reported: GenAIOperation | None, expected: GenAIOperation
): ) -> None:
url: Final = "https://doc-app.internal/search" url: Final = "https://doc-app.internal/search"
logging_obj: Final = LiteLLMLoggingObj( logging_obj: Final = LiteLLMLoggingObj(
model="unknown", model="unknown",
messages=[], messages=[],
stream=False, stream=False,
call_type="pass_through_endpoint", call_type="pass_through_endpoint",
start_time=datetime.now(), start_time=_START,
litellm_call_id="search-1", litellm_call_id="search-1",
function_id="search", function_id="search",
) )
@ -679,8 +679,8 @@ async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_nam
logging_obj=logging_obj, logging_obj=logging_obj,
url_route=url, url_route=url,
result="", result="",
start_time=datetime.now(), start_time=_START,
end_time=datetime.now(), end_time=_END,
cache_hit=False, cache_hit=False,
request_body={}, request_body={},
passthrough_logging_payload=PassthroughStandardLoggingPayload(url=url, request_body={}, request_method="POST"), passthrough_logging_payload=PassthroughStandardLoggingPayload(url=url, request_body={}, request_method="POST"),

View file

@ -1,4 +1,6 @@
from typing import Final
import httpx import httpx
import pytest import pytest
@ -131,8 +133,8 @@ def test_apply_only_overwrites_what_upstream_reported():
@pytest.mark.parametrize("operation", [GenAIOperation.RETRIEVAL, GenAIOperation.EMBEDDINGS]) @pytest.mark.parametrize("operation", [GenAIOperation.RETRIEVAL, GenAIOperation.EMBEDDINGS])
def test_apply_operation_records_the_operation_the_upstream_named(operation: GenAIOperation): def test_apply_operation_records_the_operation_the_upstream_named(operation: GenAIOperation) -> None:
logging_obj = _logging_obj() logging_obj: Final = _logging_obj()
apply_upstream_reported_operation(logging_obj, _headers(**{"x-litellm-operation-name": operation.value})) apply_upstream_reported_operation(logging_obj, _headers(**{"x-litellm-operation-name": operation.value}))
@ -143,8 +145,8 @@ def test_apply_operation_records_the_operation_the_upstream_named(operation: Gen
"headers", "headers",
[{}, {"x-litellm-operation-name": "summarize"}, {"x-litellm-operation-name": "Retrieval"}], [{}, {"x-litellm-operation-name": "summarize"}, {"x-litellm-operation-name": "Retrieval"}],
) )
def test_apply_operation_ignores_a_missing_or_unknown_name(headers: dict[str, str]): def test_apply_operation_ignores_a_missing_or_unknown_name(headers: dict[str, str]) -> None:
logging_obj = _logging_obj() logging_obj: Final = _logging_obj()
apply_upstream_reported_operation(logging_obj, _headers(**headers)) apply_upstream_reported_operation(logging_obj, _headers(**headers))