test(otel): type the pass-through operation tests and drive them with a real logging object at fixed times

This commit is contained in:
yucheng 2026-10-27 17:17:10 -07:00
parent c7b3d77056
commit ccf84e6006
6 changed files with 36 additions and 27 deletions

View file

@ -24,6 +24,7 @@ dashboard already querying that histogram.
import asyncio
import json
from datetime import datetime, timedelta
from typing import Final
import pytest
@ -504,15 +505,12 @@ def test_agent_message_is_not_labelled_as_chat(call_type):
assert dp.attributes[OPERATION_KEY] == "invoke_agent"
def test_passthrough_operation_labels_the_metrics():
reader = InMemoryMetricReader()
def test_passthrough_operation_labels_the_metrics() -> None:
reader: Final = InMemoryMetricReader()
kwargs, response_obj, start, end = _build_call(call_type="pass_through_endpoint")
labelled: Final = {**kwargs, PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}
asyncio.run(
_logger(reader, enable_metrics=True).async_log_success_event(
{**kwargs, PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}, response_obj, start, end
)
)
asyncio.run(_logger(reader, enable_metrics=True).async_log_success_event(labelled, response_obj, start, end))
for dp in _metrics_by_name(reader)[OPERATION_DURATION]:
assert dp.attributes[OPERATION_KEY] == "retrieval"

View file

@ -32,13 +32,13 @@ from litellm.integrations.otel import (
from litellm.integrations.otel.mappers.genai import GenAIMapper
from litellm.integrations.otel.model import spans as spans_mod
from litellm.integrations.otel.model.metadata import LLMCallEvent
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY
from litellm.integrations.otel.model.payloads import (
EmbeddingOutput,
LLMCallSpanData,
RequestIdentity,
_upstream_address_port,
)
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY
from litellm.integrations.otel.model.spans import (
SPAN_REGISTRY,
LiteLLMSpanKind,
@ -224,10 +224,10 @@ def test_operation_resolution():
assert resolve_operation("call_mcp_tool") is GenAIOperation.EXECUTE_TOOL
def test_passthrough_operation_wins_over_the_call_type():
def test_passthrough_operation_wins_over_the_call_type() -> None:
assert resolve_operation("pass_through_endpoint", GenAIOperation.RETRIEVAL) is GenAIOperation.RETRIEVAL
assert resolve_operation("pass_through_endpoint", "retrieval") is GenAIOperation.CHAT
event = LLMCallEvent.from_dict(
event: Final = LLMCallEvent.from_dict(
{"call_type": "pass_through_endpoint", "model": "unknown", PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}
)
assert event.operation is GenAIOperation.RETRIEVAL

View file

@ -1049,9 +1049,9 @@ class TestOpenTelemetrySemconvStability(unittest.TestCase):
kwargs = {"model": "claude-sonnet-4-5", "call_type": "unknown"}
self.assertEqual(h._get_span_name(kwargs), "chat claude-sonnet-4-5")
def test_passthrough_operation_names_the_span(self):
h = self._make(env="gen_ai_latest_experimental")
kwargs = {
def test_passthrough_operation_names_the_span(self) -> None:
h: Final = self._make(env="gen_ai_latest_experimental")
kwargs: Final = {
"model": "unknown",
"call_type": "pass_through_endpoint",
PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL,

View file

@ -1,7 +1,7 @@
import json
import logging
from collections.abc import Iterator
from datetime import datetime
from datetime import datetime, timezone
from typing import Final
from unittest.mock import MagicMock
@ -122,9 +122,18 @@ def test_streaming_generate_content_bills_against_the_requested_provider(
)
def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call(
call_type: str, expected: GenAIOperation | None
):
logging_obj = _logging_obj()
logging_obj.model_call_details["call_type"] = call_type
) -> None:
start: Final = datetime(2025, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
logging_obj: Final = LiteLLMLoggingObj(
model="unknown",
messages=[],
stream=True,
call_type=call_type,
start_time=start,
litellm_call_id="search-1",
function_id="search",
)
logging_obj.update_environment_variables(litellm_params={}, optional_params={})
PassThroughStreamingHandler._build_passthrough_logging_result(
litellm_logging_obj=logging_obj,
@ -132,9 +141,9 @@ def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call(
url_route="https://doc-app.internal/search",
request_body={},
endpoint_type=EndpointType.GENERIC,
start_time=datetime.now(),
start_time=start,
raw_bytes=[b'data: {"chunk": 1}\n\n'],
end_time=datetime.now(),
end_time=datetime(2025, 1, 2, 3, 4, 9, tzinfo=timezone.utc),
model=None,
)

View file

@ -659,14 +659,14 @@ def test_vertex_ai_live_route_without_usage_frames_yields_no_logging_response(
)
async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_named_it(
reported: GenAIOperation | None, expected: GenAIOperation
):
) -> None:
url: Final = "https://doc-app.internal/search"
logging_obj: Final = LiteLLMLoggingObj(
model="unknown",
messages=[],
stream=False,
call_type="pass_through_endpoint",
start_time=datetime.now(),
start_time=_START,
litellm_call_id="search-1",
function_id="search",
)
@ -679,8 +679,8 @@ async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_nam
logging_obj=logging_obj,
url_route=url,
result="",
start_time=datetime.now(),
end_time=datetime.now(),
start_time=_START,
end_time=_END,
cache_hit=False,
request_body={},
passthrough_logging_payload=PassthroughStandardLoggingPayload(url=url, request_body={}, request_method="POST"),

View file

@ -1,4 +1,6 @@
from typing import Final
import httpx
import pytest
@ -131,8 +133,8 @@ def test_apply_only_overwrites_what_upstream_reported():
@pytest.mark.parametrize("operation", [GenAIOperation.RETRIEVAL, GenAIOperation.EMBEDDINGS])
def test_apply_operation_records_the_operation_the_upstream_named(operation: GenAIOperation):
logging_obj = _logging_obj()
def test_apply_operation_records_the_operation_the_upstream_named(operation: GenAIOperation) -> None:
logging_obj: Final = _logging_obj()
apply_upstream_reported_operation(logging_obj, _headers(**{"x-litellm-operation-name": operation.value}))
@ -143,8 +145,8 @@ def test_apply_operation_records_the_operation_the_upstream_named(operation: Gen
"headers",
[{}, {"x-litellm-operation-name": "summarize"}, {"x-litellm-operation-name": "Retrieval"}],
)
def test_apply_operation_ignores_a_missing_or_unknown_name(headers: dict[str, str]):
logging_obj = _logging_obj()
def test_apply_operation_ignores_a_missing_or_unknown_name(headers: dict[str, str]) -> None:
logging_obj: Final = _logging_obj()
apply_upstream_reported_operation(logging_obj, _headers(**headers))