From ccf84e6006c6be3da77b366b8c1809cb73f97b55 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 17:17:10 -0700 Subject: [PATCH] test(otel): type the pass-through operation tests and drive them with a real logging object at fixed times --- .../integrations/otel/test_otel_v2_metrics.py | 12 +++++------ .../otel/test_otel_v2_sources_of_truth.py | 6 +++--- tests/unit/integrations/test_opentelemetry.py | 6 +++--- .../test_streaming_handler.py | 21 +++++++++++++------ .../test_success_handler.py | 8 +++---- .../test_upstream_usage_headers.py | 10 +++++---- 6 files changed, 36 insertions(+), 27 deletions(-) diff --git a/tests/unit/integrations/otel/test_otel_v2_metrics.py b/tests/unit/integrations/otel/test_otel_v2_metrics.py index 5acbb565889..74921512825 100644 --- a/tests/unit/integrations/otel/test_otel_v2_metrics.py +++ b/tests/unit/integrations/otel/test_otel_v2_metrics.py @@ -24,6 +24,7 @@ dashboard already querying that histogram. import asyncio import json from datetime import datetime, timedelta +from typing import Final import pytest @@ -504,15 +505,12 @@ def test_agent_message_is_not_labelled_as_chat(call_type): assert dp.attributes[OPERATION_KEY] == "invoke_agent" -def test_passthrough_operation_labels_the_metrics(): - reader = InMemoryMetricReader() +def test_passthrough_operation_labels_the_metrics() -> None: + reader: Final = InMemoryMetricReader() kwargs, response_obj, start, end = _build_call(call_type="pass_through_endpoint") + labelled: Final = {**kwargs, PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL} - asyncio.run( - _logger(reader, enable_metrics=True).async_log_success_event( - {**kwargs, PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}, response_obj, start, end - ) - ) + asyncio.run(_logger(reader, enable_metrics=True).async_log_success_event(labelled, response_obj, start, end)) for dp in _metrics_by_name(reader)[OPERATION_DURATION]: assert dp.attributes[OPERATION_KEY] == "retrieval" diff --git a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py index 37112395ba9..2d0b4701c9f 100644 --- a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py @@ -32,13 +32,13 @@ from litellm.integrations.otel import ( from litellm.integrations.otel.mappers.genai import GenAIMapper from litellm.integrations.otel.model import spans as spans_mod from litellm.integrations.otel.model.metadata import LLMCallEvent -from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY from litellm.integrations.otel.model.payloads import ( EmbeddingOutput, LLMCallSpanData, RequestIdentity, _upstream_address_port, ) +from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY from litellm.integrations.otel.model.spans import ( SPAN_REGISTRY, LiteLLMSpanKind, @@ -224,10 +224,10 @@ def test_operation_resolution(): assert resolve_operation("call_mcp_tool") is GenAIOperation.EXECUTE_TOOL -def test_passthrough_operation_wins_over_the_call_type(): +def test_passthrough_operation_wins_over_the_call_type() -> None: assert resolve_operation("pass_through_endpoint", GenAIOperation.RETRIEVAL) is GenAIOperation.RETRIEVAL assert resolve_operation("pass_through_endpoint", "retrieval") is GenAIOperation.CHAT - event = LLMCallEvent.from_dict( + event: Final = LLMCallEvent.from_dict( {"call_type": "pass_through_endpoint", "model": "unknown", PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL} ) assert event.operation is GenAIOperation.RETRIEVAL diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 7e83ff8d873..f0fca94c370 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -1049,9 +1049,9 @@ class TestOpenTelemetrySemconvStability(unittest.TestCase): kwargs = {"model": "claude-sonnet-4-5", "call_type": "unknown"} self.assertEqual(h._get_span_name(kwargs), "chat claude-sonnet-4-5") - def test_passthrough_operation_names_the_span(self): - h = self._make(env="gen_ai_latest_experimental") - kwargs = { + def test_passthrough_operation_names_the_span(self) -> None: + h: Final = self._make(env="gen_ai_latest_experimental") + kwargs: Final = { "model": "unknown", "call_type": "pass_through_endpoint", PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL, diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py index 5aaa6eac5d3..829289768f4 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py @@ -1,7 +1,7 @@ import json import logging from collections.abc import Iterator -from datetime import datetime +from datetime import datetime, timezone from typing import Final from unittest.mock import MagicMock @@ -122,9 +122,18 @@ def test_streaming_generate_content_bills_against_the_requested_provider( ) def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call( call_type: str, expected: GenAIOperation | None -): - logging_obj = _logging_obj() - logging_obj.model_call_details["call_type"] = call_type +) -> None: + start: Final = datetime(2025, 1, 2, 3, 4, 5, tzinfo=timezone.utc) + logging_obj: Final = LiteLLMLoggingObj( + model="unknown", + messages=[], + stream=True, + call_type=call_type, + start_time=start, + litellm_call_id="search-1", + function_id="search", + ) + logging_obj.update_environment_variables(litellm_params={}, optional_params={}) PassThroughStreamingHandler._build_passthrough_logging_result( litellm_logging_obj=logging_obj, @@ -132,9 +141,9 @@ def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call( url_route="https://doc-app.internal/search", request_body={}, endpoint_type=EndpointType.GENERIC, - start_time=datetime.now(), + start_time=start, raw_bytes=[b'data: {"chunk": 1}\n\n'], - end_time=datetime.now(), + end_time=datetime(2025, 1, 2, 3, 4, 9, tzinfo=timezone.utc), model=None, ) diff --git a/tests/unit/proxy/pass_through_endpoints/test_success_handler.py b/tests/unit/proxy/pass_through_endpoints/test_success_handler.py index ccf4aa22a7b..9151c9f020b 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_success_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/test_success_handler.py @@ -659,14 +659,14 @@ def test_vertex_ai_live_route_without_usage_frames_yields_no_logging_response( ) async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_named_it( reported: GenAIOperation | None, expected: GenAIOperation -): +) -> None: url: Final = "https://doc-app.internal/search" logging_obj: Final = LiteLLMLoggingObj( model="unknown", messages=[], stream=False, call_type="pass_through_endpoint", - start_time=datetime.now(), + start_time=_START, litellm_call_id="search-1", function_id="search", ) @@ -679,8 +679,8 @@ async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_nam logging_obj=logging_obj, url_route=url, result="", - start_time=datetime.now(), - end_time=datetime.now(), + start_time=_START, + end_time=_END, cache_hit=False, request_body={}, passthrough_logging_payload=PassthroughStandardLoggingPayload(url=url, request_body={}, request_method="POST"), diff --git a/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py b/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py index 9c55ef53ebf..6976f6e1d2a 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py +++ b/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py @@ -1,4 +1,6 @@ +from typing import Final + import httpx import pytest @@ -131,8 +133,8 @@ def test_apply_only_overwrites_what_upstream_reported(): @pytest.mark.parametrize("operation", [GenAIOperation.RETRIEVAL, GenAIOperation.EMBEDDINGS]) -def test_apply_operation_records_the_operation_the_upstream_named(operation: GenAIOperation): - logging_obj = _logging_obj() +def test_apply_operation_records_the_operation_the_upstream_named(operation: GenAIOperation) -> None: + logging_obj: Final = _logging_obj() apply_upstream_reported_operation(logging_obj, _headers(**{"x-litellm-operation-name": operation.value})) @@ -143,8 +145,8 @@ def test_apply_operation_records_the_operation_the_upstream_named(operation: Gen "headers", [{}, {"x-litellm-operation-name": "summarize"}, {"x-litellm-operation-name": "Retrieval"}], ) -def test_apply_operation_ignores_a_missing_or_unknown_name(headers: dict[str, str]): - logging_obj = _logging_obj() +def test_apply_operation_ignores_a_missing_or_unknown_name(headers: dict[str, str]) -> None: + logging_obj: Final = _logging_obj() apply_upstream_reported_operation(logging_obj, _headers(**headers))