diff --git a/tests/unit/integrations/otel/test_root_io_logger.py b/tests/unit/integrations/otel/test_root_io_logger.py index c13f2598955..51e03bcf54b 100644 --- a/tests/unit/integrations/otel/test_root_io_logger.py +++ b/tests/unit/integrations/otel/test_root_io_logger.py @@ -19,11 +19,9 @@ from opentelemetry.trace import Span # noqa: E402 import litellm # noqa: E402 from litellm.caching.dual_cache import DualCache # noqa: E402 from litellm.integrations.custom_logger import CustomLogger # noqa: E402 -from litellm.integrations.otel import root_io_logger # noqa: E402 from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger # noqa: E402 from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402 from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402 -from litellm.integrations.otel.model.request_io import response_output # noqa: E402 from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402 from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 @@ -177,31 +175,34 @@ def test_chat_stream_relays_chunks_untouched_and_stamps_the_assembled_output() - assert [(turn["role"], turn["content"]) for turn in json.loads(attrs["output.value"])] == [("assistant", "pong")] -def test_root_output_renders_while_the_event_loop_serves_other_requests(monkeypatch: pytest.MonkeyPatch) -> None: - logger, exporter = _logger() - loop_served_another_request: Final = threading.Event() +class _ContentThatWaitsForAnotherRequest: + def __init__(self) -> None: + self.loop_served_another_request: Final = threading.Event() - def render_after_another_request(response: object) -> str | None: - return response_output(response) if loop_served_another_request.wait(timeout=2) else None + def __str__(self) -> str: + return "pong" if self.loop_served_another_request.wait(timeout=2) else "the event loop was blocked" + + +def test_root_output_renders_while_the_event_loop_serves_other_requests() -> None: + logger, exporter = _logger() + content: Final = _ContentThatWaitsForAnotherRequest() + reply: Final = {"type": "message", "role": "assistant", "content": content} async def another_request() -> None: - loop_served_another_request.set() + content.loop_served_another_request.set() async def serve_both() -> None: await asyncio.gather( - logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=PONG), + logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=reply), another_request(), ) - monkeypatch.setattr(root_io_logger, "response_output", render_after_another_request) root: Final = _start_root(logger) asyncio.run(serve_both()) root.end() - assert [(turn["role"], turn["content"]) for turn in json.loads(_root_attrs(exporter)["output.value"])] == [ - ("assistant", "pong") - ] + assert json.loads(_root_attrs(exporter)["output.value"]) == [{"role": "assistant", "content": "pong"}] def test_a_root_another_callback_started_is_left_alone() -> None: @@ -275,7 +276,7 @@ def _logging_obj(dynamic_params: dict[str, object], *, no_log: bool = False) -> messages=CHAT_DATA["messages"], stream=False, call_type="acompletion", - start_time=datetime.now(), + start_time=datetime(2026, 1, 1), litellm_call_id="root-io", function_id="fn", ) diff --git a/tests/unit/proxy/test_proxy_logging_hook_detection.py b/tests/unit/proxy/test_proxy_logging_hook_detection.py index 9ee96833d44..c7a0192841b 100644 --- a/tests/unit/proxy/test_proxy_logging_hook_detection.py +++ b/tests/unit/proxy/test_proxy_logging_hook_detection.py @@ -1,3 +1,4 @@ +from collections.abc import AsyncGenerator, AsyncIterator from typing import Final import pytest @@ -10,7 +11,15 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse +from litellm.types.utils import ( + CallTypesLiteral, + Choices, + Delta, + Message, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) def test_has_post_call_response_headers_callbacks_ignores_empty_callbacks( @@ -280,6 +289,7 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke from fastapi import HTTPException from litellm.caching.caching import DualCache + guardrail = _content_filter_guardrail("BLOCK") monkeypatch.setattr(litellm, "callbacks", [guardrail]) @@ -295,6 +305,7 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke yield chunk delivered = [] + async def _drain(): async for chunk in proxy_logging.async_post_call_streaming_iterator_hook( response=fake_stream(), @@ -329,12 +340,8 @@ async def test_post_call_stream_guardrail_keeps_own_iterator_on_chat_completions proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) async def fake_stream(): - yield ModelResponseStream( - choices=[StreamingChoices(index=0, delta=Delta(content="the zebra runs"))] - ) - yield ModelResponseStream( - choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")] - ) + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="the zebra runs"))]) + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")]) delivered_text = "" async for chunk in proxy_logging.async_post_call_streaming_iterator_hook( @@ -404,9 +411,12 @@ async def test_post_call_stream_records_the_served_text_when_the_client_disconne class _MasksReply(CustomLogger): - async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data): - from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - + async def async_post_call_streaming_iterator_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + response: AsyncIterator[ModelResponseStream], + request_data: dict[str, object], + ) -> AsyncGenerator[ModelResponseStream, None]: async for _ in response: yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content=""))]) @@ -418,24 +428,28 @@ class _ReadsServedStream(CustomLogger): super().__init__() self.read: list[str | None] = [] - async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data): + async def async_post_call_streaming_iterator_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + response: AsyncIterator[ModelResponseStream], + request_data: dict[str, object], + ) -> AsyncGenerator[ModelResponseStream, None]: async for chunk in response: self.read.append(chunk.choices[0].delta.content) yield chunk @pytest.mark.asyncio -async def test_post_call_stream_served_stream_relay_reads_what_a_later_hook_serves(monkeypatch): - from litellm.caching.caching import DualCache - from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - - reader = _ReadsServedStream() +async def test_post_call_stream_served_stream_relay_reads_what_a_later_hook_serves( + monkeypatch: pytest.MonkeyPatch, +) -> None: + reader: Final = _ReadsServedStream() monkeypatch.setattr(litellm, "callbacks", [reader, _MasksReply()]) - async def fake_stream(): + async def fake_stream() -> AsyncIterator[ModelResponseStream]: yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="John Smith"))]) - served = [ + served: Final = [ chunk.choices[0].delta.content async for chunk in ProxyLogging(user_api_key_cache=DualCache()).async_post_call_streaming_iterator_hook( response=fake_stream(), @@ -581,6 +595,7 @@ async def test_post_call_stream_guardrail_reroutes_inherited_apply_guardrail(mon yield chunk delivered = [] + async def _drain(): async for chunk in proxy_logging.async_post_call_streaming_iterator_hook( response=fake_stream(), @@ -621,9 +636,7 @@ async def test_post_call_stream_masking_guardrail_keeps_own_iterator_on_anthropi async for item in response: yield item - guardrail = _content_filter_guardrail( - "BLOCK", guardrail_cls=_MasksViaOwnRawStreamHook, mask_response_content=True - ) + guardrail = _content_filter_guardrail("BLOCK", guardrail_cls=_MasksViaOwnRawStreamHook, mask_response_content=True) monkeypatch.setattr(litellm, "callbacks", [guardrail]) proxy_logging = ProxyLogging(user_api_key_cache=DualCache())