test(otel v2): type the served-stream test hooks, fix the logging object timestamp, and block the root render through the reply content
Some checks failed
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled

This commit is contained in:
yucheng 2026-10-27 02:37:54 -07:00
parent 93454cb7e3
commit 8e3f22429d
2 changed files with 49 additions and 35 deletions

View file

@ -19,11 +19,9 @@ from opentelemetry.trace import Span # noqa: E402
import litellm # noqa: E402
from litellm.caching.dual_cache import DualCache # noqa: E402
from litellm.integrations.custom_logger import CustomLogger # noqa: E402
from litellm.integrations.otel import root_io_logger # noqa: E402
from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger # noqa: E402
from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402
from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402
from litellm.integrations.otel.model.request_io import response_output # noqa: E402
from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402
from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402
from litellm.integrations.otel.plumbing import providers # noqa: E402
@ -177,31 +175,34 @@ def test_chat_stream_relays_chunks_untouched_and_stamps_the_assembled_output() -
assert [(turn["role"], turn["content"]) for turn in json.loads(attrs["output.value"])] == [("assistant", "pong")]
def test_root_output_renders_while_the_event_loop_serves_other_requests(monkeypatch: pytest.MonkeyPatch) -> None:
logger, exporter = _logger()
loop_served_another_request: Final = threading.Event()
class _ContentThatWaitsForAnotherRequest:
def __init__(self) -> None:
self.loop_served_another_request: Final = threading.Event()
def render_after_another_request(response: object) -> str | None:
return response_output(response) if loop_served_another_request.wait(timeout=2) else None
def __str__(self) -> str:
return "pong" if self.loop_served_another_request.wait(timeout=2) else "the event loop was blocked"
def test_root_output_renders_while_the_event_loop_serves_other_requests() -> None:
logger, exporter = _logger()
content: Final = _ContentThatWaitsForAnotherRequest()
reply: Final = {"type": "message", "role": "assistant", "content": content}
async def another_request() -> None:
loop_served_another_request.set()
content.loop_served_another_request.set()
async def serve_both() -> None:
await asyncio.gather(
logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=PONG),
logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=reply),
another_request(),
)
monkeypatch.setattr(root_io_logger, "response_output", render_after_another_request)
root: Final = _start_root(logger)
asyncio.run(serve_both())
root.end()
assert [(turn["role"], turn["content"]) for turn in json.loads(_root_attrs(exporter)["output.value"])] == [
("assistant", "pong")
]
assert json.loads(_root_attrs(exporter)["output.value"]) == [{"role": "assistant", "content": "pong"}]
def test_a_root_another_callback_started_is_left_alone() -> None:
@ -275,7 +276,7 @@ def _logging_obj(dynamic_params: dict[str, object], *, no_log: bool = False) ->
messages=CHAT_DATA["messages"],
stream=False,
call_type="acompletion",
start_time=datetime.now(),
start_time=datetime(2026, 1, 1),
litellm_call_id="root-io",
function_id="fn",
)

View file

@ -1,3 +1,4 @@
from collections.abc import AsyncGenerator, AsyncIterator
from typing import Final
import pytest
@ -10,7 +11,15 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.utils import ProxyLogging
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse
from litellm.types.utils import (
CallTypesLiteral,
Choices,
Delta,
Message,
ModelResponse,
ModelResponseStream,
StreamingChoices,
)
def test_has_post_call_response_headers_callbacks_ignores_empty_callbacks(
@ -280,6 +289,7 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke
from fastapi import HTTPException
from litellm.caching.caching import DualCache
guardrail = _content_filter_guardrail("BLOCK")
monkeypatch.setattr(litellm, "callbacks", [guardrail])
@ -295,6 +305,7 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke
yield chunk
delivered = []
async def _drain():
async for chunk in proxy_logging.async_post_call_streaming_iterator_hook(
response=fake_stream(),
@ -329,12 +340,8 @@ async def test_post_call_stream_guardrail_keeps_own_iterator_on_chat_completions
proxy_logging = ProxyLogging(user_api_key_cache=DualCache())
async def fake_stream():
yield ModelResponseStream(
choices=[StreamingChoices(index=0, delta=Delta(content="the zebra runs"))]
)
yield ModelResponseStream(
choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")]
)
yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="the zebra runs"))])
yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")])
delivered_text = ""
async for chunk in proxy_logging.async_post_call_streaming_iterator_hook(
@ -404,9 +411,12 @@ async def test_post_call_stream_records_the_served_text_when_the_client_disconne
class _MasksReply(CustomLogger):
async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data):
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
async def async_post_call_streaming_iterator_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
response: AsyncIterator[ModelResponseStream],
request_data: dict[str, object],
) -> AsyncGenerator[ModelResponseStream, None]:
async for _ in response:
yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="<PERSON>"))])
@ -418,24 +428,28 @@ class _ReadsServedStream(CustomLogger):
super().__init__()
self.read: list[str | None] = []
async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data):
async def async_post_call_streaming_iterator_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
response: AsyncIterator[ModelResponseStream],
request_data: dict[str, object],
) -> AsyncGenerator[ModelResponseStream, None]:
async for chunk in response:
self.read.append(chunk.choices[0].delta.content)
yield chunk
@pytest.mark.asyncio
async def test_post_call_stream_served_stream_relay_reads_what_a_later_hook_serves(monkeypatch):
from litellm.caching.caching import DualCache
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
reader = _ReadsServedStream()
async def test_post_call_stream_served_stream_relay_reads_what_a_later_hook_serves(
monkeypatch: pytest.MonkeyPatch,
) -> None:
reader: Final = _ReadsServedStream()
monkeypatch.setattr(litellm, "callbacks", [reader, _MasksReply()])
async def fake_stream():
async def fake_stream() -> AsyncIterator[ModelResponseStream]:
yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="John Smith"))])
served = [
served: Final = [
chunk.choices[0].delta.content
async for chunk in ProxyLogging(user_api_key_cache=DualCache()).async_post_call_streaming_iterator_hook(
response=fake_stream(),
@ -581,6 +595,7 @@ async def test_post_call_stream_guardrail_reroutes_inherited_apply_guardrail(mon
yield chunk
delivered = []
async def _drain():
async for chunk in proxy_logging.async_post_call_streaming_iterator_hook(
response=fake_stream(),
@ -621,9 +636,7 @@ async def test_post_call_stream_masking_guardrail_keeps_own_iterator_on_anthropi
async for item in response:
yield item
guardrail = _content_filter_guardrail(
"BLOCK", guardrail_cls=_MasksViaOwnRawStreamHook, mask_response_content=True
)
guardrail = _content_filter_guardrail("BLOCK", guardrail_cls=_MasksViaOwnRawStreamHook, mask_response_content=True)
monkeypatch.setattr(litellm, "callbacks", [guardrail])
proxy_logging = ProxyLogging(user_api_key_cache=DualCache())