mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
test(otel v2): type the served-stream test hooks, fix the logging object timestamp, and block the root render through the reply content
This commit is contained in:
parent
93454cb7e3
commit
8e3f22429d
2 changed files with 49 additions and 35 deletions
|
|
@ -19,11 +19,9 @@ from opentelemetry.trace import Span # noqa: E402
|
|||
import litellm # noqa: E402
|
||||
from litellm.caching.dual_cache import DualCache # noqa: E402
|
||||
from litellm.integrations.custom_logger import CustomLogger # noqa: E402
|
||||
from litellm.integrations.otel import root_io_logger # noqa: E402
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger # noqa: E402
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402
|
||||
from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402
|
||||
from litellm.integrations.otel.model.request_io import response_output # noqa: E402
|
||||
from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402
|
||||
from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402
|
||||
from litellm.integrations.otel.plumbing import providers # noqa: E402
|
||||
|
|
@ -177,31 +175,34 @@ def test_chat_stream_relays_chunks_untouched_and_stamps_the_assembled_output() -
|
|||
assert [(turn["role"], turn["content"]) for turn in json.loads(attrs["output.value"])] == [("assistant", "pong")]
|
||||
|
||||
|
||||
def test_root_output_renders_while_the_event_loop_serves_other_requests(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
logger, exporter = _logger()
|
||||
loop_served_another_request: Final = threading.Event()
|
||||
class _ContentThatWaitsForAnotherRequest:
|
||||
def __init__(self) -> None:
|
||||
self.loop_served_another_request: Final = threading.Event()
|
||||
|
||||
def render_after_another_request(response: object) -> str | None:
|
||||
return response_output(response) if loop_served_another_request.wait(timeout=2) else None
|
||||
def __str__(self) -> str:
|
||||
return "pong" if self.loop_served_another_request.wait(timeout=2) else "the event loop was blocked"
|
||||
|
||||
|
||||
def test_root_output_renders_while_the_event_loop_serves_other_requests() -> None:
|
||||
logger, exporter = _logger()
|
||||
content: Final = _ContentThatWaitsForAnotherRequest()
|
||||
reply: Final = {"type": "message", "role": "assistant", "content": content}
|
||||
|
||||
async def another_request() -> None:
|
||||
loop_served_another_request.set()
|
||||
content.loop_served_another_request.set()
|
||||
|
||||
async def serve_both() -> None:
|
||||
await asyncio.gather(
|
||||
logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=PONG),
|
||||
logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=reply),
|
||||
another_request(),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(root_io_logger, "response_output", render_after_another_request)
|
||||
root: Final = _start_root(logger)
|
||||
|
||||
asyncio.run(serve_both())
|
||||
root.end()
|
||||
|
||||
assert [(turn["role"], turn["content"]) for turn in json.loads(_root_attrs(exporter)["output.value"])] == [
|
||||
("assistant", "pong")
|
||||
]
|
||||
assert json.loads(_root_attrs(exporter)["output.value"]) == [{"role": "assistant", "content": "pong"}]
|
||||
|
||||
|
||||
def test_a_root_another_callback_started_is_left_alone() -> None:
|
||||
|
|
@ -275,7 +276,7 @@ def _logging_obj(dynamic_params: dict[str, object], *, no_log: bool = False) ->
|
|||
messages=CHAT_DATA["messages"],
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
start_time=datetime.now(),
|
||||
start_time=datetime(2026, 1, 1),
|
||||
litellm_call_id="root-io",
|
||||
function_id="fn",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
|
@ -10,7 +11,15 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse
|
||||
from litellm.types.utils import (
|
||||
CallTypesLiteral,
|
||||
Choices,
|
||||
Delta,
|
||||
Message,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
)
|
||||
|
||||
|
||||
def test_has_post_call_response_headers_callbacks_ignores_empty_callbacks(
|
||||
|
|
@ -280,6 +289,7 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke
|
|||
from fastapi import HTTPException
|
||||
|
||||
from litellm.caching.caching import DualCache
|
||||
|
||||
guardrail = _content_filter_guardrail("BLOCK")
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail])
|
||||
|
||||
|
|
@ -295,6 +305,7 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke
|
|||
yield chunk
|
||||
|
||||
delivered = []
|
||||
|
||||
async def _drain():
|
||||
async for chunk in proxy_logging.async_post_call_streaming_iterator_hook(
|
||||
response=fake_stream(),
|
||||
|
|
@ -329,12 +340,8 @@ async def test_post_call_stream_guardrail_keeps_own_iterator_on_chat_completions
|
|||
proxy_logging = ProxyLogging(user_api_key_cache=DualCache())
|
||||
|
||||
async def fake_stream():
|
||||
yield ModelResponseStream(
|
||||
choices=[StreamingChoices(index=0, delta=Delta(content="the zebra runs"))]
|
||||
)
|
||||
yield ModelResponseStream(
|
||||
choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")]
|
||||
)
|
||||
yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="the zebra runs"))])
|
||||
yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")])
|
||||
|
||||
delivered_text = ""
|
||||
async for chunk in proxy_logging.async_post_call_streaming_iterator_hook(
|
||||
|
|
@ -404,9 +411,12 @@ async def test_post_call_stream_records_the_served_text_when_the_client_disconne
|
|||
|
||||
|
||||
class _MasksReply(CustomLogger):
|
||||
async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data):
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
async def async_post_call_streaming_iterator_hook(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
response: AsyncIterator[ModelResponseStream],
|
||||
request_data: dict[str, object],
|
||||
) -> AsyncGenerator[ModelResponseStream, None]:
|
||||
async for _ in response:
|
||||
yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="<PERSON>"))])
|
||||
|
||||
|
|
@ -418,24 +428,28 @@ class _ReadsServedStream(CustomLogger):
|
|||
super().__init__()
|
||||
self.read: list[str | None] = []
|
||||
|
||||
async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data):
|
||||
async def async_post_call_streaming_iterator_hook(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
response: AsyncIterator[ModelResponseStream],
|
||||
request_data: dict[str, object],
|
||||
) -> AsyncGenerator[ModelResponseStream, None]:
|
||||
async for chunk in response:
|
||||
self.read.append(chunk.choices[0].delta.content)
|
||||
yield chunk
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_stream_served_stream_relay_reads_what_a_later_hook_serves(monkeypatch):
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
reader = _ReadsServedStream()
|
||||
async def test_post_call_stream_served_stream_relay_reads_what_a_later_hook_serves(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
reader: Final = _ReadsServedStream()
|
||||
monkeypatch.setattr(litellm, "callbacks", [reader, _MasksReply()])
|
||||
|
||||
async def fake_stream():
|
||||
async def fake_stream() -> AsyncIterator[ModelResponseStream]:
|
||||
yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="John Smith"))])
|
||||
|
||||
served = [
|
||||
served: Final = [
|
||||
chunk.choices[0].delta.content
|
||||
async for chunk in ProxyLogging(user_api_key_cache=DualCache()).async_post_call_streaming_iterator_hook(
|
||||
response=fake_stream(),
|
||||
|
|
@ -581,6 +595,7 @@ async def test_post_call_stream_guardrail_reroutes_inherited_apply_guardrail(mon
|
|||
yield chunk
|
||||
|
||||
delivered = []
|
||||
|
||||
async def _drain():
|
||||
async for chunk in proxy_logging.async_post_call_streaming_iterator_hook(
|
||||
response=fake_stream(),
|
||||
|
|
@ -621,9 +636,7 @@ async def test_post_call_stream_masking_guardrail_keeps_own_iterator_on_anthropi
|
|||
async for item in response:
|
||||
yield item
|
||||
|
||||
guardrail = _content_filter_guardrail(
|
||||
"BLOCK", guardrail_cls=_MasksViaOwnRawStreamHook, mask_response_content=True
|
||||
)
|
||||
guardrail = _content_filter_guardrail("BLOCK", guardrail_cls=_MasksViaOwnRawStreamHook, mask_response_content=True)
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail])
|
||||
|
||||
proxy_logging = ProxyLogging(user_api_key_cache=DualCache())
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue