mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
test(proxy): type the per-chunk streaming hook test helpers
This commit is contained in:
parent
c05907f867
commit
01ab5bb7a1
3 changed files with 33 additions and 23 deletions
|
|
@ -4973,16 +4973,18 @@ class TestAsyncStreamingDataGeneratorFastPath:
|
||||||
hook_spy.assert_not_awaited()
|
hook_spy.assert_not_awaited()
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_fast_path_skips_per_chunk_hook_with_a_request_only_guardrail(self, monkeypatch):
|
async def test_fast_path_skips_per_chunk_hook_with_a_request_only_guardrail(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
monkeypatch.setattr(litellm, "callbacks", [CustomGuardrail(guardrail_name="g", event_hook="pre_call")])
|
monkeypatch.setattr(litellm, "callbacks", [CustomGuardrail(guardrail_name="g", event_hook="pre_call")])
|
||||||
ProxyLogging._callback_capabilities_cache.clear()
|
ProxyLogging._callback_capabilities_cache.clear()
|
||||||
|
|
||||||
proxy_logging_obj = ProxyLogging(user_api_key_cache=MagicMock())
|
proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=MagicMock())
|
||||||
hook_spy = AsyncMock(side_effect=lambda **kw: kw["response"])
|
hook_spy: Final = AsyncMock(side_effect=lambda **kw: kw["response"])
|
||||||
monkeypatch.setattr(proxy_logging_obj, "async_post_call_streaming_hook", hook_spy)
|
monkeypatch.setattr(proxy_logging_obj, "async_post_call_streaming_hook", hook_spy)
|
||||||
|
|
||||||
chunks = [b"event: a\ndata: {}\n\n", b"event: b\ndata: {}\n\n"]
|
chunks: Final = [b"event: a\ndata: {}\n\n", b"event: b\ndata: {}\n\n"]
|
||||||
out = [
|
out: Final = [
|
||||||
c
|
c
|
||||||
async for c in ProxyBaseLLMRequestProcessing.async_streaming_data_generator(
|
async for c in ProxyBaseLLMRequestProcessing.async_streaming_data_generator(
|
||||||
response=self._aiter(chunks),
|
response=self._aiter(chunks),
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,7 @@ class _OverridesPerChunk(CustomLogger):
|
||||||
|
|
||||||
|
|
||||||
class _PerChunkGuardrail(CustomGuardrail):
|
class _PerChunkGuardrail(CustomGuardrail):
|
||||||
async def async_post_call_streaming_hook(self, *args, **kwargs):
|
async def async_post_call_streaming_hook(self, user_api_key_dict: UserAPIKeyAuth, response: str) -> None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ Covers ``_wrap_streaming_iterator_with_enrichment``,
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from collections.abc import AsyncGenerator, AsyncIterator
|
from collections.abc import AsyncGenerator, AsyncIterator, Callable
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Dict, Final, List
|
from typing import Any, Dict, Final, List
|
||||||
from unittest.mock import AsyncMock, MagicMock
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
@ -389,24 +389,26 @@ class _RequestOnlyGuardrail(CustomGuardrail):
|
||||||
super().__init__(guardrail_name="request-only", event_hook=GuardrailEventHooks.pre_call, default_on=True)
|
super().__init__(guardrail_name="request-only", event_hook=GuardrailEventHooks.pre_call, default_on=True)
|
||||||
self.asked: List[GuardrailEventHooks] = []
|
self.asked: List[GuardrailEventHooks] = []
|
||||||
|
|
||||||
def should_run_guardrail(self, data, event_type: GuardrailEventHooks) -> bool:
|
def should_run_guardrail(self, data: dict[str, object], event_type: GuardrailEventHooks) -> bool:
|
||||||
self.asked.append(event_type)
|
self.asked.append(event_type)
|
||||||
return super().should_run_guardrail(data=data, event_type=event_type)
|
return super().should_run_guardrail(data=data, event_type=event_type)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_async_post_call_streaming_hook_does_not_ask_a_request_only_guardrail(
|
async def test_async_post_call_streaming_hook_does_not_ask_a_request_only_guardrail(
|
||||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
proxy_logging: ProxyLogging,
|
||||||
):
|
make_user_api_key_auth: Callable[..., UserAPIKeyAuth],
|
||||||
guardrail = _RequestOnlyGuardrail()
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
guardrail: Final = _RequestOnlyGuardrail()
|
||||||
monkeypatch.setattr(litellm, "callbacks", [guardrail])
|
monkeypatch.setattr(litellm, "callbacks", [guardrail])
|
||||||
chunk = ModelResponseStream(
|
chunk: Final = ModelResponseStream(
|
||||||
id="rid",
|
id="rid",
|
||||||
choices=[{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None}],
|
choices=[{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None}],
|
||||||
created=0,
|
created=0,
|
||||||
model="gpt-4o-mini",
|
model="gpt-4o-mini",
|
||||||
)
|
)
|
||||||
out = await proxy_logging.async_post_call_streaming_hook(
|
out: Final = await proxy_logging.async_post_call_streaming_hook(
|
||||||
data={"messages": [{"role": "user", "content": "hi"}]},
|
data={"messages": [{"role": "user", "content": "hi"}]},
|
||||||
response=chunk,
|
response=chunk,
|
||||||
user_api_key_dict=make_user_api_key_auth(),
|
user_api_key_dict=make_user_api_key_auth(),
|
||||||
|
|
@ -416,23 +418,25 @@ async def test_async_post_call_streaming_hook_does_not_ask_a_request_only_guardr
|
||||||
|
|
||||||
|
|
||||||
class _PerChunkRewriter(CustomLogger):
|
class _PerChunkRewriter(CustomLogger):
|
||||||
async def async_post_call_streaming_hook(self, user_api_key_dict, response): # type: ignore[override]
|
async def async_post_call_streaming_hook(self, user_api_key_dict: UserAPIKeyAuth, response: str) -> str:
|
||||||
return f"modified-{response}"
|
return f"modified-{response}"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_async_post_call_streaming_hook_skips_a_request_only_guardrail_beside_a_per_chunk_callback(
|
async def test_async_post_call_streaming_hook_skips_a_request_only_guardrail_beside_a_per_chunk_callback(
|
||||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
proxy_logging: ProxyLogging,
|
||||||
):
|
make_user_api_key_auth: Callable[..., UserAPIKeyAuth],
|
||||||
guardrail = _RequestOnlyGuardrail()
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
guardrail: Final = _RequestOnlyGuardrail()
|
||||||
monkeypatch.setattr(litellm, "callbacks", [guardrail, _PerChunkRewriter()])
|
monkeypatch.setattr(litellm, "callbacks", [guardrail, _PerChunkRewriter()])
|
||||||
chunk = ModelResponseStream(
|
chunk: Final = ModelResponseStream(
|
||||||
id="rid",
|
id="rid",
|
||||||
choices=[{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None}],
|
choices=[{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None}],
|
||||||
created=0,
|
created=0,
|
||||||
model="gpt-4o-mini",
|
model="gpt-4o-mini",
|
||||||
)
|
)
|
||||||
out = await proxy_logging.async_post_call_streaming_hook(
|
out: Final = await proxy_logging.async_post_call_streaming_hook(
|
||||||
data={"messages": [{"role": "user", "content": "hi"}]},
|
data={"messages": [{"role": "user", "content": "hi"}]},
|
||||||
response=chunk,
|
response=chunk,
|
||||||
user_api_key_dict=make_user_api_key_auth(),
|
user_api_key_dict=make_user_api_key_auth(),
|
||||||
|
|
@ -474,8 +478,10 @@ async def _say_hi_through_chunk_hooks(proxy_logging: ProxyLogging, user_api_key_
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_async_post_call_streaming_hook_runs_an_instance_hook_beside_a_guardrail(
|
async def test_async_post_call_streaming_hook_runs_an_instance_hook_beside_a_guardrail(
|
||||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
proxy_logging: ProxyLogging,
|
||||||
):
|
make_user_api_key_auth: Callable[..., UserAPIKeyAuth],
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
guardrail: Final = _RequestOnlyGuardrail()
|
guardrail: Final = _RequestOnlyGuardrail()
|
||||||
monkeypatch.setattr(litellm, "callbacks", [guardrail, _instance_rewriter()])
|
monkeypatch.setattr(litellm, "callbacks", [guardrail, _instance_rewriter()])
|
||||||
assert await _say_hi_through_chunk_hooks(proxy_logging, make_user_api_key_auth()) == "modified-say hi"
|
assert await _say_hi_through_chunk_hooks(proxy_logging, make_user_api_key_auth()) == "modified-say hi"
|
||||||
|
|
@ -484,8 +490,10 @@ async def test_async_post_call_streaming_hook_runs_an_instance_hook_beside_a_gua
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_async_post_call_streaming_hook_runs_an_instance_hook_beside_a_class_hook(
|
async def test_async_post_call_streaming_hook_runs_an_instance_hook_beside_a_class_hook(
|
||||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
proxy_logging: ProxyLogging,
|
||||||
):
|
make_user_api_key_auth: Callable[..., UserAPIKeyAuth],
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
monkeypatch.setattr(litellm, "callbacks", [_ClassChunkLogger(), _instance_rewriter()])
|
monkeypatch.setattr(litellm, "callbacks", [_ClassChunkLogger(), _instance_rewriter()])
|
||||||
assert await _say_hi_through_chunk_hooks(proxy_logging, make_user_api_key_auth()) == "modified-say hi"
|
assert await _say_hi_through_chunk_hooks(proxy_logging, make_user_api_key_auth()) == "modified-say hi"
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue