mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-12 03:48:56 +00:00
8a103e26ab
* test: move routing, caching and callback tests out of local_testing Moves the 37 routing, caching and callback files in tests/local_testing into mirrored tests/unit paths and tests/integration sdk and security groups per the migration manifest, and deletes the 8 rows it marks for deletion * test: cover least-busy routing and tighten migrated router, cache and pass-through tests Add router-level least-busy tests for the three legacy rows that had no destination, move completion fallback and model alias tests to their mirror files, move router batch tests next to the router, and strengthen tests whose mutants survived (specific deployment, batch pair grouping, custom cache key, router cache ttl). Add a unit test for configured pass-through Authorization forwarding. * test: tighten migrated callback, proxy config and cache ttl tests Assert the full StandardLoggingPayload key set and the wire user on callback tests, add an async embedding failure callback test, drive the prometheus redis failure through a real RedisCache, use the real salt key in proxy config tests, pin the redis ttl value and rename fallback test helpers * test: fix semantic cache assertions and order-dependent router and callback tests Count semantic cache embeddings on the route, drain async cache writes before the second lookup, echo the requested model from the callback provider mock so response_cost does not depend on model_cost entries other tests register, and move the two-key authorization assertion to the auth failure test it belongs to * test: cover model_list fan-out and the usage-based routing tpm limit lost in the move * test: wait for the streamed cache entry in redis and drop deleted files from the hosted mock guard * test: restore sync callback, audio redaction and langfuse export coverage, keep redis-compat filter on the moved file * ci: run the remaining router-keyword legacy tests on one node * test: refuse the redis write in-process instead of resolving an invalid host --------- Co-authored-by: yuneng <yuneng@berri.ai>
88 lines
3.2 KiB
Python
88 lines
3.2 KiB
Python
import asyncio
|
|
import os
|
|
from typing import Final
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
|
|
import litellm
|
|
from litellm import Cache, acompletion
|
|
from litellm.caching.caching import LiteLLMCacheType
|
|
from litellm.caching.caching_handler import _PENDING_CACHE_WRITES
|
|
from litellm.integrations.custom_logger import CustomLogger
|
|
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
|
from litellm._service_logger import ServiceLogging
|
|
|
|
|
|
async def _drain_cache_writes() -> None:
|
|
await GLOBAL_LOGGING_WORKER.flush()
|
|
await asyncio.gather(*_PENDING_CACHE_WRITES)
|
|
|
|
|
|
@pytest.fixture
|
|
def redis_response_cache(monkeypatch: pytest.MonkeyPatch) -> Cache:
|
|
cache: Final = Cache(
|
|
type=LiteLLMCacheType.REDIS,
|
|
host=os.environ["REDIS_HOST"],
|
|
port=os.environ["REDIS_PORT"],
|
|
)
|
|
monkeypatch.setattr(litellm, "cache", cache)
|
|
return cache
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_redis_cache_completion_stream(redis_response_cache: Cache, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(litellm, "callbacks", [CustomLogger()])
|
|
messages: Final = [{"role": "user", "content": f"cache stream {uuid4().hex}"}]
|
|
first_response: Final = await acompletion(
|
|
model="gpt-4o-mini",
|
|
messages=messages,
|
|
caching=True,
|
|
stream=True,
|
|
mock_response="The same response is replayed.",
|
|
)
|
|
first_chunks: Final = tuple([chunk async for chunk in first_response])
|
|
for _ in range(10):
|
|
await asyncio.sleep(0)
|
|
await _drain_cache_writes()
|
|
second_response: Final = await acompletion(
|
|
model="gpt-4o-mini",
|
|
messages=messages,
|
|
caching=True,
|
|
stream=True,
|
|
mock_response="A cache miss would return this.",
|
|
)
|
|
second_chunks: Final = tuple([chunk async for chunk in second_response])
|
|
|
|
assert first_chunks[-1].id == second_chunks[-1].id
|
|
assert "".join(chunk.choices[0].delta.content or "" for chunk in first_chunks) == "The same response is replayed."
|
|
assert "".join(chunk.choices[0].delta.content or "" for chunk in second_chunks) == "The same response is replayed."
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_completion_with_caching(redis_response_cache: Cache, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(litellm, "service_callback", ["prometheus_system"])
|
|
service_logger: Final = ServiceLogging(mock_testing=True)
|
|
service_logger.prometheusServicesLogger.mock_testing = True
|
|
redis_response_cache.cache.service_logger_obj = service_logger
|
|
messages: Final = [{"role": "user", "content": f"prometheus cache {uuid4().hex}"}]
|
|
|
|
await acompletion(
|
|
model="gpt-4o-mini",
|
|
messages=messages,
|
|
caching=True,
|
|
mock_response="cached response",
|
|
)
|
|
await _drain_cache_writes()
|
|
await acompletion(
|
|
model="gpt-4o-mini",
|
|
messages=messages,
|
|
caching=True,
|
|
mock_response="unused cache miss response",
|
|
)
|
|
await _drain_cache_writes()
|
|
|
|
assert service_logger.mock_testing_async_success_hook == 2
|
|
assert service_logger.prometheusServicesLogger.mock_testing_success_calls == 2
|
|
assert service_logger.mock_testing_sync_failure_hook == 0
|
|
assert service_logger.mock_testing_async_failure_hook == 0
|