import asyncio import logging from collections.abc import Callable, Iterator from pathlib import Path from typing import Final import pytest import uvloop from litellm._logging import verbose_logger, verbose_proxy_logger, verbose_router_logger from litellm.proxy.collector import ( SpendEventConsumer, address_argument, apply_log_level, pod_pgbouncer_database_url, ) from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings from litellm.proxy.spend_tracking.spend_event_producer import ( AddressError, SpendEventProducer, TcpAddress, UnixAddress, open_collector_connection, ) class _Handler: def __init__(self, fail_on: bytes | None = None) -> None: self.lines: list[bytes] = [] # mutable-ok: test double records the events the consumer handed over self._fail_on = fail_on async def __call__(self, line: bytes) -> None: if line == self._fail_on: raise RuntimeError("pipeline failed") self.lines.append(line) async def _no_fallback(line: bytes) -> None: raise AssertionError(f"unexpected fallback for {line!r}") class _Fallback: def __init__(self) -> None: self.lines: list[bytes] = [] # mutable-ok: test double records the events run in-process async def __call__(self, line: bytes) -> None: self.lines.append(line) @pytest.mark.asyncio @pytest.mark.parametrize("transport", ["unix", "tcp"]) async def test_consumer_handles_each_producer_line_once_in_order(tmp_path: Path, transport: str): handler: Final = _Handler(fail_on=b"event-3\n") consumer: Final = SpendEventConsumer(handler) server: Final = await consumer.serve( UnixAddress(path=str(tmp_path / "spend.sock")) if transport == "unix" else TcpAddress("127.0.0.1", 0) ) address: Final = ( UnixAddress(path=str(tmp_path / "spend.sock")) if transport == "unix" else TcpAddress("127.0.0.1", server.sockets[0].getsockname()[1]) ) producer: Final = SpendEventProducer( address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=_no_fallback ) for i in range(6): await producer.publish(f"event-{i}\n".encode()) await producer.close(drain_timeout=5.0) server.close() assert await consumer.drain(timeout=5.0) == 0 assert handler.lines == [f"event-{i}\n".encode() for i in range(6) if i != 3] assert (consumer.received, consumer.handled, consumer.failed) == (6, 5, 1) @pytest.mark.asyncio async def test_consumer_discards_a_truncated_trailing_event(tmp_path: Path): handler: Final = _Handler() consumer: Final = SpendEventConsumer(handler) address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) server: Final = await consumer.serve(address) _, writer = await open_collector_connection(address, timeout=1.0) writer.write(b"whole\npartial-without-newline") await writer.drain() writer.close() await writer.wait_closed() await asyncio.sleep(0.05) server.close() assert await consumer.drain(timeout=5.0) == 0 assert handler.lines == [b"whole\n"] assert consumer.received == 1 @pytest.mark.asyncio async def test_drain_reports_producers_still_connected_after_the_timeout(tmp_path: Path): consumer: Final = SpendEventConsumer(_Handler()) address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) server: Final = await consumer.serve(address) _, writer = await open_collector_connection(address, timeout=1.0) await asyncio.sleep(0.05) server.close() assert await consumer.drain(timeout=0.1) == 1 writer.close() await writer.wait_closed() assert await consumer.drain(timeout=5.0) == 0 @pytest.mark.asyncio async def test_graceful_stop_hands_the_producer_over_to_its_fallback_without_losing_events(tmp_path: Path): handler: Final = _Handler() fallback: Final = _Fallback() consumer: Final = SpendEventConsumer(handler) address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) server: Final = await consumer.serve(address) producer: Final = SpendEventProducer( address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=fallback ) await producer.publish(b"event-1\n") await asyncio.sleep(0.05) server.close() draining: Final = asyncio.ensure_future(consumer.drain(timeout=5.0)) await asyncio.sleep(0.05) await producer.publish(b"event-2\n") await producer.close(drain_timeout=5.0) assert await draining == 0 assert handler.lines == [b"event-1\n"] assert fallback.lines == [b"event-2\n"] assert (producer.stats().sent, producer.stats().fallback) == (1, 1) @pytest.mark.parametrize("loop_factory", [asyncio.new_event_loop, uvloop.new_event_loop], ids=["asyncio", "uvloop"]) def test_drain_still_hands_over_live_producers_when_another_connection_already_died( tmp_path: Path, loop_factory: Callable[[], asyncio.AbstractEventLoop] ): """A transport the loop force-closed under a busy handler must not abort the half-close of the others.""" async def scenario() -> tuple[int, list[bytes]]: release: Final = asyncio.Event() async def slow_handler(line: bytes) -> None: await release.wait() consumer: Final = SpendEventConsumer(slow_handler) address: Final = UnixAddress(path=str(tmp_path / "spend.sock")) server: Final = await consumer.serve(address) _, dead = await open_collector_connection(address, timeout=1.0) dead.write(b"stuck\n") await dead.drain() await asyncio.sleep(0.05) for connection in consumer._open_connections: # pyright: ignore[reportPrivateUsage] # force-close like uvloop does on a socket error connection.transport.close() dead.close() fallback: Final = _Fallback() producer: Final = SpendEventProducer( address=address, on_unavailable="fallback", buffer_size=100, connect_timeout=1.0, fallback=fallback ) await producer.publish(b"event-1\n") await asyncio.sleep(0.05) server.close() draining: Final = asyncio.ensure_future(consumer.drain(timeout=0.5)) await asyncio.sleep(0.05) await producer.publish(b"event-2\n") await producer.close(drain_timeout=5.0) still_open: Final = await draining release.set() await asyncio.sleep(0.05) return still_open, fallback.lines with asyncio.Runner(loop_factory=loop_factory) as runner: still_open, fallback_lines = runner.run(scenario()) assert still_open == 2 assert fallback_lines == [b"event-2\n"] def test_address_argument(): assert address_argument((), default="unix:///tmp/x.sock") == "unix:///tmp/x.sock" assert address_argument(("--address", "tcp://127.0.0.1:4100"), default="unix:///tmp/x.sock") == ( "tcp://127.0.0.1:4100" ) assert isinstance(address_argument(("--listen", "x"), default="unix:///tmp/x.sock"), AddressError) def test_pod_pgbouncer_database_url_points_at_the_proxy_containers_pooler(): """With pgbouncer on, the sidecar must not open its own upstream connections but share the pod's pooler.""" upstream: Final = "postgresql://u:p@db.internal:5432/litellm?schema=public" environ: Final = {"DATABASE_URL": upstream} assert pod_pgbouncer_database_url(PgBouncerSettings(enabled=False), environ, token_auth=False) is None assert ( pod_pgbouncer_database_url(PgBouncerSettings(enabled=True, port=6543), environ, token_auth=False) == "postgresql://u:p@127.0.0.1:6543/litellm?schema=public&pgbouncer=true" ) assert isinstance(pod_pgbouncer_database_url(PgBouncerSettings(enabled=True), {}, token_auth=False), PgBouncerError) def test_pod_pgbouncer_database_url_goes_direct_under_token_auth(): """The proxy's pgbouncer only knows the token that container minted, so the sidecar must mint its own upstream.""" iam_upstream: Final = "postgresql://u@db.internal:5432/litellm?schema=public" assert ( pod_pgbouncer_database_url(PgBouncerSettings(enabled=True), {"DATABASE_URL": iam_upstream}, token_auth=True) is None ) assert pod_pgbouncer_database_url(PgBouncerSettings(enabled=True), {}, token_auth=True) is None @pytest.fixture def restore_log_levels() -> Iterator[None]: loggers: Final = (verbose_logger, verbose_router_logger, verbose_proxy_logger) levels: Final = tuple(logger.level for logger in loggers) yield for logger, level in zip(loggers, levels, strict=True): logger.setLevel(level) @pytest.mark.usefixtures("restore_log_levels") @pytest.mark.parametrize( ("litellm_log", "expected"), [("DEBUG", logging.DEBUG), ("info", logging.INFO), (None, logging.WARNING), ("loud", logging.WARNING)], ) def test_apply_log_level_mirrors_the_proxy_env_contract(litellm_log: str | None, expected: int): verbose_proxy_logger.setLevel(logging.WARNING) apply_log_level(litellm_log) assert verbose_proxy_logger.isEnabledFor(expected) assert not verbose_proxy_logger.isEnabledFor(expected - 10)