From cc8ae16bb44fce5db586d6c3c22f0f6345b832d9 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Sun, 15 Nov 2026 08:28:10 +0900 Subject: [PATCH 01/57] feat(db): add the SQLx pool foundation with the writer pool (#49798) NativeDatabase.connect opens a lazy SQLx writer pool from an Endpoint that Python resolves from the final DATABASE_URL, read the way Prisma reads it: Unix socket host, sslmode / sslaccept / sslcert with Prisma's verification rules (prefer + strict asks the server once whether it offers TLS), sslidentity / sslpassword handed over as PEM, schema, options, pgbouncer mode, pool_timeout (0 waits forever) and socket_timeout. Options SQLx cannot send are logged instead of failing boot ConnectionBudget splits each worker's connection_limit between Prisma and the native pool, so the budget an operator sized never grows. The Rust share is the hard-coded RUST_RESERVED_CONNECTIONS and no setting names Rust, SQLx or Prisma. It is 0 until a ported query needs the pool, so Prisma's URL is exactly what it was, no native pool opens and nothing connects through SQLx. A connection_limit that is not a positive integer is never split and goes to Prisma as before. Once the share is above 0 Prisma keeps at least one connection, and pools are tagged litellm-prisma and litellm-native in pg_stat_activity. Python keeps minting RDS IAM and Entra tokens and hands each one to the pool statement_cache_size sets the native statement cache (Prisma's default 100) and pgbouncer=true turns it off. sqlx-postgres 0.9.0 still names every statement when the cache is off, so Statements runs queries unnamed in that case, like Prisma, until a release carries transact-rs/sqlx#4337. An application_name the operator set is used on both pools The SQL query files are pinned to LF so Windows checkouts hash them like .sqlx, and the crate lists .sqlx in its package include so the core sdist builds offline crates/db/AGENTS.md tracks parity with the Python Prisma stack The bridge now links litellm-db, so the editable install compiles its queries. The Rust DB workflow builds that install from the checked-in .sqlx metadata, since it runs before the migrations that the online check needs --- .github/workflows/test-rust-db.yml | 2 + litellm/proxy/db/connection_budget.py | 52 +++ litellm/proxy/db/db_url_settings.py | 36 +- litellm/proxy/db/native_database.py | 281 +++++++++++++ litellm/proxy/db/pgbouncer.py | 28 +- litellm/proxy/proxy_cli.py | 21 +- litellm/proxy/proxy_server.py | 5 +- litellm/repositories/database.py | 42 +- litellm/rust/Cargo.lock | 4 + litellm/rust/crates/db/AGENTS.md | 35 ++ litellm/rust/crates/db/Cargo.toml | 7 +- litellm/rust/crates/db/queries/.gitattributes | 1 + litellm/rust/crates/db/src/deadline.rs | 51 +++ litellm/rust/crates/db/src/endpoint.rs | 332 ++++++++++++++++ litellm/rust/crates/db/src/error.rs | 2 + litellm/rust/crates/db/src/lib.rs | 6 + litellm/rust/crates/db/src/statements.rs | 152 +++++++ litellm/rust/crates/db/src/test_support.rs | 34 +- litellm/rust/crates/python-bridge/Cargo.toml | 2 + litellm/rust/crates/python-bridge/src/db.rs | 48 ++- litellm/rust_bridge/_native.pyi | 4 +- litellm/rust_bridge/database.py | 33 ++ tests/_support/native_database.py | 10 +- tests/unit/proxy/db/test_connection_budget.py | 46 +++ tests/unit/proxy/db/test_db_url_settings.py | 65 ++- tests/unit/proxy/db/test_native_database.py | 373 ++++++++++++++++++ .../test_deployment_warning_endpoints.py | 4 +- tests/unit/proxy/test_proxy_cli.py | 64 +++ ...test_proxy_server_endpoints_and_startup.py | 5 + tests/unit/repositories/test_database.py | 74 +++- 30 files changed, 1757 insertions(+), 62 deletions(-) create mode 100644 litellm/proxy/db/connection_budget.py create mode 100644 litellm/proxy/db/native_database.py create mode 100644 litellm/rust/crates/db/queries/.gitattributes create mode 100644 litellm/rust/crates/db/src/deadline.rs create mode 100644 litellm/rust/crates/db/src/endpoint.rs create mode 100644 litellm/rust/crates/db/src/statements.rs create mode 100644 litellm/rust_bridge/database.py create mode 100644 tests/unit/proxy/db/test_connection_budget.py create mode 100644 tests/unit/proxy/db/test_native_database.py diff --git a/.github/workflows/test-rust-db.yml b/.github/workflows/test-rust-db.yml index 116d6785287..3a8af7f170f 100644 --- a/.github/workflows/test-rust-db.yml +++ b/.github/workflows/test-rust-db.yml @@ -91,6 +91,8 @@ jobs: - name: Install Python dependencies working-directory: . + env: + SQLX_OFFLINE: "true" run: uv sync --frozen --extra extra-proxy - name: Deploy Prisma migrations diff --git a/litellm/proxy/db/connection_budget.py b/litellm/proxy/db/connection_budget.py new file mode 100644 index 00000000000..71b839b91ef --- /dev/null +++ b/litellm/proxy/db/connection_budget.py @@ -0,0 +1,52 @@ +"""Split each Postgres pool's connection budget between Prisma and the Rust core's SQLx pool. + +Migration scaffolding. Prisma and SQLx share a database only while the Python gateway still runs Prisma +queries next to ported ones. These invariants hold until Prisma is gone: + +- Operators set one `connection_limit` per pool (`database_connection_pool_limit`, a URL pin or + `database_extra_connection_params`). Nothing an operator sets names Prisma, SQLx or Rust. The Rust share + is carved out of that limit, never added on top, so a Rust pool does not grow the connection budget. +- The Rust share is the hard-coded `RUST_RESERVED_CONNECTIONS`. It is 0 until a ported query runs on the + native pool, so Prisma keeps its whole limit, untouched, and no native pool opens. +- Only a positive integer limit is split. Any other value goes to Prisma as it is, which judges it the same + way it did before the split existed. +- The split is Python's. It is about Prisma and Prisma lives only in the Python gateway, so this module is + deleted with Prisma. +- The Rust core takes a plain `max_connections` and never learns about the split. That is already its final + interface, since the Rust gateway will pass `database_connection_pool_limit` straight to SQLx. +""" + +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +RUST_RESERVED_CONNECTIONS: Final = 0 + + +@dataclass(frozen=True, slots=True) +class ConnectionBudget: + total: int + python: int + rust: int + + @classmethod + def split(cls, total: int, rust_reserved: int = RUST_RESERVED_CONNECTIONS) -> "ConnectionBudget": + """Prisma keeps at least one connection, so a total at or below the reserve ends up over it.""" + return cls(total=total, python=max(1, total - rust_reserved), rust=rust_reserved) + + @classmethod + def of( + cls, connection_limit: str | float, rust_reserved: int = RUST_RESERVED_CONNECTIONS + ) -> "ConnectionBudget | None": + """The split of a positive integer limit, or None for any other value Prisma is left to judge.""" + digits: Final = str(connection_limit) + if not (digits.isascii() and digits.isdigit()) or int(digits) == 0: + return None + return cls.split(int(digits), rust_reserved) + + def prisma_params(self, params: Mapping[str, str | int | float]) -> Mapping[str, str | int | float]: + """``params`` with Prisma's share as its ``connection_limit``, untouched while the Rust pool takes none.""" + if self.rust == 0: + return params + return MappingProxyType({**params, "connection_limit": self.python}) diff --git a/litellm/proxy/db/db_url_settings.py b/litellm/proxy/db/db_url_settings.py index 9db060d5c9d..231904d4d78 100644 --- a/litellm/proxy/db/db_url_settings.py +++ b/litellm/proxy/db/db_url_settings.py @@ -254,12 +254,28 @@ def translate_libpq_ssl_params(url: str, resolve_root_cert: RootCertResolver = p return urllib.parse.urlunsplit(parsed._replace(query=query)) -def postgres_connection_budget_message(writer_limit: str, reader_limit: str | None, num_workers: str) -> str: +PRISMA_APPLICATION_NAME: Final = "litellm-prisma" +NATIVE_APPLICATION_NAME: Final = "litellm-native" + + +def prisma_default_params( + lifetime_params: Mapping[str, str | int | float], native_connections: int +) -> Mapping[str, str | int | float]: + """Prisma's URL defaults, tagged ``PRISMA_APPLICATION_NAME`` only once a native pool shares the database.""" + if native_connections == 0: + return lifetime_params + return MappingProxyType({**lifetime_params, "application_name": PRISMA_APPLICATION_NAME}) + + +def postgres_connection_budget_message( + writer_limit: str, rust_writer_limit: int, reader_limit: str | None, num_workers: str +) -> str: """The startup line that states this pod's worst-case Postgres connection demand. Prisma's ``connection_limit`` is per query engine, and every uvicorn worker owns one engine per configured database (writer, plus the reader when - ``DATABASE_URL_READ_REPLICA`` is set). The limits are read off the final URLs, + ``DATABASE_URL_READ_REPLICA`` is set) plus the Rust writer pool's share + (``connection_budget``) when it is above 0. The limits are read off the final URLs, so a reader that pins its own ``connection_limit`` or an operator override in ``database_extra_connection_params`` is counted at its real value. The server-side cap is shared by every pod, so the number an operator has to keep @@ -272,16 +288,18 @@ def postgres_connection_budget_message(writer_limit: str, reader_limit: str | No reader: Final = int(reader_limit) if reader_limit is not None else 0 except ValueError: return ( - "LiteLLM Proxy: Postgres connection budget per pod = workers x (writer connection_limit + reader " - f"connection_limit) (workers={num_workers!r}, writer={writer_limit!r}, reader={reader_limit!r}); " + "LiteLLM Proxy: Postgres connection budget per pod = workers x (writer connection_limit + " + f"{'rust writer pool + ' if rust_writer_limit else ''}reader connection_limit) " + f"(workers={num_workers!r}, writer={writer_limit!r}, reader={reader_limit!r}); " "keep pods x that figure under max_connections minus superuser_reserved_connections" ) - engines: Final = ( - f"(writer connection_limit {writer} + reader connection_limit {reader})" - if reader_limit is not None - else f"writer connection_limit {writer}" + terms: Final = ( + f"writer connection_limit {writer}", + *((f"rust writer pool {rust_writer_limit}",) if rust_writer_limit else ()), + *((f"reader connection_limit {reader}",) if reader_limit is not None else ()), ) - per_pod: Final = workers * (writer + reader) + engines: Final = terms[0] if len(terms) == 1 else f"({' + '.join(terms)})" + per_pod: Final = workers * (writer + rust_writer_limit + reader) return ( f"LiteLLM Proxy: Postgres connection budget per pod = {workers} worker(s) x {engines} = up to {per_pod} " "connections; keep pods x that figure under max_connections minus superuser_reserved_connections" diff --git a/litellm/proxy/db/native_database.py b/litellm/proxy/db/native_database.py new file mode 100644 index 00000000000..eb5f5ff40b8 --- /dev/null +++ b/litellm/proxy/db/native_database.py @@ -0,0 +1,281 @@ +"""Settings for the native (SQLx) pool, resolved from the same URL Prisma connects with. + +``DATABASE_URL`` is final once ``db_url_settings`` and the CLI have assembled it: token +minted, libpq TLS translated to Prisma's dialect, timeouts in ``options``, pool params +added. Reading that URL keeps one resolver for both pools instead of a second pass over +the ``DATABASE_*`` env vars. + +Credentials follow the host. Each process runs one token refresh loop, owned by whatever +hosts it, and feeds every client in that process: here the worker's loop re-mints Prisma's +URL and pushes the same kind of token into the Rust pool through ``set_writer_password``. +The Rust core only accepts a password and never mints one, so a Rust gateway process +brings its own loop. Two processes minting at once is fine, since IAM and Entra tokens +are independent. What must not exist twice is the logic that resolves region, endpoint +and user, so when the Rust gateway needs it, it moves into the Rust core and this +loop calls it. +""" + +import asyncio +import shlex +import struct +import urllib.parse +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from types import MappingProxyType +from typing import Final, Protocol + +from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, pkcs12 + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.db.db_url_settings import NATIVE_APPLICATION_NAME, PRISMA_APPLICATION_NAME +from litellm.proxy.db.pgbouncer import database_url_is_pooled, fold_libpq_option_tokens, libpq_option_setting +from litellm.proxy.db.token_auth import ( + DEFAULT_POSTGRES_PORT, + DatabaseTokenAuth, + IAMEndpoint, + mint_database_token, + parse_database_token_expiration, + parse_iam_endpoint_from_url, + resolve_database_token_auth, +) +from litellm.rust_bridge.database import ( + NativeClientIdentity, + NativeDatabaseEndpoint, + NativePoolLimits, + SslMode, +) + +TOKEN_REFRESH_BUFFER_SECONDS: Final = 180.0 +TOKEN_FALLBACK_REFRESH_SECONDS: Final = 600.0 +TOKEN_RETRY_SECONDS: Final = 30.0 +PRISMA_DEFAULT_SCHEMA: Final = "public" +PRISMA_DEFAULT_POOL_TIMEOUT_SECONDS: Final = 10.0 +PRISMA_DEFAULT_STATEMENT_CACHE_SIZE: Final = 100 +TLS_PROBE_TIMEOUT_SECONDS: Final = 5.0 +_POSTGRES_SSL_REQUEST: Final = struct.pack("!ii", 8, 80877103) + + +@dataclass(frozen=True, slots=True) +class NativeEndpointSettings: + endpoint: NativeDatabaseEndpoint + ignored_options: tuple[str, ...] + + +def _params(url: str) -> Mapping[str, str]: + return MappingProxyType(dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query))) + + +def needs_tls_probe(url: str) -> bool: + """Prisma's ``sslmode=prefer`` with ``sslaccept=strict`` verifies TLS when the server offers it + and falls back to plaintext when it does not; SQLx has no verified-or-plaintext mode, so the + caller asks the server once and pins the answer.""" + params: Final = _params(url) + return params.get("sslmode", "prefer") == "prefer" and params.get("sslaccept") == "strict" and "host" not in params + + +async def server_offers_tls(url: str, timeout: float = TLS_PROBE_TIMEOUT_SECONDS) -> bool: + """Whether the server answers a Postgres ``SSLRequest`` with ``S``; an unreachable server counts as offering it.""" + parsed: Final = urllib.parse.urlsplit(url) + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection(parsed.hostname, parsed.port or int(DEFAULT_POSTGRES_PORT)), timeout + ) + except (OSError, asyncio.TimeoutError): + return True + try: + writer.write(_POSTGRES_SSL_REQUEST) + await writer.drain() + answer: Final = await asyncio.wait_for(reader.read(1), timeout) + except (OSError, asyncio.TimeoutError): + return True + finally: + writer.close() + return answer != b"N" + + +def _tls(params: Mapping[str, str], tls_offered: bool) -> tuple[SslMode, str | None]: + sslmode: Final = params.get("sslmode", "prefer") + strict: Final = params.get("sslaccept") == "strict" + if sslmode == "disable" or (sslmode == "prefer" and strict and not tls_offered): + return "disable", None + if strict: + return "verify-full", params.get("sslcert") + return ("require" if sslmode == "require" else "prefer"), None + + +def _client_identity( + params: Mapping[str, str], ssl_mode: SslMode, read_file: Callable[[Path], bytes] +) -> NativeClientIdentity | None: + identity_path: Final = params.get("sslidentity") + if identity_path is None or ssl_mode == "disable": + return None + password: Final = params.get("sslpassword") + key, cert, chain = pkcs12.load_key_and_certificates( + read_file(Path(identity_path)), None if password is None else password.encode() + ) + if key is None or cert is None: + raise ValueError(f"sslidentity {identity_path!r} must hold a client certificate and its private key") + return NativeClientIdentity( + cert_pem=b"".join(c.public_bytes(Encoding.PEM) for c in (cert, *chain)).decode(), + key_pem=key.private_bytes(Encoding.PEM, PrivateFormat.PKCS8, NoEncryption()).decode(), + ) + + +def _session_options(params: Mapping[str, str]) -> tuple[tuple[tuple[str, str], ...], tuple[str, ...]]: + raw: Final = params.get("options", "") + try: + tokens: Final[Sequence[str]] = fold_libpq_option_tokens(tuple(shlex.split(raw))) + except ValueError: + return (), (raw,) + readable: Final = tuple((token, libpq_option_setting(token)) for token in tokens) + schema: Final = params.get("schema", PRISMA_DEFAULT_SCHEMA) + search_path: Final = () if schema == PRISMA_DEFAULT_SCHEMA else (("search_path", f'"{schema}"'),) + settings: Final = tuple( + (name, value) for name, _, value in (setting.partition("=") for _, setting in readable if setting is not None) + ) + return (*search_path, *settings), tuple(token for token, setting in readable if setting is None) + + +def _seconds(params: Mapping[str, str], key: str, default: float | None = None) -> float | None: + value: Final = params.get(key) + if value is None: + return default + return None if float(value) == 0 else float(value) + + +def _statement_cache_capacity(params: Mapping[str, str]) -> int: + if params.get("pgbouncer") == "true": + return 0 + return int(params.get("statement_cache_size", PRISMA_DEFAULT_STATEMENT_CACHE_SIZE)) + + +def _application_name(params: Mapping[str, str]) -> str: + configured: Final = params.get("application_name", PRISMA_APPLICATION_NAME) + return NATIVE_APPLICATION_NAME if configured == PRISMA_APPLICATION_NAME else configured + + +def native_endpoint( + url: str, + *, + tls_offered: bool, + max_connections: int, + read_file: Callable[[Path], bytes] = Path.read_bytes, +) -> NativeEndpointSettings: + """The native pool's endpoint for the Prisma URL ``url``, read the way Prisma reads it. + + ``host`` is Prisma's Unix socket directory. The pool takes ``max_connections``, the + Rust share of the budget (``connection_budget``), and tags its connections + ``NATIVE_APPLICATION_NAME``. An unset ``pool_timeout`` takes Prisma's default and + 0 waits forever. + ``sslidentity`` / ``sslpassword`` are a PKCS#12 client certificate, handed over as PEM. + ``options`` tokens that are not ``-c name=value`` settings cannot be sent by SQLx and + come back in ``ignored_options`` instead of failing boot. + """ + parsed: Final = urllib.parse.urlsplit(url) + params: Final = _params(url) + socket: Final = params.get("host") + if not (parsed.hostname or socket) or not parsed.username: + raise ValueError("DATABASE_URL must name a host and a user for the native pool") + ssl_mode, ssl_root_cert = _tls(params, tls_offered) + options, ignored = _session_options(params) + return NativeEndpointSettings( + endpoint=NativeDatabaseEndpoint( + host=parsed.hostname or "localhost", + port=parsed.port or int(DEFAULT_POSTGRES_PORT), + socket=socket, + username=urllib.parse.unquote(parsed.username), + password=urllib.parse.unquote(parsed.password) if parsed.password else None, + database=urllib.parse.unquote(parsed.path.lstrip("/")) or "postgres", + ssl_mode=ssl_mode, + ssl_root_cert=ssl_root_cert, + ssl_client=_client_identity(params, ssl_mode, read_file), + application_name=_application_name(params), + options=options, + statement_cache_capacity=_statement_cache_capacity(params), + pool=NativePoolLimits( + max_connections=max_connections, + acquire_timeout_seconds=_seconds(params, "pool_timeout", PRISMA_DEFAULT_POOL_TIMEOUT_SECONDS), + idle_timeout_seconds=_seconds(params, "max_idle_connection_lifetime"), + socket_timeout_seconds=_seconds(params, "socket_timeout"), + ), + ), + ignored_options=ignored, + ) + + +async def resolve_native_endpoint(url: str, max_connections: int) -> NativeEndpointSettings: + tls_offered: Final = await server_offers_tls(url) if needs_tls_probe(url) else True + return native_endpoint(url, tls_offered=tls_offered, max_connections=max_connections) + + +class PasswordSink(Protocol): + def set_writer_password(self, password: str) -> None: ... + + +@dataclass(frozen=True, slots=True) +class TokenRotation: + auth: DatabaseTokenAuth + endpoint: IAMEndpoint + mint_token: Callable[[DatabaseTokenAuth, IAMEndpoint], str] = mint_database_token + + def mint(self) -> str: + return urllib.parse.unquote(self.mint_token(self.auth, self.endpoint)) + + def seconds_until_refresh(self, token: str, now: datetime) -> float: + expires_at: Final = parse_database_token_expiration(self.auth, token) + if expires_at is None: + return TOKEN_FALLBACK_REFRESH_SECONDS + return max(TOKEN_RETRY_SECONDS, (expires_at - now).total_seconds() - TOKEN_REFRESH_BUFFER_SECONDS) + + +def writer_token_rotation(url: str) -> TokenRotation | None: + """The token rotation the writer needs, or None for a static password. + + A URL pointed at the in-container PgBouncer carries PgBouncer's own credential, + and PgBouncer renews the upstream token itself. + """ + if database_url_is_pooled(): + return None + auth: Final = resolve_database_token_auth() + if auth is None: + return None + return TokenRotation(auth=auth, endpoint=parse_iam_endpoint_from_url(url)) + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc).replace(tzinfo=None) + + +async def _refresh(sink: PasswordSink, rotation: TokenRotation, now: Callable[[], datetime]) -> float: + try: + token: Final = await asyncio.to_thread(rotation.mint) + except Exception as error: # noqa: BLE001 # boto3 and azure-identity raise many types; any failure means retry + verbose_proxy_logger.error( + "Native database pool keeps its current %s (%r); retrying in %.0fs.", + rotation.auth.label, + error, + TOKEN_RETRY_SECONDS, + ) + return TOKEN_RETRY_SECONDS + sink.set_writer_password(token) + return rotation.seconds_until_refresh(token, now()) + + +async def keep_writer_token_fresh( + sink: PasswordSink, + rotation: TokenRotation, + current_token: str, + now: Callable[[], datetime] = _utcnow, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, +) -> None: + """Hand the native pool a fresh token before the current one expires. + + Existing connections stay authenticated; only connections opened after a refresh + use the new token, the same as a re-minted Prisma URL. + """ + delay = rotation.seconds_until_refresh(current_token, now()) # rebind-ok: each refresh schedules the next + while True: + await sleep(delay) + delay = await _refresh(sink, rotation, now) diff --git a/litellm/proxy/db/pgbouncer.py b/litellm/proxy/db/pgbouncer.py index c9fbabd3585..d9983ce774a 100644 --- a/litellm/proxy/db/pgbouncer.py +++ b/litellm/proxy/db/pgbouncer.py @@ -140,19 +140,27 @@ def _userlist_quote(value: str) -> str: return '"' + value.replace('"', '""') + '"' -def _option_settings(tokens: Sequence[str]) -> tuple[str, ...] | None: - """The ``name=value`` settings in a libpq ``options`` string, or None if it holds anything else. - - Accepts ``-c name=value``, ``-cname=value`` and ``--name=value``; a - detached ``-c`` is folded into the token that follows it first. - """ - folded: Final = tuple( +def fold_libpq_option_tokens(tokens: Sequence[str]) -> tuple[str, ...]: + """The tokens of a libpq ``options`` string with each detached ``-c`` folded into the token after it.""" + return tuple( f"-c{tokens[index + 1]}" if token == "-c" and index + 1 < len(tokens) else token for index, token in enumerate(tokens) if index == 0 or tokens[index - 1] != "-c" ) - settings: Final = tuple(token[2:] for token in folded if token.startswith(("-c", "--")) and "=" in token[2:]) - return settings if len(settings) == len(folded) else None + + +def libpq_option_setting(token: str) -> str | None: + """The ``name=value`` setting a folded ``-cname=value`` or ``--name=value`` token carries, or None.""" + return token[2:] if token.startswith(("-c", "--")) and "=" in token[2:] else None + + +def libpq_option_settings(tokens: Sequence[str]) -> tuple[str, ...] | None: + """The ``name=value`` settings in a libpq ``options`` string, or None if it holds anything else. + + Accepts ``-c name=value``, ``-cname=value`` and ``--name=value``. + """ + settings: Final = tuple(libpq_option_setting(token) for token in fold_libpq_option_tokens(tokens)) + return None if None in settings else tuple(setting for setting in settings if setting is not None) def _connect_query(options: str) -> str | PgBouncerError: @@ -163,7 +171,7 @@ def _connect_query(options: str) -> str | PgBouncerError: the settings are applied to each new server connection instead. Every client shares them, which is what the single ``DATABASE_URL`` gave anyway. """ - settings: Final = _option_settings(tuple(shlex.split(options))) + settings: Final = libpq_option_settings(tuple(shlex.split(options))) if settings is None: return PgBouncerError(f"cannot translate the DATABASE_URL options {options!r} into PgBouncer settings") return "; ".join( diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index f252e69d739..cef24e441c1 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -79,7 +79,7 @@ def _build_db_connection_url_params( socket_timeout: float | None = None, disable_prepared_statements: bool = False, extra_params: dict | None = None, -) -> dict: +) -> dict[str, str | int | float]: """Build the Prisma DATABASE_URL query params controlling connection pool behavior. `connect_timeout` / `socket_timeout` map to the Prisma URL params of the same @@ -91,7 +91,7 @@ def _build_db_connection_url_params( `extra_params` is an untyped passthrough — keys it provides win over the named arguments above, so it can be used to override any default we set here. """ - params: Final[dict] = { + params: Final[dict[str, str | int | float]] = { "connection_limit": connection_limit, } if pool_timeout is not None: @@ -1637,11 +1637,13 @@ def run_server( db_connection_timeout = LiteLLMDatabaseConnectionPool.database_connection_pool_timeout.value if os.getenv("DATABASE_URL", None) is not None or os.getenv("DIRECT_URL", None) is not None: + from litellm.proxy.db.connection_budget import ConnectionBudget from litellm.proxy.db.db_url_settings import ( DISABLE_PREPARED_STATEMENTS_ENV_VAR, add_missing_query_params, idle_lifetime_params, postgres_connection_budget_message, + prisma_default_params, reader_shareable_params, translate_libpq_ssl_params, unsupported_db_scheme, @@ -1676,7 +1678,13 @@ def run_server( disable_prepared_statements=disable_prepared_statements, extra_params=db_extra_connection_params, ) + writer_budget: Final = ConnectionBudget.of(connection_url_params["connection_limit"]) + rust_writer_connections: Final = 0 if writer_budget is None else writer_budget.rust + prisma_url_params: Final = ( + connection_url_params if writer_budget is None else writer_budget.prisma_params(connection_url_params) + ) lifetime_params: Final = idle_lifetime_params(general_settings.get("database_max_idle_connection_lifetime")) + prisma_defaults: Final = prisma_default_params(lifetime_params, rust_writer_connections) if os.getenv("DATABASE_URL", None) is not None: database_url = get_secret("DATABASE_URL", default_value=None) resolved_url: Final[str | None] = str(database_url) if database_url else None @@ -1693,10 +1701,10 @@ def run_server( ) modified_url = append_query_params( writer_url, - connection_url_params, + dict(prisma_url_params), ) os.environ["DATABASE_URL"] = translate_libpq_ssl_params( - add_missing_query_params(modified_url, lifetime_params) + add_missing_query_params(modified_url, prisma_defaults) ) if os.getenv("DIRECT_URL", None) is not None: database_url = os.getenv("DIRECT_URL") @@ -1723,15 +1731,16 @@ def run_server( _with_query_value(read_replica_url, "options", reader_options) if reader_options else read_replica_url, - reader_shareable_params(connection_url_params), + reader_shareable_params(prisma_url_params), ), - lifetime_params, + prisma_defaults, ) ) print_startup_message( postgres_connection_budget_message( writer_limit=_url_query_value(os.getenv("DATABASE_URL"), "connection_limit") or f"{db_connection_pool_limit}", + rust_writer_limit=rust_writer_connections, reader_limit=( _url_query_value(os.getenv("DATABASE_URL_READ_REPLICA"), "connection_limit") or f"{db_connection_pool_limit}" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b161fef588a..b50ab3fd571 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -681,6 +681,7 @@ from litellm.proxy.config_resolvers.settings_rules import ( from litellm.proxy.config_resolvers.sso import SSO_FIELD_ENV_VARS from litellm.proxy.container_endpoints.endpoints import router as container_router from litellm.proxy.credential_endpoints.endpoints import router as credential_router +from litellm.proxy.db.connection_budget import RUST_RESERVED_CONNECTIONS from litellm.proxy.db.create_views import SupportsRawQueries from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import ( @@ -1620,7 +1621,9 @@ async def _connect_to_count_stored_values(resources: AsyncExitStack) -> Supports async def _own_database(resources: AsyncExitStack, client: PrismaClient | None) -> database_runtime.Database | None: if client is None: return None - database: Final = await database_runtime.Database.open(client) + database: Final = await database_runtime.Database.open( + client, os.environ["DATABASE_URL"], RUST_RESERVED_CONNECTIONS + ) resources.push_async_callback(database.aclose) return database diff --git a/litellm/repositories/database.py b/litellm/repositories/database.py index d60a2099477..b577ba2bd67 100644 --- a/litellm/repositories/database.py +++ b/litellm/repositories/database.py @@ -1,9 +1,16 @@ import asyncio +import urllib.parse from collections.abc import AsyncGenerator, Coroutine from contextlib import asynccontextmanager from dataclasses import dataclass from typing import TYPE_CHECKING, Final +from litellm._logging import verbose_proxy_logger +from litellm.proxy.db.native_database import ( + keep_writer_token_fresh, + resolve_native_endpoint, + writer_token_rotation, +) from litellm.rust_bridge._native import NativeDatabase if TYPE_CHECKING: @@ -13,22 +20,47 @@ if TYPE_CHECKING: @dataclass(frozen=True, slots=True) class Database: prisma: "PrismaClient" - native: NativeDatabase + native: NativeDatabase | None + tasks: "tuple[asyncio.Task[None], ...]" = () @classmethod - async def open(cls, prisma: "PrismaClient") -> "Database": + async def open(cls, prisma: "PrismaClient", writer_url: str, native_connections: int) -> "Database": + if native_connections == 0: + return cls(prisma=prisma, native=None) try: - native: Final = await NativeDatabase.connect() + settings: Final = await resolve_native_endpoint(writer_url, native_connections) + native: Final = await NativeDatabase.connect(settings.endpoint) except BaseException: await prisma.aclose() raise - return cls(prisma=prisma, native=native) + if settings.ignored_options: + verbose_proxy_logger.error( + "Native database pool cannot send the DATABASE_URL options %s; only -c name=value settings carry over", + settings.ignored_options, + ) + rotation: Final = writer_token_rotation(writer_url) + token_refresh: Final = ( + () + if rotation is None + else ( + asyncio.create_task( + keep_writer_token_fresh( + native, rotation, urllib.parse.unquote(urllib.parse.urlsplit(writer_url).password or "") + ) + ), + ) + ) + return cls(prisma=prisma, native=native, tasks=token_refresh) async def aclose(self) -> None: + for task in self.tasks: + task.cancel() + await asyncio.gather(*self.tasks, return_exceptions=True) try: await self.prisma.aclose() finally: - await self.native.close() + if self.native is not None: + await self.native.close() @asynccontextmanager diff --git a/litellm/rust/Cargo.lock b/litellm/rust/Cargo.lock index 811bda4f798..f40a7fd3ea3 100644 --- a/litellm/rust/Cargo.lock +++ b/litellm/rust/Cargo.lock @@ -3836,6 +3836,8 @@ name = "litellm-db" version = "0.1.0" dependencies = [ "chrono", + "futures-util", + "litellm-secret-value", "macro_rules_attribute", "rstest", "serde", @@ -3844,6 +3846,7 @@ dependencies = [ "strum", "thiserror 2.0.19", "tokio", + "url", ] [[package]] @@ -4485,6 +4488,7 @@ dependencies = [ "litellm-callbacks-s3-log", "litellm-circuit-breaker", "litellm-core-utils", + "litellm-db", "litellm-error", "litellm-host", "litellm-host-python", diff --git a/litellm/rust/crates/db/AGENTS.md b/litellm/rust/crates/db/AGENTS.md index 1e08317230e..1fedf9c9a2b 100644 --- a/litellm/rust/crates/db/AGENTS.md +++ b/litellm/rust/crates/db/AGENTS.md @@ -23,3 +23,38 @@ SQLX_OFFLINE=false cargo sqlx prepare ``` Tests use transactions and leave them uncommitted so fixture rows roll back when each test finishes + +# Boundary with the hosts + +The Python gateway and the Rust gateway both host this crate during the migration, so it takes only what both can hand it: + +- Pool size is a plain `max_connections`. How a worker's `connection_limit` is split between Prisma and SQLx is decided by the Python host (`litellm/proxy/db/connection_budget.py`) and disappears with Prisma. Never add budget or Prisma knowledge here +- Credentials arrive as a password (`set_writer_password`). Each process runs one token refresh loop owned by its host, so this crate never mints or refreshes a token on its own. When the Rust gateway needs IAM or Entra tokens, the minting logic moves here as a function the host's loop calls, and the Python loop switches to it, so the resolution logic exists once + +# Parity with the Python Prisma stack + +Python and SQLx run side by side until every repository operation is native, so the SQLx pool must honor every database setting Python honors. `Endpoint` is resolved in Python from the final `DATABASE_URL` (`litellm/proxy/db/native_database.py`); Rust never reads `DATABASE_*` env vars or parses a Prisma URL. When `litellm/proxy/db/` gains a setting or behavior, add it here + +| Python behavior | Where | Native status | +| --- | --- | --- | +| URL assembled from `DATABASE_*`, token minted, libpq TLS translated | `db_url_settings.py` | Inherited: the native endpoint reads the final URL | +| `host` Unix socket directory | URL param | `Endpoint::socket` | +| `sslmode`, `sslaccept=strict`, `sslcert` | `db_url_settings.py` | `strict` is `verify-full` with `sslcert` as the root; without it nothing is verified, so `sslcert` is dropped. `prefer` + `strict` asks the server once at startup and pins `verify-full` or `disable` | +| `sslidentity`, `sslpassword` | URL params | Python opens the PKCS#12 file and hands the client certificate and key over as PEM | +| `connection_limit` | `proxy_cli.py`, `connection_budget.py` | Split, not added: `ConnectionBudget.split` gives the Rust pool the hard-coded `RUST_RESERVED_CONNECTIONS` and Prisma the rest, so an upgrade never grows the budget. No setting names Rust, SQLx or Prisma. The reserve is 0 until a ported query needs the pool, so Prisma keeps its whole limit and no native pool opens. Once it is above 0, Prisma connections are tagged `application_name=litellm-prisma` and native ones `litellm-native` | +| `pool_timeout`, `max_idle_connection_lifetime` | `proxy_cli.py`, `db_url_settings.py` | `PoolLimits`, with Prisma's 10s default. `pool_timeout=0` waits forever | +| `socket_timeout` | `proxy_cli.py` | `PoolLimits::socket_timeout_seconds`, a deadline on each native operation | +| `connect_timeout` | `proxy_cli.py` | Not carried over: SQLx has no per-connect timeout, so a black-holed connect fails at `pool_timeout` | +| `options` (`statement_timeout`, `lock_timeout`, `idle_in_transaction_session_timeout`) | `proxy_cli.py` | `Endpoint::options`, sent as startup `-c` settings. Tokens that are not `-c name=value` are logged at boot and dropped | +| `schema` | URL param | `search_path` startup setting when not `public`; a custom schema behind PgBouncer is rejected by PgBouncer | +| `statement_cache_size`, `pgbouncer=true`, `DATABASE_DISABLE_PREPARED_STATEMENTS` | URL params, `db_url_settings.py` | `statement_cache_capacity`: `statement_cache_size` (Prisma's default 100), 0 under `pgbouncer=true`. Run every query through `Statements::new(executor, endpoint.names_statements())` so a 0 capacity sends unnamed statements like Prisma, until a sqlx release carries transact-rs/sqlx#4337 | +| `application_name` | URL param | A value the operator set is used on both pools; Prisma's `litellm-prisma` default becomes `litellm-native` | +| Rotating RDS IAM / Entra token | `token_auth.py`, `PrismaWrapper` | Python mints, `NativeDatabase.set_writer_password` swaps connect options | +| In-container PgBouncer | `pgbouncer.py` | Inherited: the pooled URL carries PgBouncer's credential | +| Startup connection budget | `postgres_connection_budget_message` | Counts the Rust writer pool's share | +| Read replica routing | `routing_prisma_wrapper.py` | Planned: optional reader pool with explicit routing | +| Error classification, `lock_timeout` on rollups | `exception_handler.py`, `rollup_lock_timeout.py` | Planned | +| `ServiceTypes.DB` events per round trip | `db_span.py`, `log_db_metrics.py` | Planned | +| Auth lookup gate, fail open on DB outage, reader choice per query | `db_lookup_gate.py`, `allow_requests_on_db_unavailable` | With each ported auth read | +| Health watchdog, engine reaper, drain tracker, engine recreation | `utils.py`, `query_engine_reaper.py`, `prisma_client.py` | Not needed: SQLx pools reconnect per connection and `close` waits for checked-out connections | +| Migrations, `check_migration`, views | `prisma_client.py`, `create_views.py` | Prisma keeps them until the SQLx cutover | diff --git a/litellm/rust/crates/db/Cargo.toml b/litellm/rust/crates/db/Cargo.toml index 4f8fe4e28e2..527a0927338 100644 --- a/litellm/rust/crates/db/Cargo.toml +++ b/litellm/rust/crates/db/Cargo.toml @@ -4,16 +4,21 @@ version = "0.1.0" edition.workspace = true license.workspace = true repository.workspace = true +include = ["Cargo.toml", "src/**", "queries/**", ".sqlx/**"] [dependencies] chrono.workspace = true +futures-util.workspace = true +litellm-secret-value.workspace = true macro_rules_attribute.workspace = true serde.workspace = true serde_json.workspace = true sqlx.workspace = true strum.workspace = true thiserror.workspace = true +tokio = { workspace = true, features = ["time"] } [dev-dependencies] rstest.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["test-util"] } +url.workspace = true diff --git a/litellm/rust/crates/db/queries/.gitattributes b/litellm/rust/crates/db/queries/.gitattributes new file mode 100644 index 00000000000..cc2d335b830 --- /dev/null +++ b/litellm/rust/crates/db/queries/.gitattributes @@ -0,0 +1 @@ +*.sql text eol=lf diff --git a/litellm/rust/crates/db/src/deadline.rs b/litellm/rust/crates/db/src/deadline.rs new file mode 100644 index 00000000000..d986daa108c --- /dev/null +++ b/litellm/rust/crates/db/src/deadline.rs @@ -0,0 +1,51 @@ +use std::{io, time::Duration}; + +use crate::Error; + +pub async fn within( + socket_timeout: Option, + operation: impl Future>, +) -> Result { + let Some(limit) = socket_timeout else { + return operation.await; + }; + tokio::time::timeout(limit, operation) + .await + .unwrap_or_else(|_| { + Err(Error::Query(sqlx::Error::Io( + io::ErrorKind::TimedOut.into(), + ))) + }) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::within; + use crate::Error; + + #[tokio::test(start_paused = true)] + async fn a_slow_operation_fails_like_a_socket_timeout() { + let result = within(Some(Duration::from_secs(5)), async { + tokio::time::sleep(Duration::from_secs(6)).await; + Ok::<_, Error>(()) + }) + .await; + + assert!( + matches!(result, Err(Error::Query(sqlx::Error::Io(error))) if error.kind() == std::io::ErrorKind::TimedOut) + ); + } + + #[tokio::test(start_paused = true)] + async fn no_timeout_waits_for_the_operation() { + let result = within(None, async { + tokio::time::sleep(Duration::from_secs(3600)).await; + Ok::<_, Error>(7) + }) + .await; + + assert_eq!(result.ok(), Some(7)); + } +} diff --git a/litellm/rust/crates/db/src/endpoint.rs b/litellm/rust/crates/db/src/endpoint.rs new file mode 100644 index 00000000000..cb04943333b --- /dev/null +++ b/litellm/rust/crates/db/src/endpoint.rs @@ -0,0 +1,332 @@ +use std::{path::PathBuf, time::Duration}; + +use litellm_secret_value::SecretValue; +use serde::Deserialize; +use sqlx::{ + PgPool, + postgres::{PgConnectOptions, PgPoolOptions, PgSslMode}, +}; + +use crate::Error; + +const WAIT_FOREVER: Duration = Duration::from_secs(100 * 365 * 24 * 60 * 60); + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct Endpoint { + pub host: String, + pub port: u16, + pub socket: Option, + pub username: String, + pub password: Option, + pub database: String, + pub ssl_mode: SslMode, + pub ssl_root_cert: Option, + pub ssl_client: Option, + pub application_name: Option, + pub options: Vec<(String, String)>, + pub statement_cache_capacity: usize, + pub pool: PoolLimits, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum SslMode { + Disable, + Prefer, + Require, + VerifyCa, + VerifyFull, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ClientIdentity { + pub cert_pem: String, + pub key_pem: SecretValue, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq)] +pub struct PoolLimits { + pub max_connections: u32, + pub acquire_timeout_seconds: Option, + pub idle_timeout_seconds: Option, + pub socket_timeout_seconds: Option, +} + +impl From for PgSslMode { + fn from(mode: SslMode) -> Self { + match mode { + SslMode::Disable => Self::Disable, + SslMode::Prefer => Self::Prefer, + SslMode::Require => Self::Require, + SslMode::VerifyCa => Self::VerifyCa, + SslMode::VerifyFull => Self::VerifyFull, + } + } +} + +impl Endpoint { + pub fn with_password(&self, password: SecretValue) -> Self { + Self { + password: Some(password), + ..self.clone() + } + } + + pub fn connect_options(&self) -> PgConnectOptions { + let options = PgConnectOptions::new_without_pgpass() + .host(&self.host) + .port(self.port) + .username(&self.username) + .database(&self.database) + .ssl_mode(self.ssl_mode.into()) + .options(self.options.iter().map(|(name, value)| (name, value))) + .statement_cache_capacity(self.statement_cache_capacity); + let options = match &self.socket { + Some(directory) => options.socket(directory), + None => options, + }; + let options = match &self.password { + Some(password) => options.password(password.expose()), + None => options, + }; + let options = match &self.application_name { + Some(name) => options.application_name(name), + None => options, + }; + let options = match &self.ssl_client { + Some(identity) => options + .ssl_client_cert_from_pem(&identity.cert_pem) + .ssl_client_key_from_pem(identity.key_pem.expose()), + None => options, + }; + match &self.ssl_root_cert { + Some(cert) => options.ssl_root_cert(cert), + None => options, + } + } + + pub fn pool(&self) -> Result { + let limits = self.pool; + Ok(PgPoolOptions::new() + .min_connections(0) + .max_connections(limits.max_connections) + .acquire_timeout(seconds(limits.acquire_timeout_seconds)?.unwrap_or(WAIT_FOREVER)) + .idle_timeout(seconds(limits.idle_timeout_seconds)?) + .connect_lazy_with(self.connect_options())) + } + + pub fn names_statements(&self) -> bool { + self.statement_cache_capacity > 0 + } + + pub fn socket_timeout(&self) -> Result, Error> { + seconds(self.pool.socket_timeout_seconds) + } +} + +fn seconds(value: Option) -> Result, Error> { + value + .map(Duration::try_from_secs_f64) + .transpose() + .map_err(Error::from) +} + +#[cfg(test)] +mod tests { + use std::{error::Error as StdError, path::PathBuf, time::Duration}; + + use litellm_secret_value::SecretValue; + use rstest::rstest; + use sqlx::postgres::PgSslMode; + + use super::{Endpoint, PoolLimits, SslMode, WAIT_FOREVER}; + use crate::{Error, test_support::database_endpoint}; + + const LIMITS: PoolLimits = PoolLimits { + max_connections: 7, + acquire_timeout_seconds: Some(2.5), + idle_timeout_seconds: Some(60.0), + socket_timeout_seconds: None, + }; + + fn endpoint() -> Endpoint { + Endpoint { + host: "db.internal".to_owned(), + port: 6543, + socket: None, + username: "litellm".to_owned(), + password: Some(SecretValue::new("first")), + database: "proxy".to_owned(), + ssl_mode: SslMode::VerifyFull, + ssl_root_cert: None, + ssl_client: None, + application_name: Some("litellm-native".to_owned()), + options: vec![("statement_timeout".to_owned(), "30000".to_owned())], + statement_cache_capacity: 100, + pool: LIMITS, + } + } + + #[test] + fn maps_endpoint_to_connect_options() { + let options = endpoint().connect_options(); + + assert_eq!(options.get_host(), "db.internal"); + assert_eq!(options.get_port(), 6543); + assert_eq!(options.get_socket(), None); + assert_eq!(options.get_username(), "litellm"); + assert_eq!(options.get_database(), Some("proxy")); + assert_eq!(options.get_application_name(), Some("litellm-native")); + assert!(matches!(options.get_ssl_mode(), PgSslMode::VerifyFull)); + assert_eq!(options.get_options(), Some("-c statement_timeout=30000")); + } + + #[test] + fn connects_through_a_unix_socket_directory() { + let options = Endpoint { + socket: Some(PathBuf::from("/cloudsql/project:region:instance")), + ..endpoint() + } + .connect_options(); + + assert_eq!( + options.get_socket(), + Some(&PathBuf::from("/cloudsql/project:region:instance")) + ); + } + + #[tokio::test] + async fn applies_pool_limits() -> Result<(), Error> { + let pool = endpoint().pool()?; + + assert_eq!(pool.options().get_max_connections(), 7); + assert_eq!(pool.options().get_min_connections(), 0); + assert_eq!( + pool.options().get_acquire_timeout(), + Duration::from_millis(2500) + ); + assert_eq!( + pool.options().get_idle_timeout(), + Some(Duration::from_secs(60)) + ); + assert_eq!(pool.size(), 0); + Ok(()) + } + + #[tokio::test] + async fn missing_acquire_timeout_waits_forever() -> Result<(), Error> { + let pool = Endpoint { + pool: PoolLimits { + acquire_timeout_seconds: None, + ..LIMITS + }, + ..endpoint() + } + .pool()?; + + assert_eq!(pool.options().get_acquire_timeout(), WAIT_FOREVER); + Ok(()) + } + + #[rstest] + #[case::negative(-1.0)] + #[case::not_a_number(f64::NAN)] + #[tokio::test] + async fn rejects_invalid_durations(#[case] seconds: f64) { + let error = Endpoint { + pool: PoolLimits { + acquire_timeout_seconds: Some(seconds), + ..LIMITS + }, + ..endpoint() + } + .pool() + .expect_err("an invalid duration must not build a pool"); + + assert!(matches!(error, Error::InvalidDuration(_))); + } + + #[test] + fn redacts_password() { + assert!(!format!("{:?}", endpoint()).contains("first")); + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn waits_past_an_exhausted_pool_without_a_timeout() -> Result<(), Box> { + let endpoint = database_endpoint()?; + let pool = Endpoint { + pool: PoolLimits { + max_connections: 1, + acquire_timeout_seconds: None, + ..endpoint.pool + }, + ..endpoint + } + .pool()?; + let held = pool.acquire().await?; + let waiter = tokio::spawn({ + let pool = pool.clone(); + async move { pool.acquire().await.map(drop) } + }); + + tokio::time::sleep(Duration::from_millis(300)).await; + assert!(!waiter.is_finished()); + drop(held); + + waiter.await??; + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn applies_session_options() -> Result<(), Box> { + let pool = Endpoint { + options: vec![ + ("statement_timeout".to_owned(), "1234".to_owned()), + ( + "search_path".to_owned(), + "\"Other Schema\", public".to_owned(), + ), + ], + application_name: Some("litellm-native".to_owned()), + ..database_endpoint()? + } + .pool()?; + + #[expect( + clippy::disallowed_methods, + reason = "Session settings are not part of the compile-checked schema" + )] + let (statement_timeout, search_path, application_name): (String, String, String) = + sqlx::query_as( + "SELECT current_setting('statement_timeout'), current_setting('search_path'), \ + current_setting('application_name')", + ) + .fetch_one(&pool) + .await?; + + assert_eq!(statement_timeout, "1234ms"); + assert_eq!(search_path, "\"Other Schema\", public"); + assert_eq!(application_name, "litellm-native"); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn rotated_password_reaches_new_connections() -> Result<(), Box> { + let endpoint = database_endpoint()?; + let pool = endpoint + .with_password(SecretValue::new("not-the-password")) + .pool()?; + #[expect(clippy::disallowed_methods, reason = "A probe query needs no schema")] + let probe = || sqlx::query("SELECT 1"); + let stale = probe().execute(&pool).await; + + pool.set_connect_options(endpoint.connect_options()); + probe().execute(&pool).await?; + + assert!(stale.is_err()); + Ok(()) + } +} diff --git a/litellm/rust/crates/db/src/error.rs b/litellm/rust/crates/db/src/error.rs index af0ee030896..c4d44a3231e 100644 --- a/litellm/rust/crates/db/src/error.rs +++ b/litellm/rust/crates/db/src/error.rs @@ -6,6 +6,8 @@ pub enum Error { InvalidSpendBatch { ids: usize, costs: usize }, #[error("invalid spend timestamp")] InvalidTimestamp(#[from] chrono::ParseError), + #[error("invalid pool duration")] + InvalidDuration(#[from] std::time::TryFromFloatSecsError), } impl Error { diff --git a/litellm/rust/crates/db/src/lib.rs b/litellm/rust/crates/db/src/lib.rs index 541f97c7db9..bab2d492c92 100644 --- a/litellm/rust/crates/db/src/lib.rs +++ b/litellm/rust/crates/db/src/lib.rs @@ -7,15 +7,20 @@ macro_rules_attribute::attribute_alias! { } mod daily_spend; +mod deadline; mod end_user; +mod endpoint; mod error; mod failure; mod spend; +mod statements; #[cfg(test)] mod test_support; pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow}; +pub use deadline::within; pub use end_user::{EndUser, RateLimits, find_end_user}; +pub use endpoint::{ClientIdentity, Endpoint, PoolLimits, SslMode}; pub use error::Error; pub use failure::DbFailure; pub use spend::{ @@ -24,3 +29,4 @@ pub use spend::{ begin, insert_spend_logs, lock_team, upsert_daily_end_user_spend, upsert_daily_spend, upsert_daily_team_spend, upsert_daily_user_spend, }; +pub use statements::Statements; diff --git a/litellm/rust/crates/db/src/statements.rs b/litellm/rust/crates/db/src/statements.rs new file mode 100644 index 00000000000..a3da065fe81 --- /dev/null +++ b/litellm/rust/crates/db/src/statements.rs @@ -0,0 +1,152 @@ +use futures_util::{future::BoxFuture, stream::BoxStream}; +use sqlx::{ + Describe, Either, Execute, Executor, Postgres, SqlStr, + error::BoxDynError, + postgres::{PgArguments, PgQueryResult, PgRow, PgStatement, PgTypeInfo}, +}; + +// TODO(transact-rs/sqlx#4337): sqlx-postgres 0.9.0 still names every statement when +// statement_cache_capacity is 0, leaving one open per query on the server and colliding +// as "sqlx_s_1 already exists" behind transaction poolers. #4337 makes capacity 0 send +// unnamed statements on its own; delete this wrapper once a sqlx release carries it. +#[derive(Debug)] +pub struct Statements { + executor: E, + named: bool, +} + +impl Statements { + pub fn new(executor: E, named: bool) -> Self { + Self { executor, named } + } +} + +struct Naming { + query: Q, + named: bool, +} + +impl<'q, Q: Execute<'q, Postgres>> Execute<'q, Postgres> for Naming { + fn sql(self) -> SqlStr { + self.query.sql() + } + + fn statement(&self) -> Option<&PgStatement> { + self.query.statement() + } + + fn take_arguments(&mut self) -> Result, BoxDynError> { + self.query.take_arguments() + } + + fn persistent(&self) -> bool { + self.named && self.query.persistent() + } +} + +impl<'c, E: Executor<'c, Database = Postgres>> Executor<'c> for Statements { + type Database = Postgres; + + fn fetch_many<'e, 'q: 'e, Q>( + self, + query: Q, + ) -> BoxStream<'e, Result, sqlx::Error>> + where + 'c: 'e, + Q: 'q + Execute<'q, Postgres>, + { + self.executor.fetch_many(Naming { + query, + named: self.named, + }) + } + + fn fetch_optional<'e, 'q: 'e, Q>( + self, + query: Q, + ) -> BoxFuture<'e, Result, sqlx::Error>> + where + 'c: 'e, + Q: 'q + Execute<'q, Postgres>, + { + self.executor.fetch_optional(Naming { + query, + named: self.named, + }) + } + + fn prepare_with<'e>( + self, + sql: SqlStr, + parameters: &'e [PgTypeInfo], + ) -> BoxFuture<'e, Result> + where + 'c: 'e, + { + self.executor.prepare_with(sql, parameters) + } + + fn describe<'e>(self, sql: SqlStr) -> BoxFuture<'e, Result, sqlx::Error>> + where + 'c: 'e, + { + self.executor.describe(sql) + } +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use rstest::rstest; + use sqlx::{AssertSqlSafe, PgConnection}; + + use super::Statements; + use crate::{Endpoint, test_support::database_endpoint}; + + async fn named_statements( + connection: &mut PgConnection, + named: bool, + ) -> Result { + #[expect( + clippy::disallowed_methods, + reason = "pg_prepared_statements is not part of the compile-checked schema" + )] + sqlx::query_scalar("SELECT count(*) FROM pg_prepared_statements WHERE NOT from_sql") + .fetch_one(Statements::new(connection, named)) + .await + } + + #[rstest] + #[case::cache_off(0, 0)] + #[case::cache_on(100, 4)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn the_endpoint_cache_decides_whether_statements_stay_named( + #[case] statement_cache_capacity: usize, + #[case] expected: i64, + ) -> Result<(), Box> { + let endpoint = Endpoint { + statement_cache_capacity, + ..database_endpoint()? + }; + let pool = endpoint.pool()?; + let mut connection = pool.acquire().await?; + + for query in ["SELECT 1", "SELECT 2", "SELECT 3"] { + #[expect(clippy::disallowed_methods, reason = "probe queries need no schema")] + sqlx::query(AssertSqlSafe(query)) + .execute(Statements::new( + &mut *connection, + endpoint.names_statements(), + )) + .await?; + } + + assert_eq!( + named_statements(&mut connection, endpoint.names_statements()).await?, + expected + ); + Ok(()) + } +} diff --git a/litellm/rust/crates/db/src/test_support.rs b/litellm/rust/crates/db/src/test_support.rs index 992be4a4447..40446ca53dc 100644 --- a/litellm/rust/crates/db/src/test_support.rs +++ b/litellm/rust/crates/db/src/test_support.rs @@ -1,10 +1,40 @@ -use std::error::Error as StdError; +use std::{error::Error as StdError, str::FromStr}; +use litellm_secret_value::SecretValue; use rstest::fixture; -use sqlx::PgPool; +use sqlx::{PgPool, postgres::PgConnectOptions}; + +use crate::{Endpoint, PoolLimits, SslMode}; #[fixture] pub async fn database_pool() -> Result> { let database_url = std::env::var("DATABASE_URL")?; Ok(PgPool::connect(&database_url).await?) } + +pub fn database_endpoint() -> Result> { + let database_url = std::env::var("DATABASE_URL")?; + let options = PgConnectOptions::from_str(&database_url)?; + Ok(Endpoint { + host: options.get_host().to_owned(), + port: options.get_port(), + socket: None, + username: options.get_username().to_owned(), + password: url::Url::parse(&database_url)? + .password() + .map(SecretValue::new), + database: options.get_database().unwrap_or("postgres").to_owned(), + ssl_mode: SslMode::Prefer, + ssl_root_cert: None, + ssl_client: None, + application_name: None, + options: Vec::new(), + statement_cache_capacity: 100, + pool: PoolLimits { + max_connections: 4, + acquire_timeout_seconds: Some(10.0), + idle_timeout_seconds: None, + socket_timeout_seconds: None, + }, + }) +} diff --git a/litellm/rust/crates/python-bridge/Cargo.toml b/litellm/rust/crates/python-bridge/Cargo.toml index 4a52308cef1..06c390b820f 100644 --- a/litellm/rust/crates/python-bridge/Cargo.toml +++ b/litellm/rust/crates/python-bridge/Cargo.toml @@ -21,6 +21,7 @@ tiktoken = ["litellm-token-counter/tiktoken"] [dependencies] rustls.workspace = true litellm-secret-value.workspace = true +litellm-db.workspace = true fancy-regex.workspace = true litellm-tracing.workspace = true litellm-spend-clickhouse.workspace = true @@ -76,6 +77,7 @@ serde_json.workspace = true strum.workspace = true veil.workspace = true thiserror.workspace = true +sqlx.workspace = true tokio = { workspace = true, features = ["rt", "sync"] } [dev-dependencies] diff --git a/litellm/rust/crates/python-bridge/src/db.rs b/litellm/rust/crates/python-bridge/src/db.rs index 03fb30b986f..ca1ccc8fb2b 100644 --- a/litellm/rust/crates/python-bridge/src/db.rs +++ b/litellm/rust/crates/python-bridge/src/db.rs @@ -1,19 +1,51 @@ -use litellm_host_python::ready_future; -use pyo3::{prelude::*, types::PyNone}; +use litellm_db::Endpoint; +use litellm_secret_value::SecretValue; +use pyo3::{exceptions::PyValueError, prelude::*}; +use sqlx::PgPool; #[pyclass(frozen)] -pub(crate) struct NativeDatabase {} +pub(crate) struct NativeDatabase { + writer_endpoint: Endpoint, + writer: PgPool, +} + +fn map_error(error: litellm_db::Error) -> PyErr { + PyValueError::new_err(litellm_error::Report::new(&error).to_string()) +} -// TODO: take typed writer/reader settings and a token provider, then open a sqlx PgPool, once the first -// repository moves to native. The Prisma DATABASE_URL carries Prisma-only params and rotating IAM tokens #[pymethods] impl NativeDatabase { #[staticmethod] - fn connect(py: Python<'_>) -> PyResult> { - ready_future(py, Bound::new(py, NativeDatabase {})?.as_any()) + fn connect( + py: Python<'_>, + #[pyo3(from_py_with = litellm_host_python::from_py_argument)] writer: Endpoint, + ) -> PyResult> { + crate::execution::run_async_value(py, async move { + let pool = writer.pool().map_err(map_error)?; + Ok(NativeDatabase { + writer_endpoint: writer, + writer: pool, + }) + }) + } + + fn set_writer_password(&self, password: String) { + self.writer.set_connect_options( + self.writer_endpoint + .with_password(SecretValue::new(password)) + .connect_options(), + ); } fn close<'py>(&self, py: Python<'py>) -> PyResult> { - ready_future(py, PyNone::get(py).as_any()) + let writer = self.writer.clone(); + crate::execution::run_async( + py, + async move { + writer.close().await; + Ok(()) + }, + |error: PyErr| error, + ) } } diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 331fad62c12..be58056c8ef 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -9,6 +9,7 @@ from typing_extensions import disjoint_base from litellm.integrations.custom_logger_v2.rust import RustHandle from litellm.redis.store_settings import RedisBreakerSettings, RedisStoreSettings from litellm.redis.stores.router_session_pins.keys import PinKey +from litellm.rust_bridge.database import NativeDatabaseEndpoint from litellm.rust_bridge.execution import Execution from litellm.rust_bridge.public_call import NativeCall @@ -596,7 +597,8 @@ class NativeSpendCounters: @final class NativeDatabase: @staticmethod - def connect() -> Future[NativeDatabase]: ... + def connect(writer: NativeDatabaseEndpoint) -> Future[NativeDatabase]: ... + def set_writer_password(self, password: str) -> None: ... def close(self) -> Future[None]: ... @final diff --git a/litellm/rust_bridge/database.py b/litellm/rust_bridge/database.py new file mode 100644 index 00000000000..1888a349f68 --- /dev/null +++ b/litellm/rust_bridge/database.py @@ -0,0 +1,33 @@ +from typing import Literal, TypeAlias + +from typing_extensions import ReadOnly, TypedDict + +SslMode: TypeAlias = Literal["disable", "prefer", "require", "verify-ca", "verify-full"] + + +class NativePoolLimits(TypedDict): + max_connections: ReadOnly[int] + acquire_timeout_seconds: ReadOnly[float | None] + idle_timeout_seconds: ReadOnly[float | None] + socket_timeout_seconds: ReadOnly[float | None] + + +class NativeClientIdentity(TypedDict): + cert_pem: ReadOnly[str] + key_pem: ReadOnly[str] + + +class NativeDatabaseEndpoint(TypedDict): + host: ReadOnly[str] + port: ReadOnly[int] + socket: ReadOnly[str | None] + username: ReadOnly[str] + password: ReadOnly[str | None] + database: ReadOnly[str] + ssl_mode: ReadOnly[SslMode] + ssl_root_cert: ReadOnly[str | None] + ssl_client: ReadOnly[NativeClientIdentity | None] + application_name: ReadOnly[str | None] + options: ReadOnly[tuple[tuple[str, str], ...]] + statement_cache_capacity: ReadOnly[int] + pool: ReadOnly[NativePoolLimits] diff --git a/tests/_support/native_database.py b/tests/_support/native_database.py index 29b1fff399f..bade92d79f8 100644 --- a/tests/_support/native_database.py +++ b/tests/_support/native_database.py @@ -1,11 +1,15 @@ import asyncio +from typing import Final +from litellm.proxy.db.native_database import native_endpoint from litellm.rust_bridge._native import NativeDatabase +_NEVER_DIALED_URL: Final = "postgresql://litellm:secret@db.invalid:5432/proxy" -async def _connect() -> NativeDatabase: - return await NativeDatabase.connect() + +async def connect_native_database() -> NativeDatabase: + return await NativeDatabase.connect(native_endpoint(_NEVER_DIALED_URL, tls_offered=True, max_connections=1).endpoint) def native_database() -> NativeDatabase: - return asyncio.run(_connect()) + return asyncio.run(connect_native_database()) diff --git a/tests/unit/proxy/db/test_connection_budget.py b/tests/unit/proxy/db/test_connection_budget.py new file mode 100644 index 00000000000..115bba5d68b --- /dev/null +++ b/tests/unit/proxy/db/test_connection_budget.py @@ -0,0 +1,46 @@ +from typing import Final + +import pytest + +from litellm.proxy.db.connection_budget import ConnectionBudget + + +@pytest.mark.parametrize("total", range(1, 41)) +def test_default_split_leaves_prisma_the_whole_total(total: int) -> None: + budget: Final = ConnectionBudget.split(total) + + assert (budget.python, budget.rust) == (total, 0) + + +@pytest.mark.parametrize("total", range(3, 41)) +def test_a_reserve_comes_out_of_the_total(total: int) -> None: + budget: Final = ConnectionBudget.split(total, rust_reserved=2) + + assert (budget.python + budget.rust, budget.rust) == (total, 2) + + +@pytest.mark.parametrize(("total", "python"), [(2, 1), (1, 1)], ids=["reserve_equals_total", "single_connection"]) +def test_prisma_keeps_one_connection_when_the_reserve_does_not_fit(total: int, python: int) -> None: + budget: Final = ConnectionBudget.split(total, rust_reserved=2) + + assert (budget.python, budget.rust) == (python, 2) + + +@pytest.mark.parametrize("connection_limit", ["7", 7]) +def test_a_positive_integer_limit_is_split(connection_limit: str | int) -> None: + assert ConnectionBudget.of(connection_limit, rust_reserved=2) == ConnectionBudget.split(7, rust_reserved=2) + + +@pytest.mark.parametrize("connection_limit", ["0", 0, "-1", -1, "abc", "10.0", 10.5, "", "\u00b2"]) +def test_a_limit_that_is_not_a_positive_integer_is_left_to_prisma(connection_limit: str | float) -> None: + assert ConnectionBudget.of(connection_limit, rust_reserved=2) is None + + +def test_prisma_params_change_only_beside_a_rust_pool() -> None: + params: Final = {"connection_limit": "010", "pool_timeout": 30} + + assert ConnectionBudget.split(10).prisma_params(params) == params + assert ConnectionBudget.split(10, rust_reserved=2).prisma_params(params) == { + "connection_limit": 8, + "pool_timeout": 30, + } diff --git a/tests/unit/proxy/db/test_db_url_settings.py b/tests/unit/proxy/db/test_db_url_settings.py index 1f86f6ab5d1..a42894efa89 100644 --- a/tests/unit/proxy/db/test_db_url_settings.py +++ b/tests/unit/proxy/db/test_db_url_settings.py @@ -34,6 +34,8 @@ from pydantic import ValidationError from litellm.proxy.db.db_url_settings import ( PG_SSL_REQUEST, DatabaseURLSettings, + add_missing_query_params, + prisma_default_params, token_refresh_params_from_url, translate_libpq_ssl_params, unsupported_db_scheme, @@ -1278,32 +1280,52 @@ def test_reader_keeps_its_own_pinned_idle_lifetime(monkeypatch): @pytest.mark.parametrize( - "writer_limit, reader_limit, num_workers, expected", + "writer_limit, rust_writer_limit, reader_limit, num_workers, expected", [ - ("10", None, "4", "4 worker(s) x writer connection_limit 10 = up to 40 connections"), + ("10", 0, None, "4", "4 worker(s) x writer connection_limit 10 = up to 40 connections"), ( "10", + 0, "10", "4", "4 worker(s) x (writer connection_limit 10 + reader connection_limit 10) = up to 80 connections", ), + ("8", 2, None, "4", "4 worker(s) x (writer connection_limit 8 + rust writer pool 2) = up to 40 connections"), ( - "10", + "8", + 2, + "8", + "4", + "4 worker(s) x (writer connection_limit 8 + rust writer pool 2 + reader connection_limit 8) = up to 72 connections", + ), + ( + "8", + 2, "50", "4", - "4 worker(s) x (writer connection_limit 10 + reader connection_limit 50) = up to 240 connections", + "4 worker(s) x (writer connection_limit 8 + rust writer pool 2 + reader connection_limit 50) = up to 240 connections", ), - ("10", None, "0", "1 worker(s) x writer connection_limit 10 = up to 10 connections"), + ("8", 2, None, "0", "1 worker(s) x (writer connection_limit 8 + rust writer pool 2) = up to 10 connections"), + ], + ids=[ + "no_rust_share_writer_only", + "no_rust_share_with_reader", + "writer_only", + "reader_adds_an_engine", + "reader_pins_its_own_limit", + "worker_floor", ], - ids=["writer_only", "reader_doubles_engines", "reader_pins_its_own_limit", "worker_floor"], ) def test_connection_budget_message_sums_the_engines_each_worker_owns( - writer_limit: str, reader_limit: str | None, num_workers: str, expected: str + writer_limit: str, rust_writer_limit: int, reader_limit: str | None, num_workers: str, expected: str ) -> None: from litellm.proxy.db.db_url_settings import postgres_connection_budget_message message: Final = postgres_connection_budget_message( - writer_limit=writer_limit, reader_limit=reader_limit, num_workers=num_workers + writer_limit=writer_limit, + rust_writer_limit=rust_writer_limit, + reader_limit=reader_limit, + num_workers=num_workers, ) assert expected in message assert "max_connections minus superuser_reserved_connections" in message @@ -1312,5 +1334,30 @@ def test_connection_budget_message_sums_the_engines_each_worker_owns( def test_connection_budget_message_survives_an_unparseable_limit() -> None: from litellm.proxy.db.db_url_settings import postgres_connection_budget_message - message: Final = postgres_connection_budget_message(writer_limit="ten", reader_limit=None, num_workers="2") + message: Final = postgres_connection_budget_message( + writer_limit="ten", rust_writer_limit=2, reader_limit=None, num_workers="2" + ) assert "writer='ten'" in message + + +@pytest.mark.parametrize( + ("url", "native_connections", "expected_name"), + [ + ("postgresql://u:p@db:5432/proxy", 0, None), + ("postgresql://u:p@db:5432/proxy", 2, "litellm-prisma"), + ("postgresql://u:p@db:5432/proxy?application_name=ops", 2, "ops"), + ], + ids=["untagged_without_native_pool", "tagged_beside_native_pool", "operator_name_wins"], +) +def test_prisma_is_tagged_only_beside_a_native_pool( + url: str, native_connections: int, expected_name: str | None +) -> None: + final_url: Final = add_missing_query_params( + url, prisma_default_params({"max_idle_connection_lifetime": 60}, native_connections) + ) + + query: Final = urllib.parse.parse_qs(urllib.parse.urlsplit(final_url).query) + assert (query.get("application_name"), query["max_idle_connection_lifetime"]) == ( + None if expected_name is None else [expected_name], + ["60"], + ) diff --git a/tests/unit/proxy/db/test_native_database.py b/tests/unit/proxy/db/test_native_database.py new file mode 100644 index 00000000000..dad278ea13c --- /dev/null +++ b/tests/unit/proxy/db/test_native_database.py @@ -0,0 +1,373 @@ +import asyncio +import datetime as dt +from datetime import datetime, timedelta +from pathlib import Path +from typing import Final + +import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.serialization import BestAvailableEncryption, load_pem_private_key, pkcs12 +from cryptography.x509.oid import NameOID +from pydantic import TypeAdapter + +from litellm.proxy.db.native_database import ( + TokenRotation, + keep_writer_token_fresh, + native_endpoint, + needs_tls_probe, + server_offers_tls, + writer_token_rotation, +) +from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR +from litellm.proxy.db.token_auth import IAMEndpoint, RdsIamTokenAuth +from litellm.rust_bridge.database import NativeDatabaseEndpoint, NativePoolLimits + +NOW: Final = datetime(2026, 11, 3, 12, 0, 0) +_ADDRESS: Final = TypeAdapter(tuple[str, int]) +PRISMA_DEFAULT_POOL: Final = NativePoolLimits( + max_connections=2, acquire_timeout_seconds=10.0, idle_timeout_seconds=None, socket_timeout_seconds=None +) + + +def _rds_token(issued: datetime, lifetime_seconds: int) -> str: + return ( + "db.internal:5432/?Action=connect&DBUser=litellm" + f"&X-Amz-Date={issued:%Y%m%dT%H%M%SZ}&X-Amz-Expires={lifetime_seconds}&X-Amz-Signature=abc" + ) + + +def _endpoint(**overrides: object) -> NativeDatabaseEndpoint: + return NativeDatabaseEndpoint( + **{ # pyright: ignore[reportArgumentType] # test overrides are checked by the equality assertion + "host": "db", + "port": 5432, + "socket": None, + "username": "u", + "password": None, + "database": "proxy", + "ssl_mode": "prefer", + "ssl_root_cert": None, + "ssl_client": None, + "application_name": "litellm-native", + "options": (), + "statement_cache_capacity": 100, + "pool": PRISMA_DEFAULT_POOL, + **overrides, + } + ) + + +@pytest.mark.parametrize( + "url, tls_offered, expected", + [ + pytest.param( + "postgresql://litellm:p%40ss%3Aword@db.internal:6543/proxy?connection_limit=12&pool_timeout=20" + "&max_idle_connection_lifetime=60&socket_timeout=7&application_name=litellm-prisma", + True, + _endpoint( + host="db.internal", + port=6543, + username="litellm", + password="p@ss:word", + pool=NativePoolLimits( + max_connections=2, + acquire_timeout_seconds=20.0, + idle_timeout_seconds=60.0, + socket_timeout_seconds=7.0, + ), + ), + id="password_pool_params_and_native_share", + ), + pytest.param( + "postgresql://svc%40corp@db/proxy?sslmode=require&sslcert=/certs/root.pem&sslaccept=strict" + "&schema=Tenant&options=-c%20statement_timeout%3D30000%20-clock_timeout%3D5000&pgbouncer=true", + True, + _endpoint( + username="svc@corp", + ssl_mode="verify-full", + ssl_root_cert="/certs/root.pem", + options=(("search_path", '"Tenant"'), ("statement_timeout", "30000"), ("lock_timeout", "5000")), + statement_cache_capacity=0, + ), + id="strict_tls_schema_timeouts_and_pgbouncer", + ), + pytest.param( + "postgresql://u@db/proxy?statement_cache_size=0&application_name=billing-proxy", + True, + _endpoint(statement_cache_capacity=0, application_name="billing-proxy"), + id="statement_cache_size_zero_and_an_operator_application_name_carry_over", + ), + pytest.param( + "postgresql://u@db/proxy?statement_cache_size=250&pgbouncer=false", + True, + _endpoint(statement_cache_capacity=250), + id="statement_cache_size_sets_the_capacity", + ), + pytest.param( + "postgresql://u@db/proxy?statement_cache_size=250&pgbouncer=true", + True, + _endpoint(statement_cache_capacity=0), + id="pgbouncer_disables_the_statement_cache_whatever_its_size", + ), + pytest.param( + "postgresql://u@db/proxy?sslmode=require&sslcert=/certs/root.pem&schema=public", + True, + _endpoint(ssl_mode="require"), + id="require_without_strict_never_verifies_the_operator_cert", + ), + pytest.param( + "postgresql://u@db/proxy?sslcert=/certs/root.pem&sslaccept=strict", + True, + _endpoint(ssl_mode="verify-full", ssl_root_cert="/certs/root.pem"), + id="prefer_strict_verifies_when_the_server_offers_tls", + ), + pytest.param( + "postgresql://u@db/proxy?sslcert=/certs/root.pem&sslaccept=strict", + False, + _endpoint(ssl_mode="disable"), + id="prefer_strict_falls_back_to_plaintext_without_server_tls", + ), + pytest.param( + "postgresql://u@db/proxy?sslmode=disable&sslaccept=strict", + True, + _endpoint(ssl_mode="disable"), + id="disable_wins_over_strict", + ), + pytest.param( + "postgresql://u@localhost/proxy?host=/cloudsql/project:region:db&pool_timeout=0", + True, + _endpoint( + host="localhost", + socket="/cloudsql/project:region:db", + pool=NativePoolLimits( + max_connections=2, + acquire_timeout_seconds=None, + idle_timeout_seconds=None, + socket_timeout_seconds=None, + ), + ), + id="unix_socket_and_pool_timeout_zero_waits_forever", + ), + ], +) +def test_native_endpoint_reads_the_prisma_url(url: str, tls_offered: bool, expected: NativeDatabaseEndpoint) -> None: + settings: Final = native_endpoint(url, tls_offered=tls_offered, max_connections=2) + + assert settings.endpoint == expected + assert settings.ignored_options == () + + +@pytest.mark.parametrize( + "options, expected_options, expected_ignored", + [ + pytest.param("-c statement_timeout=5 -B 64", (("statement_timeout", "5"),), ("-B", "64"), id="flags"), + pytest.param("statement_timeout=5", (), ("statement_timeout=5",), id="bare_setting"), + pytest.param("-c 'unterminated", (), ("-c 'unterminated",), id="unbalanced_quote"), + ], +) +def test_native_endpoint_reports_options_it_cannot_send_instead_of_failing( + options: str, expected_options: tuple[tuple[str, str], ...], expected_ignored: tuple[str, ...] +) -> None: + settings: Final = native_endpoint(f"postgresql://u@db/proxy?options={options}", tls_offered=True, max_connections=2) + + assert settings.endpoint["options"] == expected_options + assert settings.ignored_options == expected_ignored + + +def test_native_endpoint_rejects_a_url_without_a_user() -> None: + with pytest.raises(ValueError, match="DATABASE_URL must name a host and a user"): + native_endpoint("postgresql://db.internal/proxy", tls_offered=True, max_connections=2) + + +def _identity(password: bytes) -> tuple[bytes, ec.EllipticCurvePrivateKey, x509.Certificate]: + key: Final = ec.generate_private_key(ec.SECP256R1()) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "litellm")]) + cert: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(1) + .not_valid_before(dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc)) + .not_valid_after(dt.datetime(2027, 1, 1, tzinfo=dt.timezone.utc)) + .sign(key, hashes.SHA256()) + ) + bundle: Final = pkcs12.serialize_key_and_certificates( + b"litellm", key, cert, None, BestAvailableEncryption(password) + ) + return bundle, key, cert + + +def test_native_endpoint_hands_over_the_pkcs12_identity_as_pem() -> None: + bundle, key, cert = _identity(b"s3cret") + read: Final = {Path("/certs/client.p12"): bundle} + + identity: Final = native_endpoint( + "postgresql://u@db/proxy?sslmode=require&sslidentity=/certs/client.p12&sslpassword=s3cret", + tls_offered=True, + max_connections=2, + read_file=read.__getitem__, + ).endpoint["ssl_client"] + + assert identity is not None + assert x509.load_pem_x509_certificate(identity["cert_pem"].encode()) == cert + assert load_pem_private_key(identity["key_pem"].encode(), None).public_key() == key.public_key() + + +def test_native_endpoint_rejects_a_wrong_pkcs12_password() -> None: + bundle, _, _ = _identity(b"s3cret") + + with pytest.raises(ValueError, match=r"Invalid password or PKCS12 data|Could not deserialize PKCS12 data"): + native_endpoint( + "postgresql://u@db/proxy?sslidentity=/certs/client.p12&sslpassword=wrong", + tls_offered=True, + max_connections=2, + read_file=lambda _path: bundle, + ) + + +@pytest.mark.parametrize( + "url, expected", + [ + pytest.param("postgresql://u@db/proxy?sslaccept=strict", True, id="default_prefer_strict"), + pytest.param("postgresql://u@db/proxy?sslmode=prefer&sslaccept=strict", True, id="prefer_strict"), + pytest.param("postgresql://u@db/proxy?sslmode=require&sslaccept=strict", False, id="require_never_falls_back"), + pytest.param("postgresql://u@db/proxy", False, id="prefer_unverified_falls_back_itself"), + pytest.param("postgresql://u@localhost/proxy?sslaccept=strict&host=/run/pg", False, id="unix_socket"), + ], +) +def test_only_prefer_strict_needs_a_tls_probe(url: str, expected: bool) -> None: + assert needs_tls_probe(url) is expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("answer, offered", [(b"S", True), (b"N", False)]) +async def test_server_offers_tls_reads_the_ssl_request_answer(answer: bytes, offered: bool) -> None: + received: Final = asyncio.Event() + + async def respond(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + if await reader.readexactly(8) == b"\x00\x00\x00\x08\x04\xd2\x16\x2f": + received.set() + writer.write(answer) + await writer.drain() + writer.close() + + server: Final = await asyncio.start_server(respond, "127.0.0.1", 0) + _, port = _ADDRESS.validate_python(server.sockets[0].getsockname()) + try: + assert await server_offers_tls(f"postgresql://u@127.0.0.1:{port}/proxy") is offered + finally: + server.close() + await server.wait_closed() + assert received.is_set() + + +@pytest.mark.asyncio +async def test_unreachable_server_counts_as_offering_tls() -> None: + assert await server_offers_tls("postgresql://u@127.0.0.1:1/proxy", timeout=1) is True + + +def test_writer_token_rotation_follows_token_auth(monkeypatch: pytest.MonkeyPatch) -> None: + url: Final = "postgresql://litellm:tok@db.internal:5432/proxy?schema=s" + monkeypatch.delenv("AZURE_POSTGRESQL_AUTH", raising=False) + monkeypatch.delenv(PGBOUNCER_POOLED_ENV_VAR, raising=False) + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "false") + assert writer_token_rotation(url) is None + + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.setenv("AWS_RDS_REGION", "us-west-2") + assert writer_token_rotation(url) == TokenRotation( + auth=RdsIamTokenAuth(region="us-west-2"), + endpoint=IAMEndpoint(host="db.internal", port="5432", user="litellm", name="proxy", schema="s"), + ) + + +def test_pooled_writer_needs_no_token_rotation(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.setenv(PGBOUNCER_POOLED_ENV_VAR, "true") + + assert writer_token_rotation("postgresql://litellm:local@127.0.0.1:6432/proxy") is None + + +@pytest.mark.parametrize( + "token, expected_seconds", + [ + pytest.param(_rds_token(NOW, 900), 720.0, id="refreshes_buffer_before_expiry"), + pytest.param(_rds_token(NOW - timedelta(seconds=890), 900), 30.0, id="never_sooner_than_retry_floor"), + pytest.param("opaque", 600.0, id="unreadable_expiry_uses_fallback"), + ], +) +def test_token_rotation_schedules_the_next_refresh(token: str, expected_seconds: float) -> None: + rotation: Final = TokenRotation( + auth=RdsIamTokenAuth(), endpoint=IAMEndpoint(host="db", port="5432", user="u", name="n") + ) + + assert rotation.seconds_until_refresh(token, NOW) == expected_seconds + + +class _Sink: + def __init__(self, wanted: int) -> None: + self.passwords: tuple[str, ...] = () + self._wanted: Final = wanted + self.done: Final = asyncio.Event() + + def set_writer_password(self, password: str) -> None: + self.passwords = (*self.passwords, password) + if len(self.passwords) >= self._wanted: + self.done.set() + + +class _Schedule: + def __init__(self) -> None: + self.delays: tuple[float, ...] = () + + async def sleep(self, seconds: float) -> None: + self.delays = (*self.delays, seconds) + await asyncio.sleep(0) + + +def _minting(*outcomes: str | Exception) -> TokenRotation: + remaining: Final = iter(outcomes) + + def mint(_auth: object, _endpoint: object) -> str: + outcome: Final = next(remaining) + if isinstance(outcome, Exception): + raise outcome + return outcome + + return TokenRotation( + auth=RdsIamTokenAuth(), endpoint=IAMEndpoint(host="db", port="5432", user="u", name="n"), mint_token=mint + ) + + +async def _run_until(sink: _Sink, rotation: TokenRotation, schedule: _Schedule) -> None: + task: Final = asyncio.create_task( + keep_writer_token_fresh(sink, rotation, "opaque", now=lambda: NOW, sleep=schedule.sleep) + ) + await asyncio.wait_for(sink.done.wait(), timeout=5) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_keep_writer_token_fresh_installs_each_minted_token() -> None: + sink: Final = _Sink(wanted=2) + schedule: Final = _Schedule() + + await _run_until(sink, _minting("first%2Ftoken", _rds_token(NOW, 900)), schedule) + + assert sink.passwords == ("first/token", _rds_token(NOW, 900)) + assert schedule.delays[:2] == (600.0, 600.0) + + +@pytest.mark.asyncio +async def test_keep_writer_token_fresh_retries_a_failed_mint() -> None: + sink: Final = _Sink(wanted=1) + schedule: Final = _Schedule() + + await _run_until(sink, _minting(RuntimeError("sts down"), "recovered"), schedule) + + assert sink.passwords == ("recovered",) + assert schedule.delays[:2] == (600.0, 30.0) diff --git a/tests/unit/proxy/health_endpoints/test_deployment_warning_endpoints.py b/tests/unit/proxy/health_endpoints/test_deployment_warning_endpoints.py index 67aea8dfeb8..456cc2005e9 100644 --- a/tests/unit/proxy/health_endpoints/test_deployment_warning_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_deployment_warning_endpoints.py @@ -32,7 +32,7 @@ from litellm.proxy.proxy_server import ( ) from litellm.repositories.database import Database from litellm.repositories.deployment_warning_ignore_repository import DEPLOYMENT_WARNING_IGNORES_ROW_ID -from litellm.rust_bridge._native import NativeDatabase +from tests._support.native_database import connect_native_database T0: Final = datetime(2026, 10, 14, 12, 0, tzinfo=timezone.utc) THIRTY_DAYS: Final = 30 * 24 * 60 * 60 @@ -254,7 +254,7 @@ class TestDeploymentWarningIgnoreStore: db=ui_settings_table(None), writer_db=ui_settings_table(SimpleNamespace(ui_settings=fresh)) ) store: Final = await get_deployment_warning_ignore_repository( - Database(prisma=prisma, native=await NativeDatabase.connect()) + Database(prisma=prisma, native=await connect_native_database()) ) assert store is not None assert await store.get_ignores() == { diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index d85d685c352..4e2ddfe859d 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -16,6 +16,7 @@ import click import fastapi import httpx import pytest +from pydantic import TypeAdapter import builtins @@ -3518,6 +3519,68 @@ def _run_server_and_capture_urls( return {k: os.environ[k] for k in _CAPTURED_DB_ENV_VARS if k in os.environ} +_CAPTURED_URLS: Final = TypeAdapter(dict[str, str]) + + +@pytest.mark.parametrize( + "general_settings, database_url, expected_limit, expected_name", + [ + pytest.param({}, "postgresql://t:t@localhost:5432/t", "10", None, id="default_limit"), + pytest.param( + {"database_connection_pool_limit": 30}, + "postgresql://t:t@localhost:5432/t?application_name=ops", + "30", + "ops", + id="configured_limit_and_operator_name", + ), + pytest.param( + {"database_extra_connection_params": {"connection_limit": 5}}, + "postgresql://t:t@localhost:5432/t", + "5", + None, + id="extra_params_limit", + ), + pytest.param( + {"database_connection_pool_limit": 0}, + "postgresql://t:t@localhost:5432/t", + "0", + None, + id="zero_limit_prisma_accepts", + ), + pytest.param( + {"database_extra_connection_params": {"connection_limit": "abc"}}, + "postgresql://t:t@localhost:5432/t", + "abc", + None, + id="non_integer_limit_left_to_prisma", + ), + ], +) +def test_prisma_keeps_the_whole_connection_limit_and_its_name( + tmp_path: Path, + general_settings: dict[str, object], + database_url: str, + expected_limit: str, + expected_name: str | None, +) -> None: + config_path: Final = tmp_path / "config.yaml" + config_path.write_text(yaml.dump({"model_list": [], "general_settings": general_settings})) + + captured: Final = _CAPTURED_URLS.validate_python( + _run_server_and_capture_urls( + str(config_path), database_url=database_url, direct_url="postgresql://t:t@localhost:5432/t" + ) + ) + + writer: Final = urlparse.parse_qs(urlparse.urlparse(captured["DATABASE_URL"]).query) + direct: Final = urlparse.parse_qs(urlparse.urlparse(captured["DIRECT_URL"]).query) + assert (writer["connection_limit"], writer.get("application_name")) == ( + [expected_limit], + None if expected_name is None else [expected_name], + ) + assert direct["connection_limit"] == [expected_limit] + + class TestReadReplicaConnectionParams: """The reader is a second Prisma client with its own pool. Without the configured params on DATABASE_URL_READ_REPLICA it sizes itself from Prisma's @@ -3552,6 +3615,7 @@ class TestReadReplicaConnectionParams: query = urlparse.parse_qs(urlparse.urlparse(captured["DATABASE_URL_READ_REPLICA"]).query) assert query["connection_limit"] == ["3"] + assert "application_name" not in query assert query["pool_timeout"] == ["20"] assert query["connect_timeout"] == ["15"] assert query["socket_timeout"] == ["120"] diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 09a896ec053..051c578b2e9 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1702,6 +1702,7 @@ async def test_aaaproxy_startup_master_key(mock_prisma, monkeypatch, tmp_path): # Mock the Prisma import monkeypatch.setattr("litellm.proxy.proxy_server.PrismaClient", MockPrisma) + monkeypatch.setenv("DATABASE_URL", "postgresql://nobody:nothing@127.0.0.1:1/unreachable") # Create test app app = FastAPI() @@ -1932,6 +1933,7 @@ async def test_proxy_startup_stops_when_the_requested_migration_fails(monkeypatc _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) prisma: Final = _PrismaClientWhoseDatabaseRejectsQueries() monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma) + monkeypatch.setenv("DATABASE_URL", "postgresql://nobody:nothing@127.0.0.1:1/unreachable") monkeypatch.setenv("LITELLM_MIGRATE_FROM_MASTER_KEY", PUBLICLY_KNOWN_KEY) with ( @@ -1955,12 +1957,14 @@ async def test_proxy_startup_after_a_failed_migration_does_not_reuse_the_closed_ _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _PrismaClientWhoseDatabaseRejectsQueries()) + monkeypatch.setenv("DATABASE_URL", "postgresql://nobody:nothing@127.0.0.1:1/unreachable") monkeypatch.setenv("LITELLM_MIGRATE_FROM_MASTER_KEY", PUBLICLY_KNOWN_KEY) with pytest.raises(RuntimeError, match="permission denied"): async with proxy_startup_event(FastAPI()): pass monkeypatch.setenv("LITELLM_MIGRATE_FROM_MASTER_KEY", "") + monkeypatch.delenv("DATABASE_URL") async with proxy_startup_event(FastAPI()) as state: assert state["database"] is None @@ -2023,6 +2027,7 @@ async def test_proxy_shutdown_stops_database_background_tasks(monkeypatch, tmp_p yaml.dump({"general_settings": {"master_key": MASTER_KEY}}, f) monkeypatch.setenv("CONFIG_FILE_PATH", str(config_path)) monkeypatch.setattr(proxy_server_module, "prisma_client", fake_prisma) + monkeypatch.setenv("DATABASE_URL", "postgresql://nobody:nothing@127.0.0.1:1/unreachable") monkeypatch.setattr(proxy_server_module, "store_model_in_db", False) async with proxy_startup_event(FastAPI()): diff --git a/tests/unit/repositories/test_database.py b/tests/unit/repositories/test_database.py index d9343a796d2..798bdc39325 100644 --- a/tests/unit/repositories/test_database.py +++ b/tests/unit/repositories/test_database.py @@ -3,16 +3,80 @@ from unittest.mock import AsyncMock, MagicMock import pytest +from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR from litellm.repositories.database import Database +WRITER_URL: Final = "postgresql://litellm:secret@db.invalid:5432/proxy?connection_limit=3" +NATIVE_CONNECTIONS: Final = 2 + + +def _prisma(aclose: AsyncMock | None = None) -> MagicMock: + return MagicMock(aclose=aclose or AsyncMock()) + + +@pytest.fixture +def static_password(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("IAM_TOKEN_DB_AUTH", raising=False) + monkeypatch.delenv("AZURE_POSTGRESQL_AUTH", raising=False) + monkeypatch.delenv(PGBOUNCER_POOLED_ENV_VAR, raising=False) + @pytest.mark.asyncio +@pytest.mark.usefixtures("static_password") async def test_closing_the_database_closes_prisma() -> None: - prisma: Final = MagicMock() - prisma.aclose = AsyncMock() - database: Final = await Database.open(prisma) + aclose: Final = AsyncMock() + database: Final = await Database.open(_prisma(aclose), WRITER_URL, NATIVE_CONNECTIONS) - prisma.aclose.assert_not_awaited() + aclose.assert_not_awaited() await database.aclose() - prisma.aclose.assert_awaited_once() + aclose.assert_awaited_once() + + +def _task_names(database: Database) -> tuple[str, ...]: + return tuple(task.get_coro().__qualname__ for task in database.tasks) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("static_password") +async def test_static_password_runs_no_background_task() -> None: + database: Final = await Database.open(_prisma(), WRITER_URL, NATIVE_CONNECTIONS) + + assert _task_names(database) == () + await database.aclose() + + +@pytest.mark.asyncio +async def test_closing_the_database_stops_the_token_refresh(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.delenv("AZURE_POSTGRESQL_AUTH", raising=False) + monkeypatch.delenv(PGBOUNCER_POOLED_ENV_VAR, raising=False) + database: Final = await Database.open(_prisma(), WRITER_URL, NATIVE_CONNECTIONS) + assert _task_names(database) == ("keep_writer_token_fresh",) + + await database.aclose() + + assert all(task.done() for task in database.tasks) + assert database.tasks[0].cancelled() + + +@pytest.mark.asyncio +async def test_rejected_native_settings_still_close_prisma() -> None: + aclose: Final = AsyncMock() + + with pytest.raises(ValueError, match="must name a host and a user"): + await Database.open(_prisma(aclose), "postgresql://db.invalid/proxy", NATIVE_CONNECTIONS) + + aclose.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_zero_native_connections_open_no_native_pool(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + aclose: Final = AsyncMock() + + database: Final = await Database.open(_prisma(aclose), "postgresql://db.invalid/proxy", 0) + + assert (database.native, database.tasks) == (None, ()) + await database.aclose() + aclose.assert_awaited_once() From f6c9c70398f1ecd3b069ab77d10ac17f48944e50 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Sun, 15 Nov 2026 08:28:10 +0900 Subject: [PATCH 02/57] feat(db): add an optional native reader pool with explicit routing (#49804) When DATABASE_URL_READ_REPLICA is set, NativeDatabase opens a reader pool resolved from that URL, with its own token rotation. Pools exposes writer(), read() and read_after_write(), matching RoutingPrismaWrapper and WriterPinnedClient. read() prefers the replica and read_after_write() the writer. Each falls back to the other side when its pool cannot connect and skips the failed side for 30 seconds, so reads keep working on the replica while the writer is down The reader pool takes its share out of the reader's connection_limit the same way the writer does, with the same hard-coded RUST_RESERVED_CONNECTIONS. While that is 0 the replica URL is built exactly as before and no native reader pool opens. A replica connection_limit that is not a positive integer is left for Prisma to judge, as it always was --- litellm/proxy/db/connection_budget.py | 10 +- litellm/proxy/db/db_url_settings.py | 13 +- litellm/proxy/db/native_database.py | 28 ++- litellm/proxy/proxy_cli.py | 20 +- litellm/proxy/proxy_server.py | 5 +- litellm/repositories/database.py | 64 +++-- litellm/rust/Cargo.lock | 1 + litellm/rust/crates/db/AGENTS.md | 6 +- litellm/rust/crates/db/Cargo.toml | 1 + litellm/rust/crates/db/src/lib.rs | 2 + litellm/rust/crates/db/src/pools.rs | 242 +++++++++++++++++++ litellm/rust/crates/python-bridge/Cargo.toml | 1 - litellm/rust/crates/python-bridge/src/db.rs | 31 ++- litellm/rust_bridge/_native.pyi | 5 +- tests/_support/native_database.py | 4 +- tests/unit/proxy/db/test_db_url_settings.py | 7 +- tests/unit/proxy/db/test_native_database.py | 29 ++- tests/unit/proxy/test_proxy_cli.py | 17 ++ tests/unit/repositories/test_database.py | 30 ++- 19 files changed, 442 insertions(+), 74 deletions(-) create mode 100644 litellm/rust/crates/db/src/pools.rs diff --git a/litellm/proxy/db/connection_budget.py b/litellm/proxy/db/connection_budget.py index 71b839b91ef..26fd7d91f33 100644 --- a/litellm/proxy/db/connection_budget.py +++ b/litellm/proxy/db/connection_budget.py @@ -45,8 +45,12 @@ class ConnectionBudget: return None return cls.split(int(digits), rust_reserved) + @property + def prisma_limit(self) -> int | None: + """Prisma's share once the Rust pool takes some of the total, or None to leave Prisma's limit untouched.""" + return None if self.rust == 0 else self.python + def prisma_params(self, params: Mapping[str, str | int | float]) -> Mapping[str, str | int | float]: """``params`` with Prisma's share as its ``connection_limit``, untouched while the Rust pool takes none.""" - if self.rust == 0: - return params - return MappingProxyType({**params, "connection_limit": self.python}) + limit: Final = self.prisma_limit + return params if limit is None else MappingProxyType({**params, "connection_limit": limit}) diff --git a/litellm/proxy/db/db_url_settings.py b/litellm/proxy/db/db_url_settings.py index 231904d4d78..217c2dca8c4 100644 --- a/litellm/proxy/db/db_url_settings.py +++ b/litellm/proxy/db/db_url_settings.py @@ -268,14 +268,14 @@ def prisma_default_params( def postgres_connection_budget_message( - writer_limit: str, rust_writer_limit: int, reader_limit: str | None, num_workers: str + writer_limit: str, rust_writer_limit: int, reader_limit: str | None, rust_reader_limit: int, num_workers: str ) -> str: """The startup line that states this pod's worst-case Postgres connection demand. Prisma's ``connection_limit`` is per query engine, and every uvicorn worker owns one engine per configured database (writer, plus the reader when - ``DATABASE_URL_READ_REPLICA`` is set) plus the Rust writer pool's share - (``connection_budget``) when it is above 0. The limits are read off the final URLs, + ``DATABASE_URL_READ_REPLICA`` is set) plus a Rust pool beside each whose + share (``connection_budget``) is above 0. The limits are read off the final URLs, so a reader that pins its own ``connection_limit`` or an operator override in ``database_extra_connection_params`` is counted at its real value. The server-side cap is shared by every pod, so the number an operator has to keep @@ -289,17 +289,20 @@ def postgres_connection_budget_message( except ValueError: return ( "LiteLLM Proxy: Postgres connection budget per pod = workers x (writer connection_limit + " - f"{'rust writer pool + ' if rust_writer_limit else ''}reader connection_limit) " + f"{'rust writer pool + ' if rust_writer_limit else ''}reader connection_limit" + f"{' + rust reader pool' if rust_reader_limit else ''}) " f"(workers={num_workers!r}, writer={writer_limit!r}, reader={reader_limit!r}); " "keep pods x that figure under max_connections minus superuser_reserved_connections" ) + rust_reader: Final = rust_reader_limit if reader_limit is not None else 0 terms: Final = ( f"writer connection_limit {writer}", *((f"rust writer pool {rust_writer_limit}",) if rust_writer_limit else ()), *((f"reader connection_limit {reader}",) if reader_limit is not None else ()), + *((f"rust reader pool {rust_reader}",) if rust_reader else ()), ) engines: Final = terms[0] if len(terms) == 1 else f"({' + '.join(terms)})" - per_pod: Final = workers * (writer + rust_writer_limit + reader) + per_pod: Final = workers * (writer + rust_writer_limit + reader + rust_reader) return ( f"LiteLLM Proxy: Postgres connection budget per pod = {workers} worker(s) x {engines} = up to {per_pod} " "connections; keep pods x that figure under max_connections minus superuser_reserved_connections" diff --git a/litellm/proxy/db/native_database.py b/litellm/proxy/db/native_database.py index eb5f5ff40b8..430755652fb 100644 --- a/litellm/proxy/db/native_database.py +++ b/litellm/proxy/db/native_database.py @@ -24,7 +24,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from types import MappingProxyType -from typing import Final, Protocol +from typing import Final from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, pkcs12 @@ -210,10 +210,6 @@ async def resolve_native_endpoint(url: str, max_connections: int) -> NativeEndpo return native_endpoint(url, tls_offered=tls_offered, max_connections=max_connections) -class PasswordSink(Protocol): - def set_writer_password(self, password: str) -> None: ... - - @dataclass(frozen=True, slots=True) class TokenRotation: auth: DatabaseTokenAuth @@ -244,11 +240,23 @@ def writer_token_rotation(url: str) -> TokenRotation | None: return TokenRotation(auth=auth, endpoint=parse_iam_endpoint_from_url(url)) +def reader_token_rotation(url: str) -> TokenRotation | None: + """The token rotation the read replica needs, or None for a static password. + + The reader never goes through the in-container PgBouncer, so only the token toggles + decide, with the reader's own region, the same as its Prisma client. + """ + auth: Final = resolve_database_token_auth(read_replica=True) + if auth is None: + return None + return TokenRotation(auth=auth, endpoint=parse_iam_endpoint_from_url(url)) + + def _utcnow() -> datetime: return datetime.now(timezone.utc).replace(tzinfo=None) -async def _refresh(sink: PasswordSink, rotation: TokenRotation, now: Callable[[], datetime]) -> float: +async def _refresh(install: Callable[[str], None], rotation: TokenRotation, now: Callable[[], datetime]) -> float: try: token: Final = await asyncio.to_thread(rotation.mint) except Exception as error: # noqa: BLE001 # boto3 and azure-identity raise many types; any failure means retry @@ -259,12 +267,12 @@ async def _refresh(sink: PasswordSink, rotation: TokenRotation, now: Callable[[] TOKEN_RETRY_SECONDS, ) return TOKEN_RETRY_SECONDS - sink.set_writer_password(token) + install(token) return rotation.seconds_until_refresh(token, now()) -async def keep_writer_token_fresh( - sink: PasswordSink, +async def keep_token_fresh( + install: Callable[[str], None], rotation: TokenRotation, current_token: str, now: Callable[[], datetime] = _utcnow, @@ -278,4 +286,4 @@ async def keep_writer_token_fresh( delay = rotation.seconds_until_refresh(current_token, now()) # rebind-ok: each refresh schedules the next while True: await sleep(delay) - delay = await _refresh(sink, rotation, now) + delay = await _refresh(install, rotation, now) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index cef24e441c1..a56f6d9f00d 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -1718,6 +1718,14 @@ def run_server( # on the writer. Anything pinned on the replica URL wins, unlike the # writer where the config is applied on top. read_replica_url: Final[str | None] = os.getenv("DATABASE_URL_READ_REPLICA") + reader_budget: Final = ( + ConnectionBudget.of( + _url_query_value(read_replica_url, "connection_limit") or connection_url_params["connection_limit"] + ) + if read_replica_url + else None + ) + reader_prisma_limit: Final = None if reader_budget is None else reader_budget.prisma_limit if read_replica_url: reader_options: Final[str] = _pg_options_with_timeouts( _url_query_value(read_replica_url, "options"), @@ -1725,12 +1733,17 @@ def run_server( db_lock_timeout, db_idle_in_transaction_timeout, ) + reader_url: Final = ( + _with_query_value(read_replica_url, "options", reader_options) + if reader_options + else read_replica_url + ) os.environ["DATABASE_URL_READ_REPLICA"] = translate_libpq_ssl_params( add_missing_query_params( add_missing_query_params( - _with_query_value(read_replica_url, "options", reader_options) - if reader_options - else read_replica_url, + reader_url + if reader_prisma_limit is None + else _with_query_value(reader_url, "connection_limit", str(reader_prisma_limit)), reader_shareable_params(prisma_url_params), ), prisma_defaults, @@ -1747,6 +1760,7 @@ def run_server( ) if read_replica_url else None, + rust_reader_limit=0 if reader_budget is None else reader_budget.rust, num_workers=f"{1 if run_hypercorn else num_workers}", ) ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b50ab3fd571..eb238a7aa9d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1622,7 +1622,10 @@ async def _own_database(resources: AsyncExitStack, client: PrismaClient | None) if client is None: return None database: Final = await database_runtime.Database.open( - client, os.environ["DATABASE_URL"], RUST_RESERVED_CONNECTIONS + client, + os.environ["DATABASE_URL"], + os.environ.get("DATABASE_URL_READ_REPLICA") or None, + RUST_RESERVED_CONNECTIONS, ) resources.push_async_callback(database.aclose) return database diff --git a/litellm/repositories/database.py b/litellm/repositories/database.py index b577ba2bd67..0f4dd33a2f8 100644 --- a/litellm/repositories/database.py +++ b/litellm/repositories/database.py @@ -1,13 +1,16 @@ import asyncio import urllib.parse -from collections.abc import AsyncGenerator, Coroutine +from collections.abc import AsyncGenerator, Callable, Coroutine from contextlib import asynccontextmanager from dataclasses import dataclass from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger from litellm.proxy.db.native_database import ( - keep_writer_token_fresh, + NativeEndpointSettings, + TokenRotation, + keep_token_fresh, + reader_token_rotation, resolve_native_endpoint, writer_token_rotation, ) @@ -17,6 +20,24 @@ if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient +def _token_refresh( + install: Callable[[str], None], rotation: TokenRotation | None, url: str +) -> "tuple[asyncio.Task[None], ...]": + if rotation is None: + return () + current: Final = urllib.parse.unquote(urllib.parse.urlsplit(url).password or "") + return (asyncio.create_task(keep_token_fresh(install, rotation, current)),) + + +def _log_ignored_options(env_var: str, settings: NativeEndpointSettings) -> None: + if settings.ignored_options: + verbose_proxy_logger.error( + "Native database pool cannot send the %s options %s; only -c name=value settings carry over", + env_var, + settings.ignored_options, + ) + + @dataclass(frozen=True, slots=True) class Database: prisma: "PrismaClient" @@ -24,33 +45,32 @@ class Database: tasks: "tuple[asyncio.Task[None], ...]" = () @classmethod - async def open(cls, prisma: "PrismaClient", writer_url: str, native_connections: int) -> "Database": + async def open( + cls, prisma: "PrismaClient", writer_url: str, reader_url: str | None, native_connections: int + ) -> "Database": if native_connections == 0: return cls(prisma=prisma, native=None) try: - settings: Final = await resolve_native_endpoint(writer_url, native_connections) - native: Final = await NativeDatabase.connect(settings.endpoint) + writer: Final = await resolve_native_endpoint(writer_url, native_connections) + reader: Final = ( + None if reader_url is None else await resolve_native_endpoint(reader_url, native_connections) + ) + native: Final = await NativeDatabase.connect(writer.endpoint, None if reader is None else reader.endpoint) except BaseException: await prisma.aclose() raise - if settings.ignored_options: - verbose_proxy_logger.error( - "Native database pool cannot send the DATABASE_URL options %s; only -c name=value settings carry over", - settings.ignored_options, - ) - rotation: Final = writer_token_rotation(writer_url) - token_refresh: Final = ( - () - if rotation is None - else ( - asyncio.create_task( - keep_writer_token_fresh( - native, rotation, urllib.parse.unquote(urllib.parse.urlsplit(writer_url).password or "") - ) - ), - ) + _log_ignored_options("DATABASE_URL", writer) + if reader is not None: + _log_ignored_options("DATABASE_URL_READ_REPLICA", reader) + writer_refresh: Final = _token_refresh( + native.set_writer_password, writer_token_rotation(writer_url), writer_url ) - return cls(prisma=prisma, native=native, tasks=token_refresh) + reader_refresh: Final = ( + () + if reader_url is None + else _token_refresh(native.set_reader_password, reader_token_rotation(reader_url), reader_url) + ) + return cls(prisma=prisma, native=native, tasks=(*writer_refresh, *reader_refresh)) async def aclose(self) -> None: for task in self.tasks: diff --git a/litellm/rust/Cargo.lock b/litellm/rust/Cargo.lock index f40a7fd3ea3..d0f6e37e642 100644 --- a/litellm/rust/Cargo.lock +++ b/litellm/rust/Cargo.lock @@ -3837,6 +3837,7 @@ version = "0.1.0" dependencies = [ "chrono", "futures-util", + "litellm-circuit-breaker", "litellm-secret-value", "macro_rules_attribute", "rstest", diff --git a/litellm/rust/crates/db/AGENTS.md b/litellm/rust/crates/db/AGENTS.md index 1fedf9c9a2b..2ffecedd789 100644 --- a/litellm/rust/crates/db/AGENTS.md +++ b/litellm/rust/crates/db/AGENTS.md @@ -49,10 +49,10 @@ Python and SQLx run side by side until every repository operation is native, so | `schema` | URL param | `search_path` startup setting when not `public`; a custom schema behind PgBouncer is rejected by PgBouncer | | `statement_cache_size`, `pgbouncer=true`, `DATABASE_DISABLE_PREPARED_STATEMENTS` | URL params, `db_url_settings.py` | `statement_cache_capacity`: `statement_cache_size` (Prisma's default 100), 0 under `pgbouncer=true`. Run every query through `Statements::new(executor, endpoint.names_statements())` so a 0 capacity sends unnamed statements like Prisma, until a sqlx release carries transact-rs/sqlx#4337 | | `application_name` | URL param | A value the operator set is used on both pools; Prisma's `litellm-prisma` default becomes `litellm-native` | -| Rotating RDS IAM / Entra token | `token_auth.py`, `PrismaWrapper` | Python mints, `NativeDatabase.set_writer_password` swaps connect options | +| Rotating RDS IAM / Entra token | `token_auth.py`, `PrismaWrapper` | Python mints per side, `NativeDatabase.set_writer_password` / `set_reader_password` swap connect options | | In-container PgBouncer | `pgbouncer.py` | Inherited: the pooled URL carries PgBouncer's credential | -| Startup connection budget | `postgres_connection_budget_message` | Counts the Rust writer pool's share | -| Read replica routing | `routing_prisma_wrapper.py` | Planned: optional reader pool with explicit routing | +| Startup connection budget | `postgres_connection_budget_message` | Counts the Rust writer and reader pools' shares | +| Read replica routing | `routing_prisma_wrapper.py`, `DATABASE_URL_READ_REPLICA` | `Pools`: callers pick `writer()`, `read()` or `read_after_write()`. `read()` prefers the reader and `read_after_write()` the writer (`WriterPinnedClient`); either falls back to the other side when its pool cannot connect and skips that side for 30s, so reads keep working on the reader while the writer is down. Writes have no fallback | | Error classification, `lock_timeout` on rollups | `exception_handler.py`, `rollup_lock_timeout.py` | Planned | | `ServiceTypes.DB` events per round trip | `db_span.py`, `log_db_metrics.py` | Planned | | Auth lookup gate, fail open on DB outage, reader choice per query | `db_lookup_gate.py`, `allow_requests_on_db_unavailable` | With each ported auth read | diff --git a/litellm/rust/crates/db/Cargo.toml b/litellm/rust/crates/db/Cargo.toml index 527a0927338..c95f982433e 100644 --- a/litellm/rust/crates/db/Cargo.toml +++ b/litellm/rust/crates/db/Cargo.toml @@ -9,6 +9,7 @@ include = ["Cargo.toml", "src/**", "queries/**", ".sqlx/**"] [dependencies] chrono.workspace = true futures-util.workspace = true +litellm-circuit-breaker.workspace = true litellm-secret-value.workspace = true macro_rules_attribute.workspace = true serde.workspace = true diff --git a/litellm/rust/crates/db/src/lib.rs b/litellm/rust/crates/db/src/lib.rs index bab2d492c92..cea4e689768 100644 --- a/litellm/rust/crates/db/src/lib.rs +++ b/litellm/rust/crates/db/src/lib.rs @@ -12,6 +12,7 @@ mod end_user; mod endpoint; mod error; mod failure; +mod pools; mod spend; mod statements; #[cfg(test)] @@ -23,6 +24,7 @@ pub use end_user::{EndUser, RateLimits, find_end_user}; pub use endpoint::{ClientIdentity, Endpoint, PoolLimits, SslMode}; pub use error::Error; pub use failure::DbFailure; +pub use pools::Pools; pub use spend::{ DailySpend, DailyTable, EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend, add_end_user_spend, add_key_spend, add_team_member_spend, add_team_spend, add_user_spend, diff --git a/litellm/rust/crates/db/src/pools.rs b/litellm/rust/crates/db/src/pools.rs new file mode 100644 index 00000000000..561c1bad914 --- /dev/null +++ b/litellm/rust/crates/db/src/pools.rs @@ -0,0 +1,242 @@ +use std::time::{Duration, Instant}; + +use litellm_circuit_breaker::{Breaker, Config, Failure}; +use sqlx::{PgPool, Postgres, pool::PoolConnection}; + +use crate::{Endpoint, Error}; + +const RECOVERY: Duration = Duration::from_secs(30); + +type Connection = PoolConnection; + +#[derive(Clone)] +struct Side { + pool: PgPool, + breaker: Breaker, +} + +impl Side { + fn new(endpoint: &Endpoint) -> Result { + Ok(Self { + pool: endpoint.pool()?, + breaker: Breaker::new( + Config { + failure_threshold: 1, + recovery_timeout: RECOVERY, + ..Config::default() + }, + Box::new(|_| {}), + ), + }) + } + + async fn acquire_unless_skipped(&self) -> Option { + let admission = self.breaker.admit(Instant::now()).ok()?; + match self.pool.acquire().await { + Ok(connection) => { + admission.succeeded(); + Some(connection) + } + Err(_) => { + admission.failed(Failure::Hard, Instant::now()); + None + } + } + } + + async fn acquire(&self) -> Result { + let result = self.pool.acquire().await; + if result.is_err() { + self.breaker.failed(Failure::Hard, Instant::now()); + } + Ok(result?) + } +} + +async fn acquire_preferring( + preferred: &Side, + fallback: Option<&Side>, +) -> Result { + let Some(fallback) = fallback else { + return preferred.acquire().await; + }; + match preferred.acquire_unless_skipped().await { + Some(connection) => Ok(connection), + None => fallback.acquire().await, + } +} + +#[derive(Clone)] +pub struct Pools { + writer: Side, + reader: Option, +} + +impl Pools { + pub fn new(writer: &Endpoint, reader: Option<&Endpoint>) -> Result { + Ok(Self { + writer: Side::new(writer)?, + reader: reader.map(Side::new).transpose()?, + }) + } + + pub fn writer(&self) -> &PgPool { + &self.writer.pool + } + + pub fn reader_pool(&self) -> Option<&PgPool> { + self.reader.as_ref().map(|reader| &reader.pool) + } + + pub async fn read(&self) -> Result { + match &self.reader { + Some(reader) => acquire_preferring(reader, Some(&self.writer)).await, + None => self.writer.acquire().await, + } + } + + pub async fn read_after_write(&self) -> Result { + acquire_preferring(&self.writer, self.reader.as_ref()).await + } + + pub async fn close(&self) { + self.writer.pool.close().await; + if let Some(reader) = &self.reader { + reader.pool.close().await; + } + } +} + +#[cfg(test)] +mod tests { + use std::{error::Error as StdError, time::Instant}; + + use sqlx::PgConnection; + + use super::Pools; + use crate::{Endpoint, PoolLimits, test_support::database_endpoint}; + + fn read_only(endpoint: &Endpoint) -> Endpoint { + Endpoint { + options: vec![("default_transaction_read_only".to_owned(), "on".to_owned())], + ..endpoint.clone() + } + } + + fn unreachable(endpoint: &Endpoint) -> Endpoint { + Endpoint { + host: "127.0.0.1".to_owned(), + port: 1, + pool: PoolLimits { + acquire_timeout_seconds: Some(2.0), + ..endpoint.pool + }, + ..endpoint.clone() + } + } + + async fn is_read_only(connection: &mut PgConnection) -> Result { + #[expect( + clippy::disallowed_methods, + reason = "Session settings are not part of the compile-checked schema" + )] + let setting: String = sqlx::query_scalar("SHOW default_transaction_read_only") + .fetch_one(connection) + .await?; + Ok(setting == "on") + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn reads_go_to_the_reader_and_read_after_write_to_the_writer() + -> Result<(), Box> { + let writer = database_endpoint()?; + let pools = Pools::new(&writer, Some(&read_only(&writer)))?; + + assert!(is_read_only(&mut *pools.read().await?).await?); + assert!(!is_read_only(&mut *pools.read_after_write().await?).await?); + assert!(!is_read_only(&mut *pools.writer().acquire().await?).await?); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn writes_sent_to_the_reader_fail_read_only() -> Result<(), Box> { + let writer = database_endpoint()?; + let pools = Pools::new(&writer, Some(&read_only(&writer)))?; + let mut connection = pools.read().await?; + + #[expect(clippy::disallowed_methods, reason = "A scratch table needs no schema")] + let error = sqlx::query("CREATE TABLE native_reader_write_probe (id int)") + .execute(&mut *connection) + .await + .expect_err("a read-only session must reject writes"); + + assert_eq!( + error.as_database_error().and_then(|error| error.code()), + Some("25006".into()) + ); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn every_read_uses_the_writer_without_a_reader() -> Result<(), Box> { + let pools = Pools::new(&database_endpoint()?, None)?; + + assert!(pools.reader_pool().is_none()); + assert!(!is_read_only(&mut *pools.read().await?).await?); + assert!(!is_read_only(&mut *pools.read_after_write().await?).await?); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn unreachable_reader_falls_back_once_then_skips_it() -> Result<(), Box> { + let writer = database_endpoint()?; + let pools = Pools::new(&writer, Some(&unreachable(&writer)))?; + + assert!(!is_read_only(&mut *pools.read().await?).await?); + let started = Instant::now(); + assert!(!is_read_only(&mut *pools.read().await?).await?); + + assert!(started.elapsed().as_secs_f64() < 1.0); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn reads_stay_on_the_reader_while_the_writer_is_down() -> Result<(), Box> { + let reader = read_only(&database_endpoint()?); + let pools = Pools::new(&unreachable(&reader), Some(&reader))?; + + assert!(is_read_only(&mut *pools.read().await?).await?); + assert!(pools.writer().acquire().await.is_err()); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn read_after_write_yields_to_the_reader_while_the_writer_is_down() + -> Result<(), Box> { + let reader = read_only(&database_endpoint()?); + let pools = Pools::new(&unreachable(&reader), Some(&reader))?; + + assert!(is_read_only(&mut *pools.read_after_write().await?).await?); + let started = Instant::now(); + assert!(is_read_only(&mut *pools.read_after_write().await?).await?); + + assert!(started.elapsed().as_secs_f64() < 1.0); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn read_after_write_fails_when_the_only_database_is_down() -> Result<(), Box> + { + let pools = Pools::new(&unreachable(&database_endpoint()?), None)?; + + assert!(pools.read_after_write().await.is_err()); + Ok(()) + } +} diff --git a/litellm/rust/crates/python-bridge/Cargo.toml b/litellm/rust/crates/python-bridge/Cargo.toml index 06c390b820f..90ae7c7f530 100644 --- a/litellm/rust/crates/python-bridge/Cargo.toml +++ b/litellm/rust/crates/python-bridge/Cargo.toml @@ -77,7 +77,6 @@ serde_json.workspace = true strum.workspace = true veil.workspace = true thiserror.workspace = true -sqlx.workspace = true tokio = { workspace = true, features = ["rt", "sync"] } [dev-dependencies] diff --git a/litellm/rust/crates/python-bridge/src/db.rs b/litellm/rust/crates/python-bridge/src/db.rs index ca1ccc8fb2b..d17f5fd8099 100644 --- a/litellm/rust/crates/python-bridge/src/db.rs +++ b/litellm/rust/crates/python-bridge/src/db.rs @@ -1,12 +1,12 @@ -use litellm_db::Endpoint; +use litellm_db::{Endpoint, Pools}; use litellm_secret_value::SecretValue; use pyo3::{exceptions::PyValueError, prelude::*}; -use sqlx::PgPool; #[pyclass(frozen)] pub(crate) struct NativeDatabase { writer_endpoint: Endpoint, - writer: PgPool, + reader_endpoint: Option, + pools: Pools, } fn map_error(error: litellm_db::Error) -> PyErr { @@ -16,33 +16,48 @@ fn map_error(error: litellm_db::Error) -> PyErr { #[pymethods] impl NativeDatabase { #[staticmethod] + #[pyo3(signature = (writer, reader=None))] fn connect( py: Python<'_>, #[pyo3(from_py_with = litellm_host_python::from_py_argument)] writer: Endpoint, + #[pyo3(from_py_with = litellm_host_python::from_py_argument)] reader: Option, ) -> PyResult> { crate::execution::run_async_value(py, async move { - let pool = writer.pool().map_err(map_error)?; + let pools = Pools::new(&writer, reader.as_ref()).map_err(map_error)?; Ok(NativeDatabase { writer_endpoint: writer, - writer: pool, + reader_endpoint: reader, + pools, }) }) } fn set_writer_password(&self, password: String) { - self.writer.set_connect_options( + self.pools.writer().set_connect_options( self.writer_endpoint .with_password(SecretValue::new(password)) .connect_options(), ); } + fn set_reader_password(&self, password: String) -> PyResult<()> { + let (Some(endpoint), Some(pool)) = (&self.reader_endpoint, self.pools.reader_pool()) else { + return Err(PyValueError::new_err("no read replica is configured")); + }; + pool.set_connect_options( + endpoint + .with_password(SecretValue::new(password)) + .connect_options(), + ); + Ok(()) + } + fn close<'py>(&self, py: Python<'py>) -> PyResult> { - let writer = self.writer.clone(); + let pools = self.pools.clone(); crate::execution::run_async( py, async move { - writer.close().await; + pools.close().await; Ok(()) }, |error: PyErr| error, diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index be58056c8ef..906b6e0549b 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -597,8 +597,11 @@ class NativeSpendCounters: @final class NativeDatabase: @staticmethod - def connect(writer: NativeDatabaseEndpoint) -> Future[NativeDatabase]: ... + def connect( + writer: NativeDatabaseEndpoint, reader: NativeDatabaseEndpoint | None = None + ) -> Future[NativeDatabase]: ... def set_writer_password(self, password: str) -> None: ... + def set_reader_password(self, password: str) -> None: ... def close(self) -> Future[None]: ... @final diff --git a/tests/_support/native_database.py b/tests/_support/native_database.py index bade92d79f8..8e7ea8feb18 100644 --- a/tests/_support/native_database.py +++ b/tests/_support/native_database.py @@ -8,7 +8,9 @@ _NEVER_DIALED_URL: Final = "postgresql://litellm:secret@db.invalid:5432/proxy" async def connect_native_database() -> NativeDatabase: - return await NativeDatabase.connect(native_endpoint(_NEVER_DIALED_URL, tls_offered=True, max_connections=1).endpoint) + return await NativeDatabase.connect( + native_endpoint(_NEVER_DIALED_URL, tls_offered=True, max_connections=1).endpoint, None + ) def native_database() -> NativeDatabase: diff --git a/tests/unit/proxy/db/test_db_url_settings.py b/tests/unit/proxy/db/test_db_url_settings.py index a42894efa89..abd4ed14b2e 100644 --- a/tests/unit/proxy/db/test_db_url_settings.py +++ b/tests/unit/proxy/db/test_db_url_settings.py @@ -1296,14 +1296,14 @@ def test_reader_keeps_its_own_pinned_idle_lifetime(monkeypatch): 2, "8", "4", - "4 worker(s) x (writer connection_limit 8 + rust writer pool 2 + reader connection_limit 8) = up to 72 connections", + "4 worker(s) x (writer connection_limit 8 + rust writer pool 2 + reader connection_limit 8 + rust reader pool 2) = up to 80 connections", ), ( "8", 2, "50", "4", - "4 worker(s) x (writer connection_limit 8 + rust writer pool 2 + reader connection_limit 50) = up to 240 connections", + "4 worker(s) x (writer connection_limit 8 + rust writer pool 2 + reader connection_limit 50 + rust reader pool 2) = up to 248 connections", ), ("8", 2, None, "0", "1 worker(s) x (writer connection_limit 8 + rust writer pool 2) = up to 10 connections"), ], @@ -1325,6 +1325,7 @@ def test_connection_budget_message_sums_the_engines_each_worker_owns( writer_limit=writer_limit, rust_writer_limit=rust_writer_limit, reader_limit=reader_limit, + rust_reader_limit=rust_writer_limit, num_workers=num_workers, ) assert expected in message @@ -1335,7 +1336,7 @@ def test_connection_budget_message_survives_an_unparseable_limit() -> None: from litellm.proxy.db.db_url_settings import postgres_connection_budget_message message: Final = postgres_connection_budget_message( - writer_limit="ten", rust_writer_limit=2, reader_limit=None, num_workers="2" + writer_limit="ten", rust_writer_limit=2, reader_limit=None, rust_reader_limit=0, num_workers="2" ) assert "writer='ten'" in message diff --git a/tests/unit/proxy/db/test_native_database.py b/tests/unit/proxy/db/test_native_database.py index dad278ea13c..edd4cf7fa2d 100644 --- a/tests/unit/proxy/db/test_native_database.py +++ b/tests/unit/proxy/db/test_native_database.py @@ -14,9 +14,10 @@ from pydantic import TypeAdapter from litellm.proxy.db.native_database import ( TokenRotation, - keep_writer_token_fresh, + keep_token_fresh, native_endpoint, needs_tls_probe, + reader_token_rotation, server_offers_tls, writer_token_rotation, ) @@ -284,6 +285,24 @@ def test_writer_token_rotation_follows_token_auth(monkeypatch: pytest.MonkeyPatc ) +def test_reader_token_rotation_uses_the_reader_region_even_when_the_writer_is_pooled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + url: Final = "postgresql://reader:tok@replica.internal:5433/proxy" + monkeypatch.delenv("AZURE_POSTGRESQL_AUTH", raising=False) + monkeypatch.setenv(PGBOUNCER_POOLED_ENV_VAR, "true") + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "false") + assert reader_token_rotation(url) is None + + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.setenv("AWS_RDS_REGION", "us-west-2") + monkeypatch.setenv("AWS_RDS_READ_REPLICA_REGION", "eu-west-1") + assert reader_token_rotation(url) == TokenRotation( + auth=RdsIamTokenAuth(region="eu-west-1"), + endpoint=IAMEndpoint(host="replica.internal", port="5433", user="reader", name="proxy"), + ) + + def test_pooled_writer_needs_no_token_rotation(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") monkeypatch.setenv(PGBOUNCER_POOLED_ENV_VAR, "true") @@ -313,7 +332,7 @@ class _Sink: self._wanted: Final = wanted self.done: Final = asyncio.Event() - def set_writer_password(self, password: str) -> None: + def install(self, password: str) -> None: self.passwords = (*self.passwords, password) if len(self.passwords) >= self._wanted: self.done.set() @@ -344,7 +363,7 @@ def _minting(*outcomes: str | Exception) -> TokenRotation: async def _run_until(sink: _Sink, rotation: TokenRotation, schedule: _Schedule) -> None: task: Final = asyncio.create_task( - keep_writer_token_fresh(sink, rotation, "opaque", now=lambda: NOW, sleep=schedule.sleep) + keep_token_fresh(sink.install, rotation, "opaque", now=lambda: NOW, sleep=schedule.sleep) ) await asyncio.wait_for(sink.done.wait(), timeout=5) task.cancel() @@ -352,7 +371,7 @@ async def _run_until(sink: _Sink, rotation: TokenRotation, schedule: _Schedule) @pytest.mark.asyncio -async def test_keep_writer_token_fresh_installs_each_minted_token() -> None: +async def test_keep_token_fresh_installs_each_minted_token() -> None: sink: Final = _Sink(wanted=2) schedule: Final = _Schedule() @@ -363,7 +382,7 @@ async def test_keep_writer_token_fresh_installs_each_minted_token() -> None: @pytest.mark.asyncio -async def test_keep_writer_token_fresh_retries_a_failed_mint() -> None: +async def test_keep_token_fresh_retries_a_failed_mint() -> None: sink: Final = _Sink(wanted=1) schedule: Final = _Schedule() diff --git a/tests/unit/proxy/test_proxy_cli.py b/tests/unit/proxy/test_proxy_cli.py index 4e2ddfe859d..f964782939d 100644 --- a/tests/unit/proxy/test_proxy_cli.py +++ b/tests/unit/proxy/test_proxy_cli.py @@ -3650,6 +3650,23 @@ class TestReadReplicaConnectionParams: assert query["connection_limit"] == ["50"] assert query["pool_timeout"] == ["20"] + @pytest.mark.parametrize("pinned_limit", ["0", "abc"]) + def test_replica_limit_that_is_not_a_positive_integer_is_left_to_prisma( + self, tmp_path: Path, pinned_limit: str + ) -> None: + config_path: Final = tmp_path / "config.yaml" + config_path.write_text(yaml.dump({"model_list": [], "general_settings": {}})) + + captured: Final = _CAPTURED_URLS.validate_python( + _run_server_and_capture_urls( + str(config_path), + read_replica_url=f"postgresql://t:t@reader:5432/t?connection_limit={pinned_limit}", + ) + ) + + query: Final = urlparse.parse_qs(urlparse.urlparse(captured["DATABASE_URL_READ_REPLICA"]).query) + assert query["connection_limit"] == [pinned_limit] + def test_connection_budget_line_counts_the_limits_the_final_urls_carry( self, tmp_path: Path, diff --git a/tests/unit/repositories/test_database.py b/tests/unit/repositories/test_database.py index 798bdc39325..2497c9237e9 100644 --- a/tests/unit/repositories/test_database.py +++ b/tests/unit/repositories/test_database.py @@ -7,6 +7,7 @@ from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR from litellm.repositories.database import Database WRITER_URL: Final = "postgresql://litellm:secret@db.invalid:5432/proxy?connection_limit=3" +READER_URL: Final = "postgresql://reader:secret@replica.invalid:5432/proxy?connection_limit=3" NATIVE_CONNECTIONS: Final = 2 @@ -25,7 +26,7 @@ def static_password(monkeypatch: pytest.MonkeyPatch) -> None: @pytest.mark.usefixtures("static_password") async def test_closing_the_database_closes_prisma() -> None: aclose: Final = AsyncMock() - database: Final = await Database.open(_prisma(aclose), WRITER_URL, NATIVE_CONNECTIONS) + database: Final = await Database.open(_prisma(aclose), WRITER_URL, None, NATIVE_CONNECTIONS) aclose.assert_not_awaited() await database.aclose() @@ -40,24 +41,24 @@ def _task_names(database: Database) -> tuple[str, ...]: @pytest.mark.asyncio @pytest.mark.usefixtures("static_password") async def test_static_password_runs_no_background_task() -> None: - database: Final = await Database.open(_prisma(), WRITER_URL, NATIVE_CONNECTIONS) + database: Final = await Database.open(_prisma(), WRITER_URL, READER_URL, NATIVE_CONNECTIONS) assert _task_names(database) == () await database.aclose() @pytest.mark.asyncio -async def test_closing_the_database_stops_the_token_refresh(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_closing_the_database_stops_both_token_refreshes(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") monkeypatch.delenv("AZURE_POSTGRESQL_AUTH", raising=False) monkeypatch.delenv(PGBOUNCER_POOLED_ENV_VAR, raising=False) - database: Final = await Database.open(_prisma(), WRITER_URL, NATIVE_CONNECTIONS) - assert _task_names(database) == ("keep_writer_token_fresh",) + database: Final = await Database.open(_prisma(), WRITER_URL, READER_URL, NATIVE_CONNECTIONS) + assert _task_names(database) == ("keep_token_fresh", "keep_token_fresh") await database.aclose() assert all(task.done() for task in database.tasks) - assert database.tasks[0].cancelled() + assert database.tasks[0].cancelled() and database.tasks[1].cancelled() @pytest.mark.asyncio @@ -65,7 +66,18 @@ async def test_rejected_native_settings_still_close_prisma() -> None: aclose: Final = AsyncMock() with pytest.raises(ValueError, match="must name a host and a user"): - await Database.open(_prisma(aclose), "postgresql://db.invalid/proxy", NATIVE_CONNECTIONS) + await Database.open(_prisma(aclose), "postgresql://db.invalid/proxy", None, NATIVE_CONNECTIONS) + + aclose.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("static_password") +async def test_rejected_reader_settings_still_close_prisma() -> None: + aclose: Final = AsyncMock() + + with pytest.raises(ValueError, match="must name a host and a user"): + await Database.open(_prisma(aclose), WRITER_URL, "postgresql://replica.invalid/proxy", NATIVE_CONNECTIONS) aclose.assert_awaited_once() @@ -75,7 +87,9 @@ async def test_zero_native_connections_open_no_native_pool(monkeypatch: pytest.M monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") aclose: Final = AsyncMock() - database: Final = await Database.open(_prisma(aclose), "postgresql://db.invalid/proxy", 0) + database: Final = await Database.open( + _prisma(aclose), "postgresql://db.invalid/proxy", "postgresql://replica.invalid/proxy", 0 + ) assert (database.native, database.tasks) == (None, ()) await database.aclose() From bd27f4e34c0190fb9386ea992b9c336e10e0bf6a Mon Sep 17 00:00:00 2001 From: yujonglee Date: Sun, 15 Nov 2026 08:28:10 +0900 Subject: [PATCH 03/57] feat(db): emit DB service events from native operations (#49810) crates/db gains observe(), which records one event per round trip with the call type, table and operation a Prisma call reports. NativeDatabase forwards those events to a Python ServiceLoggingEvents that Database.open injects, and it calls the same ServiceTypes.DB success and failure hooks as db_span on the proxy's event loop. A native health_check, which pings one pooled connection without running SQL (call type health_check), is the first observed operation The native pool is lazy, so a pool that could not connect stayed silent until the first ported query failed on a request path. Database.open now pings it once in the background and logs a warning on failure, without failing or delaying boot The crate AGENTS.md records the Prometheus and OTel names a Rust host sink must match --- litellm/proxy/db/db_span.py | 10 +- litellm/proxy/db/native_database.py | 84 ++++++++++++- litellm/repositories/database.py | 32 ++++- litellm/rust/Cargo.lock | 1 + litellm/rust/crates/db/AGENTS.md | 16 ++- litellm/rust/crates/db/Cargo.toml | 1 + litellm/rust/crates/db/src/health.rs | 29 +++++ litellm/rust/crates/db/src/lib.rs | 4 + litellm/rust/crates/db/src/observe.rs | 128 ++++++++++++++++++++ litellm/rust/crates/python-bridge/src/db.rs | 67 +++++++++- litellm/rust_bridge/_native.pyi | 5 +- litellm/rust_bridge/database.py | 14 ++- tests/_support/native_database.py | 6 +- tests/unit/proxy/db/test_native_database.py | 87 +++++++++++++ tests/unit/repositories/test_database.py | 22 ++++ 15 files changed, 488 insertions(+), 18 deletions(-) create mode 100644 litellm/rust/crates/db/src/health.rs create mode 100644 litellm/rust/crates/db/src/observe.rs diff --git a/litellm/proxy/db/db_span.py b/litellm/proxy/db/db_span.py index 06b1a6aa11b..514c97c02da 100644 --- a/litellm/proxy/db/db_span.py +++ b/litellm/proxy/db/db_span.py @@ -32,7 +32,7 @@ from litellm.proxy.db.log_db_metrics import ( # noqa: F401 # legacy module exp _T = TypeVar("_T") -def _service_logging() -> ServiceLogging | None: +def proxy_service_logging() -> ServiceLogging | None: try: from litellm.proxy.proxy_server import proxy_logging_obj except ImportError: @@ -40,7 +40,9 @@ def _service_logging() -> ServiceLogging | None: return proxy_logging_obj.service_logging_obj -def _event_metadata(table: str | None, operation: str | None) -> dict[str, str]: +def db_event_metadata( + table: str | None, operation: str | None +) -> dict[str, str]: # mutable-ok: ServiceLogging hooks take event_metadata as a dict pairs: Final = (("table_name", table), ("db_operation", operation)) return {key: value for key, value in pairs if value is not None} @@ -73,9 +75,9 @@ async def db_span(call_type: str, table: str | None, operation: str | None = Non if db_io_claimed(): yield return - service_logging: Final = _service_logging() + service_logging: Final = proxy_service_logging() start_time: Final = datetime.now() - event_metadata: Final = _event_metadata(table, operation) + event_metadata: Final = db_event_metadata(table, operation) with claim_db_io() as witness: try: yield diff --git a/litellm/proxy/db/native_database.py b/litellm/proxy/db/native_database.py index 430755652fb..5c322f0afec 100644 --- a/litellm/proxy/db/native_database.py +++ b/litellm/proxy/db/native_database.py @@ -21,14 +21,16 @@ import struct import urllib.parse from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path from types import MappingProxyType -from typing import Final +from typing import Final, Protocol from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, pkcs12 from litellm._logging import verbose_proxy_logger +from litellm._service_logger import ServiceTypes +from litellm.proxy.db.db_span import db_event_metadata from litellm.proxy.db.db_url_settings import NATIVE_APPLICATION_NAME, PRISMA_APPLICATION_NAME from litellm.proxy.db.pgbouncer import database_url_is_pooled, fold_libpq_option_tokens, libpq_option_setting from litellm.proxy.db.token_auth import ( @@ -287,3 +289,81 @@ async def keep_token_fresh( while True: await sleep(delay) delay = await _refresh(install, rotation, now) + + +class DatabaseServiceHooks(Protocol): + async def async_service_success_hook( + self, + service: ServiceTypes, + call_type: str, + duration: float, + parent_otel_span: None = None, + start_time: datetime | None = None, + end_time: datetime | None = None, + event_metadata: dict[str, str] | None = None, # mutable-ok: matches the ServiceLogging hook signature + ) -> None: ... + + async def async_service_failure_hook( + self, + service: ServiceTypes, + duration: float, + error: Exception, + call_type: str, + parent_otel_span: None = None, + start_time: datetime | None = None, + end_time: datetime | None = None, + event_metadata: dict[str, str] | None = None, # mutable-ok: matches the ServiceLogging hook signature + ) -> None: ... + + +class NativeDatabaseError(Exception): + """A native (SQLx) query failure, reported to service logging like a Prisma one.""" + + +@dataclass(frozen=True, slots=True) +class ServiceLoggingEvents: + """Turns native round trips into the ``ServiceTypes.DB`` events ``db_span`` emits. + + Rust calls ``record`` from its own threads once per round trip, so the hook is + handed to the proxy's event loop rather than awaited here. + """ + + loop: asyncio.AbstractEventLoop + service_logging: Callable[[], DatabaseServiceHooks | None] + + def record( + self, + call_type: str, + table: str | None, + operation: str | None, + started_at: float, + duration_seconds: float, + failure: str | None, + ) -> None: + service_logging: Final = self.service_logging() + if service_logging is None: + return + start_time: Final = datetime.fromtimestamp(started_at, tz=timezone.utc).astimezone().replace(tzinfo=None) + hook: Final = ( + service_logging.async_service_success_hook( + service=ServiceTypes.DB, + call_type=call_type, + parent_otel_span=None, + duration=duration_seconds, + start_time=start_time, + end_time=start_time + timedelta(seconds=duration_seconds), + event_metadata=db_event_metadata(table, operation), + ) + if failure is None + else service_logging.async_service_failure_hook( + error=NativeDatabaseError(failure), + service=ServiceTypes.DB, + call_type=call_type, + parent_otel_span=None, + duration=duration_seconds, + start_time=start_time, + end_time=start_time + timedelta(seconds=duration_seconds), + event_metadata=db_event_metadata(table, operation), + ) + ) + _ = self.loop.call_soon_threadsafe(self.loop.create_task, hook) diff --git a/litellm/repositories/database.py b/litellm/repositories/database.py index 0f4dd33a2f8..5ee0accab8c 100644 --- a/litellm/repositories/database.py +++ b/litellm/repositories/database.py @@ -6,8 +6,10 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger +from litellm.proxy.db.db_span import proxy_service_logging from litellm.proxy.db.native_database import ( NativeEndpointSettings, + ServiceLoggingEvents, TokenRotation, keep_token_fresh, reader_token_rotation, @@ -38,11 +40,23 @@ def _log_ignored_options(env_var: str, settings: NativeEndpointSettings) -> None ) +async def _warn_if_unreachable(native: NativeDatabase) -> None: + try: + await native.health_check() + except ValueError as error: + verbose_proxy_logger.warning( + "Native database pool cannot reach the database, so operations ported to it will fail until it" + " connects. Prisma is checked separately. Cause: %s", + error, + ) + + @dataclass(frozen=True, slots=True) class Database: prisma: "PrismaClient" native: NativeDatabase | None tasks: "tuple[asyncio.Task[None], ...]" = () + boot_probe: "asyncio.Task[None] | None" = None @classmethod async def open( @@ -55,7 +69,11 @@ class Database: reader: Final = ( None if reader_url is None else await resolve_native_endpoint(reader_url, native_connections) ) - native: Final = await NativeDatabase.connect(writer.endpoint, None if reader is None else reader.endpoint) + native: Final = await NativeDatabase.connect( + ServiceLoggingEvents(loop=asyncio.get_running_loop(), service_logging=proxy_service_logging), + writer.endpoint, + None if reader is None else reader.endpoint, + ) except BaseException: await prisma.aclose() raise @@ -70,12 +88,18 @@ class Database: if reader_url is None else _token_refresh(native.set_reader_password, reader_token_rotation(reader_url), reader_url) ) - return cls(prisma=prisma, native=native, tasks=(*writer_refresh, *reader_refresh)) + return cls( + prisma=prisma, + native=native, + tasks=(*writer_refresh, *reader_refresh), + boot_probe=asyncio.create_task(_warn_if_unreachable(native)), + ) async def aclose(self) -> None: - for task in self.tasks: + background: Final = (*self.tasks, *(() if self.boot_probe is None else (self.boot_probe,))) + for task in background: task.cancel() - await asyncio.gather(*self.tasks, return_exceptions=True) + await asyncio.gather(*background, return_exceptions=True) try: await self.prisma.aclose() finally: diff --git a/litellm/rust/Cargo.lock b/litellm/rust/Cargo.lock index d0f6e37e642..58e7d9d389f 100644 --- a/litellm/rust/Cargo.lock +++ b/litellm/rust/Cargo.lock @@ -3838,6 +3838,7 @@ dependencies = [ "chrono", "futures-util", "litellm-circuit-breaker", + "litellm-error", "litellm-secret-value", "macro_rules_attribute", "rstest", diff --git a/litellm/rust/crates/db/AGENTS.md b/litellm/rust/crates/db/AGENTS.md index 2ffecedd789..b23a379d798 100644 --- a/litellm/rust/crates/db/AGENTS.md +++ b/litellm/rust/crates/db/AGENTS.md @@ -54,7 +54,21 @@ Python and SQLx run side by side until every repository operation is native, so | Startup connection budget | `postgres_connection_budget_message` | Counts the Rust writer and reader pools' shares | | Read replica routing | `routing_prisma_wrapper.py`, `DATABASE_URL_READ_REPLICA` | `Pools`: callers pick `writer()`, `read()` or `read_after_write()`. `read()` prefers the reader and `read_after_write()` the writer (`WriterPinnedClient`); either falls back to the other side when its pool cannot connect and skips that side for 30s, so reads keep working on the reader while the writer is down. Writes have no fallback | | Error classification, `lock_timeout` on rollups | `exception_handler.py`, `rollup_lock_timeout.py` | Planned | -| `ServiceTypes.DB` events per round trip | `db_span.py`, `log_db_metrics.py` | Planned | +| `ServiceTypes.DB` events per round trip | `db_span.py`, `log_db_metrics.py`, `prometheus_services.py`, `otel/model/spans.py` | Python host only. Wrap each native operation in `observe` with the call type and table its Prisma version uses, and the bridge forwards events to `ServiceLoggingEvents`. Nesting under an outer Python `db_span` is not detected. A Rust host has no sink yet and must emit the names below | | Auth lookup gate, fail open on DB outage, reader choice per query | `db_lookup_gate.py`, `allow_requests_on_db_unavailable` | With each ported auth read | +| Connect at boot, so a database that cannot be reached is reported before traffic | `PrismaClient.connect` | The pool stays lazy and never fails boot. `Database.open` pings it once in the background and logs a warning when it cannot connect, so a mistranslated endpoint is not first found by a ported query | | Health watchdog, engine reaper, drain tracker, engine recreation | `utils.py`, `query_engine_reaper.py`, `prisma_client.py` | Not needed: SQLx pools reconnect per connection and `close` waits for checked-out connections | | Migrations, `check_migration`, views | `prisma_client.py`, `create_views.py` | Prisma keeps them until the SQLx cutover | + +## DB event names a Rust host sink must match + +Prometheus and OTel dashboards aggregate across the Python proxy and any Rust server, so a Rust sink emits the same names and labels Python does, pinned by inline cases on both sides + +- Prometheus + - `litellm_postgres_latency`, a histogram with label `postgres="postgres"` + - `litellm_postgres_total_requests` + - `litellm_postgres_failed_requests`, adding `error_class` (`NativeDatabaseError` for native failures) and `function_name` (the call type) +- OTel span name, from `service_span_name` + - `postgres.{verb} {table}` when the call type maps to a known Prisma helper and its table resolves + - `postgres.{verb}` for targetless verbs, for example `postgres.ping` for `health_check` + - `postgres {call_type}` otherwise diff --git a/litellm/rust/crates/db/Cargo.toml b/litellm/rust/crates/db/Cargo.toml index c95f982433e..a9cebd9d5f5 100644 --- a/litellm/rust/crates/db/Cargo.toml +++ b/litellm/rust/crates/db/Cargo.toml @@ -10,6 +10,7 @@ include = ["Cargo.toml", "src/**", "queries/**", ".sqlx/**"] chrono.workspace = true futures-util.workspace = true litellm-circuit-breaker.workspace = true +litellm-error.workspace = true litellm-secret-value.workspace = true macro_rules_attribute.workspace = true serde.workspace = true diff --git a/litellm/rust/crates/db/src/health.rs b/litellm/rust/crates/db/src/health.rs new file mode 100644 index 00000000000..bd734b267fb --- /dev/null +++ b/litellm/rust/crates/db/src/health.rs @@ -0,0 +1,29 @@ +use sqlx::{Connection, PgPool}; + +use crate::Error; + +pub async fn health_check(pool: &PgPool) -> Result<(), Error> { + pool.acquire().await?.ping().await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use rstest::rstest; + use sqlx::PgPool; + + use super::health_check; + use crate::test_support::database_pool; + + #[rstest] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn answers_when_the_database_is_up( + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + health_check(&database_pool?).await?; + Ok(()) + } +} diff --git a/litellm/rust/crates/db/src/lib.rs b/litellm/rust/crates/db/src/lib.rs index cea4e689768..4ded5231a8e 100644 --- a/litellm/rust/crates/db/src/lib.rs +++ b/litellm/rust/crates/db/src/lib.rs @@ -12,6 +12,8 @@ mod end_user; mod endpoint; mod error; mod failure; +mod health; +mod observe; mod pools; mod spend; mod statements; @@ -24,6 +26,8 @@ pub use end_user::{EndUser, RateLimits, find_end_user}; pub use endpoint::{ClientIdentity, Endpoint, PoolLimits, SslMode}; pub use error::Error; pub use failure::DbFailure; +pub use health::health_check; +pub use observe::{Call, Observer, Outcome, QueryEvent, observe}; pub use pools::Pools; pub use spend::{ DailySpend, DailyTable, EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend, diff --git a/litellm/rust/crates/db/src/observe.rs b/litellm/rust/crates/db/src/observe.rs new file mode 100644 index 00000000000..bc02a0d5f21 --- /dev/null +++ b/litellm/rust/crates/db/src/observe.rs @@ -0,0 +1,128 @@ +use std::time::{Duration, Instant, SystemTime}; + +use crate::Error; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Call { + pub call_type: &'static str, + pub table: Option<&'static str>, + pub operation: Option<&'static str>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + Succeeded, + Failed(String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct QueryEvent { + pub call: Call, + pub started_at: SystemTime, + pub duration: Duration, + pub outcome: Outcome, +} + +pub trait Observer: Send + Sync { + fn record(&self, event: QueryEvent); +} + +pub async fn observe( + observer: &dyn Observer, + call: Call, + query: impl Future>, +) -> Result { + let started_at = SystemTime::now(); + let clock = Instant::now(); + let result = query.await; + let outcome = match &result { + Ok(_) => Some(Outcome::Succeeded), + Err(error @ Error::Query(_)) => Some(Outcome::Failed( + litellm_error::Report::new(error).to_string(), + )), + Err(_) => None, + }; + if let Some(outcome) = outcome { + observer.record(QueryEvent { + call, + started_at, + duration: clock.elapsed(), + outcome, + }); + } + result +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::{Call, Observer, Outcome, QueryEvent, observe}; + use crate::Error; + + const CALL: Call = Call { + call_type: "add_key_spend", + table: Some("LiteLLM_VerificationToken"), + operation: Some("update"), + }; + + #[derive(Default)] + struct Recording(Mutex>); + + impl Observer for Recording { + fn record(&self, event: QueryEvent) { + self.0.lock().expect("recording lock").push(event); + } + } + + impl Recording { + fn outcomes(&self) -> Vec<(Call, Outcome)> { + self.0 + .lock() + .expect("recording lock") + .iter() + .map(|event| (event.call, event.outcome.clone())) + .collect() + } + } + + #[tokio::test] + async fn records_a_successful_round_trip() { + let recording = Recording::default(); + + let value = observe(&recording, CALL, async { Ok::<_, Error>(7) }).await; + + assert_eq!(value.ok(), Some(7)); + assert_eq!(recording.outcomes(), vec![(CALL, Outcome::Succeeded)]); + } + + #[tokio::test] + async fn records_a_database_failure() { + let recording = Recording::default(); + + let result = observe(&recording, CALL, async { + Err::<(), _>(Error::from(sqlx::Error::PoolTimedOut)) + }) + .await; + + assert!(result.is_err()); + let outcomes = recording.outcomes(); + assert_eq!(outcomes.len(), 1); + assert!( + matches!(&outcomes[0], (call, Outcome::Failed(message)) if *call == CALL && message.contains("pool timed out")) + ); + } + + #[tokio::test] + async fn skips_failures_that_never_reached_the_database() { + let recording = Recording::default(); + + let result = observe(&recording, CALL, async { + Err::<(), _>(Error::InvalidSpendBatch { ids: 1, costs: 2 }) + }) + .await; + + assert!(result.is_err()); + assert!(recording.outcomes().is_empty()); + } +} diff --git a/litellm/rust/crates/python-bridge/src/db.rs b/litellm/rust/crates/python-bridge/src/db.rs index d17f5fd8099..3d7f3ffca36 100644 --- a/litellm/rust/crates/python-bridge/src/db.rs +++ b/litellm/rust/crates/python-bridge/src/db.rs @@ -1,12 +1,53 @@ -use litellm_db::{Endpoint, Pools}; +use std::{sync::Arc, time::UNIX_EPOCH}; + +use litellm_db::{Call, Endpoint, Observer, Outcome, Pools, QueryEvent}; use litellm_secret_value::SecretValue; use pyo3::{exceptions::PyValueError, prelude::*}; +const HEALTH_CHECK: Call = Call { + call_type: "health_check", + table: None, + operation: None, +}; + +struct PythonEvents(Py); + +impl Observer for PythonEvents { + fn record(&self, event: QueryEvent) { + let started_at = event + .started_at + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(); + let failure = match event.outcome { + Outcome::Succeeded => None, + Outcome::Failed(message) => Some(message), + }; + Python::attach(|py| { + let events = self.0.bind(py); + if let Err(error) = events.call_method1( + "record", + ( + event.call.call_type, + event.call.table, + event.call.operation, + started_at, + event.duration.as_secs_f64(), + failure, + ), + ) { + error.write_unraisable(py, Some(events)); + } + }); + } +} + #[pyclass(frozen)] pub(crate) struct NativeDatabase { writer_endpoint: Endpoint, reader_endpoint: Option, pools: Pools, + events: Arc, } fn map_error(error: litellm_db::Error) -> PyErr { @@ -16,9 +57,10 @@ fn map_error(error: litellm_db::Error) -> PyErr { #[pymethods] impl NativeDatabase { #[staticmethod] - #[pyo3(signature = (writer, reader=None))] + #[pyo3(signature = (events, writer, reader=None))] fn connect( py: Python<'_>, + events: Py, #[pyo3(from_py_with = litellm_host_python::from_py_argument)] writer: Endpoint, #[pyo3(from_py_with = litellm_host_python::from_py_argument)] reader: Option, ) -> PyResult> { @@ -28,6 +70,7 @@ impl NativeDatabase { writer_endpoint: writer, reader_endpoint: reader, pools, + events: Arc::new(PythonEvents(events)), }) }) } @@ -52,6 +95,26 @@ impl NativeDatabase { Ok(()) } + fn health_check<'py>(&self, py: Python<'py>) -> PyResult> { + let writer = self.pools.writer().clone(); + let events = Arc::clone(&self.events); + let socket_timeout = self.writer_endpoint.socket_timeout().map_err(map_error)?; + // Boxed so each future wrapper between here and the tokio worker holds a pointer instead of a copy of + // the SQLx connect state, which overflowed the worker's 2 MiB stack in unoptimized builds + crate::execution::run_async( + py, + Box::pin(async move { + litellm_db::observe( + events.as_ref(), + HEALTH_CHECK, + litellm_db::within(socket_timeout, litellm_db::health_check(&writer)), + ) + .await + }), + map_error, + ) + } + fn close<'py>(&self, py: Python<'py>) -> PyResult> { let pools = self.pools.clone(); crate::execution::run_async( diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 906b6e0549b..dbcaabc3e7b 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -9,7 +9,7 @@ from typing_extensions import disjoint_base from litellm.integrations.custom_logger_v2.rust import RustHandle from litellm.redis.store_settings import RedisBreakerSettings, RedisStoreSettings from litellm.redis.stores.router_session_pins.keys import PinKey -from litellm.rust_bridge.database import NativeDatabaseEndpoint +from litellm.rust_bridge.database import NativeDatabaseEndpoint, NativeDatabaseEvents from litellm.rust_bridge.execution import Execution from litellm.rust_bridge.public_call import NativeCall @@ -598,10 +598,11 @@ class NativeSpendCounters: class NativeDatabase: @staticmethod def connect( - writer: NativeDatabaseEndpoint, reader: NativeDatabaseEndpoint | None = None + events: NativeDatabaseEvents, writer: NativeDatabaseEndpoint, reader: NativeDatabaseEndpoint | None = None ) -> Future[NativeDatabase]: ... def set_writer_password(self, password: str) -> None: ... def set_reader_password(self, password: str) -> None: ... + def health_check(self) -> Future[None]: ... def close(self) -> Future[None]: ... @final diff --git a/litellm/rust_bridge/database.py b/litellm/rust_bridge/database.py index 1888a349f68..c4971003821 100644 --- a/litellm/rust_bridge/database.py +++ b/litellm/rust_bridge/database.py @@ -1,4 +1,4 @@ -from typing import Literal, TypeAlias +from typing import Literal, Protocol, TypeAlias from typing_extensions import ReadOnly, TypedDict @@ -31,3 +31,15 @@ class NativeDatabaseEndpoint(TypedDict): options: ReadOnly[tuple[tuple[str, str], ...]] statement_cache_capacity: ReadOnly[int] pool: ReadOnly[NativePoolLimits] + + +class NativeDatabaseEvents(Protocol): + def record( + self, + call_type: str, + table: str | None, + operation: str | None, + started_at: float, + duration_seconds: float, + failure: str | None, + ) -> None: ... diff --git a/tests/_support/native_database.py b/tests/_support/native_database.py index 8e7ea8feb18..6428dc4b789 100644 --- a/tests/_support/native_database.py +++ b/tests/_support/native_database.py @@ -1,7 +1,7 @@ import asyncio from typing import Final -from litellm.proxy.db.native_database import native_endpoint +from litellm.proxy.db.native_database import ServiceLoggingEvents, native_endpoint from litellm.rust_bridge._native import NativeDatabase _NEVER_DIALED_URL: Final = "postgresql://litellm:secret@db.invalid:5432/proxy" @@ -9,7 +9,9 @@ _NEVER_DIALED_URL: Final = "postgresql://litellm:secret@db.invalid:5432/proxy" async def connect_native_database() -> NativeDatabase: return await NativeDatabase.connect( - native_endpoint(_NEVER_DIALED_URL, tls_offered=True, max_connections=1).endpoint, None + ServiceLoggingEvents(loop=asyncio.get_running_loop(), service_logging=lambda: None), + native_endpoint(_NEVER_DIALED_URL, tls_offered=True, max_connections=1).endpoint, + None, ) diff --git a/tests/unit/proxy/db/test_native_database.py b/tests/unit/proxy/db/test_native_database.py index edd4cf7fa2d..dbe24da55c0 100644 --- a/tests/unit/proxy/db/test_native_database.py +++ b/tests/unit/proxy/db/test_native_database.py @@ -1,5 +1,6 @@ import asyncio import datetime as dt +from collections.abc import Mapping from datetime import datetime, timedelta from pathlib import Path from typing import Final @@ -12,7 +13,9 @@ from cryptography.hazmat.primitives.serialization import BestAvailableEncryption from cryptography.x509.oid import NameOID from pydantic import TypeAdapter +from litellm._service_logger import ServiceLogging, ServiceTypes from litellm.proxy.db.native_database import ( + ServiceLoggingEvents, TokenRotation, keep_token_fresh, native_endpoint, @@ -23,6 +26,7 @@ from litellm.proxy.db.native_database import ( ) from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR from litellm.proxy.db.token_auth import IAMEndpoint, RdsIamTokenAuth +from litellm.rust_bridge._native import NativeDatabase from litellm.rust_bridge.database import NativeDatabaseEndpoint, NativePoolLimits NOW: Final = datetime(2026, 11, 3, 12, 0, 0) @@ -390,3 +394,86 @@ async def test_keep_token_fresh_retries_a_failed_mint() -> None: assert sink.passwords == ("recovered",) assert schedule.delays[:2] == (600.0, 30.0) + + +class _RecordingServiceLogging(ServiceLogging): + def __init__(self) -> None: + super().__init__(mock_testing=True) + self.events: tuple[tuple[str, str, float, Mapping[str, str] | None, str | None], ...] = () + self.arrived: Final = asyncio.Event() + + async def async_service_success_hook( + self, + service: ServiceTypes, + call_type: str, + duration: float, + parent_otel_span: object = None, + start_time: datetime | float | None = None, + end_time: datetime | float | None = None, + event_metadata: dict[str, str] | None = None, + caller: str | None = None, + ) -> None: + self.events = (*self.events, (service.value, call_type, duration, event_metadata, None)) + self.arrived.set() + + async def async_service_failure_hook( + self, + service: ServiceTypes, + duration: float, + error: str | Exception, + call_type: str, + parent_otel_span: object = None, + start_time: datetime | float | None = None, + end_time: datetime | float | None = None, + event_metadata: dict[str, str] | None = None, + caller: str | None = None, + ) -> None: + self.events = (*self.events, (service.value, call_type, duration, event_metadata, str(error))) + self.arrived.set() + + +@pytest.mark.asyncio +async def test_service_logging_events_reach_the_loop_from_another_thread() -> None: + logging: Final = _RecordingServiceLogging() + events: Final = ServiceLoggingEvents(loop=asyncio.get_running_loop(), service_logging=lambda: logging) + + await asyncio.to_thread( + events.record, "add_key_spend", "LiteLLM_VerificationToken", "update", NOW.timestamp(), 0.25, None + ) + await asyncio.wait_for(logging.arrived.wait(), timeout=5) + logging.arrived.clear() + await asyncio.to_thread(events.record, "health_check", None, None, NOW.timestamp(), 1.5, "pool timed out") + await asyncio.wait_for(logging.arrived.wait(), timeout=5) + + assert logging.events == ( + ( + "postgres", + "add_key_spend", + 0.25, + {"table_name": "LiteLLM_VerificationToken", "db_operation": "update"}, + None, + ), + ("postgres", "health_check", 1.5, {}, "pool timed out"), + ) + + +@pytest.mark.asyncio +async def test_native_health_check_failure_reports_a_db_event() -> None: + logging: Final = _RecordingServiceLogging() + native: Final = await NativeDatabase.connect( + ServiceLoggingEvents(loop=asyncio.get_running_loop(), service_logging=lambda: logging), + native_endpoint( + "postgresql://litellm:secret@127.0.0.1:1/proxy?pool_timeout=1", tls_offered=True, max_connections=2 + ).endpoint, + ) + + with pytest.raises(ValueError, match="database query failed"): + await native.health_check() + await asyncio.wait_for(logging.arrived.wait(), timeout=5) + await native.close() + + assert len(logging.events) == 1 + service, call_type, duration, metadata, failure = logging.events[0] + assert (service, call_type, metadata) == ("postgres", "health_check", {}) + assert duration >= 0.9 + assert failure is not None and "database query failed" in failure diff --git a/tests/unit/repositories/test_database.py b/tests/unit/repositories/test_database.py index 2497c9237e9..f5a3323b87c 100644 --- a/tests/unit/repositories/test_database.py +++ b/tests/unit/repositories/test_database.py @@ -1,8 +1,11 @@ +import asyncio +import logging from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from litellm._logging import verbose_proxy_logger from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR from litellm.repositories.database import Database @@ -94,3 +97,22 @@ async def test_zero_native_connections_open_no_native_pool(monkeypatch: pytest.M assert (database.native, database.tasks) == (None, ()) await database.aclose() aclose.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("static_password") +async def test_an_unreachable_native_pool_is_reported_without_failing_boot(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + database: Final = await Database.open( + _prisma(), "postgresql://litellm:secret@127.0.0.1:1/proxy?pool_timeout=1", None, NATIVE_CONNECTIONS + ) + await asyncio.wait_for(database.boot_probe, timeout=10) + await database.aclose() + + assert [ + record.getMessage() for record in caplog.records if "Native database pool cannot reach" in record.getMessage() + ] == [ + "Native database pool cannot reach the database, so operations ported to it will fail until it connects." + " Prisma is checked separately. Cause: database query failed: pool timed out while waiting for an open" + " connection" + ] From 1bc3190a942bdf84c7efd7fc87d6fe98080f8089 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Sun, 15 Nov 2026 08:28:11 +0900 Subject: [PATCH 04/57] feat(db): classify errors and recycle poisoned pools like Prisma (#49801) db::Error now reports a kind that matches PrismaDBExceptionHandler's buckets: unreachable, capacity, deadlock, serialization failure, lock timeout, read-only after failover, stale cached plan and rejected data. is_unavailable gates fail-open and is_requeue_safe says whether a failed write provably never applied Pools::recycle_after is the native counterpart of recreating the Prisma client: after a 25006 (the writer endpoint now points at a read-only node) or "cached plan must not change result type" (a migration changed a table under a prepared statement), every pooled connection opened before the error is dropped on its next acquire, so the pool reconnects and prepares fresh Native failures reached Python as a ValueError, so allow_requests_on_db_unavailable treated a native outage as a permanent fault and a write on a dropped connection was requeued. The bridge now raises NativeDatabaseFailure with the kind and the Rust answers for is_unavailable and is_requeue_safe, and the two Python gates return them --- litellm/proxy/db/exception_handler.py | 5 + litellm/repositories/database.py | 4 +- litellm/rust/crates/db/AGENTS.md | 4 +- litellm/rust/crates/db/src/endpoint.rs | 9 +- litellm/rust/crates/db/src/error.rs | 127 +++++++++++--- litellm/rust/crates/db/src/lib.rs | 2 +- litellm/rust/crates/db/src/pools.rs | 169 ++++++++++++++++++- litellm/rust/crates/python-bridge/src/db.rs | 20 ++- litellm/rust/crates/python-bridge/src/lib.rs | 3 +- litellm/rust_bridge/_native.pyi | 6 + tests/unit/proxy/db/test_native_database.py | 51 +++++- 11 files changed, 365 insertions(+), 35 deletions(-) diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index d9d0ece4468..cdafa2cf52f 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -14,6 +14,7 @@ from litellm.proxy._types import ( ProxyException, ) from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded +from litellm.rust_bridge._native import NativeDatabaseFailure from litellm.secret_managers.main import str_to_bool # Bounds the __cause__/__context__ walk in find_database_service_unavailable_error_in_chain. @@ -175,6 +176,8 @@ class PrismaDBExceptionHandler: """ import prisma.engine.errors + if isinstance(e, NativeDatabaseFailure): + return e.unavailable if isinstance(e, (*DB_CONNECTION_ERROR_TYPES, DBLookupDeadlineExceeded)): return True if isinstance(e, _exception_types(prisma.engine.errors.EngineConnectionError)): @@ -769,6 +772,8 @@ _DATA_REJECTED_SQLSTATE_CLASSES: Final = frozenset({"22", "23"}) def db_write_failure_is_requeue_safe(e: Exception) -> bool: """Whether a failed write provably never applied, so its rows can be sent again without double counting.""" + if isinstance(e, NativeDatabaseFailure): + return e.requeue_safe if isinstance(e, DB_CONNECTION_ERROR_TYPES): return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e) diff --git a/litellm/repositories/database.py b/litellm/repositories/database.py index 5ee0accab8c..45849a17a4b 100644 --- a/litellm/repositories/database.py +++ b/litellm/repositories/database.py @@ -16,7 +16,7 @@ from litellm.proxy.db.native_database import ( resolve_native_endpoint, writer_token_rotation, ) -from litellm.rust_bridge._native import NativeDatabase +from litellm.rust_bridge._native import NativeDatabase, NativeDatabaseFailure if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -43,7 +43,7 @@ def _log_ignored_options(env_var: str, settings: NativeEndpointSettings) -> None async def _warn_if_unreachable(native: NativeDatabase) -> None: try: await native.health_check() - except ValueError as error: + except NativeDatabaseFailure as error: verbose_proxy_logger.warning( "Native database pool cannot reach the database, so operations ported to it will fail until it" " connects. Prisma is checked separately. Cause: %s", diff --git a/litellm/rust/crates/db/AGENTS.md b/litellm/rust/crates/db/AGENTS.md index b23a379d798..5fdee482cf8 100644 --- a/litellm/rust/crates/db/AGENTS.md +++ b/litellm/rust/crates/db/AGENTS.md @@ -53,7 +53,9 @@ Python and SQLx run side by side until every repository operation is native, so | In-container PgBouncer | `pgbouncer.py` | Inherited: the pooled URL carries PgBouncer's credential | | Startup connection budget | `postgres_connection_budget_message` | Counts the Rust writer and reader pools' shares | | Read replica routing | `routing_prisma_wrapper.py`, `DATABASE_URL_READ_REPLICA` | `Pools`: callers pick `writer()`, `read()` or `read_after_write()`. `read()` prefers the reader and `read_after_write()` the writer (`WriterPinnedClient`); either falls back to the other side when its pool cannot connect and skips that side for 30s, so reads keep working on the reader while the writer is down. Writes have no fallback | -| Error classification, `lock_timeout` on rollups | `exception_handler.py`, `rollup_lock_timeout.py` | Planned | +| `lock_timeout` on rollups | `rollup_lock_timeout.py` | Planned | +| Error classification | `exception_handler.py` | `Error::kind`, `is_unavailable`, `is_requeue_safe`. The bridge raises `NativeDatabaseFailure` with the kind and both answers, and `is_database_connection_error` and `db_write_failure_is_requeue_safe` return them instead of classifying the error again | +| Recreate the client after 25006 (writer became read-only) or a stale cached plan | `exception_handler.py`, `_query_first_with_cached_plan_fallback` | `Pools::recycle_after` drops every pooled connection opened before the error, so the next acquire reconnects and prepares fresh. Callers retry a `StalePlan` once | | `ServiceTypes.DB` events per round trip | `db_span.py`, `log_db_metrics.py`, `prometheus_services.py`, `otel/model/spans.py` | Python host only. Wrap each native operation in `observe` with the call type and table its Prisma version uses, and the bridge forwards events to `ServiceLoggingEvents`. Nesting under an outer Python `db_span` is not detected. A Rust host has no sink yet and must emit the names below | | Auth lookup gate, fail open on DB outage, reader choice per query | `db_lookup_gate.py`, `allow_requests_on_db_unavailable` | With each ported auth read | | Connect at boot, so a database that cannot be reached is reported before traffic | `PrismaClient.connect` | The pool stays lazy and never fails boot. `Database.open` pings it once in the background and logs a warning when it cannot connect, so a mistranslated endpoint is not first found by a ported query | diff --git a/litellm/rust/crates/db/src/endpoint.rs b/litellm/rust/crates/db/src/endpoint.rs index cb04943333b..4eba6a9942c 100644 --- a/litellm/rust/crates/db/src/endpoint.rs +++ b/litellm/rust/crates/db/src/endpoint.rs @@ -105,13 +105,18 @@ impl Endpoint { } } - pub fn pool(&self) -> Result { + pub fn pool_options(&self) -> Result { let limits = self.pool; Ok(PgPoolOptions::new() .min_connections(0) .max_connections(limits.max_connections) .acquire_timeout(seconds(limits.acquire_timeout_seconds)?.unwrap_or(WAIT_FOREVER)) - .idle_timeout(seconds(limits.idle_timeout_seconds)?) + .idle_timeout(seconds(limits.idle_timeout_seconds)?)) + } + + pub fn pool(&self) -> Result { + Ok(self + .pool_options()? .connect_lazy_with(self.connect_options())) } diff --git a/litellm/rust/crates/db/src/error.rs b/litellm/rust/crates/db/src/error.rs index c4d44a3231e..22ddb643d9c 100644 --- a/litellm/rust/crates/db/src/error.rs +++ b/litellm/rust/crates/db/src/error.rs @@ -10,21 +10,76 @@ pub enum Error { InvalidDuration(#[from] std::time::TryFromFloatSecsError), } +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr)] +#[strum(serialize_all = "snake_case")] +pub enum ErrorKind { + Unreachable, + Capacity, + Deadlock, + SerializationFailure, + LockTimeout, + ReadOnly, + StalePlan, + DataRejected, + Other, +} + impl Error { - pub fn is_retryable(&self) -> bool { + pub fn kind(&self) -> ErrorKind { match self { + Self::Query(sqlx::Error::Io(_)) => ErrorKind::Unreachable, + Self::Query(sqlx::Error::PoolTimedOut) => ErrorKind::Capacity, Self::Query(sqlx::Error::Database(error)) => { - matches!( - error.code().as_deref(), - Some("40P01" | "55P03" | "40001" | "53300") - ) + sqlstate_kind(error.code().as_deref(), error.message()) } - Self::Query(sqlx::Error::Io(_) | sqlx::Error::PoolTimedOut) => true, + _ => ErrorKind::Other, + } + } + + pub fn is_retryable(&self) -> bool { + matches!( + self.kind(), + ErrorKind::Unreachable + | ErrorKind::Capacity + | ErrorKind::Deadlock + | ErrorKind::SerializationFailure + | ErrorKind::LockTimeout + | ErrorKind::StalePlan + ) + } + + pub fn needs_fresh_connections(&self) -> bool { + matches!(self.kind(), ErrorKind::ReadOnly | ErrorKind::StalePlan) + } + + pub fn is_unavailable(&self) -> bool { + matches!(self.kind(), ErrorKind::Unreachable | ErrorKind::Capacity) + } + + pub fn is_requeue_safe(&self) -> bool { + match self { + Self::Query(sqlx::Error::PoolTimedOut | sqlx::Error::PoolClosed) => true, + Self::Query(sqlx::Error::Database(_)) => self.kind() != ErrorKind::DataRejected, _ => false, } } } +fn sqlstate_kind(code: Option<&str>, message: &str) -> ErrorKind { + match code { + Some("0A000") if message.contains("cached plan must not change result type") => { + ErrorKind::StalePlan + } + Some("53300") => ErrorKind::Capacity, + Some("40P01") => ErrorKind::Deadlock, + Some("40001") => ErrorKind::SerializationFailure, + Some("55P03") => ErrorKind::LockTimeout, + Some("25006") => ErrorKind::ReadOnly, + Some(code) if code.starts_with("22") || code.starts_with("23") => ErrorKind::DataRejected, + _ => ErrorKind::Other, + } +} + #[cfg(test)] mod tests { use std::error::Error as StdError; @@ -32,29 +87,59 @@ mod tests { use rstest::rstest; use sqlx::PgPool; - use super::Error; + use super::{Error, ErrorKind}; use crate::test_support::database_pool; #[rstest] - #[case::closed_pool(sqlx::Error::PoolClosed, false)] - #[case::pool_timeout(sqlx::Error::PoolTimedOut, true)] - #[case::io(sqlx::Error::Io(std::io::ErrorKind::ConnectionReset.into()), true)] - #[case::missing_row(sqlx::Error::RowNotFound, false)] - fn classifies_client_errors(#[case] error: sqlx::Error, #[case] retryable: bool) { - assert_eq!(Error::from(error).is_retryable(), retryable); + #[case::closed_pool(sqlx::Error::PoolClosed, ErrorKind::Other, false, false, true)] + #[case::pool_timeout(sqlx::Error::PoolTimedOut, ErrorKind::Capacity, true, true, true)] + #[case::io( + sqlx::Error::Io(std::io::ErrorKind::ConnectionReset.into()), + ErrorKind::Unreachable, + true, + true, + false + )] + #[case::missing_row(sqlx::Error::RowNotFound, ErrorKind::Other, false, false, false)] + fn classifies_client_errors( + #[case] error: sqlx::Error, + #[case] kind: ErrorKind, + #[case] retryable: bool, + #[case] unavailable: bool, + #[case] requeue_safe: bool, + ) { + let error = Error::from(error); + + assert_eq!(error.kind(), kind); + assert_eq!(error.is_retryable(), retryable); + assert_eq!(error.is_unavailable(), unavailable); + assert_eq!(error.is_requeue_safe(), requeue_safe); } #[rstest] - #[case::connection_capacity("53300", true)] - #[case::deadlock("40P01", true)] - #[case::lock_unavailable("55P03", true)] - #[case::serialization_failure("40001", true)] - #[case::unique_violation("23505", false)] + #[case::connection_capacity("53300", ErrorKind::Capacity, true, true, true)] + #[case::deadlock("40P01", ErrorKind::Deadlock, true, false, true)] + #[case::lock_unavailable("55P03", ErrorKind::LockTimeout, true, false, true)] + #[case::serialization_failure("40001", ErrorKind::SerializationFailure, true, false, true)] + #[case::read_only_after_failover("25006", ErrorKind::ReadOnly, false, false, true)] + #[case::unique_violation("23505", ErrorKind::DataRejected, false, false, false)] + #[case::numeric_overflow("22003", ErrorKind::DataRejected, false, false, false)] + #[case::undefined_table("42P01", ErrorKind::Other, false, false, true)] + #[case::feature_not_supported_without_a_stale_plan( + "0A000", + ErrorKind::Other, + false, + false, + true + )] #[tokio::test] #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] async fn classifies_server_errors( #[case] code: &str, + #[case] kind: ErrorKind, #[case] retryable: bool, + #[case] unavailable: bool, + #[case] requeue_safe: bool, #[future(awt)] database_pool: Result>, ) -> Result<(), Box> { let pool = database_pool?; @@ -74,7 +159,11 @@ mod tests { error.as_database_error().and_then(|error| error.code()), Some(code.into()) ); - assert_eq!(Error::from(error).is_retryable(), retryable); + let error = Error::from(error); + assert_eq!(error.kind(), kind); + assert_eq!(error.is_retryable(), retryable); + assert_eq!(error.is_unavailable(), unavailable); + assert_eq!(error.is_requeue_safe(), requeue_safe); tx.rollback().await?; Ok(()) } diff --git a/litellm/rust/crates/db/src/lib.rs b/litellm/rust/crates/db/src/lib.rs index 4ded5231a8e..89d655a8502 100644 --- a/litellm/rust/crates/db/src/lib.rs +++ b/litellm/rust/crates/db/src/lib.rs @@ -24,7 +24,7 @@ pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow}; pub use deadline::within; pub use end_user::{EndUser, RateLimits, find_end_user}; pub use endpoint::{ClientIdentity, Endpoint, PoolLimits, SslMode}; -pub use error::Error; +pub use error::{Error, ErrorKind}; pub use failure::DbFailure; pub use health::health_check; pub use observe::{Call, Observer, Outcome, QueryEvent, observe}; diff --git a/litellm/rust/crates/db/src/pools.rs b/litellm/rust/crates/db/src/pools.rs index 561c1bad914..59bcd68e5c5 100644 --- a/litellm/rust/crates/db/src/pools.rs +++ b/litellm/rust/crates/db/src/pools.rs @@ -1,4 +1,7 @@ -use std::time::{Duration, Instant}; +use std::{ + sync::{Arc, Mutex, PoisonError}, + time::{Duration, Instant}, +}; use litellm_circuit_breaker::{Breaker, Config, Failure}; use sqlx::{PgPool, Postgres, pool::PoolConnection}; @@ -9,16 +12,46 @@ const RECOVERY: Duration = Duration::from_secs(30); type Connection = PoolConnection; +#[derive(Default)] +struct Recycled(Mutex>); + +impl Recycled { + fn mark(&self, now: Instant) { + *self.0.lock().unwrap_or_else(PoisonError::into_inner) = Some(now); + } + + fn predates(&self, now: Instant, age: Duration) -> bool { + let Some(recycled_at) = *self.0.lock().unwrap_or_else(PoisonError::into_inner) else { + return false; + }; + now.checked_sub(age) + .is_none_or(|opened_at| opened_at <= recycled_at) + } +} + #[derive(Clone)] struct Side { pool: PgPool, breaker: Breaker, + recycled: Arc, } impl Side { fn new(endpoint: &Endpoint) -> Result { + let recycled = Arc::new(Recycled::default()); + let pool = endpoint + .pool_options()? + .before_acquire({ + let recycled = Arc::clone(&recycled); + move |_, metadata| { + let fresh = !recycled.predates(Instant::now(), metadata.age); + Box::pin(async move { Ok(fresh) }) + } + }) + .connect_lazy_with(endpoint.connect_options()); Ok(Self { - pool: endpoint.pool()?, + pool, + recycled, breaker: Breaker::new( Config { failure_threshold: 1, @@ -99,6 +132,18 @@ impl Pools { acquire_preferring(&self.writer, self.reader.as_ref()).await } + pub fn recycle_after(&self, error: &Error) -> bool { + if !error.needs_fresh_connections() { + return false; + } + let now = Instant::now(); + self.writer.recycled.mark(now); + if let Some(reader) = &self.reader { + reader.recycled.mark(now); + } + true + } + pub async fn close(&self) { self.writer.pool.close().await; if let Some(reader) = &self.reader { @@ -109,12 +154,124 @@ impl Pools { #[cfg(test)] mod tests { - use std::{error::Error as StdError, time::Instant}; + use std::{ + error::Error as StdError, + time::{Duration, Instant}, + }; - use sqlx::PgConnection; + use rstest::rstest; + use sqlx::{Connection as _, PgConnection}; - use super::Pools; - use crate::{Endpoint, PoolLimits, test_support::database_endpoint}; + use super::{Pools, Recycled}; + use crate::{Endpoint, Error, ErrorKind, PoolLimits, test_support::database_endpoint}; + + #[rstest] + #[case::never_recycled(None, 5, false)] + #[case::opened_before_the_recycle(Some(10), 30, true)] + #[case::opened_after_the_recycle(Some(10), 5, false)] + fn discards_connections_opened_before_a_recycle( + #[case] recycled_seconds_ago: Option, + #[case] age_seconds: u64, + #[case] stale: bool, + ) { + let now = Instant::now() + Duration::from_secs(60); + let recycled = Recycled::default(); + if let Some(ago) = recycled_seconds_ago { + recycled.mark(now - Duration::from_secs(ago)); + } + + assert_eq!( + recycled.predates(now, Duration::from_secs(age_seconds)), + stale + ); + } + + async fn backend_pid(pools: &Pools) -> Result { + #[expect(clippy::disallowed_methods, reason = "A probe query needs no schema")] + sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *pools.writer().acquire().await?) + .await + } + + async fn raised(pools: &Pools, code: &str) -> Result { + #[expect( + clippy::disallowed_methods, + reason = "Test query must raise the requested SQLSTATE" + )] + let error = sqlx::query(sqlx::AssertSqlSafe(format!( + "DO $$ BEGIN RAISE EXCEPTION USING ERRCODE = '{code}'; END $$" + ))) + .execute(pools.writer()) + .await + .expect_err("PostgreSQL must raise the requested error"); + Ok(Error::from(error)) + } + + #[rstest] + #[case::read_only_writer("25006", true)] + #[case::lock_timeout("55P03", false)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn recycles_pooled_connections_only_after_errors_that_poison_them( + #[case] code: &str, + #[case] recycled: bool, + ) -> Result<(), Box> { + let pools = Pools::new(&database_endpoint()?, None)?; + let before = backend_pid(&pools).await?; + + assert_eq!(pools.recycle_after(&raised(&pools, code).await?), recycled); + + assert_eq!(backend_pid(&pools).await? != before, recycled); + Ok(()) + } + + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn a_schema_change_under_a_cached_plan_is_recovered_by_a_recycle() + -> Result<(), Box> { + let endpoint = database_endpoint()?; + let pools = Pools::new( + &Endpoint { + pool: PoolLimits { + max_connections: 1, + ..endpoint.pool + }, + ..endpoint.clone() + }, + None, + )?; + let table = format!("native_stale_plan_probe_{}", std::process::id()); + let select = format!("SELECT * FROM {table}"); + #[expect(clippy::disallowed_methods, reason = "A scratch table needs no schema")] + let run = |sql: &str| sqlx::query(sqlx::AssertSqlSafe(sql.to_owned())); + let mut admin = PgConnection::connect_with(&endpoint.connect_options()).await?; + run(&format!("CREATE TABLE {table} (id int)")) + .execute(&mut admin) + .await?; + + let outcome = async { + run(&select).fetch_all(pools.writer()).await?; + run(&format!("ALTER TABLE {table} ADD COLUMN extra int")) + .execute(&mut admin) + .await?; + let stale = Error::from( + run(&select) + .fetch_all(pools.writer()) + .await + .expect_err("the cached plan must be stale"), + ); + let recycled = pools.recycle_after(&stale); + run(&select).fetch_all(pools.writer()).await?; + Ok::<_, Box>((stale.kind(), recycled)) + } + .await; + run(&format!("DROP TABLE {table}")) + .execute(&mut admin) + .await?; + + assert_eq!(outcome?, (ErrorKind::StalePlan, true)); + Ok(()) + } fn read_only(endpoint: &Endpoint) -> Endpoint { Endpoint { diff --git a/litellm/rust/crates/python-bridge/src/db.rs b/litellm/rust/crates/python-bridge/src/db.rs index 3d7f3ffca36..c14317219c8 100644 --- a/litellm/rust/crates/python-bridge/src/db.rs +++ b/litellm/rust/crates/python-bridge/src/db.rs @@ -50,8 +50,26 @@ pub(crate) struct NativeDatabase { events: Arc, } +pyo3::create_exception!( + _native, + NativeDatabaseFailure, + pyo3::exceptions::PyException, + "A native database operation failed. The kind attribute is litellm-db's ErrorKind, and unavailable and requeue_safe are the error's is_unavailable and is_requeue_safe." +); + fn map_error(error: litellm_db::Error) -> PyErr { - PyValueError::new_err(litellm_error::Report::new(&error).to_string()) + let kind: &'static str = error.kind().into(); + let failure = NativeDatabaseFailure::new_err(litellm_error::Report::new(&error).to_string()); + let decided = Python::attach(|py| { + let value = failure.value(py); + value.setattr("kind", kind)?; + value.setattr("unavailable", error.is_unavailable())?; + value.setattr("requeue_safe", error.is_requeue_safe()) + }); + match decided { + Ok(()) => failure, + Err(error) => error, + } } #[pymethods] diff --git a/litellm/rust/crates/python-bridge/src/lib.rs b/litellm/rust/crates/python-bridge/src/lib.rs index cdf3ace6cca..dc3584c7e45 100644 --- a/litellm/rust/crates/python-bridge/src/lib.rs +++ b/litellm/rust/crates/python-bridge/src/lib.rs @@ -27,7 +27,7 @@ mod _native { use pyo3::{prelude::*, types::PyModule}; #[pymodule_export] - use crate::db::NativeDatabase; + use crate::db::{NativeDatabase, NativeDatabaseFailure}; #[cfg(feature = "panic-test")] #[pymodule_export] use crate::diagnostics::_panic_for_test; @@ -184,6 +184,7 @@ mod tests { "NativeClickHouseSpendConfig", "NativeClickHouseSpendStorage", "NativeDatabase", + "NativeDatabaseFailure", "trace_encode_error", "bundled_model_cost_map", "NativeModelCostMap", diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index dbcaabc3e7b..bf4df6a39ea 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -21,6 +21,11 @@ class ForkedAfterNativeRuntimeStarted(RuntimeError): ... class ProcessReservedForForking(RuntimeError): ... class NativeRedisStoreError(Exception): ... +class NativeDatabaseFailure(Exception): + kind: str + unavailable: bool + requeue_safe: bool + def trace_encode_error(message: str) -> bytes: ... def bundled_model_cost_map() -> bytes: ... def native_redis_breaker_states() -> list[str]: ... @@ -803,6 +808,7 @@ __all__ = [ "NativeClickHouseSpendStorage", "NativeCostMapSource", "NativeDatabase", + "NativeDatabaseFailure", "NativeDeploymentHealth", "NativeDiagnosticProcessor", "NativeFetchFailed", diff --git a/tests/unit/proxy/db/test_native_database.py b/tests/unit/proxy/db/test_native_database.py index dbe24da55c0..575661fab0b 100644 --- a/tests/unit/proxy/db/test_native_database.py +++ b/tests/unit/proxy/db/test_native_database.py @@ -14,6 +14,7 @@ from cryptography.x509.oid import NameOID from pydantic import TypeAdapter from litellm._service_logger import ServiceLogging, ServiceTypes +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, db_write_failure_is_requeue_safe from litellm.proxy.db.native_database import ( ServiceLoggingEvents, TokenRotation, @@ -26,7 +27,7 @@ from litellm.proxy.db.native_database import ( ) from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR from litellm.proxy.db.token_auth import IAMEndpoint, RdsIamTokenAuth -from litellm.rust_bridge._native import NativeDatabase +from litellm.rust_bridge._native import NativeDatabase, NativeDatabaseFailure from litellm.rust_bridge.database import NativeDatabaseEndpoint, NativePoolLimits NOW: Final = datetime(2026, 11, 3, 12, 0, 0) @@ -467,7 +468,7 @@ async def test_native_health_check_failure_reports_a_db_event() -> None: ).endpoint, ) - with pytest.raises(ValueError, match="database query failed"): + with pytest.raises(NativeDatabaseFailure, match="database query failed"): await native.health_check() await asyncio.wait_for(logging.arrived.wait(), timeout=5) await native.close() @@ -477,3 +478,49 @@ async def test_native_health_check_failure_reports_a_db_event() -> None: assert (service, call_type, metadata) == ("postgres", "health_check", {}) assert duration >= 0.9 assert failure is not None and "database query failed" in failure + + +async def _native_failure(url: str) -> NativeDatabaseFailure: + native: Final = await NativeDatabase.connect( + ServiceLoggingEvents(loop=asyncio.get_running_loop(), service_logging=lambda: None), + native_endpoint(url, tls_offered=True, max_connections=2).endpoint, + ) + try: + with pytest.raises(NativeDatabaseFailure) as raised: + await native.health_check() + finally: + await native.close() + return raised.value + + +async def _drop_connection(_: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + writer.close() + + +@pytest.mark.asyncio +async def test_native_outage_passes_the_db_unavailable_gate_and_is_requeue_safe() -> None: + error: Final = await _native_failure("postgresql://litellm:secret@127.0.0.1:1/proxy?pool_timeout=1") + + assert (error.kind, str(error)) == ( + "capacity", + "database query failed: pool timed out while waiting for an open connection", + ) + assert PrismaDBExceptionHandler.is_database_connection_error(error) + assert db_write_failure_is_requeue_safe(error) + + +@pytest.mark.asyncio +async def test_native_dropped_connection_is_unavailable_but_not_requeue_safe() -> None: + server: Final = await asyncio.start_server(_drop_connection, "127.0.0.1", 0) + _, port = _ADDRESS.validate_python(server.sockets[0].getsockname()) + try: + error: Final = await _native_failure( + f"postgresql://litellm:secret@127.0.0.1:{port}/proxy?pool_timeout=5&sslmode=disable" + ) + finally: + server.close() + await server.wait_closed() + + assert error.kind == "unreachable" + assert PrismaDBExceptionHandler.is_database_connection_error(error) + assert not db_write_failure_is_requeue_safe(error) From ac23812b4b124318997dcf1634e2e7861cd47fc0 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Sun, 15 Nov 2026 08:28:11 +0900 Subject: [PATCH 05/57] feat(db): take the rollup lock_timeout from the caller and keep team lock statements unnamed under PgBouncer (#51549) begin() takes the rollup lock_timeout from the caller instead of hardcoding 5000ms, so SPEND_ROLLUP_LOCK_TIMEOUT_MS applies on both paths. It takes whole positive milliseconds, like that setting, so it can never send 0ms, which Postgres reads as no timeout begin, lock_team and add_team_member_spend take whether statements stay named and run through Statements, so they send unnamed statements when the cache is off --- ...021f3a07befaedd77d4949e9401b120a5322.json} | 8 +++-- litellm/rust/crates/db/AGENTS.md | 2 +- litellm/rust/crates/db/src/spend/team_lock.rs | 35 +++++++++++++------ .../crates/db/src/spend/team_member_spend.rs | 24 ++++++++----- 4 files changed, 46 insertions(+), 23 deletions(-) rename litellm/rust/crates/db/.sqlx/{query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json => query-805611b4198a8f11e66e29b77afa021f3a07befaedd77d4949e9401b120a5322.json} (59%) diff --git a/litellm/rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json b/litellm/rust/crates/db/.sqlx/query-805611b4198a8f11e66e29b77afa021f3a07befaedd77d4949e9401b120a5322.json similarity index 59% rename from litellm/rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json rename to litellm/rust/crates/db/.sqlx/query-805611b4198a8f11e66e29b77afa021f3a07befaedd77d4949e9401b120a5322.json index 7eef5278cb5..7fb7d2aed5f 100644 --- a/litellm/rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json +++ b/litellm/rust/crates/db/.sqlx/query-805611b4198a8f11e66e29b77afa021f3a07befaedd77d4949e9401b120a5322.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT set_config('lock_timeout', '5000ms', true)", + "query": "SELECT set_config('lock_timeout', $1::text, true)", "describe": { "columns": [ { @@ -11,11 +11,13 @@ } ], "parameters": { - "Left": [] + "Left": [ + "Text" + ] }, "nullable": [ null ] }, - "hash": "5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4" + "hash": "805611b4198a8f11e66e29b77afa021f3a07befaedd77d4949e9401b120a5322" } diff --git a/litellm/rust/crates/db/AGENTS.md b/litellm/rust/crates/db/AGENTS.md index 5fdee482cf8..c7ebedc6948 100644 --- a/litellm/rust/crates/db/AGENTS.md +++ b/litellm/rust/crates/db/AGENTS.md @@ -53,7 +53,7 @@ Python and SQLx run side by side until every repository operation is native, so | In-container PgBouncer | `pgbouncer.py` | Inherited: the pooled URL carries PgBouncer's credential | | Startup connection budget | `postgres_connection_budget_message` | Counts the Rust writer and reader pools' shares | | Read replica routing | `routing_prisma_wrapper.py`, `DATABASE_URL_READ_REPLICA` | `Pools`: callers pick `writer()`, `read()` or `read_after_write()`. `read()` prefers the reader and `read_after_write()` the writer (`WriterPinnedClient`); either falls back to the other side when its pool cannot connect and skips that side for 30s, so reads keep working on the reader while the writer is down. Writes have no fallback | -| `lock_timeout` on rollups | `rollup_lock_timeout.py` | Planned | +| `lock_timeout` on rollups | `rollup_lock_timeout.py` | `begin` takes the caller's `lock_timeout` as whole positive milliseconds, like `SPEND_ROLLUP_LOCK_TIMEOUT_MS`, so it can never send `0ms`, which Postgres reads as no timeout | | Error classification | `exception_handler.py` | `Error::kind`, `is_unavailable`, `is_requeue_safe`. The bridge raises `NativeDatabaseFailure` with the kind and both answers, and `is_database_connection_error` and `db_write_failure_is_requeue_safe` return them instead of classifying the error again | | Recreate the client after 25006 (writer became read-only) or a stale cached plan | `exception_handler.py`, `_query_first_with_cached_plan_fallback` | `Pools::recycle_after` drops every pooled connection opened before the error, so the next acquire reconnects and prepares fresh. Callers retry a `StalePlan` once | | `ServiceTypes.DB` events per round trip | `db_span.py`, `log_db_metrics.py`, `prometheus_services.py`, `otel/model/spans.py` | Python host only. Wrap each native operation in `observe` with the call type and table its Prisma version uses, and the bridge forwards events to `ServiceLoggingEvents`. Nesting under an outer Python `db_span` is not detected. A Rust host has no sink yet and must emit the names below | diff --git a/litellm/rust/crates/db/src/spend/team_lock.rs b/litellm/rust/crates/db/src/spend/team_lock.rs index 7a6b9c7f7d0..5f958d6b9ce 100644 --- a/litellm/rust/crates/db/src/spend/team_lock.rs +++ b/litellm/rust/crates/db/src/spend/team_lock.rs @@ -1,28 +1,38 @@ +use std::num::NonZeroU32; + use sqlx::{PgPool, Postgres, Transaction}; -use crate::Error; +use crate::{Error, Statements}; -pub async fn begin(pool: &PgPool) -> Result, Error> { +pub async fn begin( + pool: &PgPool, + lock_timeout_ms: NonZeroU32, + named: bool, +) -> Result, Error> { let mut transaction = pool.begin().await?; - sqlx::query!("SELECT set_config('lock_timeout', '5000ms', true)") - .fetch_one(&mut *transaction) - .await?; + sqlx::query!( + "SELECT set_config('lock_timeout', $1::text, true)", + format!("{lock_timeout_ms}ms") + ) + .fetch_one(Statements::new(&mut *transaction, named)) + .await?; Ok(transaction) } pub async fn lock_team( transaction: &mut Transaction<'_, Postgres>, team_id: &str, + named: bool, ) -> Result<(), Error> { sqlx::query_file!("queries/spend/lock_team.sql", team_id) - .fetch_one(&mut **transaction) + .fetch_one(Statements::new(&mut **transaction, named)) .await?; Ok(()) } #[cfg(test)] mod tests { - use std::error::Error as StdError; + use std::{error::Error as StdError, num::NonZeroU32}; use super::{begin, lock_team}; use crate::test_support::database_pool; @@ -42,7 +52,12 @@ mod tests { .max_connections(2) .connect_with(database_pool.connect_options().as_ref().clone()) .await?; - let mut first = begin(&pool).await?; + let mut first = begin( + &pool, + NonZeroU32::new(1500).ok_or("1500 is not zero")?, + true, + ) + .await?; let mut second = pool.begin().await?; #[expect( clippy::disallowed_methods, @@ -51,7 +66,7 @@ mod tests { let original_timeout: String = sqlx::query_scalar("SELECT current_setting('lock_timeout')") .fetch_one(&mut *second) .await?; - lock_team(&mut first, team_id).await?; + lock_team(&mut first, team_id, true).await?; #[expect( clippy::disallowed_methods, reason = "Test assertions are outside the compile-checked query set" @@ -60,7 +75,7 @@ mod tests { sqlx::query_as("SELECT pg_backend_pid(), current_setting('lock_timeout')") .fetch_one(&mut *first) .await?; - assert_eq!(timeout, "5s"); + assert_eq!(timeout, "1500ms"); #[expect( clippy::disallowed_methods, reason = "Test assertions are outside the compile-checked query set" diff --git a/litellm/rust/crates/db/src/spend/team_member_spend.rs b/litellm/rust/crates/db/src/spend/team_member_spend.rs index 965a39ada4f..9b4317fd137 100644 --- a/litellm/rust/crates/db/src/spend/team_member_spend.rs +++ b/litellm/rust/crates/db/src/spend/team_member_spend.rs @@ -1,7 +1,7 @@ use sqlx::{Postgres, Transaction}; use super::lock_team; -use crate::Error; +use crate::{Error, Statements}; #[macro_rules_attribute::apply(crate::db_input)] pub struct TeamMemberSpend { @@ -13,6 +13,7 @@ pub struct TeamMemberSpend { pub async fn add_team_member_spend( transaction: &mut Transaction<'_, Postgres>, members: &[TeamMemberSpend], + named: bool, ) -> Result<(), Error> { if members.is_empty() { return Ok(()); @@ -27,7 +28,7 @@ pub async fn add_team_member_spend( .collect(); teams.dedup(); for team in teams { - lock_team(transaction, team).await?; + lock_team(transaction, team, named).await?; } let rows = serde_json::Value::Array( members @@ -42,14 +43,14 @@ pub async fn add_team_member_spend( .collect(), ); sqlx::query_file!("queries/spend/upsert_team_member_spend.sql", rows) - .execute(&mut **transaction) + .execute(Statements::new(&mut **transaction, named)) .await?; Ok(()) } #[cfg(test)] mod tests { - use std::error::Error as StdError; + use std::{error::Error as StdError, num::NonZeroU32}; use super::super::begin; use super::{TeamMemberSpend, add_team_member_spend}; @@ -90,7 +91,12 @@ mod tests { #[future(awt)] database_pool: Result>, ) -> Result<(), Box> { let pool = database_pool?; - let mut tx = begin(&pool).await?; + let mut tx = begin( + &pool, + NonZeroU32::new(5000).ok_or("5000 is not zero")?, + true, + ) + .await?; #[expect( clippy::disallowed_methods, reason = "Fixture writes are outside the compile-checked query set" @@ -113,10 +119,10 @@ mod tests { ) .execute(&mut *tx) .await?; - add_team_member_spend(&mut tx, &[]).await?; - add_team_member_spend(&mut tx, &members(0.0)).await?; - add_team_member_spend(&mut tx, &members(cost)).await?; - add_team_member_spend(&mut tx, &members(cost)).await?; + add_team_member_spend(&mut tx, &[], true).await?; + add_team_member_spend(&mut tx, &members(0.0), true).await?; + add_team_member_spend(&mut tx, &members(cost), true).await?; + add_team_member_spend(&mut tx, &members(cost), true).await?; #[expect( clippy::disallowed_methods, reason = "Test assertions are outside the compile-checked query set" From da558936569b2bd13ce533e3115d3284cb775cb8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 14 Nov 2026 15:41:30 -0800 Subject: [PATCH 06/57] ci(unit): cut unit check time with cheaper per-test resets and fewer non-unit tests (#51864) * test(unit): reset module state only when a test changed it The proxy conftest deep-copied every snapshotted litellm attribute before each test, and the copy of litellm.model_cost alone was a median 0.12 s for each of 56,031 proxy tests on the runner. The reset now copies an attribute back only when it no longer equals the snapshot. The enterprise and MCP server conftests reloaded litellm before every test, 3.27 s and 0.21 s per test. They now use the same reset. * test(unit): remove six flaky tests They failed on main without a code change behind them between 2026-11-05 and 2026-11-14: the vector store hook SDK fallback, the concurrent user config TPM count, three Prometheus metrics server port tests and the OpenCode list cost billing test. * ci: run four heavy unit shards on 16-core runners and keep two smoke jobs off pull requests proxy-feature-endpoints, proxy-endpoints, root and llms-providers ran 1.9 to 2.2 times faster with 8 workers on a 16-core runner in a benchmark run. windows-sdk and proxy-build-from-pip still run on pushes to main and manual runs. proxy-build-from-pip installs a published release and never runs the pull request's code. * test: move 109 tests that start a process or open a socket into tests/integration tests/unit is in-process by its own rules, and these tests spawn a Python interpreter, a shell or a local TCP or HTTP server. They took 921 s of unit shard time per run, 396 s of it in the MCP client transport tests. Each one now lives in a *_real_io.py file under the integration directory for its domain and runs with that directory's group. The tests themselves are unchanged apart from one repository-root path in the CLI test. * test(unit): copy back only the dict entries a test changed A test module that registers models at import leaves litellm.model_cost four entries larger than the snapshot before every test in its directory, so the reset still deep-copied the whole price map for each of them. It now rebuilds a changed dict from the live entries that still match and copies only the rest. * ci(smoke): keep the two jobs that skip on pull requests out of the smoke gate A skipped job passing the gate is what test_unit_passed_gate guards against, so the gate goes back to requiring success from every job it needs and windows-sdk and proxy-build-from-pip stop being part of it. They still run and report on pushes to main and manual runs. --- .github/workflows/test-merge-smoke.yml | 4 +- .github/workflows/test-unit.yml | 14 +- tests/_module_state.py | 107 ++ .../test_unit_passed_gate.py | 2 +- tests/integration/AGENTS.md | 4 + .../test_litellm_http_client_sso_real_io.py} | 0 .../test_master_key_boot_check_real_io.py | 124 ++ ..._user_api_key_auth_request_flow_real_io.py | 155 ++ .../test_evicted_client_closer_real_io.py | 119 ++ .../test_valkey_semantic_cache_real_io.py | 32 + .../test_claude_settings_real_io.py | 272 +++ .../configuration/test_main_real_io.py | 20 + .../configuration/test_runtime_real_io.py | 83 + .../test_litellm_proxy_mcp_handler_real_io.py | 88 + .../mcp/test_mcp_client_real_io.py | 1527 +++++++++++++++++ .../test_openapi_to_mcp_generator_real_io.py | 44 + ...st_anthropic_cache_control_hook_real_io.py | 65 + .../test_anthropic_common_utils_real_io.py | 41 + .../sdk/test_azure_common_utils_real_io.py | 204 +++ .../sdk/test_configuration_real_io.py | 39 + tests/integration/sdk/test_context_real_io.py | 146 ++ tests/integration/sdk/test_docker_real_io.py | 50 + .../sdk/test_drop_params_env_var_real_io.py} | 0 .../sdk/test_eager_tiktoken_load_real_io.py} | 0 .../sdk/test_http_handler_real_io.py | 296 ++++ .../sdk/test_jwt_signing_real_io.py | 37 + .../sdk/test_lazy_imports_real_io.py | 23 + .../sdk/test_litellm_cli_real_io.py | 384 +++++ .../sdk/test_long_decode_real_io.py | 323 ++++ ...uter_fuzz_regressions_rust_only_real_io.py | 193 +++ .../sdk/test_router_host_real_io.py | 28 + .../sdk/test_streaming_handler_real_io.py | 367 ++++ .../sdk/test_token_counter_real_io.py | 207 +++ .../integration/sdk/test_tokenizer_real_io.py | 119 ++ .../caching/test_evicted_client_closer.py | 87 - .../caching/test_valkey_semantic_cache.py | 29 - tests/unit/enterprise/conftest.py | 40 +- tests/unit/harness/sandbox/test_docker.py | 45 - .../test_anthropic_cache_control_hook.py | 52 +- .../test_vector_store_pre_call_hook.py | 30 - .../test_streaming_handler.py | 66 - .../litellm_core_utils/test_token_counter.py | 175 +- .../unit/litellm_core_utils/test_tokenizer.py | 111 -- .../anthropic/test_anthropic_common_utils.py | 38 - .../llms/azure/test_azure_common_utils.py | 23 - .../llms/base_llm/auth/test_jwt_signing.py | 29 - .../llms/custom_httpx/test_http_handler.py | 243 --- .../llms/opencode/chat/test_transformation.py | 20 - .../vertex_ai/count_tokens/test_context.py | 132 -- .../llms/vertex_ai/gemini/test_long_decode.py | 220 +-- tests/unit/mcp_client/test_mcp_client.py | 1379 +-------------- .../proxy/auth/test_master_key_boot_check.py | 63 - .../test_user_api_key_auth_request_flow.py | 127 -- .../proxy/client/cli/test_claude_settings.py | 62 - tests/unit/proxy/conftest.py | 111 +- .../proxy/hooks/litellm_skills/test_main.py | 20 - tests/unit/proxy/mcp_server/conftest.py | 13 +- .../test_openapi_to_mcp_generator.py | 18 - .../rust_forward/supervisor/test_runtime.py | 60 +- .../proxy/test_prometheus_metrics_server.py | 47 - tests/unit/proxy/test_route_llm_request.py | 27 - .../mcp/test_litellm_proxy_mcp_handler.py | 48 - tests/unit/rust_bridge/test_configuration.py | 22 - tests/unit/test_lazy_imports.py | 19 - tests/unit/test_litellm_cli.py | 369 ---- .../test_router_fuzz_regressions_rust_only.py | 106 -- tests/unit/test_router/test_router_host.py | 14 - tests/windows_tests/AGENTS.md | 2 +- 68 files changed, 5143 insertions(+), 3821 deletions(-) create mode 100644 tests/_module_state.py rename tests/{unit/proxy/management_endpoints/sso/test_litellm_http_client_sso.py => integration/authorization/test_litellm_http_client_sso_real_io.py} (100%) create mode 100644 tests/integration/authorization/test_master_key_boot_check_real_io.py create mode 100644 tests/integration/authorization/test_user_api_key_auth_request_flow_real_io.py create mode 100644 tests/integration/caching/test_evicted_client_closer_real_io.py create mode 100644 tests/integration/caching/test_valkey_semantic_cache_real_io.py create mode 100644 tests/integration/configuration/test_claude_settings_real_io.py create mode 100644 tests/integration/configuration/test_main_real_io.py create mode 100644 tests/integration/configuration/test_runtime_real_io.py create mode 100644 tests/integration/mcp/test_litellm_proxy_mcp_handler_real_io.py create mode 100644 tests/integration/mcp/test_mcp_client_real_io.py create mode 100644 tests/integration/mcp/test_openapi_to_mcp_generator_real_io.py create mode 100644 tests/integration/observability/test_anthropic_cache_control_hook_real_io.py create mode 100644 tests/integration/sdk/test_anthropic_common_utils_real_io.py create mode 100644 tests/integration/sdk/test_azure_common_utils_real_io.py create mode 100644 tests/integration/sdk/test_configuration_real_io.py create mode 100644 tests/integration/sdk/test_context_real_io.py create mode 100644 tests/integration/sdk/test_docker_real_io.py rename tests/{unit/test_drop_params_env_var.py => integration/sdk/test_drop_params_env_var_real_io.py} (100%) rename tests/{unit/test_eager_tiktoken_load.py => integration/sdk/test_eager_tiktoken_load_real_io.py} (100%) create mode 100644 tests/integration/sdk/test_http_handler_real_io.py create mode 100644 tests/integration/sdk/test_jwt_signing_real_io.py create mode 100644 tests/integration/sdk/test_lazy_imports_real_io.py create mode 100644 tests/integration/sdk/test_litellm_cli_real_io.py create mode 100644 tests/integration/sdk/test_long_decode_real_io.py create mode 100644 tests/integration/sdk/test_router_fuzz_regressions_rust_only_real_io.py create mode 100644 tests/integration/sdk/test_router_host_real_io.py create mode 100644 tests/integration/sdk/test_streaming_handler_real_io.py create mode 100644 tests/integration/sdk/test_token_counter_real_io.py create mode 100644 tests/integration/sdk/test_tokenizer_real_io.py diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index 8ed1876a081..88acf50837f 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -178,6 +178,7 @@ jobs: windows-sdk: name: windows-sdk + if: github.event_name != 'pull_request' runs-on: windows-2025 timeout-minutes: 30 env: @@ -213,6 +214,7 @@ jobs: proxy-build-from-pip: name: proxy-build-from-pip + if: github.event_name != 'pull_request' runs-on: ubuntu-24.04 timeout-minutes: 10 services: @@ -352,7 +354,7 @@ jobs: smoke-passed: name: smoke passed - needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip] + needs: [dashboard-build, core-checks, all-extras] if: always() runs-on: ubuntu-latest timeout-minutes: 2 diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f27b58149ab..50bcc6c2b42 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -126,7 +126,7 @@ jobs: name: ${{ matrix.shard }} needs: [rust-bridge, assert-shard-coverage] if: ${{ !cancelled() }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner || 'ubuntu-latest' }} timeout-minutes: ${{ matrix.job-timeout-minutes }} permissions: contents: read @@ -220,6 +220,7 @@ jobs: dist: loadscope - shard: llms-providers + runner: ubuntu-latest-16-cores test-path: |- tests/unit/llms/test_api_base_trailing_slash_routes.py tests/unit/llms/test_cache_control_and_reasoning.py @@ -380,7 +381,7 @@ jobs: tests/unit/llms/zai --ignore=tests/unit/llms/base_llm/batches/base_batches_config_test.py python-version: "3.12" - workers: 4 + workers: 8 reruns: 0 timeout-minutes: 20 job-timeout-minutes: 60 @@ -398,10 +399,11 @@ jobs: dist: loadscope - shard: root + runner: ubuntu-latest-16-cores test-path: >- tests/unit/test_*.py python-version: "3.12" - workers: 4 + workers: 8 reruns: 2 timeout-minutes: 20 job-timeout-minutes: 60 @@ -522,6 +524,7 @@ jobs: dist: loadscope - shard: proxy-endpoints + runner: ubuntu-latest-16-cores test-path: |- tests/unit/proxy/management_endpoints tests/unit/proxy/management_helpers @@ -539,13 +542,14 @@ jobs: --ignore=tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py --ignore=tests/unit/proxy/management_helpers/test_audit_logs_proxy.py python-version: "3.12" - workers: 4 + workers: 8 reruns: 0 timeout-minutes: 20 job-timeout-minutes: 60 dist: loadscope - shard: proxy-feature-endpoints + runner: ubuntu-latest-16-cores test-path: |- tests/unit/proxy/guardrails --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py @@ -578,7 +582,7 @@ jobs: tests/unit/proxy/config_resolvers tests/unit/proxy/utils python-version: "3.12" - workers: 4 + workers: 8 reruns: 0 timeout-minutes: 20 job-timeout-minutes: 60 diff --git a/tests/_module_state.py b/tests/_module_state.py new file mode 100644 index 00000000000..b168616d4c9 --- /dev/null +++ b/tests/_module_state.py @@ -0,0 +1,107 @@ +import copy +import inspect +import itertools +import warnings +from collections.abc import Mapping +from types import ModuleType +from typing import Final + +_SNAPSHOT_TYPES: Final = (list, dict, set, tuple, str, int, float, bool, bytes) +_MUTABLE_CONTAINERS: Final = (list, dict, set, bytearray) +_ABSENT: Final = object() + + +def _holds_mutable_container(value: object) -> bool: + if isinstance(value, _MUTABLE_CONTAINERS): + return True + if isinstance(value, tuple): + return any(_holds_mutable_container(element) for element in value) + return False + + +def _restored_value(value: object) -> object: + return copy.deepcopy(value) if _holds_mutable_container(value) else value + + +def _readable_attributes(module: ModuleType) -> tuple[tuple[str, object], ...]: + return tuple((attr, value) for attr, value in tuple(vars(module).items()) if not attr.startswith("_")) + + +def _snapshot_of(module: ModuleType, attr: str, value: object) -> object: + try: + return _restored_value(value) + except Exception as exc: + warnings.warn(f"conftest: could not snapshot {module.__name__}.{attr}: {exc}", stacklevel=2) + return _ABSENT + + +def snapshot_mutable_state(module: ModuleType) -> Mapping[str, object]: + candidates: Final = ( + (attr, _snapshot_of(module, attr, value)) + for attr, value in _readable_attributes(module) + if value is None or isinstance(value, _SNAPSHOT_TYPES) + ) + return {attr: value for attr, value in candidates if value is not _ABSENT} + + +def _still_matches(current: object, default: object) -> bool: + if current is default: + return True + if type(current) is not type(default): + return False + try: + return bool(current == default) + except Exception: + return False + + +def _restored_entries(current: dict[object, object], default: dict[object, object]) -> dict[object, object]: + return { + key: current[key] + if key in current and type(current[key]) is type(value) and current[key] == value + else _restored_value(value) + for key, value in default.items() + } + + +def _restored_from(current: object, default: object) -> object: + if type(current) is dict and type(default) is dict: + return _restored_entries(current, default) + return _restored_value(default) + + +def restore_mutable_state(module: ModuleType, snapshot: Mapping[str, object]) -> None: + live: Final = vars(module) + for attr, default in snapshot.items(): + current = live.get(attr, _ABSENT) + if _still_matches(current, default): + continue + try: + setattr(module, attr, _restored_from(current, default)) + except Exception as exc: + warnings.warn(f"conftest: could not restore {module.__name__}.{attr}: {exc}", stacklevel=2) + + +def _is_flushable(value: object) -> bool: + if inspect.isclass(value) or inspect.ismodule(value): + return False + return callable(getattr(value, "flush_cache", None)) + + +def _flushable_attributes(module: ModuleType) -> tuple[tuple[ModuleType, str], ...]: + return tuple((module, attr) for attr, value in _readable_attributes(module) if _is_flushable(value)) + + +def collect_flushable_caches(modules: tuple[ModuleType, ...]) -> tuple[tuple[ModuleType, str], ...]: + return tuple(itertools.chain.from_iterable(_flushable_attributes(module) for module in modules)) + + +def flush_caches(targets: tuple[tuple[ModuleType, str], ...]) -> None: + for module, attr in targets: + flush = getattr(vars(module).get(attr), "flush_cache", None) + if not callable(flush): + continue + try: + flush() + except Exception as exc: + warnings.warn(f"conftest: flush_cache failed on {module.__name__}.{attr}: {exc}", stacklevel=2) diff --git a/tests/code_coverage_tests/test_unit_passed_gate.py b/tests/code_coverage_tests/test_unit_passed_gate.py index 5a0d04681d4..2bb480d67c1 100644 --- a/tests/code_coverage_tests/test_unit_passed_gate.py +++ b/tests/code_coverage_tests/test_unit_passed_gate.py @@ -11,7 +11,7 @@ _WORKFLOWS_DIR: Final = Path(__file__).resolve().parents[2] / ".github" / "workf _TIERS: Final = ( ("test-linting.yml", "lint-passed", frozenset()), ("test-unit.yml", "unit-passed", frozenset({"coverage"})), - ("test-merge-smoke.yml", "smoke-passed", frozenset()), + ("test-merge-smoke.yml", "smoke-passed", frozenset({"windows-sdk", "proxy-build-from-pip"})), ) _Tier = tuple[str, str, frozenset[str]] diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index b430ab553a5..96ff87994a1 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -26,6 +26,10 @@ directory that a `GROUPS` entry in `run.py` selects. There is no manifest and no tests. A product bug the test exposes is `pytest.skip("BUG: ")` at the top of the body, not a fix in the test and not a deletion. Needs no network, proxy, DB or Redis: `tests/unit` +A test that starts its own process or opens its own socket but needs no proxy also lives here, in a +`*_real_io.py` file next to the domain it covers, because `tests/unit` is in-process. It asks for none of +the gateway fixtures, so it runs as plain pytest inside its group + GitHub Actions integration groups run every file here except the OCI and sandbox tests, which no CI runs, and the files in `GITHUB_FILES` in `run.py`. `test-integration.yml` runs the same groups through the same `run_integration.sh` on GitHub Actions, on main every three hours from 8am to 8pm Pacific Monday to Saturday, diff --git a/tests/unit/proxy/management_endpoints/sso/test_litellm_http_client_sso.py b/tests/integration/authorization/test_litellm_http_client_sso_real_io.py similarity index 100% rename from tests/unit/proxy/management_endpoints/sso/test_litellm_http_client_sso.py rename to tests/integration/authorization/test_litellm_http_client_sso_real_io.py diff --git a/tests/integration/authorization/test_master_key_boot_check_real_io.py b/tests/integration/authorization/test_master_key_boot_check_real_io.py new file mode 100644 index 00000000000..68d62ed8316 --- /dev/null +++ b/tests/integration/authorization/test_master_key_boot_check_real_io.py @@ -0,0 +1,124 @@ +import re +import shutil +import subprocess +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest + +from litellm.proxy.auth.master_key_boot_check import ( + GENERATE_MASTER_KEY_COMMAND, + MASTER_KEY_ENV_VAR, + MIGRATE_FROM_MASTER_KEY_ENV_VAR, + PRINT_NEW_MASTER_KEY_COMMAND, + ConfigFileSource, + EnvironmentSource, + MasterKeyBootVerdict, + SafeMasterKey, + StoredSecretsMigration, + UnsafeMasterKeyReason, + UnsafeMasterKeyRefused, + announce_on_stderr_at_exit, + master_key_boot_verdict, + render_refusal, +) + +PUBLICLY_KNOWN_KEY: Final = "sk-" + "1234" + + +def _verdict( + master_key: str | None, + general_settings: Mapping[str, object] | None = None, + *, + environment_master_key: str | None = None, + config_file_path: str | None = None, + override_env_is_on: bool = False, + salt_key_is_set: bool = False, + database_is_configured: bool = False, +) -> MasterKeyBootVerdict: + return master_key_boot_verdict( + master_key=master_key, + environment_master_key=environment_master_key, + general_settings=general_settings or {}, + config_file_path=config_file_path, + override_env_is_on=override_env_is_on, + salt_key_is_set=salt_key_is_set, + database_is_configured=database_is_configured, + ) + + +def _refusal( + reason: UnsafeMasterKeyReason = UnsafeMasterKeyReason.PUBLICLY_KNOWN, + source: ConfigFileSource | EnvironmentSource = EnvironmentSource(), + environment_variable_is_set: bool = False, + migration: StoredSecretsMigration | None = None, +) -> UnsafeMasterKeyRefused: + return UnsafeMasterKeyRefused( + reason=reason, + source=source, + environment_variable_is_set=environment_variable_is_set, + migration=migration, + ) + + +_MIGRATION = StoredSecretsMigration(from_master_key=PUBLICLY_KNOWN_KEY, encrypted_value_count=3) + + +@pytest.mark.skipif(shutil.which("openssl") is None, reason="the printed command shells out to openssl") +@pytest.mark.parametrize("from_master_key", [PUBLICLY_KNOWN_KEY, "", " " + PUBLICLY_KNOWN_KEY]) +def test_printed_migration_commands_save_both_keys_to_the_env_file(tmp_path: Path, from_master_key: str): + from dotenv import dotenv_values + + text = render_refusal( + _refusal(migration=StoredSecretsMigration(from_master_key=from_master_key, encrypted_value_count=1)) + ) + commands = [line.strip() for line in text.splitlines() if line.strip().startswith("echo ")] + + subprocess.run(["bash", "-c", "\n".join(commands)], cwd=tmp_path, capture_output=True, text=True, check=True) + + saved = dotenv_values(tmp_path / ".env") + assert saved[MIGRATE_FROM_MASTER_KEY_ENV_VAR] == from_master_key + assert _verdict(saved[MASTER_KEY_ENV_VAR]) == SafeMasterKey() + assert len(commands) == 2 + + +@pytest.mark.skipif(shutil.which("openssl") is None, reason="the printed command shells out to openssl") +def test_printed_command_saves_a_key_the_boot_check_accepts(tmp_path: Path): + completed = subprocess.run( + ["bash", "-c", GENERATE_MASTER_KEY_COMMAND], cwd=tmp_path, capture_output=True, text=True, check=True + ) + + saved = (tmp_path / ".env").read_text() + match = re.fullmatch(rf"{MASTER_KEY_ENV_VAR}=(sk-[0-9a-f]{{64}})\n", saved) + assert match is not None + assert completed.stdout == saved + assert _verdict(match.group(1)) == SafeMasterKey() + + +@pytest.mark.skipif(shutil.which("openssl") is None, reason="the printed command shells out to openssl") +def test_rotation_command_prints_a_key_the_boot_check_accepts_and_saves_nothing(tmp_path: Path): + completed = subprocess.run( + ["bash", "-c", PRINT_NEW_MASTER_KEY_COMMAND], cwd=tmp_path, capture_output=True, text=True, check=True + ) + + assert re.fullmatch(r"sk-[0-9a-f]{64}\n", completed.stdout) is not None + assert _verdict(completed.stdout.strip()) == SafeMasterKey() + assert list(tmp_path.iterdir()) == [] + + +def test_announced_fix_is_the_last_thing_a_crashing_process_prints(): + crash_after_announcing = ( + "from litellm.proxy.auth.master_key_boot_check import announce_on_stderr_at_exit\n" + "announce_on_stderr_at_exit('THE FIX')\n" + "print('buffered stdout')\n" + "raise RuntimeError('lifespan failed')\n" + ) + + completed = subprocess.run([sys.executable, "-c", crash_after_announcing], capture_output=True, text=True) + + assert completed.returncode != 0 + assert "RuntimeError: lifespan failed" in completed.stderr + assert completed.stderr.endswith("THE FIX") + assert completed.stdout == "buffered stdout\n" diff --git a/tests/integration/authorization/test_user_api_key_auth_request_flow_real_io.py b/tests/integration/authorization/test_user_api_key_auth_request_flow_real_io.py new file mode 100644 index 00000000000..42e77e5fa07 --- /dev/null +++ b/tests/integration/authorization/test_user_api_key_auth_request_flow_real_io.py @@ -0,0 +1,155 @@ +import json +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from textwrap import dedent +from types import SimpleNamespace +from typing import Final +from unittest.mock import ANY + +import litellm +import litellm.proxy.proxy_server +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.types.videos.utils import encode_video_id_with_provider + + +class _RoutingRequest: + def __init__(self, headers=None, query_params=None): + self.headers = headers or {} + self.query_params = query_params or {} + self.state = SimpleNamespace() + + +# --------------------------------------------------------------------------- + +# _run_centralized_common_checks — centralized authz gate +# --------------------------------------------------------------------------- + + +_RECORDING_DDTRACE = dedent( + """ + import functools + import inspect + + + class _Span: + def __enter__(self): + return self + + def __exit__(self, *exc): + return None + + + class _Tracer: + def __init__(self): + self.spans = [] + + def wrap(self, name=None, **kwargs): + def decorator(f): + span_name = name or f"{f.__module__}.{f.__name__}" + if inspect.iscoroutinefunction(f): + + @functools.wraps(f) + async def async_wrapped(*args, **kw): + self.spans.append(span_name) + return await f(*args, **kw) + + return async_wrapped + + @functools.wraps(f) + def wrapped(*args, **kw): + self.spans.append(span_name) + return f(*args, **kw) + + return wrapped + + return decorator + + def trace(self, name, **kwargs): + return _Span() + + def current_span(self): + return None + + def current_root_span(self): + return None + + + tracer = _Tracer() + """ +) + +_DDTRACE_AUTH_PROBE = dedent( + """ + import asyncio + import json + + import ddtrace + from fastapi import Request + from starlette.datastructures import URL + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + proxy_server.master_key = "sk-probe" + + + async def auth(api_key): + request = Request(scope={"type": "http", "headers": [], "method": "POST", "path": "/chat/completions"}) + request._url = URL(url="/chat/completions") + try: + await user_api_key_auth( + request=request, + api_key=api_key, + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + custom_litellm_key_header=None, + ) + return "accepted" + except ProxyException: + return "rejected" + + + async def main(): + outcomes = [await auth("Bearer sk-probe"), await auth("Bearer sk-wrong")] + print(json.dumps({"outcomes": outcomes, "spans": ddtrace.tracer.spans})) + + + asyncio.run(main()) + """ +) + + +def test_user_api_key_auth_opens_a_datadog_span_for_accepted_and_rejected_keys(tmp_path: Path): + stub_root = tmp_path / "site" + (stub_root / "ddtrace").mkdir(parents=True) + (stub_root / "ddtrace" / "__init__.py").write_text(_RECORDING_DDTRACE) + probe = tmp_path / "probe.py" + probe.write_text(_DDTRACE_AUTH_PROBE) + repo_root = Path(litellm.__file__).resolve().parent.parent + env = { + **os.environ, + "USE_DDTRACE": "true", + "PYTHONPATH": os.pathsep.join( + [str(stub_root), str(repo_root)] + [p for p in (os.environ.get("PYTHONPATH"),) if p] + ), + } + + result = subprocess.run( + [sys.executable, str(probe)], env=env, cwd=repo_root, capture_output=True, text=True, check=False + ) + + assert result.returncode == 0, result.stderr[-4000:] + report = json.loads(result.stdout.strip().splitlines()[-1]) + assert report["outcomes"] == ["accepted", "rejected"] + auth_span = "litellm.proxy.auth.user_api_key_auth.user_api_key_auth" + assert [span for span in report["spans"] if span == auth_span] == [auth_span, auth_span] + + +_VIDEO_ID: Final = encode_video_id_with_provider(video_id="video_raw", provider="openai", model_id="sora-2") +_EXPIRED_BEFORE_ANY_TEST_RUN: Final = datetime(2000, 1, 1, tzinfo=timezone.utc) diff --git a/tests/integration/caching/test_evicted_client_closer_real_io.py b/tests/integration/caching/test_evicted_client_closer_real_io.py new file mode 100644 index 00000000000..c747d4bd798 --- /dev/null +++ b/tests/integration/caching/test_evicted_client_closer_real_io.py @@ -0,0 +1,119 @@ +""" +Tests for EvictedClientCloser. + +An evicted client must stay open long enough for a request that already holds it +to finish, and must then actually be closed, otherwise its connection pool is +retained until a generational collection runs. A client the caller supplied is +never closed, because litellm does not own its lifecycle. +""" + +import asyncio + +import httpx +import pytest + +from litellm.caching.evicted_client_closer import EvictedClientCloser +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + +class FakeClock: + """Hand-advanced monotonic clock, so grace windows need no real waiting.""" + + def __init__(self) -> None: + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +def make_closer(clock: FakeClock, grace_seconds: float = 60.0) -> EvictedClientCloser: + return EvictedClientCloser(grace_seconds=grace_seconds, clock=clock) + + +async def _trickling_upstream(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + """Serves a chunked body slowly, so a request stays on the wire long enough to observe.""" + await reader.read(4096) + writer.write(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n") + await writer.drain() + for _ in range(6): + writer.write(b"5\r\nhello\r\n") + await writer.drain() + await asyncio.sleep(0.1) + writer.write(b"0\r\n\r\n") + await writer.drain() + + +@pytest.mark.asyncio +async def test_a_client_serving_a_request_is_not_closed_when_its_grace_window_ends(): + """The grace window on its own cannot promise that a request has finished. + + ``litellm.request_timeout`` defaults to 6000 seconds and a streaming response + is bounded only by how long the upstream keeps sending, so a client past its + deadline is closed only once its own pool reports nothing in flight. + """ + server = await asyncio.start_server(_trickling_upstream, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + clock = FakeClock() + closer = make_closer(clock) + client = httpx.AsyncClient() + + closer.mark_owned(client) + closer.schedule(client) + + async def read_the_stream() -> int: + received = 0 + async with client.stream("GET", f"http://127.0.0.1:{port}/") as response: + async for chunk in response.aiter_bytes(): + received += len(chunk) + return received + + streaming = asyncio.create_task(read_the_stream()) + await asyncio.sleep(0.25) # the request is on the wire + clock.advance(3600.0) # and its grace window is long gone + closer.reap() + await asyncio.sleep(0.05) + + assert client.is_closed is False, "closed a client that was serving a request" + assert await streaming > 0, "the in-flight request did not survive the reap" + + clock.advance(3600.0) + closer.reap() + await asyncio.sleep(0.05) + + assert client.is_closed is True, "an idle client past its grace window must be closed" + assert closer.pending_count == 0 + server.close() + + +@pytest.mark.asyncio +async def test_the_aiohttp_backed_handler_is_not_closed_mid_request(): + """The default async path is aiohttp-backed, whose pool accounts for its own leases.""" + server = await asyncio.start_server(_trickling_upstream, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + clock = FakeClock() + closer = make_closer(clock) + handler = AsyncHTTPHandler() + held_client = handler.client + + closer.mark_owned(handler) + closer.schedule(handler) + + request = asyncio.create_task(handler.get(f"http://127.0.0.1:{port}/")) + await asyncio.sleep(0.25) + clock.advance(3600.0) + closer.reap() + await asyncio.sleep(0.05) + + assert held_client.is_closed is False, "closed a handler that was serving a request" + assert (await request).status_code == 200 + + clock.advance(3600.0) + closer.reap() + await asyncio.sleep(0.05) + + assert held_client.is_closed is True + assert handler.client.is_closed is False, "a held handler must self-heal after its evicted client is closed" + server.close() diff --git a/tests/integration/caching/test_valkey_semantic_cache_real_io.py b/tests/integration/caching/test_valkey_semantic_cache_real_io.py new file mode 100644 index 00000000000..0bd3f8b47e2 --- /dev/null +++ b/tests/integration/caching/test_valkey_semantic_cache_real_io.py @@ -0,0 +1,32 @@ +import os +import subprocess +import sys +import textwrap + +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) + + +def test_importing_caching_does_not_require_redis(): + # redis is an optional dependency (extra_proxy), so the base SDK can be + # installed without it. Selecting valkey-semantic needs redis, but merely + # importing litellm.caching.caching must not, or `import litellm` breaks for + # every base-SDK user. This runs in a subprocess with redis blocked so the + # check is not polluted by redis already being imported in this session. + code = textwrap.dedent(""" + import sys + for name in ("redis", "redis.asyncio", "redis.commands", + "redis.commands.search"): + sys.modules[name] = None + import litellm.caching.caching # must not import redis at module top + from litellm.types.caching import LiteLLMCacheType + assert LiteLLMCacheType.VALKEY_SEMANTIC == "valkey-semantic" + print("ok") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env={**os.environ, "PYTHONPATH": _REPO_ROOT}, + ) + assert result.returncode == 0, result.stderr + assert "ok" in result.stdout diff --git a/tests/integration/configuration/test_claude_settings_real_io.py b/tests/integration/configuration/test_claude_settings_real_io.py new file mode 100644 index 00000000000..5f2ed05021a --- /dev/null +++ b/tests/integration/configuration/test_claude_settings_real_io.py @@ -0,0 +1,272 @@ +import os +import pathlib +import shlex +import stat +import subprocess +import sys +import time + +import pytest +from click.testing import CliRunner +from litellm.proxy.client.cli.commands.claude_settings import ( + ApiKeyHelper, + KeepModel, + StartOn, + StaticToken, + UnpinModel, + install_statusline_script, + installed_executable, + lite_api_key_helper, + statusline_command, +) + + +class TestLiteApiKeyHelper: + @pytest.mark.skipif(sys.platform.startswith("win"), reason="runs the command through sh, as Claude Code does on POSIX") + def test_sh_runs_the_lite_on_path_against_this_proxy(self, tmp_path): + lite = tmp_path / "my tools" / "lite" + lite.parent.mkdir() + lite.write_text('#!/bin/sh\nprintf "%s\\n" "$0" "$@"\n') + lite.chmod(0o700) + + helper = lite_api_key_helper("https://proxy.example.com/", platform="linux", which=lambda name: str(lite)) + + assert helper is not None + ran = subprocess.run(["sh", "-c", helper.command], capture_output=True, text=True, check=True) + assert ran.stdout.splitlines() == [str(lite), "--base-url", "https://proxy.example.com", "auth", "print-token"] + + @pytest.mark.skipif(sys.platform.startswith("win"), reason="runs the command through sh, as Claude Code does on POSIX") + def test_a_lite_found_through_a_relative_path_entry_keeps_running_from_any_directory(self, tmp_path, monkeypatch): + for repo, name in ((tmp_path / "login-dir", "mine"), (tmp_path / "other-repo", "planted")): + lite = repo / "bin" / "lite" + lite.parent.mkdir(parents=True) + lite.write_text(f"#!/bin/sh\necho {name}\n") + lite.chmod(0o700) + monkeypatch.chdir(tmp_path / "login-dir") + + helper = lite_api_key_helper("https://proxy.example.com", platform="linux", which=lambda name: "bin/lite") + + assert helper is not None + ran = subprocess.run( + ["sh", "-c", helper.command], cwd=tmp_path / "other-repo", capture_output=True, text=True, check=True + ) + assert ran.stdout == "mine\n" + + +@pytest.mark.skipif(sys.platform.startswith("win"), reason="runs the command through sh, as Claude Code does on POSIX") +class TestInstalledExecutable: + @staticmethod + def _script(path, body): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"#!/bin/sh\n{body}\n") + path.chmod(0o700) + + def test_the_helper_runs_the_lite_installed_beside_this_python_over_a_shim_earlier_on_path( + self, tmp_path, monkeypatch + ): + self._script(tmp_path / "shims" / "lite", "echo 'pyenv: lite: command not found' >&2; exit 127") + self._script(tmp_path / "venv" / "bin" / "lite", "echo real") + monkeypatch.setenv("PATH", f"{tmp_path / 'shims'}{os.pathsep}{os.environ['PATH']}") + + helper = lite_api_key_helper( + "https://proxy.example.com", + platform="linux", + which=lambda name: installed_executable(name, scripts_dir=str(tmp_path / "venv" / "bin")), + ) + + assert helper is not None + ran = subprocess.run(["sh", "-c", helper.command], capture_output=True, text=True) + assert (ran.returncode, ran.stdout) == (0, "real\n") + + +PROXY = "http://127.0.0.1:4000" +ANTHROPIC = "https://api.anthropic.com" +RELOGIN = StaticToken("sk-fresh-login") +HELPER = ApiKeyHelper("/usr/local/bin/lite --base-url http://127.0.0.1:4000 auth print-token") +ORIGINAL = { + "theme": "dark", + "permissions": {"allow": ["Bash"]}, + "env": {"KEEP_ME": "1", "ANTHROPIC_API_KEY": "sk-ant-mine", "ANTHROPIC_BASE_URL": ANTHROPIC}, + "apiKeyHelper": "/usr/local/bin/lite auth print-token", + "model": "claude-opus-5", +} + + +def _set(path, value): + """A user edit: set (or with `_ABSENT`, remove) the key at a dotted path in the settings file.""" + + def edit(settings): + section, _, key = path.rpartition(".") + container = settings.setdefault(section, {}) if section else settings + if value is _ABSENT: + container.pop(key, None) + else: + container[key] = value + return settings + + return edit + + +_ABSENT = object() + + +# Each row: initial file, steps (configure kwargs dicts or edit callables) between the first configure +# and unconfigure, the expected file afterwards, and the expected outcome fields. Sequences that used +# to be one test each; the receipt's rules are what make them all come out right. +UNDO_SCENARIOS = { + "plain round trip": (ORIGINAL, [], ORIGINAL, {"kept": ()}), + "helper round trip": (ORIGINAL, [], ORIGINAL, {"kept": ()}, {"credential": HELPER, "model": KeepModel()}), + "helper with no file before": (None, [], None, {"file_removed": True}, {"credential": HELPER}), + "re-logins switching between the token and the helper keep the first snapshot": ( + ORIGINAL, + [{"credential": HELPER, "model": KeepModel()}, {"credential": RELOGIN, "model": KeepModel()}], + ORIGINAL, + {"kept": ()}, + {"credential": HELPER, "model": KeepModel()}, + ), + "no file before": (None, [], None, {"file_removed": True}), + "no env before": ({"theme": "dark"}, [], {"theme": "dark"}, {}), + "null env before": ({"theme": "dark", "env": None}, [], {"theme": "dark", "env": None}, {}), + "empty env before": ({"theme": "dark", "env": {}}, [], {"theme": "dark", "env": {}}, {}), + "user edits stay and are named": ( + ORIGINAL, + [_set("env.ENABLE_TOOL_SEARCH", "false"), _set("model", "claude-sonnet-4-6")], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "claude-sonnet-4-6"}, + {"kept": {"env.ENABLE_TOOL_SEARCH", "model"}, "withheld": ()}, + ), + "user filled an env configure created": (None, [_set("env.MY_VAR", "mine")], {"env": {"MY_VAR": "mine"}}, {}), + "user deleted the file": (None, [lambda s: None], None, {"file_removed": True, "restored": (), "kept": ()}), + "user removed our key: neither restored nor kept": ( + ORIGINAL, + [_set("env.ANTHROPIC_AUTH_TOKEN", _ABSENT)], + ORIGINAL, + {"not_restored": {"env.ANTHROPIC_AUTH_TOKEN"}, "kept": ()}, + ), + "restored names only what changed": ( + {"model": "claude-opus-5"}, + [], + {"model": "claude-opus-5"}, + { + "restored": { + "env.ANTHROPIC_BASE_URL", + "env.ANTHROPIC_AUTH_TOKEN", + "env.ENABLE_TOOL_SEARCH", + "env.CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY", + "statusLine", + }, + "kept": (), + }, + {"credential": RELOGIN, "model": KeepModel()}, + ), + "repeat across credential kinds keeps the first snapshot": ( + ORIGINAL, + [ + {"credential": RELOGIN, "model": UnpinModel()}, + {"credential": StaticToken("sk-rotated"), "model": StartOn("claude-sonnet-4-6")}, + ], + ORIGINAL, + {}, + ), + "repeat without a model lets go of our pin, user had none": ({}, [{"model": UnpinModel()}], {}, {}), + "repeat without a model lets go of our pin, user had one": ( + {"model": "claude-opus-5"}, + [{"model": UnpinModel()}], + {"model": "claude-opus-5"}, + {}, + ), + "re-login keeps our pin": (None, [{"credential": RELOGIN, "model": KeepModel()}], None, {"file_removed": True}), + "edit between configures survives an unpin repeat": ( + ORIGINAL, + [ + _set("model", "my-favourite"), + _set("env.ENABLE_TOOL_SEARCH", "false"), + {"credential": RELOGIN, "model": UnpinModel()}, + ], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "my-favourite"}, + {"kept": {"env.ENABLE_TOOL_SEARCH", "model"}}, + ), + "edit between configures survives a re-login": ( + ORIGINAL, + [ + _set("model", "my-favourite"), + _set("env.ENABLE_TOOL_SEARCH", "false"), + {"credential": RELOGIN, "model": KeepModel()}, + ], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "my-favourite"}, + {"kept": {"env.ENABLE_TOOL_SEARCH", "model"}}, + ), + "edit between configures: a same-model repeat displaces it, so it is what comes back": ( + ORIGINAL, + [_set("model", "my-favourite"), _set("env.ENABLE_TOOL_SEARCH", "false"), {"credential": RELOGIN}], + {**ORIGINAL, "env": {**ORIGINAL["env"], "ENABLE_TOOL_SEARCH": "false"}, "model": "my-favourite"}, + {"kept": {"env.ENABLE_TOOL_SEARCH"}, "restored_includes": {"model"}}, + ), + "base URL changed since: credentials withheld, receipt kept": ( + ORIGINAL, + [_set("env.ANTHROPIC_BASE_URL", "http://other-proxy:4000")], + {**ORIGINAL, "env": {"KEEP_ME": "1", "ANTHROPIC_BASE_URL": "http://other-proxy:4000"}, "apiKeyHelper": _ABSENT}, + { + "withheld": {("env.ANTHROPIC_API_KEY", ANTHROPIC), ("apiKeyHelper", ANTHROPIC)}, + "kept": {"env.ANTHROPIC_BASE_URL"}, + "receipt_kept": True, + }, + ), + "base URL changed and back: judged against the URL the restored file holds": ( + ORIGINAL, + [_set("env.ANTHROPIC_BASE_URL", ANTHROPIC)], + ORIGINAL, + {"withheld": ()}, + ), + "credential captured beside no URL goes back only beside no URL": ( + {"env": {"ANTHROPIC_API_KEY": "sk-default-endpoint"}}, + [_set("env.ANTHROPIC_BASE_URL", "http://other-proxy:4000")], + {"env": {"ANTHROPIC_BASE_URL": "http://other-proxy:4000"}}, + { + "withheld": {("env.ANTHROPIC_API_KEY", "no ANTHROPIC_BASE_URL (Anthropic's default endpoint)")}, + "receipt_kept": True, + }, + ), + "restored document empty while a credential is withheld: file goes, receipt stays": ( + None, + [ + _set("env.ANTHROPIC_API_KEY", "sk-user"), + {"credential": RELOGIN, "model": KeepModel()}, + _set("env.ANTHROPIC_BASE_URL", _ABSENT), + ], + None, + {"withheld": {("env.ANTHROPIC_API_KEY", PROXY)}, "file_removed": True, "receipt_kept": True}, + {"credential": RELOGIN, "model": KeepModel()}, + ), + "a credential the user changed is kept, never also withheld": ( + ORIGINAL, + [_set("env.ANTHROPIC_BASE_URL", "http://other-proxy:4000"), _set("apiKeyHelper", "/opt/mine/helper")], + { + **ORIGINAL, + "env": {"KEEP_ME": "1", "ANTHROPIC_BASE_URL": "http://other-proxy:4000"}, + "apiKeyHelper": "/opt/mine/helper", + }, + { + "withheld": {("env.ANTHROPIC_API_KEY", ANTHROPIC)}, + "kept": {"env.ANTHROPIC_BASE_URL", "apiKeyHelper"}, + "receipt_kept": True, + }, + ), +} + + +class TestStatusLine: + """Every configure registers the status line, and only while the slot is empty or already ours.""" + + COMMAND = "/opt/lite/bin/python /Users/me/.litellm/statusline.py" + + def test_the_installed_script_is_the_bundled_one_and_the_command_runs_this_interpreter(self, tmp_path): + from litellm.proxy.client.cli.commands import statusline_script + + script = tmp_path / "lite" / "statusline.py" + command = install_statusline_script(script) + assert script.read_bytes().split(b"\n", 1)[1] == pathlib.Path(statusline_script.__file__).read_bytes() + assert shlex.split(command) == [sys.executable, str(script)] + assert command == statusline_command(script) + assert stat.S_IMODE(script.stat().st_mode) == 0o600 + assert stat.S_IMODE(script.parent.stat().st_mode) == 0o700 + assert install_statusline_script(script) == command diff --git a/tests/integration/configuration/test_main_real_io.py b/tests/integration/configuration/test_main_real_io.py new file mode 100644 index 00000000000..1181ee1eec3 --- /dev/null +++ b/tests/integration/configuration/test_main_real_io.py @@ -0,0 +1,20 @@ +import os +import subprocess +import sys +from typing import Final + + +def test_importing_proxy_hooks_does_not_mutate_litellm_callbacks(): + script: Final = ( + "import litellm; " + "litellm.callbacks = []; " + "import litellm.proxy.hooks; " + "assert len(litellm.callbacks) == 0, f'mutated: {litellm.callbacks}'" + ) + result: Final = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, + ) + assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}" diff --git a/tests/integration/configuration/test_runtime_real_io.py b/tests/integration/configuration/test_runtime_real_io.py new file mode 100644 index 00000000000..23ca074ef7b --- /dev/null +++ b/tests/integration/configuration/test_runtime_real_io.py @@ -0,0 +1,83 @@ +import signal +import subprocess +import sys +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import anyio +import pytest + +from litellm.proxy.rust_forward.platform import Platform, current_platform +from litellm.proxy.rust_forward.platform.windows import WindowsPlatform +from litellm.proxy.rust_forward.supervisor import supervise +from litellm.proxy.rust_forward.supervisor.children import ChildSpec +from litellm.proxy.rust_forward.supervisor.lifecycle import Child, ChildExited +from litellm.proxy.rust_forward.supervisor.runtime import ( + SpawnFailed, + exit_like, + open_children, + spawn_subprocess, +) +from tests.unit.proxy.rust_forward.supervisor._support import python_server + + +PYTHON_CHILD: Final = """ +import json, os, pathlib, socket, time +time.sleep(0.5) +pathlib.Path(os.environ["MARKERS"], "listening").touch() +private = json.loads(os.environ["_LITELLM_GATEWAY_SUPERVISED"])["listener"] +listener = socket.socket(fileno=private["fd"]) if "fd" in private else socket.create_server(("127.0.0.1", private["port"])) +listener.listen() +time.sleep(10) +""" +RUST_CHILD: Final = """ +import os, pathlib +markers = pathlib.Path(os.environ["MARKERS"]) +(markers / "rust_saw_python_listening").write_text(str((markers / "listening").exists())) +""" + + +def _fake_gateway(scripts: Path) -> Platform: + if sys.platform != "win32": + gateway: Final = scripts / "litellm-gateway" + gateway.write_text(f"#!{sys.executable} -I\n{RUST_CHILD}") + gateway.chmod(0o755) + return current_platform() + script: Final = scripts / "rust_child.py" + script.write_text(RUST_CHILD) + (scripts / "litellm-gateway.cmd").write_text(f'@"{sys.executable}" -I "{script}"\r\n') + return WindowsPlatform(gateway_name="litellm-gateway.cmd") + + +def test_rust_opens_the_public_port_only_once_python_listens(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MARKERS", str(tmp_path)) + command: Final = (sys.executable, "-I", "-c", PYTHON_CHILD) + supervise(python_server(), command, scripts_dir=str(tmp_path), platform=_fake_gateway(tmp_path)) + assert (tmp_path / "rust_saw_python_listening").read_text() == "True" +POSIX_ONLY: Final = pytest.mark.skipif(sys.platform == "win32", reason="the child traps SIGTERM") +RUST_MISSING: Final = SpawnFailed(FileNotFoundError("litellm-gateway")) + + +def _children(argv: tuple[str, ...]) -> MappingProxyType[Child, ChildSpec]: + spec: Final = ChildSpec(argv, MappingProxyType({}), current_platform().spawn_options(())) + return MappingProxyType({Child.PYTHON: spec, Child.RUST: spec}) + + +async def test_exiting_children_are_reported_as_events() -> None: + async with open_children( + current_platform(), spawn_subprocess, _children((sys.executable, "-c", "exit(7)")), 1 + ) as children: + assert await children.spawn(Child.RUST) is None + with anyio.fail_after(5): + assert await children.next_event() == ChildExited(Child.RUST, 7) + + +@POSIX_ONLY +@pytest.mark.parametrize("signum", [signal.SIGTERM, signal.SIGKILL]) +def test_the_supervisor_dies_by_the_signal_that_killed_python(signum: signal.Signals) -> None: + script: Final = ( + "from litellm.proxy.rust_forward.supervisor.runtime import exit_like\n" + f"raise SystemExit(exit_like({-signum}, frozenset()))" + ) + assert subprocess.run((sys.executable, "-I", "-c", script), check=False, timeout=60).returncode == -signum diff --git a/tests/integration/mcp/test_litellm_proxy_mcp_handler_real_io.py b/tests/integration/mcp/test_litellm_proxy_mcp_handler_real_io.py new file mode 100644 index 00000000000..9fda7608e26 --- /dev/null +++ b/tests/integration/mcp/test_litellm_proxy_mcp_handler_real_io.py @@ -0,0 +1,88 @@ +import subprocess +import sys +import textwrap +from types import MappingProxyType +from typing import Final +from mcp.types import Tool as MCPTool +from litellm.types.mcp_server.mcp_toolset import MCPToolset +from litellm.types.mcp import MCPTransport +from litellm.types.mcp_server.mcp_server_manager import MCPServer + + +def test_completion_with_function_tools_works_without_fastapi_installed(): + script = textwrap.dedent( + """ + import sys + + class _FastapiBlocker: + def find_spec(self, fullname, path=None, target=None): + if fullname == "fastapi" or fullname.startswith("fastapi."): + raise ModuleNotFoundError("No module named 'fastapi'") + return None + + sys.meta_path.insert(0, _FastapiBlocker()) + + import litellm + + response = litellm.completion( + model="openai/gpt-5.5", + messages=[{"role": "user", "content": "What is the weather in SF?"}], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ], + mock_response="sunny", + ) + assert response.choices[0].message.content == "sunny" + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stderr + + +TOOLSET_ROWS: Final = ( + MCPToolset( + toolset_id="ts-granted", toolset_name="team-toolset", tools=[{"server_id": "srv-1", "tool_name": "add"}] + ), + MCPToolset( + toolset_id="ts-other", toolset_name="other-toolset", tools=[{"server_id": "srv-2", "tool_name": "lookup"}] + ), + MCPToolset(toolset_id="ts-shouty", toolset_name="CALC", tools=[{"server_id": "srv-2", "tool_name": "lookup"}]), + MCPToolset(toolset_id="ts-group", toolset_name="reference", tools=[{"server_id": "srv-1", "tool_name": "add"}]), + MCPToolset( + toolset_id="ts-prefixed", + toolset_name="prefixed-toolset", + tools=[{"server_id": "srv-1", "tool_name": "add"}, {"server_id": "srv-1", "tool_name": "calc-subtract"}], + ), + MCPToolset( + toolset_id="ts-subtract", toolset_name="subtract-toolset", tools=[{"server_id": "srv-1", "tool_name": "subtract"}] + ), +) +GATEWAY_SERVERS: Final = ( + MCPServer(server_id="srv-1", name="calc", alias="calc", transport=MCPTransport.http, access_groups=["wiki"]), + MCPServer(server_id="srv-2", name="docs", alias="docs", transport=MCPTransport.http, access_groups=["reference"]), +) +UPSTREAM_TOOLS: Final = MappingProxyType( + { + "calc": ( + MCPTool(name="add", inputSchema={"type": "object"}), + MCPTool(name="subtract", inputSchema={"type": "object"}), + ), + "docs": (MCPTool(name="lookup", inputSchema={"type": "object"}),), + } +) diff --git a/tests/integration/mcp/test_mcp_client_real_io.py b/tests/integration/mcp/test_mcp_client_real_io.py new file mode 100644 index 00000000000..f957c6d71e5 --- /dev/null +++ b/tests/integration/mcp/test_mcp_client_real_io.py @@ -0,0 +1,1527 @@ +import asyncio +import json +import sys +from collections.abc import AsyncIterator, Callable +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from pathlib import Path +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import anyio +import httpx2 +import pytest +from mcp import ClientSession, MCPError +from mcp.client.streamable_http import streamable_http_client +from mcp.server.mcpserver import Context +from mcp.server.mcpserver import MCPServer as UpstreamServer +from mcp.server.transport_security import TransportSecuritySettings +from mcp.types import ( + CONNECTION_CLOSED, + CallToolRequestParams, + CallToolResult, + DiscoverResult, + JSONRPCMessage, + JSONRPCRequest, + LoggingMessageNotificationParams, + ServerCapabilities, +) +from pydantic import TypeAdapter, ValidationError +from starlette.applications import Starlette +from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from starlette.requests import Request +from starlette.responses import Response +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +# Add the parent directory to the path so we can import litellm +import litellm.mcp_client.client as mcp_client_module +from litellm.mcp_client.client import ( + MCPClient, + PersistentMCPSession, + UpstreamSessionClosedError, + _TransportContext, + as_mcp_read_timeout, +) +from litellm.types.mcp import MCPTransport +from tests._support.clock_loops import AutojumpClockLoop, ManualClockLoop + +_JSONRPC_MESSAGE_ADAPTER: Final = TypeAdapter(JSONRPCMessage) + + +class _MockTransportClient(MCPClient): + """An MCPClient whose streamable-HTTP transport runs on an httpx2 MockTransport.""" + + def __init__( + self, + respond, + *, + http_transport: httpx2.AsyncBaseTransport | None = None, + transport_context: Callable[[str, httpx2.AsyncClient], _TransportContext] | None = None, + **kwargs, + ): + super().__init__(**kwargs) + self._respond = respond + self._http_transport = http_transport + self._transport_context = transport_context + + def _create_transport_context(self) -> tuple[_TransportContext, httpx2.AsyncClient]: + transport: Final = self._http_transport or httpx2.MockTransport(self._respond) + http_client: Final = self._create_httpx_client_factory(transport=transport)( + headers=self._get_auth_headers(), timeout=httpx2.Timeout(self.timeout) + ) + if self._transport_context is not None: + return self._transport_context(self.server_url, http_client), http_client + return streamable_http_client(self.server_url, http_client=http_client), http_client + + +# --------------------------------------------------------------------------- +# _last_initialize_instructions capture +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# Transport error surfacing +# --------------------------------------------------------------------------- + + +if __name__ == "__main__": + pytest.main([__file__]) + + +@pytest.mark.parametrize("module", ["mcp", "mcp_types", "httpx2", "httpcore2"]) +def test_base_sdk_guard_rejects_mcp_dependencies(tmp_path: Path, module: str) -> None: + import subprocess + import sys + + (tmp_path / f"{module}.py").write_text("") + checker = Path(__file__).parents[2] / "base_sdk_tests" / "check_base_sdk_install.py" + result = subprocess.run( + [ + sys.executable, + "-S", + "-c", + "import runpy, sys; sys.path.insert(0, sys.argv[2]); " + "runpy.run_path(sys.argv[1])['check_environment_is_base_only']()", + str(checker), + str(tmp_path), + ], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode != 0, f"base-only guard accepted installed {module}" + assert f"{module} installed" in result.stderr + + +class _DiagnosticSSEStream(httpx2.AsyncByteStream): + def __init__(self, messages: asyncio.Queue[bytes | Exception | None]) -> None: + self.messages = messages + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b"event: endpoint\ndata: /messages\n\n" + while True: + message: Final = await self.messages.get() + if message is None: + return + if isinstance(message, Exception): + raise message + yield b"event: message\ndata: " + message + b"\n\n" + + +_DIAGNOSTIC_STDIO_SERVER: Final = """ +import json, sys +mode, failure_method = sys.argv[1:] +for line in sys.stdin: + request = json.loads(line) + if "method" not in request or "id" not in request: + continue + if request["method"] == failure_method: + if mode == "bad-json": + print("secret-invalid-json", flush=True) + continue + if mode == "closed": + sys.exit(0) + if mode == "silent": + print(json.dumps({"jsonrpc": "2.0", "method": "notifications/message", "params": {"level": "info", "data": "Waiting"}}), flush=True) + continue + if request["method"] == "initialize": + result = {"protocolVersion": request["params"]["protocolVersion"], "capabilities": {"tools": {}, "logging": {}}, "serverInfo": {"name": "diagnostic", "version": "1"}} + elif request["method"] == "tools/list": + print(json.dumps({"jsonrpc": "2.0", "method": "notifications/message", "params": {"level": "info", "data": "Listing tools"}}), flush=True) + print(json.dumps({"jsonrpc": "2.0", "id": "unmatched", "result": {}}), flush=True) + print(json.dumps({"jsonrpc": "2.0", "id": "server-ping", "method": "ping"}), flush=True) + result = {"tools": [{"name": "ping", "inputSchema": {"type": "object"}}]} + else: + result = {"content": [{"type": "text", "text": "pong"}], "isError": False} + print(json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": result}), flush=True) +""" + + +def _diagnostic_transport(transport: MCPTransport, mode: str, failure_method: str) -> _TransportContext: + from mcp import StdioServerParameters + from mcp.client.sse import sse_client + from mcp.client.stdio import stdio_client + + if transport == MCPTransport.stdio: + return stdio_client( + StdioServerParameters( + command=sys.executable, args=["-u", "-c", _DIAGNOSTIC_STDIO_SERVER, mode, failure_method] + ) + ) + messages: Final[asyncio.Queue[bytes | Exception | None]] = asyncio.Queue() + + async def respond(request: httpx2.Request) -> httpx2.Response: + if request.method == "GET": + return httpx2.Response( + 200, headers={"Content-Type": "text/event-stream"}, stream=_DiagnosticSSEStream(messages) + ) + payload: Final = json.loads(request.content) + if "method" not in payload or "id" not in payload: + return httpx2.Response(202) + if payload["method"] == failure_method and mode != "ok": + if mode == "bad-json": + await messages.put(b"secret-invalid-json") + elif mode == "io-error": + await messages.put(httpx2.ReadError("secret-read-error")) + elif mode == "closed": + await messages.put(None) + elif mode == "silent": + await messages.put( + b'{"jsonrpc":"2.0","method":"notifications/message","params":{"level":"info","data":"Waiting"}}' + ) + return httpx2.Response(202) + if payload["method"] == "tools/list": + for message in ( + { + "jsonrpc": "2.0", + "method": "notifications/message", + "params": {"level": "info", "data": "Listing tools"}, + }, + {"jsonrpc": "2.0", "id": "unmatched", "result": {}}, + {"jsonrpc": "2.0", "id": "server-ping", "method": "ping"}, + ): + await messages.put(json.dumps(message).encode()) + result: Final = ( + { + "protocolVersion": payload["params"]["protocolVersion"], + "capabilities": {"tools": {}, "logging": {}}, + "serverInfo": {"name": "diagnostic", "version": "1"}, + } + if payload["method"] == "initialize" + else {"tools": [{"name": "ping", "inputSchema": {"type": "object"}}]} + if payload["method"] == "tools/list" + else {"content": [{"type": "text", "text": "pong"}], "isError": False} + ) + await messages.put(json.dumps({"jsonrpc": "2.0", "id": payload["id"], "result": result}).encode()) + return httpx2.Response(202) + + def factory( + headers: dict[str, str] | None = None, + timeout: httpx2.Timeout | None = None, + auth: httpx2.Auth | None = None, + ) -> httpx2.AsyncClient: + return httpx2.AsyncClient(transport=httpx2.MockTransport(respond), headers=headers, timeout=timeout, auth=auth) + + return sse_client("https://example.com/sse", httpx_client_factory=factory) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", [MCPTransport.sse, MCPTransport.stdio]) +@pytest.mark.parametrize("failure_method", ["initialize", "tools/list"]) +async def test_transport_parsing_failure_is_preserved(transport: MCPTransport, failure_method: str) -> None: + client: Final = MCPClient(server_url="https://example.com/sse", transport_type=transport, timeout=0.2) + with pytest.raises(ValidationError): + await asyncio.wait_for( + client._execute_session_operation( + _diagnostic_transport(transport, "bad-json", failure_method), lambda session: session.list_tools() + ), + timeout=3, + ) + + +def test_sse_read_failure_is_preserved() -> None: + loop: Final = AutojumpClockLoop() + + async def run() -> None: + client: Final = MCPClient(server_url="https://example.com/sse", transport_type=MCPTransport.sse, timeout=0.2) + with pytest.raises(httpx2.ReadError, match="secret-read-error"): + await asyncio.wait_for( + client._execute_session_operation( + _diagnostic_transport(MCPTransport.sse, "io-error", "tools/list"), + lambda session: session.list_tools(), + ), + timeout=3, + ) + + try: + loop.run_until_complete(run()) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.run_until_complete(loop.shutdown_default_executor()) + loop.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("protocol_version", ["auto", "2025-06-18"]) +@pytest.mark.parametrize("transport", [MCPTransport.sse, MCPTransport.stdio]) +@pytest.mark.parametrize("mode", ["ok", "closed", "silent"]) +async def test_transport_completion_and_normal_messages( + transport: MCPTransport, mode: str, protocol_version: str +) -> None: + from mcp import ClientSession + + from litellm.proxy.mcp_server.rest_endpoints import _connection_error_message + + logging_callback: Final = AsyncMock() + read_timeout: Final = 0.2 if mode == "silent" else 30 + client: Final = MCPClient( + server_url="https://example.com/sse", + transport_type=transport, + timeout=read_timeout, + logging_callback=logging_callback, + protocol_version=protocol_version, + ) + + async def operation(session: ClientSession) -> CallToolResult: + tools: Final = await session.list_tools() + assert [tool.name for tool in tools.tools] == ["ping"] + return await session.call_tool("ping", {}) + + pending: Final = client._execute_session_operation(_diagnostic_transport(transport, mode, "tools/list"), operation) + if mode == "ok": + result: Final = await asyncio.wait_for(pending, timeout=3) + assert result.is_error is False + assert result.content[0].text == "pong" + logging_callback.assert_awaited_once_with(LoggingMessageNotificationParams(level="info", data="Listing tools")) + else: + with pytest.raises(MCPError) as caught: + await asyncio.wait_for(pending, timeout=3) + if mode == "closed": + assert "connection was closed" in _connection_error_message(caught.value, client.server_url, read_timeout) + else: + assert isinstance(as_mcp_read_timeout(caught.value), TimeoutError) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", [MCPTransport.sse, MCPTransport.stdio]) +async def test_transport_cancellation_cleans_up_a_pending_request(transport: MCPTransport) -> None: + ready: Final = asyncio.Event() + + async def on_log(message: LoggingMessageNotificationParams) -> None: + if message.data == "Waiting": + ready.set() + + client: Final = MCPClient( + server_url="https://example.com/sse", transport_type=transport, timeout=30, logging_callback=on_log + ) + task: Final = asyncio.create_task( + client._execute_session_operation( + _diagnostic_transport(transport, "silent", "tools/list"), lambda session: session.list_tools() + ) + ) + try: + await asyncio.wait_for(ready.wait(), timeout=3) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=3) + + +@pytest.mark.parametrize("method", ("prompts/list", "resources/list", "resources/templates/list")) +@pytest.mark.parametrize( + "failure", ("repeat", "cycle", "cap", "method_not_found", "internal_error", "unauthorized", "deadline") +) +@pytest.mark.parametrize("strict", (False, True)) +def test_optional_discovery_rejects_incomplete_walks( + method: str, failure: str, strict: bool, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr( + mcp_client_module, + "MCP_TOOL_LISTING_MAX_PAGES", + 3 if failure in ("cycle", "repeat") else 2, + raising=False, + ) + monkeypatch.setattr(mcp_client_module, "MCP_TOOL_LISTING_TIMEOUT", 0.05) + field: Final = { + "prompts/list": "prompts", + "resources/list": "resources", + "resources/templates/list": "resourceTemplates", + }[method] + entry: Final = { + "prompts/list": {"name": "first"}, + "resources/list": {"name": "first", "uri": "test://first"}, + "resources/templates/list": {"name": "first", "uriTemplate": "test://{name}"}, + }[method] + loop: Final = ManualClockLoop() + + async def run() -> None: + cancelled: Final = asyncio.Event() + + async def respond(request: httpx2.Request) -> httpx2.Response: + payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) + if not isinstance(payload, JSONRPCRequest): + return httpx2.Response(202) + if payload.method == "initialize": + return httpx2.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": { + "protocolVersion": (payload.params or {})["protocolVersion"], + "capabilities": {"prompts": {}, "resources": {}}, + "serverInfo": {"name": "interrupted", "version": "1"}, + }, + }, + ) + assert payload.method == method + cursor: Final = (payload.params or {}).get("cursor") + if cursor is not None: + if failure == "deadline": + loop.advance(0.15) + try: + for _ in range(1_000): + await asyncio.sleep(0) + except asyncio.CancelledError: + cancelled.set() + raise + return httpx2.Response(500) + if failure == "unauthorized": + return httpx2.Response(401) + if failure in ("method_not_found", "internal_error"): + return httpx2.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "error": { + "code": -32601 if failure == "method_not_found" else -32603, + "message": "Later page unavailable", + }, + }, + ) + if failure == "deadline" and cursor is None: + loop.advance(0.1) + next_cursor: Final = ( + "private-cursor-2" if cursor == "private-cursor-1" and failure != "repeat" else "private-cursor-1" + ) + return httpx2.Response( + 200, json={"jsonrpc": "2.0", "id": payload.id, "result": {field: [entry], "nextCursor": next_cursor}} + ) + + responder: Final = AsyncMock(side_effect=respond) + client: Final = _MockTransportClient(responder, server_url="https://example.com/mcp", timeout=0.2) + operation: Final = { + "prompts/list": client.list_prompts, + "resources/list": client.list_resources, + "resources/templates/list": client.list_resource_templates, + }[method] + if strict: + error_type: Final = { + "internal_error": MCPError, + "unauthorized": httpx2.HTTPStatusError, + "deadline": TimeoutError, + }.get(failure, RuntimeError) + with pytest.raises(error_type): + await operation(raise_on_error=True) + else: + assert await operation() == [] + assert len( + tuple( + payload + for call in responder.call_args_list + if isinstance(payload := _JSONRPC_MESSAGE_ADAPTER.validate_json(call.args[0].content), JSONRPCRequest) + and payload.method == method + ) + ) == (3 if failure == "cycle" else 2) + assert "private-cursor" not in caplog.text + if failure == "deadline": + assert cancelled.is_set() + + try: + loop.run_until_complete(run()) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.run_until_complete(loop.shutdown_default_executor()) + loop.close() + + +def test_client_import_before_proxy_credentials_succeeds_in_fresh_process(): + import subprocess + + result = subprocess.run( + [ + sys.executable, + "-c", + "import litellm.mcp_client.client; from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager; print(MCPServerManager.__name__)", + ], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "MCPServerManager" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("protocol_version", ("auto", "2025-06-18")) +@pytest.mark.parametrize("cancel_mode", ("scope", "task", "wait_for", "read_timeout")) +@pytest.mark.parametrize("concurrency", (1, 5)) +@pytest.mark.parametrize("termination", ("ok", "hang", "hang_body")) +@pytest.mark.parametrize("raise_on_error", (False, True)) +async def test_cancellation_delivers_termination_over_tcp( + cancel_mode: str, concurrency: int, termination: str, raise_on_error: bool, protocol_version: str +) -> None: + started: Final = asyncio.Event() + scope_ready: Final[asyncio.Future[anyio.CancelScope]] = asyncio.get_running_loop().create_future() + terminations: Final[list[bytes]] = [] + starts: Final[list[bytes]] = [] + stop: Final = asyncio.Event() + connections: Final[list[asyncio.Task[None]]] = [] + + async def handle_connection(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + connection: Final = asyncio.current_task() + assert connection is not None + connections.append(connection) + try: + request_line: Final = await reader.readline() + if not request_line: + return + method: Final = request_line.split()[0] + headers: Final = await reader.readuntil(b"\r\n\r\n") + length: Final = next( + ( + int(line.split(b":", 1)[1]) + for line in headers.splitlines() + if line.lower().startswith(b"content-length:") + ), + 0, + ) + try: + body: Final = await reader.readexactly(length) + except asyncio.IncompleteReadError: + return + if method == b"DELETE": + terminations.append(body) + if termination != "ok": + await stop.wait() + return + writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + elif method == b"GET": + writer.write(b"HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + else: + payload: Final = json.loads(body) + if payload["method"] == "tools/call": + if termination == "hang_body": + writer.write( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Content-Length: 200\r\nConnection: close\r\n\r\n" + ) + await writer.drain() + starts.append(body) + if len(starts) == concurrency: + started.set() + await stop.wait() + return + if payload["method"] == "initialize": + if cancel_mode != "read_timeout": + await asyncio.sleep(0.75) + response: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": payload["id"], + "result": { + "protocolVersion": "2025-06-18", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "tcp-peer", "version": "1"}, + }, + } + ).encode() + writer.write( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nMcp-Session-Id: tcp-session\r\n" + + f"Content-Length: {len(response)}\r\nConnection: close\r\n\r\n".encode() + + response + ) + else: + writer.write(b"HTTP/1.1 202 Accepted\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + await writer.drain() + finally: + writer.close() + await writer.wait_closed() + + listener: Final = await asyncio.start_server(handle_connection, "127.0.0.1", 0) + port: Final = listener.sockets[0].getsockname()[1] + client_timeout: Final = 2 if cancel_mode == "read_timeout" else 30 + client: Final = MCPClient( + server_url=f"http://127.0.0.1:{port}/mcp", + protocol_version=protocol_version, + timeout=client_timeout, + ) + + async def calls(): + results: Final = await asyncio.gather( + *( + client.call_tool(CallToolRequestParams(name="slow", arguments={}), raise_on_error=raise_on_error) + for _ in range(concurrency) + ), + return_exceptions=cancel_mode == "read_timeout", + ) + if cancel_mode == "read_timeout": + if raise_on_error: + assert all(isinstance(result, TimeoutError) for result in results) + else: + assert all(isinstance(result, CallToolResult) and result.is_error for result in results) + return results + + async def invoke(): + if cancel_mode == "scope": + with anyio.fail_after(None) as scope: + scope_ready.set_result(scope) + return await calls() + return await calls() + + try: + task: Final = asyncio.create_task(invoke()) + await asyncio.wait_for(started.wait(), 30) + if cancel_mode == "scope": + (await scope_ready).deadline = anyio.current_time() + 0.2 + if cancel_mode == "task": + task.cancel() + expected_error: Final = ( + TimeoutError + if cancel_mode == "read_timeout" + else asyncio.CancelledError + if cancel_mode == "task" + else TimeoutError + ) + if cancel_mode == "read_timeout": + done, _ = await asyncio.wait((task,), timeout=8) + assert task in done, "Read timeout and bounded cleanup must complete without external cancellation" + await task + elif cancel_mode == "wait_for": + with pytest.raises(expected_error): + await asyncio.wait_for(task, 0.2) + else: + with pytest.raises(expected_error): + await task + assert len(starts) == concurrency + assert len(terminations) == concurrency, "Each cancelled call must send DELETE over a fresh TCP connection" + finally: + stop.set() + if not task.done(): + task.cancel() + await asyncio.wait((task,), timeout=8) + listener.close() + for connection in connections: + connection.cancel() + closed: Final = await asyncio.wait_for(asyncio.gather(*connections, return_exceptions=True), 2) + assert all(result is None or isinstance(result, asyncio.CancelledError) for result in closed), closed + await asyncio.wait_for(listener.wait_closed(), 2) + + +def test_modern_call_uses_discovery_deadline_for_multiple_pages() -> None: + from mcp.types import ToolsCapability + + loop: Final = AutojumpClockLoop() + + async def respond(request: httpx2.Request) -> httpx2.Response: + payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) + assert isinstance(payload, JSONRPCRequest) + if payload.method == "server/discover": + return httpx2.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": DiscoverResult( + supported_versions=["2026-07-28"], capabilities=ServerCapabilities(tools=ToolsCapability()) + ).model_dump(by_alias=True, exclude_none=True), + }, + ) + if payload.method == "tools/list": + await asyncio.sleep(0.06) + assert payload.params is not None + page: Final = ( + {"tools": [], "nextCursor": "second"} + if "cursor" not in payload.params + else { + "tools": [ + { + "name": "quote", + "inputSchema": { + "type": "object", + "properties": {"workspace": {"type": "string", "x-mcp-header": "Workspace"}}, + }, + } + ] + } + ) + return httpx2.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": {"resultType": "complete", "cacheScope": "private", "ttlMs": 0, **page}, + }, + ) + assert payload.method == "tools/call" + assert request.headers["mcp-param-workspace"] == "engineering" + return httpx2.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": {"resultType": "complete", "content": [{"type": "text", "text": "quoted"}], "isError": False}, + }, + ) + + async def run() -> None: + client: Final = _MockTransportClient( + respond, server_url="https://example.com/mcp", protocol_version="2026-07-28", timeout=0.1 + ) + assert [tool.name for tool in await client.list_tools(raise_on_error=True)] == ["quote"] + result: Final = await client.call_tool( + CallToolRequestParams(name="quote", arguments={"workspace": "engineering"}), raise_on_error=True + ) + assert result.is_error is False + assert result.content[0].text == "quoted" + + try: + loop.run_until_complete(run()) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + + +class _StatefulUpstreamClient(MCPClient): + """An MCPClient whose streamable-HTTP transport talks to an in-process stateful MCP server.""" + + def __init__(self, app: ASGIApp, *, server_url: str, transport_type: MCPTransport) -> None: + super().__init__(server_url=server_url, transport_type=transport_type) + self._app = app + + def _create_transport_context(self) -> tuple[_TransportContext, httpx2.AsyncClient]: + http_client: Final = self._create_httpx_client_factory(transport=httpx2.ASGITransport(app=self._app))( + headers=self._get_auth_headers(), timeout=httpx2.Timeout(self.timeout) + ) + return streamable_http_client(self.server_url, http_client=http_client), http_client + + +class _CleanupHoldingASGITransport(httpx2.ASGITransport): + def __init__(self, app: ASGIApp, cleanup_started: asyncio.Event, release_cleanup: asyncio.Event) -> None: + super().__init__(app=app) + self._cleanup_started: Final = cleanup_started + self._release_cleanup: Final = release_cleanup + + async def aclose(self) -> None: + self._cleanup_started.set() + await self._release_cleanup.wait() + await super().aclose() + + +class _CleanupHoldingStatefulUpstreamClient(_StatefulUpstreamClient): + def __init__( + self, + app: ASGIApp, + *, + server_url: str, + transport_type: MCPTransport, + cleanup_started: asyncio.Event, + release_cleanup: asyncio.Event, + ) -> None: + super().__init__(app, server_url=server_url, transport_type=transport_type) + self._cleanup_started: Final = cleanup_started + self._release_cleanup: Final = release_cleanup + + def _create_transport_context(self) -> tuple[_TransportContext, httpx2.AsyncClient]: + transport: Final = _CleanupHoldingASGITransport( + self._app, + self._cleanup_started, + self._release_cleanup, + ) + http_client: Final = self._create_httpx_client_factory(transport=transport)( + headers=self._get_auth_headers(), + timeout=httpx2.Timeout(self.timeout), + ) + return streamable_http_client(self.server_url, http_client=http_client), http_client + + +def _stateful_upstream() -> Starlette: + service: Final = UpstreamServer("stateful") + selected: Final[dict[str, str]] = {} + + @service.tool() + def select_project(name: str, ctx: Context) -> str: + selected[ctx.headers["mcp-session-id"]] = name + return f"selected {name}" + + @service.tool() + def create_feature(title: str, ctx: Context) -> str: + return f"{selected[ctx.headers['mcp-session-id']]}/{title}" + + return service.streamable_http_app( + stateless_http=False, + json_response=True, + transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False), + ) + + +def _upstream_with_status_for_method(method: str, status_code: int) -> Starlette: + service: Final = UpstreamServer("status upstream") + + @service.tool() + def protected_tool() -> str: + return "protected" + + app: Final = service.streamable_http_app( + stateless_http=False, + json_response=True, + transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False), + ) + + async def reject_method(request: Request, call_next: RequestResponseEndpoint) -> Response: + if request.method == "POST": + payload: Final = await request.json() + if isinstance(payload, dict) and payload.get("method") == method: + return Response(status_code=status_code, headers={"WWW-Authenticate": "Bearer"}) + return await call_next(request) + + app.add_middleware(BaseHTTPMiddleware, dispatch=reject_method) + return app + + +def _upstream_with_malformed_response_for_method(method: str) -> Starlette: + service: Final = UpstreamServer("malformed response upstream") + + @service.tool() + def malformed_tool() -> str: + return "unreachable" + + app: Final = service.streamable_http_app( + stateless_http=False, + json_response=True, + transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False), + ) + + async def malformed_method(request: Request, call_next: RequestResponseEndpoint) -> Response: + if request.method == "GET": + return Response(content="data: {}\n\n", media_type="text/event-stream") + if request.method == "POST": + payload: Final = await request.json() + if isinstance(payload, dict) and payload.get("method") == method: + return Response(content="not a JSON-RPC message", media_type="text/event-stream") + return await call_next(request) + + app.add_middleware(BaseHTTPMiddleware, dispatch=malformed_method) + return app + + +def _client_with_session( + app: Starlette, + admission: Callable[[], AbstractAsyncContextManager[None]] | None = None, +) -> tuple[_StatefulUpstreamClient, PersistentMCPSession]: + client: Final = _StatefulUpstreamClient(app, server_url="http://upstream/mcp", transport_type=MCPTransport.http) + return client, client.open_persistent_session(admission) + + +@pytest.mark.asyncio +async def test_cancelling_one_waiter_for_session_readiness_does_not_cancel_other_waiters() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + initialize_started: Final = asyncio.Event() + release_initialize: Final = asyncio.Event() + + async def gated_server(scope: Scope, receive: Receive, send: Send) -> None: + async def gated_send(message: Message) -> None: + if message["type"] == "http.response.body" and not initialize_started.is_set(): + initialize_started.set() + await release_initialize.wait() + await send(message) + + await app(scope, receive, gated_send) + + gated_app: Final[ASGIApp] = gated_server + client: Final = _StatefulUpstreamClient( + gated_app, server_url="http://upstream/mcp", transport_type=MCPTransport.http + ) + session: Final = client.open_persistent_session() + + async def operation(_: ClientSession) -> str: + return "ready" + + cancelled_waiter: Final = asyncio.create_task(session.run(operation)) + remaining_waiter: Final = asyncio.create_task(session.run(operation)) + try: + await asyncio.wait_for(initialize_started.wait(), 5) + await asyncio.sleep(0) + cancelled_waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await cancelled_waiter + release_initialize.set() + assert await asyncio.wait_for(remaining_waiter, 5) == "ready" + assert not session.closed + finally: + release_initialize.set() + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_survives_recoverable_mcp_errors_and_closes_on_connection_closed() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + client, session = _client_with_session(app) + + async def invalid_operation(_: ClientSession) -> str: + raise MCPError(code=-32602, message="invalid params") + + try: + with pytest.raises(MCPError): + await session.run(invalid_operation) + assert not session.closed + selected: Final = await client.call_tool( + CallToolRequestParams(name="select_project", arguments={"name": "recoverable"}), + persistent_session=session, + ) + created: Final = await client.call_tool( + CallToolRequestParams(name="create_feature", arguments={"title": "still-open"}), + persistent_session=session, + ) + finally: + session.close() + assert selected.is_error is False + assert created.content[0].text == "recoverable/still-open" + await asyncio.wait_for(session.wait_closed(), 5) + + connection_app: Final = _stateful_upstream() + async with connection_app.router.lifespan_context(connection_app): + _, session = _client_with_session(connection_app) + started: Final = asyncio.Event() + release: Final = asyncio.Event() + + async def disconnected_operation(_: ClientSession) -> str: + started.set() + await release.wait() + raise MCPError(code=CONNECTION_CLOSED, message="upstream connection closed") + + async def pending_operation(_: ClientSession) -> str: + return "unused" + + active: Final = asyncio.create_task(session.run(disconnected_operation)) + await asyncio.wait_for(started.wait(), 5) + pending: Final = asyncio.create_task(session.run(pending_operation)) + await asyncio.sleep(0) + release.set() + with pytest.raises(MCPError) as caught: + await asyncio.wait_for(active, 5) + assert caught.value.error.code == CONNECTION_CLOSED + with pytest.raises((MCPError, UpstreamSessionClosedError)): + await asyncio.wait_for(pending, 5) + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed + + +@pytest.mark.asyncio +async def test_ended_persistent_session_is_not_reusable_during_transport_cleanup() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + cleanup_started: Final = asyncio.Event() + release_cleanup: Final = asyncio.Event() + client: Final = _CleanupHoldingStatefulUpstreamClient( + app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + cleanup_started=cleanup_started, + release_cleanup=release_cleanup, + ) + session: Final = client.open_persistent_session() + + async def disconnected_operation(_: ClientSession) -> str: + raise MCPError(code=CONNECTION_CLOSED, message="upstream connection closed") + + try: + with pytest.raises(MCPError) as caught: + await asyncio.wait_for(session.run(disconnected_operation), 5) + assert caught.value.error.code == CONNECTION_CLOSED + await asyncio.wait_for(cleanup_started.wait(), 5) + + assert not session.reusable + assert not session.closed + session.retire() + + release_cleanup.set() + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed + finally: + release_cleanup.set() + if not session.closed: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_remaps_stream_error_over_mcp_error() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + client: Final = _StatefulUpstreamClient( + app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + ) + session: Final = client.open_persistent_session() + request: Final = httpx2.Request("POST", "http://upstream/mcp") + response: Final = httpx2.Response(401, request=request, headers={"WWW-Authenticate": "Bearer"}) + http_error: Final = httpx2.HTTPStatusError("401 Unauthorized", request=request, response=response) + + async def operation(_: ClientSession) -> str: + stream_error: Final[asyncio.Future[Exception] | None] = session._stream_error + assert stream_error is not None + stream_error.set_result(http_error) + raise MCPError(code=-32603, message="response stream closed") + + try: + with pytest.raises(httpx2.HTTPStatusError) as caught: + await session.run(operation) + assert caught.value is http_error + assert not session.reusable + finally: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_ends_when_a_malformed_response_is_remapped() -> None: + app: Final = _upstream_with_malformed_response_for_method("tools/call") + async with app.router.lifespan_context(app): + client: Final = _StatefulUpstreamClient( + app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + ) + session: Final = client.open_persistent_session() + try: + + async def ready(_: ClientSession) -> str: + return "ready" + + assert await asyncio.wait_for(session.run(ready), 5) == "ready" + stream_error: Final[asyncio.Future[Exception] | None] = session._stream_error + assert stream_error is not None + recorded_stream_error: Final = await asyncio.wait_for(asyncio.shield(stream_error), 5) + assert isinstance(recorded_stream_error, ValidationError) + + with pytest.raises(ValidationError) as caught: + await client.call_tool( + CallToolRequestParams(name="malformed_tool", arguments={}), + raise_on_error=True, + persistent_session=session, + ) + assert caught.value is recorded_stream_error + assert not session.reusable + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed + finally: + if not session.closed: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_failed_persistent_initialize_is_not_reusable_during_transport_cleanup() -> None: + app: Final = _upstream_with_status_for_method("initialize", 500) + async with app.router.lifespan_context(app): + cleanup_started: Final = asyncio.Event() + release_cleanup: Final = asyncio.Event() + client: Final = _CleanupHoldingStatefulUpstreamClient( + app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + cleanup_started=cleanup_started, + release_cleanup=release_cleanup, + ) + session: Final = client.open_persistent_session() + pending: Final = asyncio.create_task(session.run(lambda _: "ready")) + try: + await asyncio.wait_for(cleanup_started.wait(), 5) + assert not session.reusable + assert not session.closed + release_cleanup.set() + with pytest.raises(httpx2.HTTPStatusError): + await asyncio.wait_for(pending, 5) + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed + finally: + release_cleanup.set() + if not session.closed: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_cancelled_queued_operation_does_not_run_and_session_accepts_a_later_operation() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + first_started: Final = asyncio.Event() + release_first: Final = asyncio.Event() + cancelled_operation_ran: Final = asyncio.Event() + + async def first_operation(_: ClientSession) -> str: + first_started.set() + await release_first.wait() + return "first" + + async def cancelled_operation(_: ClientSession) -> str: + cancelled_operation_ran.set() + return "cancelled" + + async def third_operation(_: ClientSession) -> str: + return "third" + + first_task: Final = asyncio.create_task(session.run(first_operation)) + try: + await asyncio.wait_for(first_started.wait(), 5) + second_task: Final = asyncio.create_task(session.run(cancelled_operation)) + await asyncio.sleep(0) + assert session._queue.qsize() == 1 + second_task.cancel() + with pytest.raises(asyncio.CancelledError): + await second_task + third_task: Final = asyncio.create_task(session.run(third_operation)) + await asyncio.sleep(0) + assert session._queue.qsize() == 2 + release_first.set() + assert await asyncio.wait_for(first_task, 5) == "first" + assert await asyncio.wait_for(third_task, 5) == "third" + assert not cancelled_operation_ran.is_set() + assert not session.closed + finally: + release_first.set() + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_retiring_persistent_session_drains_in_flight_and_queued_operations() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + first_started: Final = asyncio.Event() + release_first: Final = asyncio.Event() + + async def first_operation(_: ClientSession) -> str: + first_started.set() + await release_first.wait() + return "first" + + async def queued_operation(_: ClientSession) -> str: + return "queued" + + first_waiter: Final = asyncio.create_task(session.run(first_operation)) + try: + await asyncio.wait_for(first_started.wait(), 5) + queued_waiter: Final = asyncio.create_task(session.run(queued_operation)) + await asyncio.sleep(0) + assert session._queue.qsize() == 1 + session.retire() + release_first.set() + + results: Final = await asyncio.wait_for(asyncio.gather(first_waiter, queued_waiter), 5) + assert results == ["first", "queued"] + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed + finally: + release_first.set() + if not session.closed: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_timed_out_persistent_operation_waiting_for_admission_is_skipped() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + admission_slots: Final = asyncio.Semaphore(1) + admission_requested: Final = asyncio.Event() + + def admission() -> AbstractAsyncContextManager[None]: + admission_requested.set() + return admission_slots + + _, session = _client_with_session(app, admission) + operation_ran: Final = asyncio.Event() + + async def warmup(_: ClientSession) -> str: + return "ready" + + async def timed_out_operation(_: ClientSession) -> str: + operation_ran.set() + return "expired" + + async def later_operation(_: ClientSession) -> str: + return "later" + + try: + assert await session.run(warmup) == "ready" + admission_requested.clear() + async with admission_slots: + timed_out_call: Final = asyncio.create_task(session.run(timed_out_operation)) + await asyncio.wait_for(admission_requested.wait(), 1) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(timed_out_call, 0.05) + assert not operation_ran.is_set() + + later_result: Final = await asyncio.wait_for(session.run(later_operation), 1) + + assert later_result == "later" + assert session.reusable + assert not operation_ran.is_set() + finally: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_holds_admission_until_initialize_completes() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + initialize_seen: Final = asyncio.Event() + admission_slots: Final = asyncio.Semaphore(1) + + async def observing_app(scope: Scope, receive: Receive, send: Send) -> None: + async def observe_send(message: Message) -> None: + if message["type"] == "http.response.body": + initialize_seen.set() + await send(message) + + await app(scope, receive, observe_send) + + client: Final = _StatefulUpstreamClient( + observing_app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + ) + session: Final = client.open_persistent_session(lambda: admission_slots) + + async def operation(_: ClientSession) -> str: + return "ready" + + run_task: Final = asyncio.create_task(session.run(operation)) + try: + async with admission_slots: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(initialize_seen.wait(), 0.05) + assert not session._ready.done() + assert not run_task.done() + + result: Final = await asyncio.wait_for(run_task, 5) + assert result == "ready" + assert initialize_seen.is_set() + finally: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + await asyncio.gather(run_task, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_persistent_session_releases_admission_when_connect_fails() -> None: + async def failing_app(_: Scope, __: Receive, ___: Send) -> None: + raise httpx2.ConnectError("connect failed") + + client: Final = _StatefulUpstreamClient( + failing_app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + ) + admission_slots: Final = asyncio.Semaphore(1) + session: Final = client.open_persistent_session(lambda: admission_slots) + + async def operation(_: ClientSession) -> str: + return "unreachable" + + try: + with pytest.raises(httpx2.ConnectError, match="connect failed"): + await asyncio.wait_for(session.run(operation), 1) + await asyncio.wait_for(session.wait_closed(), 1) + + await asyncio.wait_for(admission_slots.acquire(), 0.1) + admission_slots.release() + finally: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_failed_connect_is_not_reusable_during_admission_teardown() -> None: + connect_failed: Final = asyncio.Event() + admission_exit_started: Final = asyncio.Event() + release_admission_exit: Final = asyncio.Event() + + async def failing_app(_: Scope, __: Receive, ___: Send) -> None: + connect_failed.set() + raise httpx2.ConnectError("connect failed") + + @asynccontextmanager + async def blocking_admission() -> AsyncIterator[None]: + try: + yield + finally: + admission_exit_started.set() + await release_admission_exit.wait() + + client: Final = _StatefulUpstreamClient( + failing_app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + ) + session: Final = client.open_persistent_session(blocking_admission) + + async def operation(_: ClientSession) -> str: + return "unreachable" + + run_task: Final = asyncio.create_task(session.run(operation)) + try: + await asyncio.wait_for(connect_failed.wait(), 1) + await asyncio.wait_for(admission_exit_started.wait(), 1) + + assert not run_task.done() + assert not session.reusable + assert not session.closed + + release_admission_exit.set() + with pytest.raises(httpx2.ConnectError, match="connect failed"): + await asyncio.wait_for(run_task, 1) + await asyncio.wait_for(session.wait_closed(), 1) + assert session.closed + finally: + release_admission_exit.set() + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + await asyncio.gather(run_task, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_persistent_session_admission_is_acquired_only_when_operation_runs() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + admission_slots: Final = asyncio.Semaphore(2) + _, session = _client_with_session(app, lambda: admission_slots) + first_started: Final = asyncio.Event() + release_first: Final = asyncio.Event() + operation_order: Final[list[str]] = [] + + async def first_operation(_: ClientSession) -> str: + operation_order.append("first") + first_started.set() + await release_first.wait() + return "first" + + async def second_operation(_: ClientSession) -> str: + operation_order.append("second") + return "second" + + async def third_operation(_: ClientSession) -> str: + operation_order.append("third") + return "third" + + first_task: Final = asyncio.create_task(session.run(first_operation)) + second_task: Final = asyncio.create_task(session.run(second_operation)) + third_task: Final = asyncio.create_task(session.run(third_operation)) + try: + await asyncio.wait_for(first_started.wait(), 5) + await asyncio.sleep(0) + assert session._queue.qsize() == 2 + await asyncio.wait_for(admission_slots.acquire(), 0.1) + try: + assert admission_slots.locked() + finally: + admission_slots.release() + assert operation_order == ["first"] + + release_first.set() + results: Final = await asyncio.wait_for( + asyncio.gather(first_task, second_task, third_task), + 5, + ) + + assert results == ["first", "second", "third"] + assert operation_order == ["first", "second", "third"] + finally: + release_first.set() + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + await asyncio.gather(first_task, second_task, third_task, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_retiring_idle_persistent_session_closes_and_rejects_new_operations() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + + async def operation(_: ClientSession) -> str: + return "ready" + + try: + assert await session.run(operation) == "ready" + session.retire() + await asyncio.wait_for(session.wait_closed(), 1) + + assert session.closed + with pytest.raises(UpstreamSessionClosedError): + await session.run(operation) + finally: + if not session.closed: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_closing_persistent_session_mid_operation_fails_the_waiter_instead_of_hanging() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + started: Final = asyncio.Event() + + async def slow_operation(_: object) -> str: + started.set() + await asyncio.sleep(30) + return "never" + + waiter: Final = asyncio.ensure_future(session.run(slow_operation)) + await asyncio.wait_for(started.wait(), 5) + session.close() + with pytest.raises(RuntimeError, match="upstream MCP session closed"): + await asyncio.wait_for(waiter, 5) + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_reports_an_upstream_cancellation_as_a_runtime_error_not_a_cancelled_caller() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + client, session = _client_with_session(app) + + async def cancelled_upstream(_: object) -> str: + raise asyncio.CancelledError() + + try: + with pytest.raises(RuntimeError, match="cancelled"): + await asyncio.wait_for(session.run(cancelled_upstream), 5) + selected: Final = await client.call_tool( + CallToolRequestParams(name="select_project", arguments={"name": "e"}), persistent_session=session + ) + finally: + session.close() + assert selected.is_error is False, "an upstream cancellation must not be mistaken for a cancelled caller" + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_closing_persistent_session_fails_a_caller_blocked_on_a_full_queue_instead_of_hanging() -> None: + from litellm.mcp_client.client import _MAX_PENDING_OPERATIONS + + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + started: Final = asyncio.Event() + + async def slow_operation(_: object) -> str: + started.set() + await asyncio.sleep(30) + return "never" + + waiters: Final = tuple( + asyncio.ensure_future(session.run(slow_operation)) for _ in range(_MAX_PENDING_OPERATIONS + 2) + ) + await asyncio.wait_for(started.wait(), 5) + await asyncio.sleep(0) + session.close() + outcomes: Final = await asyncio.wait_for(asyncio.gather(*waiters, return_exceptions=True), 5) + assert all(isinstance(outcome, RuntimeError) for outcome in outcomes), outcomes + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_ends_after_a_broken_stream_so_the_next_call_gets_a_fresh_one() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + + async def broken_stream(_: object) -> str: + raise anyio.BrokenResourceError() + + with pytest.raises(anyio.BrokenResourceError): + await asyncio.wait_for(session.run(broken_stream), 5) + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed, "a dead transport must end the session instead of being reused" + +@pytest.mark.asyncio +@pytest.mark.parametrize("legacy_callback", [False, True]) +async def test_tool_progress_preserves_message_and_legacy_callback(legacy_callback: bool) -> None: + received = AsyncMock() + + async def legacy(progress: float, total: float | None) -> None: + await received(progress, total) + + async def modern(progress: float, total: float | None, message: str | None) -> None: + await received(progress, total, message) + + async def call_tool(**kwargs): + await kwargs["progress_callback"](1.0, 2.0, "halfway") + return CallToolResult(content=[]) + + session = AsyncMock(spec=ClientSession, call_tool=AsyncMock(side_effect=call_tool)) + + async def run(operation, **kwargs): + return await operation(session) + + persistent = MagicMock(spec=PersistentMCPSession, run=AsyncMock(side_effect=run)) + client = MCPClient(server_url="https://example.com/mcp") + result = await client.call_tool( + CallToolRequestParams(name="slow"), + host_progress_callback=legacy if legacy_callback else modern, + persistent_session=persistent, + ) + assert not result.is_error + if legacy_callback: + received.assert_awaited_once_with(1.0, 2.0) + else: + received.assert_awaited_once_with(1.0, 2.0, "halfway") + + +@pytest.mark.asyncio +async def test_cancelling_persistent_operation_stops_work_and_keeps_session_usable() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + client, session = _client_with_session(app) + started: Final = asyncio.Event() + stopped: Final = asyncio.Event() + + async def slow_operation(_: ClientSession) -> None: + started.set() + try: + await asyncio.Event().wait() + finally: + stopped.set() + + waiter: Final = asyncio.create_task(session.run(slow_operation)) + try: + await asyncio.wait_for(started.wait(), 5) + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + await asyncio.sleep(0.1) + assert stopped.is_set(), "Caller cancellation must stop the running upstream operation" + result: Final = await asyncio.wait_for( + client.call_tool( + CallToolRequestParams(name="select_project", arguments={"name": "after-cancellation"}), + persistent_session=session, + ), + 5, + ) + assert not result.is_error + finally: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + await asyncio.gather(waiter, return_exceptions=True) diff --git a/tests/integration/mcp/test_openapi_to_mcp_generator_real_io.py b/tests/integration/mcp/test_openapi_to_mcp_generator_real_io.py new file mode 100644 index 00000000000..acdeef06898 --- /dev/null +++ b/tests/integration/mcp/test_openapi_to_mcp_generator_real_io.py @@ -0,0 +1,44 @@ +""" +Tests for OpenAPI to MCP generator, focusing on security and edge cases. + +This test suite ensures that: +1. Parameter names with invalid Python identifiers are handled safely +2. No exec() is used (security) +3. All edge cases (hyphens, dots, keywords, special chars) work correctly +4. Path traversal attacks are prevented +5. Path parameters are properly URL encoded +""" + +import pytest + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) + + +# --------------------------------------------------------------------------- +# Tool name sanitization for OpenAPI -> MCP +# Repro: GitHub's REST OpenAPI uses tag-namespaced operationIds like +# "actions/download-job-logs-for-workflow-run". Without sanitization the +# generated MCP tool name contains '/', which Anthropic/OpenAI/Bedrock all +# reject (^[a-zA-Z0-9_-]+$). This block guards the registration + preview +# paths against that. +# --------------------------------------------------------------------------- + + +def test_openapi_generator_import_does_not_require_mcp_sdk() -> None: + import subprocess + import sys + + script = """ +import builtins +original_import = builtins.__import__ +def without_mcp(name, *args, **kwargs): + if name == 'mcp' or name.startswith('mcp.'): + raise ModuleNotFoundError('MCP SDK unavailable') + return original_import(name, *args, **kwargs) +builtins.__import__ = without_mcp +import litellm.proxy.mcp_server.openapi_to_mcp_generator +""" + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr diff --git a/tests/integration/observability/test_anthropic_cache_control_hook_real_io.py b/tests/integration/observability/test_anthropic_cache_control_hook_real_io.py new file mode 100644 index 00000000000..ab412de705e --- /dev/null +++ b/tests/integration/observability/test_anthropic_cache_control_hook_real_io.py @@ -0,0 +1,65 @@ +import json +import os +import subprocess +import sys +import textwrap +from typing import Optional, Tuple + +import pytest + +import litellm + + +@pytest.fixture(autouse=True) +def _no_openai_api_base_override(monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + +class TestAnthropicPromptCachingEnvVars: + """Both settings are read from the environment at import, so an admin can enable + auto-caching without a config file. Each case re-imports litellm in a subprocess + so the env is read fresh without contaminating this process's module graph. + """ + + @staticmethod + def _import_litellm_with_env(env_override: dict) -> Tuple[bool, Optional[str]]: + env = os.environ.copy() + env.pop("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", None) + env.pop("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL", None) + env.update(env_override) + script = textwrap.dedent( + """ + import json, litellm + print(json.dumps([litellm.enable_anthropic_prompt_caching, litellm.anthropic_prompt_caching_ttl])) + """ + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, env=env, timeout=300) + assert result.returncode == 0, result.stderr + enabled, ttl = json.loads(result.stdout.strip().splitlines()[-1]) + return enabled, ttl + + def test_unset_env_leaves_auto_caching_off(self): + assert self._import_litellm_with_env({}) == (False, None) + + @pytest.mark.parametrize("value", ["true", "True", "TRUE"]) + def test_env_enables_auto_caching_case_insensitively(self, value): + enabled, _ = self._import_litellm_with_env({"LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING": value}) + assert enabled is True + + @pytest.mark.parametrize("value", ["false", "0", "yes", ""]) + def test_env_only_enables_on_true(self, value): + enabled, _ = self._import_litellm_with_env({"LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING": value}) + assert enabled is False + + @pytest.mark.parametrize("value", ["5m", "1h"]) + def test_ttl_env_is_applied(self, value): + _, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value}) + assert ttl == value + + @pytest.mark.parametrize("value", ["10m", "1H", "3600", "ephemeral"]) + def test_unsupported_ttl_env_falls_back_to_provider_default(self, value): + """An unparseable TTL must fall back to Anthropic's 5m default, never reach the provider verbatim.""" + _, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value}) + assert ttl is None diff --git a/tests/integration/sdk/test_anthropic_common_utils_real_io.py b/tests/integration/sdk/test_anthropic_common_utils_real_io.py new file mode 100644 index 00000000000..c19dcbb51d1 --- /dev/null +++ b/tests/integration/sdk/test_anthropic_common_utils_real_io.py @@ -0,0 +1,41 @@ +import json +import subprocess +import sys +from typing import Final + + +_MASKED_HOSTED_WEB_CALL_PROBE: Final = """ +import json + +from litellm.llms.anthropic.common_utils import write_request_hosted_web_call_arguments + +message = { + "role": "assistant", + "provider_specific_fields": { + "web_search_calls": [ + { + "id": "ws_srvtoolu_1", + "type": "web_search_call", + "status": "completed", + "action": {"type": "search", "query": "deploy token 12345", "queries": ["deploy token 12345"]}, + } + ] + }, +} +write_request_hosted_web_call_arguments(message, {"srvtoolu_1": json.dumps({"query": "deploy token [REDACTED]"})}) +print(json.dumps(message["provider_specific_fields"]["web_search_calls"])) +""" + + +def test_a_masked_replayed_hosted_web_call_serializes_before_its_openai_model_was_ever_built() -> None: + result: Final = subprocess.run( + (sys.executable, "-I", "-c", _MASKED_HOSTED_WEB_CALL_PROBE), + capture_output=True, + text=True, + check=True, + timeout=120, + ) + hosted: Final = json.loads(result.stdout.strip().splitlines()[-1]) + assert [call["action"] for call in hosted] == [ + {"type": "search", "query": "deploy token [REDACTED]", "queries": ["deploy token [REDACTED]"]} + ] diff --git a/tests/integration/sdk/test_azure_common_utils_real_io.py b/tests/integration/sdk/test_azure_common_utils_real_io.py new file mode 100644 index 00000000000..5c988435194 --- /dev/null +++ b/tests/integration/sdk/test_azure_common_utils_real_io.py @@ -0,0 +1,204 @@ +import os +from collections.abc import Mapping +from typing import Final, Optional +from unittest.mock import MagicMock +import openai +import requests + +import litellm +from litellm.llms.azure.common_utils import BaseAzureLLM + + +def test_default_max_retries_env_var_reaches_azure_sdk_client(): + import subprocess + import sys + + code = ( + "from litellm.llms.azure.common_utils import BaseAzureLLM\n" + "client = BaseAzureLLM().get_azure_openai_client(" + "api_key='test-api-key', api_base='https://test.openai.azure.com', api_version='2024-02-01'," + " client=None, _is_async=True, litellm_params={}, model='gpt-4')\n" + "print(client.max_retries)" + ) + completed = subprocess.run( + [sys.executable, "-c", code], + env={**os.environ, "DEFAULT_MAX_RETRIES": "0"}, + capture_output=True, + text=True, + check=True, + ) + + assert completed.stdout.strip() == "0" + + +# Test parameters for different API functions with Azure models +AZURE_API_FUNCTION_PARAMS = [ + # (function_name, is_async, args) + ( + "completion", + False, + { + "model": "azure/gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "max_tokens": 10, + "api_key": "test-api-key", + "api_base": "https://test.openai.azure.com", + "api_version": "2023-05-15", + }, + ), + ( + "completion", + True, + { + "model": "azure/gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "max_tokens": 10, + "stream": True, + "api_key": "test-api-key", + "api_base": "https://test.openai.azure.com", + "api_version": "2023-05-15", + }, + ), + ( + "embedding", + False, + { + "model": "azure/text-embedding-ada-002", + "input": "Hello world", + "api_key": "test-api-key", + "api_base": "https://test.openai.azure.com", + "api_version": "2023-05-15", + }, + ), + ( + "embedding", + True, + { + "model": "azure/text-embedding-ada-002", + "input": "Hello world", + "api_key": "test-api-key", + "api_base": "https://test.openai.azure.com", + "api_version": "2023-05-15", + }, + ), + ( + "speech", + False, + { + "model": "azure/tts", + "input": "Hello, this is a test of text to speech", + "voice": "alloy", + "api_key": "test-api-key", + "api_base": "https://test.openai.azure.com", + "api_version": "2023-05-15", + }, + ), + ( + "speech", + True, + { + "model": "azure/tts-1", + "input": "Hello, this is a test of text to speech", + "voice": "alloy", + "api_key": "test-api-key", + "api_base": "https://test.openai.azure.com", + "api_version": "2023-05-15", + }, + ), + ( + "transcription", + False, + { + "model": "azure/whisper-1", + "file": MagicMock(), + "api_key": "test-api-key", + "api_base": "https://test.openai.azure.com", + "api_version": "2023-05-15", + }, + ), + ( + "transcription", + True, + { + "model": "azure/whisper-1", + "file": MagicMock(), + "api_key": "test-api-key", + "api_base": "https://test.openai.azure.com", + "api_version": "2023-05-15", + }, + ), +] +_ENTRA_TRACE_ID: Final = "f0c149d3-ef20-4194-8b81-c81a5e002500" + + +def _entra_error(aadsts_code: str, description: str) -> Mapping[str, object]: + return { + "error": "invalid_client", + "error_description": f"{aadsts_code}: {description} Trace ID: {_ENTRA_TRACE_ID}", + "error_codes": [int(aadsts_code.removeprefix("AADSTS"))], + } + + +_REAL_CREDENTIAL_NON_REJECTIONS: Final = { + "unreachable_through_a_credential_chain": ("default_azure_credential_chain", None, ""), + "html_503_through_a_credential_chain": ( + "default_azure_credential_chain", + 503, + "Service Unavailable", + ), + "json_503_through_a_credential_chain": ( + "default_azure_credential_chain", + 503, + _entra_error("AADSTS90033", "A transient error has occurred."), + ), + "throttled_client_secret": ( + "client_secret", + 429, + _entra_error("AADSTS50196", "The server terminated an operation because it encountered a client request loop."), + ), + "tenant_throttled_client_secret": ( + "client_secret", + 429, + _entra_error("AADSTS90055", "There are too many incoming requests."), + ), + "request_budget_exceeded_client_secret": ( + "client_secret", + 400, + _entra_error("AADSTS90024", "A transient error has occurred. Try again."), + ), + "retryable_error_client_secret": ( + "client_secret", + 400, + _entra_error("AADSTS50033", "A transient error has occurred."), + ), + "external_server_retryable_client_secret": ( + "client_secret", + 400, + _entra_error("AADSTS90006", "The service is temporarily unavailable."), + ), + "graph_retryable_client_secret": ( + "client_secret", + 400, + _entra_error("AADSTS90090", "The service is temporarily unavailable."), + ), + "token_issuance_error_client_secret": ( + "client_secret", + 400, + _entra_error("AADSTS50000", "There was an error issuing a token or an issue with our sign-in service."), + ), + "throttled_without_a_code_through_a_credential_chain": ( + "default_azure_credential_chain", + 429, + {"error": "temporarily_unavailable", "error_description": "Too many requests."}, + ), + "json_503_through_workload_identity": ( + "workload_identity", + 503, + _entra_error("AADSTS90033", "A transient error has occurred."), + ), + "throttled_workload_identity_naming_a_rejection_code": ( + "workload_identity", + 429, + _entra_error("AADSTS7000215", "Invalid client secret provided."), + ), +} diff --git a/tests/integration/sdk/test_configuration_real_io.py b/tests/integration/sdk/test_configuration_real_io.py new file mode 100644 index 00000000000..2821bbab722 --- /dev/null +++ b/tests/integration/sdk/test_configuration_real_io.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from collections.abc import Generator +from typing import Final + +import pytest + +from litellm.rust_bridge import configuration + + +@pytest.fixture(autouse=True) +def _isolated_configuration( # pyright: ignore[reportUnusedFunction] # pytest discovers fixtures dynamically + monkeypatch: pytest.MonkeyPatch, +) -> Generator[None]: + configuration.reset_rust_configuration() + monkeypatch.delenv("LITELLM_RUST", raising=False) + yield + configuration.reset_rust_configuration() + + +@pytest.mark.parametrize(("value", "expected"), (("1", "True"), ("0", "False"))) +def test_environment_controls_startup(value: str, expected: str) -> None: + environment: Final = {**os.environ, "LITELLM_RUST": value} + result: Final = subprocess.run( + ( + sys.executable, + "-c", + "from litellm.rust_bridge.configuration import rust_enabled; print(rust_enabled())", + ), + check=True, + capture_output=True, + text=True, + env=environment, + ) + + assert result.stdout.strip() == expected diff --git a/tests/integration/sdk/test_context_real_io.py b/tests/integration/sdk/test_context_real_io.py new file mode 100644 index 00000000000..233aeda04f6 --- /dev/null +++ b/tests/integration/sdk/test_context_real_io.py @@ -0,0 +1,146 @@ +import asyncio +import base64 +import uuid +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Event, Thread +from typing import Final, Literal + +import pytest + +import litellm +from litellm.llms.anthropic.messages.context_management.inference_view import CountContextEdits +from litellm.llms.vertex_ai.count_tokens.context import VertexGeminiCountContextError, gemini_count_contents +_NO_EDITS: Final = CountContextEdits(spec=None, model="gemini/gemini-2.5-flash", tools=None) + + +_PNG_BYTES: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" +) +_LOOP_PROOF_TIMEOUT_SECONDS: Final = 5 + + +@dataclass(frozen=True, slots=True) +class _LoopProbe: + request_arrived: Event = field(default_factory=Event) + loop_ran: Event = field(default_factory=Event) + loop_ran_while_download_pending: Event = field(default_factory=Event) + + +def _png_handler_waiting_for_loop(probe: _LoopProbe) -> type[BaseHTTPRequestHandler]: + class _PngHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + probe.request_arrived.set() + if probe.loop_ran.wait(_LOOP_PROOF_TIMEOUT_SECONDS): + probe.loop_ran_while_download_pending.set() + self.send_response(200) + self.send_header("Content-Type", "image/png") + self.send_header("Content-Length", str(len(_PNG_BYTES))) + self.end_headers() + self.wfile.write(_PNG_BYTES) + + def log_message(self, format: str, *args: object) -> None: + return None + + return _PngHandler + + +@contextmanager +def _png_server_waiting_for_loop(probe: _LoopProbe) -> Generator[str, None, None]: + with ThreadingHTTPServer(("127.0.0.1", 0), _png_handler_waiting_for_loop(probe)) as server: + thread: Final = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + + +async def _run_loop_once_download_starts(probe: _LoopProbe) -> None: + assert await asyncio.to_thread(probe.request_arrived.wait, _LOOP_PROOF_TIMEOUT_SECONDS) + probe.loop_ran.set() + + +@pytest.mark.parametrize("custom_llm_provider", ("vertex_ai", "gemini")) +async def test_remote_image_downloads_without_blocking_the_event_loop( + custom_llm_provider: Literal["vertex_ai", "gemini"], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + probe: Final = _LoopProbe() + with _png_server_waiting_for_loop(probe) as base_url: + image_url: Final = f"{base_url}/{uuid.uuid4().hex}.png" + _, contents = await asyncio.gather( + _run_loop_once_download_starts(probe), + gemini_count_contents( + ( + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe it"}, + {"type": "image", "source": {"type": "url", "url": image_url}}, + ], + }, + ), + "anthropic", + "gemini-2.5-flash", + custom_llm_provider, + {}, + ), + ) + + assert contents == [ + { + "role": "user", + "parts": [ + {"text": "Describe it"}, + {"inline_data": {"data": base64.b64encode(_PNG_BYTES).decode(), "mime_type": "image/png"}}, + ], + } + ] + assert probe.loop_ran_while_download_pending.is_set() + + +class _MissingImageHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, format: str, *args: object) -> None: + return None + + +@pytest.mark.parametrize("custom_llm_provider", ("vertex_ai", "gemini")) +async def test_image_that_cannot_be_fetched_returns_an_error_value( + custom_llm_provider: Literal["vertex_ai", "gemini"], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + with ThreadingHTTPServer(("127.0.0.1", 0), _MissingImageHandler) as server: + Thread(target=server.serve_forever, daemon=True).start() + try: + contents = await gemini_count_contents( + ( + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "url", + "url": f"http://127.0.0.1:{server.server_address[1]}/{uuid.uuid4().hex}.png", + }, + } + ], + }, + ), + "anthropic", + "gemini-2.5-flash", + custom_llm_provider, + {}, + ) + finally: + server.shutdown() + + assert isinstance(contents, VertexGeminiCountContextError) diff --git a/tests/integration/sdk/test_docker_real_io.py b/tests/integration/sdk/test_docker_real_io.py new file mode 100644 index 00000000000..d0d6d9bedf2 --- /dev/null +++ b/tests/integration/sdk/test_docker_real_io.py @@ -0,0 +1,50 @@ +import hashlib +import shutil +import subprocess + +import pytest + +from litellm import sandbox + +DOCKER_IMAGE = "alpine:3.20" + + +def _docker_usable() -> bool: + if shutil.which("docker") is None: + return False + try: + return ( + subprocess.run( + ["docker", "info"], capture_output=True, timeout=20 + ).returncode + == 0 + ) + except (OSError, subprocess.SubprocessError): + return False + + +@pytest.mark.skipif(not _docker_usable(), reason="docker daemon not available") +async def test_real_docker_roundtrip(): + box = sandbox.docker(DOCKER_IMAGE, workdir="/workspace") + try: + result = await box.run(["echo", "hello"], timeout=120) + assert result.stdout.strip() == "hello" + assert result.exit_code == 0 + + await box.write("seed.txt", b"seed") + await box.write("sub/out.txt", b"from host") + assert await box.read("sub/out.txt") == b"from host" + await box.write("node_modules/skip.js", b"x") + + assert await box.which("sh") is not None + assert await box.which("definitely-not-a-binary-xyz") is None + tmp = await box.tempdir() + assert tmp.startswith("/") + + snap = await box.snapshot() + assert snap == { + "seed.txt": hashlib.sha256(b"seed").hexdigest(), + "sub/out.txt": hashlib.sha256(b"from host").hexdigest(), + } + finally: + await box.close() diff --git a/tests/unit/test_drop_params_env_var.py b/tests/integration/sdk/test_drop_params_env_var_real_io.py similarity index 100% rename from tests/unit/test_drop_params_env_var.py rename to tests/integration/sdk/test_drop_params_env_var_real_io.py diff --git a/tests/unit/test_eager_tiktoken_load.py b/tests/integration/sdk/test_eager_tiktoken_load_real_io.py similarity index 100% rename from tests/unit/test_eager_tiktoken_load.py rename to tests/integration/sdk/test_eager_tiktoken_load_real_io.py diff --git a/tests/integration/sdk/test_http_handler_real_io.py b/tests/integration/sdk/test_http_handler_real_io.py new file mode 100644 index 00000000000..b7c0e9d748b --- /dev/null +++ b/tests/integration/sdk/test_http_handler_real_io.py @@ -0,0 +1,296 @@ +import asyncio +import importlib +import json +import os +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final +import httpx +import pytest + +import litellm +from litellm.exceptions import Timeout as LitellmTimeout +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.litellm_core_utils.model_cost_table import publish +from litellm.llms.base_llm.auth.types import TokenExchangeSpec +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + MaskedHTTPStatusError, + get_httpx_client, +) +from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome + + +async def _read_http_request(reader: asyncio.StreamReader) -> None: + raw = b"" + while b"\r\n\r\n" not in raw: + chunk = await reader.read(1024) + if not chunk: + return + raw += chunk + head, _, body = raw.partition(b"\r\n\r\n") + content_length = next( + (int(line.split(b":", 1)[1]) for line in head.split(b"\r\n") if line.lower().startswith(b"content-length")), + 0, + ) + while len(body) < content_length: + body += await reader.read(content_length - len(body)) + + +@pytest.mark.asyncio +async def test_init_held_async_handler_survives_external_client_close(): + handler = AsyncHTTPHandler(timeout=42.5) + held_client = handler.client + await held_client.aclose() + assert held_client.is_closed + + async def respond(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + await _read_http_request(reader) + writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + await writer.drain() + writer.close() + + server = await asyncio.start_server(respond, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + try: + response = await handler.post(f"http://127.0.0.1:{port}/v1/compress", json={"messages": []}) + finally: + server.close() + await server.wait_closed() + + assert response.status_code == 200 + assert handler.client is not held_client + assert handler.client.timeout == httpx.Timeout(42.5) + await handler.close() + + +@pytest.mark.asyncio +async def test_init_held_async_handler_survives_evicted_client_close(): + from litellm.caching.evicted_client_closer import EvictedClientCloser + from litellm.caching.llm_caching_handler import LLMClientCache + + cache = LLMClientCache(evicted_client_closer=EvictedClientCloser(grace_seconds=0)) + handler = AsyncHTTPHandler(timeout=42.5) + held_client = handler.client + cache.set_cache("init-held-handler", handler, litellm_owned_client=True, ttl=0) + await asyncio.sleep(0.02) + assert cache.get_cache("init-held-handler") is None + await asyncio.sleep(0.05) + assert held_client.is_closed + + async def respond(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + await _read_http_request(reader) + writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + await writer.drain() + writer.close() + + server = await asyncio.start_server(respond, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + try: + response = await handler.post(f"http://127.0.0.1:{port}/v1/compress", json={"messages": []}) + finally: + server.close() + await server.wait_closed() + + assert response.status_code == 200 + assert handler.client is not held_client + assert handler.client.timeout == httpx.Timeout(42.5) + await handler.close() + + +def test_init_held_sync_handler_recreates_closed_client(): + from http.server import BaseHTTPRequestHandler, HTTPServer + + class OkRequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Length", "2") + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, format, *args): + pass + + handler = HTTPHandler(timeout=7) + held_client = handler.client + held_client.close() + + server = HTTPServer(("127.0.0.1", 0), OkRequestHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + response = handler.get(f"http://127.0.0.1:{server.server_port}/") + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert response.status_code == 200 + assert handler.client is not held_client + assert handler.client.timeout == httpx.Timeout(7) + handler.close() + + +_WIF_SPEC: Final = TokenExchangeSpec( + token_url="https://api.anthropic.com/v1/oauth/token", + assertion_ref="oidc/env/ANTHROPIC_IDENTITY_TOKEN", + assertion_field="assertion", + static_body={"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", "federation_rule_id": "fdrl_1"}, + body_encoding="json", + request_headers={}, + cache_key_identity=("fdrl_1", "org-1", "", ""), + timeout_seconds=2.0, +) + + +@pytest.fixture() +def _vcr_outcome_gate(request, vcr): + install_live_call_probe(request, vcr) + yield + record_vcr_outcome(request, vcr) + + +@pytest.fixture(scope="function") +def isolate_litellm_state(): + """ + Per-function isolation fixture. + + Resets litellm globals to their true defaults before each test and + restores them afterward, so tests don't leak side effects. + Works safely under pytest-xdist parallel execution. + """ + original_state = {} + for attr in ( + "callbacks", + "success_callback", + "failure_callback", + "_async_success_callback", + "_async_failure_callback", + ): + if hasattr(litellm, attr): + val = getattr(litellm, attr) + original_state[attr] = val.copy() if val else [] + for attr in ("pre_call_rules", "post_call_rules"): + if hasattr(litellm, attr): + val = getattr(litellm, attr) + original_state[attr] = val.copy() if val else [] + for attr in _SCALAR_DEFAULTS: + if hasattr(litellm, attr): + original_state[attr] = getattr(litellm, attr) + if hasattr(litellm, "in_memory_llm_clients_cache"): + litellm.in_memory_llm_clients_cache.flush_cache() + for attr in ( + "callbacks", + "success_callback", + "failure_callback", + "_async_success_callback", + "_async_failure_callback", + "pre_call_rules", + "post_call_rules", + ): + if hasattr(litellm, attr): + setattr(litellm, attr, []) + for attr, default_val in _SCALAR_DEFAULTS.items(): + if hasattr(litellm, attr): + setattr(litellm, attr, default_val) + yield + asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue()) + if hasattr(litellm, "in_memory_llm_clients_cache"): + litellm.in_memory_llm_clients_cache.flush_cache() + for attr, original_value in original_state.items(): + if hasattr(litellm, attr): + setattr(litellm, attr, original_value) + publish(litellm.model_cost) + + +_SCALAR_DEFAULTS = { + "num_retries": getattr(litellm, "num_retries", None), + "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), + "request_timeout": getattr(litellm, "request_timeout", None), + "set_verbose": getattr(litellm, "set_verbose", False), + "cache": getattr(litellm, "cache", None), + "allowed_fails": getattr(litellm, "allowed_fails", 3), + "default_fallbacks": getattr(litellm, "default_fallbacks", None), + "enable_azure_ad_token_refresh": getattr(litellm, "enable_azure_ad_token_refresh", None), + "tag_budget_config": getattr(litellm, "tag_budget_config", None), + "model_cost": getattr(litellm, "model_cost", None), + "token_counter": getattr(litellm, "token_counter", None), + "disable_aiohttp_transport": getattr(litellm, "disable_aiohttp_transport", False), + "force_ipv4": getattr(litellm, "force_ipv4", False), + "drop_params": getattr(litellm, "drop_params", None), + "modify_params": getattr(litellm, "modify_params", False), + "api_base": getattr(litellm, "api_base", None), + "api_key": getattr(litellm, "api_key", None), +} + + +@pytest.fixture(scope="module") +def setup_and_teardown(): + """ + Module-scoped setup. Reloads litellm only in single-process mode + (skipped under xdist to avoid cross-worker interference). + """ + import litellm + + worker_id = os.environ.get("PYTEST_XDIST_WORKER", None) + if worker_id is None: + importlib.reload(litellm) + try: + if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): + import litellm.proxy.proxy_server + + importlib.reload(litellm.proxy.proxy_server) + except Exception: + pass + if hasattr(litellm, "in_memory_llm_clients_cache"): + litellm.in_memory_llm_clients_cache.flush_cache() + yield + + +_SERVER_DELAY_S = 5 + +_PER_REQUEST_TIMEOUT_S = 1.0 + +_CLIENT_DEFAULT_TIMEOUT_S = 60.0 + + +class _SlowHandler(BaseHTTPRequestHandler): + def do_POST(self): + time.sleep(_SERVER_DELAY_S) + try: + self.send_response(200) + self.end_headers() + self.wfile.write(b"{}") + except OSError: + pass + + def log_message(self, *args): + pass + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") +def test_post_delay_exceeds_per_request_timeout_raises(): + server = ThreadingHTTPServer(("127.0.0.1", 0), _SlowHandler) + threading.Thread(target=server.serve_forever, daemon=True).start() + host, port = server.server_address + + handler = get_httpx_client(params={"timeout": _CLIENT_DEFAULT_TIMEOUT_S}) + try: + with pytest.raises(LitellmTimeout): + handler.post( + f"http://{host}:{port}/delay", + headers={"content-type": "application/json"}, + data=json.dumps({"model": "claude", "messages": []}), + timeout=_PER_REQUEST_TIMEOUT_S, + ) + except MaskedHTTPStatusError as e: + pytest.skip(f"httpbin.org unavailable: {e}") + finally: + handler.close() + server.shutdown() + server.server_close() + + +_TIMEOUT_CLIENT_LIMITS: Final = httpx.Timeout(42.0, connect=7.0) diff --git a/tests/integration/sdk/test_jwt_signing_real_io.py b/tests/integration/sdk/test_jwt_signing_real_io.py new file mode 100644 index 00000000000..37c60fd55ff --- /dev/null +++ b/tests/integration/sdk/test_jwt_signing_real_io.py @@ -0,0 +1,37 @@ +import subprocess +import sys +import textwrap +from typing import Final + +from litellm.llms.base_llm.auth.jwt_signing import MISSING_SIGNING_DEPENDENCIES_MESSAGE + + +class TestBaseSdkImport: + """A base ``pip install litellm`` has neither PyJWT nor cryptography (both are proxy extras), + and ``litellm/__init__`` reaches this module through the Anthropic provider, so a + module-level import of either would break ``import litellm`` for every base SDK user.""" + + def test_module_imports_with_pyjwt_and_cryptography_absent(self): + script: Final = textwrap.dedent( + """ + import sys + + class Blocker: + def find_spec(self, name, path=None, target=None): + if name.split(".")[0] in {"jwt", "cryptography"}: + raise ModuleNotFoundError(f"No module named {name!r}") + + sys.meta_path.insert(0, Blocker()) + import litellm + from litellm.llms.base_llm.auth.jwt_signing import jwks_document_json + try: + jwks_document_json("not a key") + except ImportError as e: + print(e) + """ + ) + result: Final = subprocess.run( + [sys.executable, "-I", "-c", script], capture_output=True, text=True, check=False + ) + assert result.returncode == 0, result.stderr[-2000:] + assert result.stdout.strip() == MISSING_SIGNING_DEPENDENCIES_MESSAGE diff --git a/tests/integration/sdk/test_lazy_imports_real_io.py b/tests/integration/sdk/test_lazy_imports_real_io.py new file mode 100644 index 00000000000..6c1b54eb704 --- /dev/null +++ b/tests/integration/sdk/test_lazy_imports_real_io.py @@ -0,0 +1,23 @@ +import os +import subprocess +import sys +from typing import Final + +import litellm +import litellm.utils + + +def test_import_litellm_does_not_load_fastapi_or_bpe_table(): + result: Final = subprocess.run( + [ + sys.executable, + "-c", + "import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.proxy.proxy_cli','litellm.litellm_core_utils.default_encoding','openai.types.realtime') if m in sys.modules))", + ], + check=True, + capture_output=True, + text=True, + env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, + ) + + assert result.stdout.strip() == "" diff --git a/tests/integration/sdk/test_litellm_cli_real_io.py b/tests/integration/sdk/test_litellm_cli_real_io.py new file mode 100644 index 00000000000..9abf19ff9bd --- /dev/null +++ b/tests/integration/sdk/test_litellm_cli_real_io.py @@ -0,0 +1,384 @@ +import json +import os +import subprocess +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest + +import litellm +from litellm.cli.main import cli + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +_HELP_ARGV: Final = ( + ("--help",), + ("logout", "--help"), + ("login", "--help"), + ("models", "list", "--help"), + ("claude", "--help"), + ("opencode", "--help"), + ("codex", "--help"), + ("pi", "--help"), + ("keys", "list", "--help"), +) + +_COMMAND_ARGV: Final = (("logout",), ("whoami",)) + +_RUN_EACH_ARGV: Final = """ +import contextlib, io, json, sys +from {module} import cli +runs = {{}} +for argv in json.loads(sys.argv[1]): + sys.argv = ["lite", *argv] + output = io.StringIO() + with contextlib.redirect_stdout(output): + try: + cli() + except SystemExit as stopped: + code = stopped.code + else: + code = 0 + runs[" ".join(argv)] = {{"exit": code, "output": output.getvalue()}} +print(json.dumps({{"runs": runs, "sdk_initialized": "completion" in vars(sys.modules["litellm"])}})) +""" + +_USE_SDK_AFTER_HELP: Final = """ +import builtins, importlib, json, sys +interpreter_imports = (builtins.__import__, importlib.import_module) +from {module} import cli +sys.argv = ["lite", "--help"] +try: + cli() +except SystemExit: + pass +before = "completion" in vars(sys.modules["litellm"]) +{first_use} +from litellm.router import Router +import litellm +import litellm.responses.main +from litellm import acompletion +print(json.dumps({{ + "before": before, + "router": Router.__name__, + "completion": callable(litellm.completion), + "acompletion": acompletion is litellm.acompletion, + "responses_is_function": callable(litellm.responses) and not isinstance(litellm.responses, type(litellm)), + "namespace": sorted(vars(litellm)), + "module_type": type(litellm).__name__, + "imports_restored": (builtins.__import__, importlib.import_module) == interpreter_imports, +}})) +""" + +_FIRST_USE_FROM_THREADS: Final = """ +import json, sys, threading +from litellm_cli.main import cli +sys.argv = ["lite", "--help"] +try: + cli() +except SystemExit: + pass +import litellm +start = threading.Barrier(4) +seen = [] +def use_sdk(): + start.wait() + try: + seen.append(callable(litellm.completion)) + except AttributeError as error: + seen.append(str(error)) +threads = [threading.Thread(target=use_sdk) for _ in range(4)] +for thread in threads: + thread.start() +for thread in threads: + thread.join() +print(json.dumps({"seen": seen})) +""" + +_STAR_IMPORT_AFTER_HELP: Final = """ +import json, sys +from litellm_cli.main import cli +sys.argv = ["lite", "--help"] +try: + cli() +except SystemExit: + pass +before = "completion" in vars(sys.modules["litellm"]) +from litellm import * +print(json.dumps({"before": before, "completion": callable(globals().get("completion")), "acompletion": callable(globals().get("acompletion"))})) +""" + +_SDK_WHOSE_SUBMODULE_READS_IT_WHILE_LOADING: Final = { + "sdk_sync.py": """ +import threading +initializing = threading.Event() +submodule_loading = threading.Event() +""", + "litellm/__init__.py": """ +import sdk_sync +sdk_sync.initializing.set() +sdk_sync.submodule_loading.wait(2) +from litellm import submodule +ready = True +""", + "litellm/submodule.py": """ +import litellm +import sdk_sync +sdk_sync.submodule_loading.set() +ready_when_loaded = getattr(litellm, "ready", False) +""", + "litellm/_process_defaults.py": "def apply_process_defaults() -> None: ...\n", + "litellm/proxy/__init__.py": "", + "litellm/proxy/client/__init__.py": "", + "litellm/proxy/client/cli/__init__.py": "", + "litellm/proxy/client/cli/main.py": """ +import json, threading +import litellm +import sdk_sync + +def cli(): + def initialize_sdk(): + litellm.ready + + def import_submodule_once_initialization_starts(): + sdk_sync.initializing.wait() + import litellm.submodule + + threads = [threading.Thread(target=initialize_sdk, daemon=True), + threading.Thread(target=import_submodule_once_initialization_starts, daemon=True)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(10) + print(json.dumps({"finished": [not thread.is_alive() for thread in threads]})) + +litellm_proxy_cli = cli +""", +} + +_SDK_WHOSE_INIT_FAILS: Final = { + "litellm/__init__.py": 'raise ImportError("a dependency of the SDK is broken")\n', + "litellm/utils.py": "", + "litellm/_process_defaults.py": "def apply_process_defaults() -> None: ...\n", + "litellm/proxy/__init__.py": "", + "litellm/proxy/client/__init__.py": "", + "litellm/proxy/client/cli/__init__.py": "", + "litellm/proxy/client/cli/main.py": """ +import json +import litellm + +def error_from(use): + try: + use() + except Exception as error: + return f"{type(error).__name__}: {error}" + return None + +def import_sdk_code(): + import litellm.utils + +def cli(): + print(json.dumps([error_from(lambda: litellm.completion), error_from(lambda: litellm.completion), error_from(import_sdk_code)])) + +litellm_proxy_cli = cli +""", +} + +_RUN_LITE_AGAINST_SDK_AT: Final = """ +import sys +sys.path.insert(0, sys.argv[1]) +from litellm_cli.main import cli +cli() +""" + +_PYPROJECT: Final = Path(__file__).resolve().parents[3] / "pyproject.toml" + +_RUN_ENTRY_POINT_HELP: Final = """ +import contextlib, io, json, sys +from {module} import {function} +imported_sdk = "litellm" in sys.modules +sys.argv = ["litellm", "--help"] +output = io.StringIO() +with contextlib.redirect_stdout(output): + try: + {function}() + except SystemExit as stopped: + code = stopped.code + else: + code = 0 +print(json.dumps({{"imported_sdk": imported_sdk, "exit": code, "help": output.getvalue()}})) +""" + +_WORKER_LOGGERS_UNDER_A_DEFAULT_LOG_CONFIG: Final = """ +import json, logging, os +os.environ["LITELLM_WORKER_LOG_CONFIG"] = json.dumps({{"version": 1, "root": {{"level": "INFO"}}}}) +from litellm_cli.main import proxy_worker_app +proxy_worker_app() +print(json.dumps({{name: logging.getLogger(name).disabled for name in {names!r}}})) +""" + +_COMMAND_MODULES_LOADED_BY: Final = """ +import json, sys +from litellm_cli.main import cli +sys.argv = ["lite", *sys.argv[1:]] +try: + cli() +except SystemExit: + pass +prefix = "litellm.proxy.client.cli.commands." +print(json.dumps({"modules": sorted(name[len(prefix):] for name in sys.modules if name.startswith(prefix))})) +""" + + +def _run(source: str, argv: tuple[str, ...], home: Path) -> dict[str, object]: + checkout: Final = str(_PYPROJECT.parent) + bootstrap: Final = ( + "import pathlib,sys\n" + f"sys.path.insert(0, {checkout!r})\n" + f"sys.path.insert(0, str(pathlib.Path({checkout!r}) / '.github/packaging'))\n" + "from cli_package import editable_support\n" + f"exec(editable_support(pathlib.Path({checkout!r}) / 'litellm/cli')[0][1])\n" + ) + result: Final = subprocess.run( + [sys.executable, "-I", "-c", bootstrap + source, *argv], + capture_output=True, + text=True, + timeout=120, + env={**os.environ, "HOME": str(home), "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.rstrip("\n").rpartition("\n")[2]) + + +@pytest.fixture(scope="module") +def cli_runs(tmp_path_factory: pytest.TempPathFactory) -> Mapping[str, dict[str, object]]: + argv: Final = (json.dumps(_HELP_ARGV + _COMMAND_ARGV),) + return { + "thin": _run(_RUN_EACH_ARGV.format(module="litellm_cli.main"), argv, tmp_path_factory.mktemp("thin")), + "full": _run( + _RUN_EACH_ARGV.format(module="litellm.proxy.client.cli.main"), argv, tmp_path_factory.mktemp("full") + ), + } + + +def test_litellm_command_imports_the_proxy_only_when_it_runs(tmp_path: Path): + module, _, function = tomllib.loads(_PYPROJECT.read_text())["project"]["scripts"]["litellm"].partition(":") + run: Final = _run(_RUN_ENTRY_POINT_HELP.format(module=module, function=function), (), tmp_path) + assert run["imported_sdk"] is False + assert run["exit"] == 0 + assert "--num_workers" in str(run["help"]) + + +def test_a_worker_log_config_silences_the_sdk_loggers_but_not_the_ones_the_proxy_app_creates(tmp_path: Path): + disabled: Final = _run( + _WORKER_LOGGERS_UNDER_A_DEFAULT_LOG_CONFIG.format(names=("LiteLLM Proxy", "litellm.proxy.health_check")), + (), + tmp_path, + ) + assert disabled == {"LiteLLM Proxy": True, "litellm.proxy.health_check": False} + + +def test_lite_runs_commands_without_initializing_the_sdk(cli_runs: Mapping[str, dict[str, object]]): + assert cli_runs["thin"]["sdk_initialized"] is False + assert cli_runs["full"]["sdk_initialized"] is True + + +@pytest.mark.parametrize("argv", _HELP_ARGV + _COMMAND_ARGV, ids=" ".join) +def test_lite_prints_what_the_full_sdk_cli_prints(argv: tuple[str, ...], cli_runs: Mapping[str, dict[str, object]]): + thin: Final = cli_runs["thin"]["runs"] + full: Final = cli_runs["full"]["runs"] + assert isinstance(thin, dict) and isinstance(full, dict) + + assert thin[" ".join(argv)] == full[" ".join(argv)] + assert thin[" ".join(argv)]["output"] + + +@pytest.mark.parametrize( + ("argv", "loaded", "not_loaded"), + [ + (("logout", "--help"), {"auth"}, {"agents", "autoroute.commands", "models", "keys"}), + (("models", "list", "--help"), {"models"}, {"agents", "autoroute.commands", "keys"}), + (("claude", "--help"), {"agents"}, {"autoroute.commands", "models", "keys"}), + ], +) +def test_lite_imports_only_the_command_it_runs( + argv: tuple[str, ...], loaded: set[str], not_loaded: set[str], tmp_path: Path +): + modules: Final = _run(_COMMAND_MODULES_LOADED_BY, argv, tmp_path)["modules"] + assert isinstance(modules, list) + + assert loaded <= set(modules) + assert not not_loaded & set(modules) + + +@pytest.fixture(scope="module") +def sdk_used_after_full_cli(tmp_path_factory: pytest.TempPathFactory) -> dict[str, object]: + return _run( + _USE_SDK_AFTER_HELP.format(module="litellm.proxy.client.cli.main", first_use="pass"), + (), + tmp_path_factory.mktemp("full"), + ) + + +@pytest.mark.parametrize( + "first_use", + [ + "import litellm; litellm.completion", + "import litellm; litellm.types.utils.ModelResponse", + "from litellm import acompletion", + "from litellm.router import Router", + "from litellm.types import utils", + "import litellm.responses.main", + 'importlib.import_module("litellm.router")', + 'importlib.import_module(".utils", "litellm")', + 'exec("from .types import utils", {"__name__": "litellm.lazy_import", "__package__": "litellm"})', + ], +) +def test_sdk_initializes_in_place_on_first_use_after_lite_imports( + first_use: str, sdk_used_after_full_cli: dict[str, object], tmp_path: Path +): + thin: Final = _run(_USE_SDK_AFTER_HELP.format(module="litellm_cli.main", first_use=first_use), (), tmp_path) + + assert thin["before"] is False + assert sdk_used_after_full_cli["before"] is True + assert {**thin, "before": True} == sdk_used_after_full_cli + assert (thin["router"], thin["completion"], thin["acompletion"], thin["responses_is_function"]) == ( + "Router", + True, + True, + True, + ) + + +def test_star_import_after_lite_imports_gets_the_whole_sdk(tmp_path: Path): + assert _run(_STAR_IMPORT_AFTER_HELP, (), tmp_path) == {"before": False, "completion": True, "acompletion": True} + + +def test_threads_that_use_the_sdk_at_once_all_wait_for_one_initialization(tmp_path: Path): + used: Final = _run(_FIRST_USE_FROM_THREADS, (), tmp_path) + + assert used == {"seen": [True, True, True, True]} + + +def _write_sdk(root: Path, files: Mapping[str, str]) -> Path: + for relative_path, source in files.items(): + (root / relative_path).parent.mkdir(parents=True, exist_ok=True) + (root / relative_path).write_text(source) + return root + + +def test_importing_sdk_code_while_another_thread_initializes_the_sdk_does_not_hang(tmp_path: Path): + sdk_root: Final = _write_sdk(tmp_path / "sdk", _SDK_WHOSE_SUBMODULE_READS_IT_WHILE_LOADING) + + assert _run(_RUN_LITE_AGAINST_SDK_AT, (str(sdk_root),), tmp_path) == {"finished": [True, True]} + + +def test_every_use_after_a_failed_sdk_init_raises_the_init_error(tmp_path: Path): + sdk_root: Final = _write_sdk(tmp_path / "sdk", _SDK_WHOSE_INIT_FAILS) + + assert _run(_RUN_LITE_AGAINST_SDK_AT, (str(sdk_root),), tmp_path) == ["ImportError: a dependency of the SDK is broken"] * 3 diff --git a/tests/integration/sdk/test_long_decode_real_io.py b/tests/integration/sdk/test_long_decode_real_io.py new file mode 100644 index 00000000000..13147860cd2 --- /dev/null +++ b/tests/integration/sdk/test_long_decode_real_io.py @@ -0,0 +1,323 @@ +import json +import threading +from collections.abc import Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from itertools import count +from typing import Final + +import httpx +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +MESSAGES: Final = ({"role": "user", "content": "Prove it"},) + +HOP_1: Final = { + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": "hello"}]}, + "finishReason": "CONTINUATION", + "continuationToken": "dG9rLTE=", + } + ], + "usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 30, "totalTokenCount": 40}, +} +HOP_2: Final = { + "candidates": [{"content": {"role": "model", "parts": [{"text": " world"}]}, "finishReason": "STOP"}], + "usageMetadata": { + "promptTokenCount": 10, + "cachedContentTokenCount": 9, + "candidatesTokenCount": 20, + "totalTokenCount": 30, + }, +} +STREAM_HOP_1_END: Final = { + "candidates": [{**HOP_1["candidates"][0], "content": {"role": "model", "parts": [{"text": "lo"}]}}], + "usageMetadata": HOP_1["usageMetadata"], +} + + +def _replay(hops: Sequence[httpx.Response], sent: list[dict[str, object]]) -> Callable[[httpx.Request], httpx.Response]: + def handler(request: httpx.Request) -> httpx.Response: + sent.append(json.loads(request.content)) + return hops[len(sent) - 1] + + return handler + + +def _sse(*chunks: Mapping[str, object]) -> httpx.Response: + return httpx.Response( + 200, + content="".join(f"data: {json.dumps(chunk)}\r\n\r\n" for chunk in chunks).encode(), + headers={"content-type": "text/event-stream"}, + ) + + +NO_CITATIONS_OR_URL_CONTEXT: Final = ( + pytest.param({}, id="missing"), + pytest.param({"citationMetadata": None, "urlContextMetadata": None}, id="null"), + pytest.param({"citationMetadata": {}, "urlContextMetadata": {}}, id="empty"), +) + + +STREAM_FIRST_CHUNK: Final = {"candidates": [{"content": {"role": "model", "parts": [{"text": "hel"}]}}]} + + +def _stream_hops() -> tuple[httpx.Response, httpx.Response]: + return (_sse(STREAM_FIRST_CHUNK, STREAM_HOP_1_END), _sse(HOP_2)) + + +def _stream_text(chunks: list) -> str: + return "".join(c.choices[0].delta.content or "" for c in chunks if c.choices) + + +def _stream_finish_reasons(chunks: list) -> list[str]: + return [c.choices[0].finish_reason for c in chunks if c.choices and c.choices[0].finish_reason] + + +def _delta_fields(chunks: list) -> list[Mapping[str, object]]: + return [ + choice.delta.provider_specific_fields + for chunk in chunks + for choice in chunk.choices # comprehension-ok: flattening stream chunks into their choices + if choice.delta.provider_specific_fields + ] + + +def _client_tokens(chunks: list) -> list[object]: + return [fields["continuation_token"] for fields in _delta_fields(chunks) if "continuation_token" in fields] + + +@dataclass(frozen=True, slots=True) +class _TokenEndpoint: + credentials: str + down: threading.Event + + +def _service_account_json(token_uri: str) -> str: + private_key: Final = ( + rsa.generate_private_key(public_exponent=65537, key_size=2048) + .private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + .decode() + ) + return json.dumps( + { + "type": "service_account", + "project_id": "p", + "private_key_id": "k", + "private_key": private_key, + "client_email": "sa@p.iam.gserviceaccount.com", + "token_uri": token_uri, + } + ) + + +@pytest.fixture +def token_endpoint() -> Iterator[_TokenEndpoint]: + down: Final = threading.Event() + minted: Final = count(1) + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + self.rfile.read(int(self.headers["Content-Length"])) + refused: Final = down.is_set() + payload: Final = json.dumps( + {"error": "invalid_grant"} if refused else {"access_token": f"token-{next(minted)}", "expires_in": 0} + ).encode() + self.send_response(400 if refused else 200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_args: object) -> None: + return None + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + yield _TokenEndpoint(_service_account_json(f"http://127.0.0.1:{server.server_port}/token"), down) + server.shutdown() + server.server_close() + + +def _google_taking_each_access_token_once( + hops: Sequence[httpx.Response], used: list[str], on_answer: Callable[[], object] +) -> Callable[[httpx.Request], httpx.Response]: + def handler(request: httpx.Request) -> httpx.Response: + token: Final = request.headers["authorization"] + if token in used: + return httpx.Response(401, json={"error": {"code": 401, "status": "UNAUTHENTICATED"}}) + used.append(token) + on_answer() + return hops[len(used) - 1] + + return handler + + +async def _vertex_answer( + is_async: bool, + stream: bool, + handler: Callable[[httpx.Request], httpx.Response], + credentials: str, + extra_headers: Mapping[str, str] | None = None, +) -> tuple[str, list[str], list[object]]: + kwargs: Final = { + "model": "vertex_ai/gemini-3.8-flash", + "extra_headers": extra_headers, + "messages": list(MESSAGES), + "max_tokens": 100_000, + "stream": stream, + "vertex_project": "p", + "vertex_location": "global", + "vertex_credentials": credentials, + } + transport: Final = httpx.MockTransport(handler) + response: Final = ( + await litellm.acompletion(**kwargs, client=AsyncHTTPHandler(transport=transport)) + if is_async + else litellm.completion(**kwargs, client=HTTPHandler(client=httpx.Client(transport=transport))) + ) + if stream: + chunks: Final = [chunk async for chunk in response] if is_async else list(response) + return _stream_text(chunks), _stream_finish_reasons(chunks), _client_tokens(chunks) + choice: Final = response.choices[0] + fields: Final = choice.message.provider_specific_fields or {} + return choice.message.content, [choice.finish_reason], [fields["continuation_token"]] if fields else [] + + +def _two_hops(stream: bool) -> tuple[httpx.Response, httpx.Response]: + return _stream_hops() if stream else (httpx.Response(200, json=HOP_1), httpx.Response(200, json=HOP_2)) + + +@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.parametrize("is_async", [True, False]) +@pytest.mark.asyncio +async def test_vertex_long_decode_finishes_when_each_continuation_outlives_the_access_token_before_it( + is_async: bool, stream: bool, token_endpoint: _TokenEndpoint +): + used: Final[list[str]] = [] + google: Final = _google_taking_each_access_token_once(_two_hops(stream), used, lambda: None) + + answer: Final = await _vertex_answer(is_async, stream, google, token_endpoint.credentials) + + assert answer == ("hello world", ["stop"], []) + assert len(used) == 2 + + +@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.parametrize("is_async", [True, False]) +@pytest.mark.asyncio +async def test_vertex_long_decode_returns_the_answer_so_far_with_its_token_when_no_new_access_token_can_be_had( + is_async: bool, stream: bool, token_endpoint: _TokenEndpoint +): + used: Final[list[str]] = [] + google: Final = _google_taking_each_access_token_once(_two_hops(stream), used, token_endpoint.down.set) + + answer: Final = await _vertex_answer(is_async, stream, google, token_endpoint.credentials) + + assert answer == ("hello", ["length"], ["dG9rLTE="]) + assert len(used) == 1 + + +@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.parametrize("is_async", [True, False]) +@pytest.mark.asyncio +async def test_vertex_long_decode_keeps_the_callers_own_authorization_header_on_every_continuation( + is_async: bool, stream: bool, token_endpoint: _TokenEndpoint +): + hops: Final = _two_hops(stream) + seen: Final[list[str]] = [] + + def google(request: httpx.Request) -> httpx.Response: + seen.append(request.headers["authorization"]) + return hops[len(seen) - 1] + + answer: Final = await _vertex_answer( + is_async, stream, google, token_endpoint.credentials, {"Authorization": "Bearer mine"} + ) + + assert answer == ("hello world", ["stop"], []) + assert seen == ["Bearer mine", "Bearer mine"] + + +BUDGET_ENV: Final = "LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS" +ARGON: Final = "gemini/gemini-4-argon" +OUTPUT_LIMIT_KEYS: Final = ("max_output_tokens", "maxOutputTokens") + + +def _output_limits(request: Mapping[str, object]) -> dict[str, object]: + generation_config: Final = request.get("generationConfig", {}) + assert isinstance(generation_config, dict) + return {key: value for key, value in generation_config.items() if key in OUTPUT_LIMIT_KEYS} + + +def _cost_map_max_output_tokens(model: str) -> int: + provider, name = model.split("/", 1) + max_output_tokens: Final = litellm.get_model_info(model=name, custom_llm_provider=provider)["max_output_tokens"] + assert isinstance(max_output_tokens, int) + return max_output_tokens + + +def _gemini_chat_models_answering_no_text() -> tuple[str, ...]: + rows: Final = ( + (name, row) + for name, row in litellm.model_cost.items() + if row.get("mode") == "chat" and "text" not in row.get("supported_output_modalities", ("text",)) + ) + return tuple( + name if name.startswith("gemini/") else f"vertex_ai/{name}" + for name, row in rows + if row.get("litellm_provider") in ("gemini", "vertex_ai-language-models") + ) + + +@pytest.mark.asyncio +async def test_env_budget_skips_every_chat_model_the_cost_map_lists_as_answering_no_text( + monkeypatch: pytest.MonkeyPatch, token_endpoint: _TokenEndpoint +): + monkeypatch.setenv(BUDGET_ENV, "50") + models: Final = _gemini_chat_models_answering_no_text() + sent: Final[list[dict[str, object]]] = [] + replies: Final = tuple(httpx.Response(200, json=HOP_2) for _ in models) + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_replay(replies, sent))) + + for model in models: + await litellm.acompletion( + model=model, + messages=list(MESSAGES), + api_key="fake-gemini-key", + vertex_project="p", + vertex_location="us-central1", + vertex_credentials=token_endpoint.credentials, + client=client, + ) + + assert models + assert {model: _output_limits(request) for model, request in zip(models, sent, strict=True)} == dict.fromkeys( + models, {} + ) + + +@pytest.mark.parametrize("model", ["gemini/gemini-2.5-flash", "vertex_ai/gemini-2.5-flash", ARGON]) +@pytest.mark.asyncio +async def test_env_budget_above_a_models_output_limit_sends_that_limit_instead( + monkeypatch: pytest.MonkeyPatch, token_endpoint: _TokenEndpoint, model: str +): + budget: Final = 2 * _cost_map_max_output_tokens(model) + monkeypatch.setenv(BUDGET_ENV, str(budget)) + sent: Final[list[dict[str, object]]] = [] + + await litellm.acompletion( + model=model, + messages=list(MESSAGES), + api_key="fake-gemini-key", + vertex_project="p", + vertex_location="us-central1", + vertex_credentials=token_endpoint.credentials, + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), + ) + + assert _output_limits(sent[0]) == {"max_output_tokens": _cost_map_max_output_tokens(model)} diff --git a/tests/integration/sdk/test_router_fuzz_regressions_rust_only_real_io.py b/tests/integration/sdk/test_router_fuzz_regressions_rust_only_real_io.py new file mode 100644 index 00000000000..13548b34fd3 --- /dev/null +++ b/tests/integration/sdk/test_router_fuzz_regressions_rust_only_real_io.py @@ -0,0 +1,193 @@ +""" +Router behavior the router-code differential fuzz run found changed once the Rust core became the +only core: count settings below zero, an allowed_fails_policy write that is no policy, a model +response object as a deployment's mock_response, and endpoints read or patched on the Router class. +""" + +import asyncio +import json +import socket +import threading +from collections.abc import Iterator, Mapping +from datetime import datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final + +import pytest + +import litellm +from litellm import Router + +MESSAGES: Final = [{"role": "user", "content": "hi"}] + + +class _Upstream(BaseHTTPRequestHandler): + """An OpenAI upstream whose path's first segment names how it answers: ``cut`` sends two + Responses stream events then cuts the chunked body, ``filtered`` answers a chat completion the + provider blocked, ``down`` answers 502, and anything else a plain chat completion.""" + + protocol_version = "HTTP/1.1" + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 # the base class's name + del format, args + + def _json(self, status: int, body: Mapping[str, object]) -> None: + data: Final = json.dumps(body).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + _ = self.wfile.write(data) + + def _chunk(self, event: Mapping[str, object]) -> None: + data: Final = f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() + _ = self.wfile.write(f"{len(data):x}\r\n".encode() + data + b"\r\n") + self.wfile.flush() + + def _cut(self) -> None: + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("transfer-encoding", "chunked") + self.end_headers() + response: Final = {"id": "r1", "object": "response", "created_at": 1, "status": "in_progress", "output": []} + self._chunk({"type": "response.created", "sequence_number": 0, "response": {**response, "model": "gpt-4o"}}) + self._chunk( + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "m1", + "output_index": 0, + "content_index": 0, + "delta": "hello ", + } + ) + self.connection.shutdown(socket.SHUT_RDWR) + self.close_connection = True + + def do_POST(self) -> None: + _ = self.rfile.read(int(self.headers.get("content-length") or 0)) + mode: Final = self.path.split("/")[1] + if mode == "cut": + self._cut() + return + if mode == "down": + self._json(502, {"error": {"message": "provider error", "type": "api_error"}}) + return + finish: Final = "content_filter" if mode == "filtered" else "stop" + message: Final = {"role": "assistant", "content": mode} + choice: Final = {"index": 0, "finish_reason": finish, "message": message} + usage: Final = {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + self._json(200, {"id": "c1", "object": "chat.completion", "created": 1, "choices": [choice], "usage": usage}) + + +@pytest.fixture +def upstream() -> Iterator[str]: + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), _Upstream) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{server.server_address[1]}" + server.shutdown() + thread.join() + + +def _served(name: str, deployment_id: str, model: str, api_base: str) -> dict[str, object]: + return { + "model_name": name, + "litellm_params": {"model": model, "api_key": "k", "api_base": api_base}, + "model_info": {"id": deployment_id}, + } + + +def _request_logging(call_type: str, *, stream: bool) -> object: + from litellm.litellm_core_utils.litellm_logging import Logging + + return Logging( + model="g", + messages=MESSAGES, + stream=stream, + call_type=call_type, + start_time=datetime.now(), # noqa: DTZ005 # litellm's own naive start time + litellm_call_id="call-1", + function_id="f", + ) + + +async def _recorded() -> None: + """Let the background recordings of a stream's outcome run.""" + for _ in range(50): + await asyncio.sleep(0.02) + + +@pytest.mark.asyncio +async def test_a_responses_stream_cut_after_it_opened_counts_one_fail_with_a_shared_logging_object( + upstream: str, +) -> None: + router: Final = Router( + model_list=[_served("g", "d1", "openai/gpt-4o", f"{upstream}/cut/v1")], num_retries=0, disable_cooldowns=True + ) + + stream: Final = await router.aresponses( + model="g", input="hi", stream=True, litellm_logging_obj=_request_logging("aresponses", stream=True) + ) + with pytest.raises(Exception): # noqa: B017, PT011 # the transport's read error, whatever class maps it + async for _ in stream: # pyright: ignore[reportGeneralTypeIssues] # a stream when stream=True + pass + await _recorded() + + assert router.health().store.get_cache(key="d1:fails") == 1 + + +@pytest.mark.asyncio +async def test_a_blocked_response_raised_for_a_fallback_counts_no_success_with_a_shared_logging_object( + upstream: str, +) -> None: + router: Final = Router( + model_list=[ + _served("a", "a1", "openai/gpt-4o", f"{upstream}/filtered/v1"), + _served("safe", "s1", "openai/gpt-4o", f"{upstream}/ok/v1"), + ], + content_policy_fallbacks=[{"a": ["safe"]}], + num_retries=0, + disable_cooldowns=True, + ) + + response: Final = await router.acompletion( + model="a", messages=MESSAGES, litellm_logging_obj=_request_logging("acompletion", stream=False) + ) + await _recorded() + + assert response.choices[0].message.content == "ok" + assert router.health().store.get_cache(key="a1:successes") is None + assert router.health().store.get_cache(key="s1:successes") == 1 + + +@pytest.mark.asyncio +async def test_a_text_completion_stream_that_fails_before_its_first_chunk_counts_nothing(upstream: str) -> None: + router: Final = Router( + model_list=[_served("g", "d1", "openai/gpt-3.5-turbo-instruct", f"{upstream}/down/v1")], + num_retries=0, + disable_cooldowns=True, + ) + + stream: Final = await router.atext_completion(model="g", prompt="hi", stream=True) + with pytest.raises(litellm.BadGatewayError): + async for _ in stream: # pyright: ignore[reportGeneralTypeIssues] # a stream when stream=True + pass + await _recorded() + + assert router.health().store.get_cache(key="d1:successes") is None + assert router.health().store.get_cache(key="d1:fails") is None + + +@pytest.mark.asyncio +async def test_a_request_refused_before_the_provider_counts_against_no_deployment(upstream: str) -> None: + router: Final = Router( + model_list=[_served("g", "d1", "openai/gpt-4o", f"{upstream}/ok/v1")], num_retries=0, disable_cooldowns=True + ) + + with pytest.raises(litellm.UnsupportedParamsError): + await router.acompletion(model="g", messages=MESSAGES, reasoning_effort="low") + await _recorded() + + assert router.health().store.get_cache(key="d1:fails") is None + assert not [key for key in router.health().store.in_memory_cache.cache_dict if ":rpm" in key] diff --git a/tests/integration/sdk/test_router_host_real_io.py b/tests/integration/sdk/test_router_host_real_io.py new file mode 100644 index 00000000000..6ad80a90aa1 --- /dev/null +++ b/tests/integration/sdk/test_router_host_real_io.py @@ -0,0 +1,28 @@ +import subprocess +import sys + +import pytest + +import litellm.router_host as router_host_module +from litellm.router_host import router_host + + +def test_router_uses_the_proxy_host_without_importing_the_proxy_first(): + # A fresh interpreter, so no earlier test has imported litellm.proxy. + script = ( + "import sys\n" + "from litellm.router import Router\n" + "from litellm.router_host import router_host\n" + "assert 'litellm.proxy' not in sys.modules, 'importing the router imported the proxy'\n" + "print(type(router_host()).__name__)\n" + ) + result = subprocess.run([sys.executable, "-P", "-c", script], capture_output=True, text=True, check=True) + assert result.stdout.strip() == "ProxyRouterHost" + + +@pytest.fixture(autouse=True) +def nothing_installed(): + saved = router_host_module._installed[0] + router_host_module._installed[0] = None + yield + router_host_module._installed[0] = saved diff --git a/tests/integration/sdk/test_streaming_handler_real_io.py b/tests/integration/sdk/test_streaming_handler_real_io.py new file mode 100644 index 00000000000..5acd6597d99 --- /dev/null +++ b/tests/integration/sdk/test_streaming_handler_real_io.py @@ -0,0 +1,367 @@ +import itertools +import json +import os +import subprocess +import sys +from collections.abc import Mapping +from pathlib import Path +from types import MappingProxyType +from typing import Final +import pytest +from pydantic import TypeAdapter +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage +from litellm.utils import ModelResponseListIterator +from litellm import completion + + +bedrock_chunks = [ + ModelResponseStream( + id="chatcmpl-d249def8-a78b-464c-87b5-3a6f43565292", + created=1742056047, + model=None, + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + provider_specific_fields=None, + content="I'm Claude", + role="assistant", + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields={}, + usage=None, + ), + ModelResponseStream( + id="chatcmpl-fe559823-b383-4249-ab87-52f6ad9d08c2", + created=1742056047, + model=None, + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + provider_specific_fields=None, + content=", an AI", + role="assistant", + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields={}, + usage=None, + ), + ModelResponseStream( + id="chatcmpl-c1c6cc2f-75b9-4a24-88b9-4e5aacd0268b", + created=1742056047, + model=None, + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta( + provider_specific_fields=None, + content="", + role="assistant", + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields={}, + usage=None, + ), +] + + +FINISH_ONLY_LOGPROBS_STREAM_SCRIPT: Final = """ +import json +import sys + +sys.path.insert(0, sys.argv[1]) + +import httpx +from openai import OpenAI + +import litellm + +upstream_choices = ( + {"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None}, + { + "index": 0, + "delta": {}, + "logprobs": {"content": [{"token": "<|return|>", "logprob": -0.5, "top_logprobs": []}]}, + "finish_reason": "stop", + }, +) +body = "".join( + "data: " + + json.dumps({"id": "c", "object": "chat.completion.chunk", "created": 1, "model": "m", "choices": [choice]}) + + "\\n\\n" + for choice in upstream_choices +) + "data: [DONE]\\n\\n" +client = OpenAI( + api_key="unused", + base_url="https://provider.example/v1", + http_client=httpx.Client( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"}) + ) + ), +) +stream = litellm.completion( + model="openai/m", messages=[{"role": "user", "content": "hi"}], stream=True, logprobs=True, client=client +) +print(json.dumps([json.loads(chunk.model_dump_json(exclude_none=True)) for chunk in stream])) +""" + + +def test_stream_with_logprobs_only_on_finish_chunk_serializes_in_a_fresh_process() -> None: + result: Final = subprocess.run( + [sys.executable, "-I", "-c", FINISH_ONLY_LOGPROBS_STREAM_SCRIPT, str(Path(litellm.__file__).parent.parent)], + capture_output=True, + text=True, + timeout=120, + env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, + ) + + assert result.returncode == 0, result.stdout + result.stderr + wire_choices: Final = tuple( + itertools.chain.from_iterable(chunk["choices"] for chunk in json.loads(result.stdout.strip().splitlines()[-1])) + ) + finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")] + assert [choice["finish_reason"] for choice in finish_choices] == ["stop"] + assert [entry["token"] for entry in finish_choices[0]["logprobs"]["content"]] == ["<|return|>"] + + +_REPETITION_TEST_CASES = [ + # Basic cases + pytest.param( + ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + True, + id="all_identical_raises", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 1), + False, + id="below_threshold_no_raise", + ), + pytest.param( + [None] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="none_content_no_raise", + ), + pytest.param( + [""] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="empty_content_no_raise", + ), + # Short content (len <= 2) should not raise + pytest.param( + ["##"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="short_content_2chars_no_raise", + ), + pytest.param( + ["{"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="short_content_1char_no_raise", + ), + pytest.param( + ["ab"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="short_content_2chars_ab_no_raise", + ), + # All different chunks + pytest.param( + ["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="all_different_no_raise", + ), + # One chunk different at various positions + pytest.param( + ["different_first"] + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 1), + False, + id="first_chunk_different_no_raise", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 1) + ["different_last"], + False, + id="last_chunk_different_no_raise", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1) + + ["different_mid"] + + ["same"] + * ( + litellm.REPEATED_STREAMING_CHUNK_LIMIT + - litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + + 1 + ), + False, + id="middle_chunk_different_no_raise", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 2) + ["diff", "diff"], + False, + id="last_two_different_no_raise", + ), + pytest.param( + ["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + + ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + + ["diff"], + True, + id="in_between_same_and_diff_raise", + ), +] + + +# Azure streaming chunks that reproduce issue #24221: +# Azure sends an initial chunk with prompt_filter_results and choices=[], +# then a chunk with role='assistant' and content='', then content chunks. +# With stream_options.include_usage=True, the empty-choices chunk was +# forwarded with an inflated default choice, consuming the sent_first_chunk +# flag and causing strip_role_from_delta to strip the role from the real +# first chunk. +_AZURE_CHUNKS_WITH_PROMPT_FILTER = [ + # Chunk 1: prompt_filter_results, no choices (Azure-specific) + ModelResponseStream( + id="chatcmpl-abc123", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[], + usage=None, + ), + # Chunk 2: first real chunk with role='assistant' and empty content + ModelResponseStream( + id="chatcmpl-abc123", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="", role="assistant"), + ) + ], + usage=None, + ), + # Chunk 3: content + ModelResponseStream( + id="chatcmpl-abc123", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="Hello!"), + ) + ], + usage=None, + ), + # Chunk 4: finish_reason + ModelResponseStream( + id="chatcmpl-abc123", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ], + usage=None, + ), + # Chunk 5: final usage chunk, no choices + ModelResponseStream( + id="chatcmpl-abc123", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[], + usage=Usage( + completion_tokens=10, + prompt_tokens=20, + total_tokens=30, + ), + ), +] + + +_AZURE_CHOICE_EXTRAS: Final = MappingProxyType( + {"content_filter_results": {"hate": {"filtered": False, "severity": "safe"}}} +) +_OPENAI_AND_AZURE_CHOICE_EXTRAS: Final = ( + pytest.param("openai", MappingProxyType({}), id="openai"), + pytest.param("azure", _AZURE_CHOICE_EXTRAS, id="azure"), +) + + +_TOKEN_IDS: Final = TypeAdapter(list[int] | None) +_CHOICE_INDEX: Final = TypeAdapter(int) +_DELTA: Final = TypeAdapter(Mapping[str, object]) + + +def _streaming_wrapper_for_unit_test( + model: str, + custom_llm_provider: str, + chunks: list[ModelResponseStream], +) -> CustomStreamWrapper: + return CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=chunks), + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=Logging( + model=model, + messages=[{"role": "user", "content": "unit test"}], + stream=True, + call_type="completion", + start_time=0.0, + litellm_call_id="unit-test-call", + function_id="unit-test-function", + ), + ) + + +def _unit_test_streaming_chunk( + content: str | None, + finish_reason: str | None = None, + role: str | None = "assistant", +) -> ModelResponseStream: + return ModelResponseStream( + id="unit-test-stream", + created=1, + model="unit-test-model", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content=content, role=role), + finish_reason=finish_reason, + ) + ], + ) diff --git a/tests/integration/sdk/test_token_counter_real_io.py b/tests/integration/sdk/test_token_counter_real_io.py new file mode 100644 index 00000000000..fd58cb29d24 --- /dev/null +++ b/tests/integration/sdk/test_token_counter_real_io.py @@ -0,0 +1,207 @@ +#### What this tests #### +# This tests litellm.token_counter.token_counter() function +import json +import os +import subprocess +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Final +import tiktoken +from tokenizers import Regex, Tokenizer, models, pre_tokenizers + +from unittest.mock import AsyncMock + +import litellm +from litellm import decode, encode +import litellm.constants +from litellm.types.utils import SelectTokenizerResponse + + +## Choose which token_counter the test will use. + + +# test_token_counter_normal_plus_function_calling() + + +# test_tokenizers() + + +def test_encoding_and_decoding(tmp_path: Path): + sample_text = "Hellö World, this is my input string!" + + # openai encoding + decoding + openai_tokens = encode(model="gpt-3.5-turbo", text=sample_text) + openai_text = decode(model="gpt-3.5-turbo", tokens=openai_tokens) + + assert openai_text == sample_text + + # claude encoding + decoding + claude_tokens = encode(model="claude-3-5-haiku-20241022", text=sample_text) + + claude_text = decode(model="claude-3-5-haiku-20241022", tokens=claude_tokens) + + assert claude_text == sample_text + + # cohere encoding + decoding + cohere_tokens = encode(model="command-nightly", text=sample_text) + cohere_text = decode(model="command-nightly", tokens=cohere_tokens) + + assert cohere_text == sample_text + + # llama2 encoding + decoding + words = sample_text.split() + result = _run_in_memory_hub( + HUB_ROUND_TRIP_SCRIPT, + { + "hf-internal-testing/llama-tokenizer": _word_level_tokenizer_json( + pre_tokenizers.WhitespaceSplit(), + vocab={"[UNK]": 0, **{word: i + 1 for i, word in enumerate(words)}}, + ) + }, + sample_text, + tmp_path, + ) + + assert result["decoded"] == sample_text + assert result["requested"] == ["hf-internal-testing/llama-tokenizer"] + assert len(result["tokens"]) == len(words) + assert len(result["tokens"]) != len(encode(model="gpt-3.5-turbo", text=sample_text)) + + +# test_encoding_and_decoding() + + +# test_gpt_vision_token_counting() + +from litellm.utils import _HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES, claude_json_str + +# Clear the cache at module load to ensure clean state +_HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES.flush_cache() + + +HUB_SETUP_SCRIPT: Final = """ +import json +import sys +sys.path.insert(0, sys.argv[1]) +import httpx +import huggingface_hub +import litellm +served = json.loads(sys.argv[2]) +text = sys.argv[3] +requested = [] +def handle(request): + repo = request.url.path.lstrip("/").split("/resolve/")[0] + if repo not in served or not request.url.path.endswith("/tokenizer.json"): + return httpx.Response(404) + requested.append(repo) + payload = served[repo].encode() + headers = {"content-length": str(len(payload)), "etag": '"fixture"', "x-repo-commit": "a" * 40} + return httpx.Response(200, headers=headers, content=payload if request.method == "GET" else b"") +huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) +""" + +HUB_TOKENIZER_SCRIPT: Final = HUB_SETUP_SCRIPT + """ +litellm.cohere_models = {"command-r-v1"} +litellm.anthropic_models = {"claude-2"} +custom = litellm.create_pretrained_tokenizer("Xenova/llama-3-tokenizer") +print(json.dumps({ + "llama2": litellm.token_counter(model="meta-llama/Llama-2-7b-chat", text=text), + "llama3": litellm.token_counter(model="meta-llama/llama-3-70b-instruct", text=text), + "cohere": litellm.token_counter(model="command-r-v1", text=text), + "anthropic": litellm.token_counter(model="claude-2", text=text), + "custom": litellm.token_counter(custom_tokenizer=custom, text=text), + "requested": sorted(set(requested)), +})) +""" + +HUB_ROUND_TRIP_SCRIPT: Final = HUB_SETUP_SCRIPT + """ +tokens = litellm.encode(model="meta-llama/Llama-2-7b-chat", text=text) +print(json.dumps({"tokens": tokens, "decoded": litellm.decode(model="meta-llama/Llama-2-7b-chat", tokens=tokens), "requested": sorted(set(requested))})) +""" + +HUB_RETRY_DURING_EXIT_SCRIPT: Final = HUB_SETUP_SCRIPT + """ +import threading +import time +from litellm.utils import _HuggingFaceTokenizerLoadOutcomes, _return_huggingface_tokenizer +retry_reached_hub = threading.Event() +def answer_once_the_process_is_exiting(request): + retry_reached_hub.set() + while threading.main_thread().is_alive(): + time.sleep(0.01) + time.sleep(0.5) + return handle(request) +now = [0.0] +outcomes = _HuggingFaceTokenizerLoadOutcomes(clock=lambda: now[0]) +model = "meta-llama/llama-3-70b-instruct" +_return_huggingface_tokenizer(model, outcomes) +huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(answer_once_the_process_is_exiting))) +now[0] = 3600.0 +_return_huggingface_tokenizer(model, outcomes) +print(json.dumps({"retry_reached_hub": retry_reached_hub.wait(timeout=60)})) +""" + + +def _word_level_tokenizer_json( + pre_tokenizer: pre_tokenizers.PreTokenizer, vocab: Mapping[str, int] | None = None +) -> str: + tokenizer: Final = Tokenizer( + models.WordLevel(vocab=dict(vocab) if vocab is not None else {"[UNK]": 0}, unk_token="[UNK]") + ) + tokenizer.pre_tokenizer = pre_tokenizer + return tokenizer.to_str() + + +def _run_in_memory_hub(script: str, served: dict[str, str], text: str, tmp_path: Path) -> dict: + result: Final = subprocess.run( + [ + sys.executable, + "-I", + "-c", + script, + str(Path(litellm.__file__).parent.parent), + json.dumps(served), + text, + ], + capture_output=True, + text=True, + timeout=60, + env={ + **os.environ, + "HF_HOME": str(tmp_path / "home"), + "HF_HUB_CACHE": str(tmp_path / "cache"), + "HF_ENDPOINT": "http://127.0.0.1:9", + "HF_HUB_OFFLINE": "0", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + }, + ) + + assert result.returncode == 0, result.stdout + result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_tokenizer(tmp_path: Path) -> None: + sample: Final = "Tokenizers disagree: anthropic, tiktoken; llama-2 & llama-3!" + served: Final = { + "hf-internal-testing/llama-tokenizer": _word_level_tokenizer_json(pre_tokenizers.WhitespaceSplit()), + "Xenova/llama-3-tokenizer": _word_level_tokenizer_json(pre_tokenizers.Split(Regex("."), "isolated")), + "Xenova/c4ai-command-r-v01-tokenizer": _word_level_tokenizer_json(pre_tokenizers.Whitespace()), + } + expected: Final = {repo: len(Tokenizer.from_str(payload).encode(sample).ids) for repo, payload in served.items()} + anthropic_count: Final = len(Tokenizer.from_str(claude_json_str).encode(sample).ids) + tiktoken_count: Final = litellm.token_counter(model="gpt-3.5-turbo", text=sample) + assert len({*expected.values(), anthropic_count, tiktoken_count}) == len(expected) + 2 + + counts: Final = _run_in_memory_hub(HUB_TOKENIZER_SCRIPT, served, sample, tmp_path) + assert counts == { + "llama2": expected["hf-internal-testing/llama-tokenizer"], + "llama3": expected["Xenova/llama-3-tokenizer"], + "cohere": expected["Xenova/c4ai-command-r-v01-tokenizer"], + "anthropic": anthropic_count, + "custom": expected["Xenova/llama-3-tokenizer"], + "requested": sorted(served), + } + + +def test_process_exits_cleanly_while_a_background_tokenizer_retry_is_downloading(tmp_path: Path) -> None: + assert _run_in_memory_hub(HUB_RETRY_DURING_EXIT_SCRIPT, {}, "", tmp_path) == {"retry_reached_hub": True} diff --git a/tests/integration/sdk/test_tokenizer_real_io.py b/tests/integration/sdk/test_tokenizer_real_io.py new file mode 100644 index 00000000000..b6b598b81ea --- /dev/null +++ b/tests/integration/sdk/test_tokenizer_real_io.py @@ -0,0 +1,119 @@ +import os +import subprocess +import sys +from pathlib import Path +from typing import Final + +import pytest + +import litellm +from tests.unit.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + + +@pytest.mark.parametrize("offline", ("0", "1")) +def test_hub_loader_preserves_environment_auth_cache_and_offline(tmp_path: Path, offline: str) -> None: + script: Final = """ +import json +import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import httpx +import huggingface_hub +from huggingface_hub.errors import LocalEntryNotFoundError +import litellm +payload = sys.argv[2].encode() +offline = sys.argv[3] == "1" +observed = [] +def handle(request): + assert not offline, "offline loading issued a request" + if request.url.path.endswith("/tokenizer.json"): + observed.append(request.headers.get("authorization")) + if request.headers.get("authorization") != "Bearer audit-fixture-token": + return httpx.Response(401) + return httpx.Response(200, headers={"content-length": str(len(payload)), "etag": '"fixture"', "x-repo-commit": "a" * 40}, content=payload if request.method == "GET" else b"") +if not offline: + huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) +try: + tokenizer = litellm.create_pretrained_tokenizer("test-fixture/tokenizer")["tokenizer"] +except LocalEntryNotFoundError: + assert offline + assert observed == [] +else: + assert not offline + assert "Bearer audit-fixture-token" in observed + assert tokenizer.decode(tokenizer.encode("Hello World").ids) == "Hello World" + assert tuple(Path(sys.argv[4]).rglob("tokenizer.json")) +print("compatible") +""" + result: Final = subprocess.run( + [ + sys.executable, + "-I", + "-c", + script, + str(Path(litellm.__file__).parent.parent), + TOKENIZER_JSON, + offline, + str(tmp_path / "cache"), + ], + capture_output=True, + text=True, + timeout=30, + env={ + **os.environ, + "HF_HOME": str(tmp_path / "home"), + "HF_HUB_CACHE": str(tmp_path / "cache"), + "HF_ENDPOINT": "http://127.0.0.1:9", + "HF_TOKEN": "audit-fixture-token", + "HF_HUB_OFFLINE": offline, + "HF_HUB_DISABLE_IMPLICIT_TOKEN": "0", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + + +@pytest.mark.parametrize("rust", (None, "0", "1")) +def test_tokenization_stays_offline(tmp_path: Path, rust: str | None) -> None: + script: Final = """ +import sys +sys.path.insert(0, sys.argv[1]) +def reject_network(event, args): + if event == "socket.connect": + raise AssertionError("tokenizer attempted a network connection") +sys.addaudithook(reject_network) +import litellm +from litellm.rust_bridge.tokenizer import get_encoding +import tiktoken +from tokenizers import Tokenizer +assert isinstance(litellm.encoding, tiktoken.Encoding) +for name in ("cl100k_base", "o200k_base", "o200k_harmony", "p50k_base", "p50k_edit"): + encoding = get_encoding(name) + text = "offline café 漢字 🙂" + " " * 64 + assert encoding.decode(encoding.encode(text)) == text +ids = litellm.encode(text="hello world") +assert litellm.decode(tokens=ids) == "hello world" +assert litellm.token_counter(model=None, text="hello world") == len(ids) +custom = litellm.create_tokenizer(sys.argv[2]) +assert isinstance(custom["tokenizer"], Tokenizer) +custom["tokenizer"].enable_padding(pad_id=0, pad_token="[UNK]") +assert litellm.decode(tokens=litellm.encode(text="Hello World", custom_tokenizer=custom), custom_tokenizer=custom) == "Hello World" +print("compatible") +""" + result: Final = subprocess.run( + [sys.executable, "-I", "-c", script, str(Path(litellm.__file__).parent.parent), TOKENIZER_JSON], + capture_output=True, + text=True, + timeout=30, + cwd=tmp_path, + env={ + **{key: value for key, value in os.environ.items() if key != "LITELLM_RUST"}, + **({"LITELLM_RUST": rust} if rust is not None else {}), + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "TIKTOKEN_CACHE_DIR": str(tmp_path / "unused-tokenizer-cache"), + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + assert not (tmp_path / "unused-tokenizer-cache").exists() diff --git a/tests/unit/caching/test_evicted_client_closer.py b/tests/unit/caching/test_evicted_client_closer.py index 936eb37a733..5c6c9cb92b0 100644 --- a/tests/unit/caching/test_evicted_client_closer.py +++ b/tests/unit/caching/test_evicted_client_closer.py @@ -18,7 +18,6 @@ import pytest from redis.asyncio import ConnectionPool, Redis from litellm.caching.evicted_client_closer import EvictedClientCloser -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from tests.unit.caching._event_loop_reuse import runs_until_a_loop_address_repeats @@ -111,19 +110,6 @@ async def test_redis_client_is_closed_only_after_its_subscription_releases_the_c assert closer.pending_count == 0 -async def _trickling_upstream(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - """Serves a chunked body slowly, so a request stays on the wire long enough to observe.""" - await reader.read(4096) - writer.write(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n") - await writer.drain() - for _ in range(6): - writer.write(b"5\r\nhello\r\n") - await writer.drain() - await asyncio.sleep(0.1) - writer.write(b"0\r\n\r\n") - await writer.drain() - - @pytest.mark.asyncio async def test_owned_client_is_closed_once_the_grace_window_elapses(): clock = FakeClock() @@ -416,79 +402,6 @@ async def test_a_client_evicted_on_a_collected_loop_is_dropped(): assert closer.pending_count == 0 -@pytest.mark.asyncio -async def test_a_client_serving_a_request_is_not_closed_when_its_grace_window_ends(): - """The grace window on its own cannot promise that a request has finished. - - ``litellm.request_timeout`` defaults to 6000 seconds and a streaming response - is bounded only by how long the upstream keeps sending, so a client past its - deadline is closed only once its own pool reports nothing in flight. - """ - server = await asyncio.start_server(_trickling_upstream, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - clock = FakeClock() - closer = make_closer(clock) - client = httpx.AsyncClient() - - closer.mark_owned(client) - closer.schedule(client) - - async def read_the_stream() -> int: - received = 0 - async with client.stream("GET", f"http://127.0.0.1:{port}/") as response: - async for chunk in response.aiter_bytes(): - received += len(chunk) - return received - - streaming = asyncio.create_task(read_the_stream()) - await asyncio.sleep(0.25) # the request is on the wire - clock.advance(3600.0) # and its grace window is long gone - closer.reap() - await asyncio.sleep(0.05) - - assert client.is_closed is False, "closed a client that was serving a request" - assert await streaming > 0, "the in-flight request did not survive the reap" - - clock.advance(3600.0) - closer.reap() - await asyncio.sleep(0.05) - - assert client.is_closed is True, "an idle client past its grace window must be closed" - assert closer.pending_count == 0 - server.close() - - -@pytest.mark.asyncio -async def test_the_aiohttp_backed_handler_is_not_closed_mid_request(): - """The default async path is aiohttp-backed, whose pool accounts for its own leases.""" - server = await asyncio.start_server(_trickling_upstream, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - clock = FakeClock() - closer = make_closer(clock) - handler = AsyncHTTPHandler() - held_client = handler.client - - closer.mark_owned(handler) - closer.schedule(handler) - - request = asyncio.create_task(handler.get(f"http://127.0.0.1:{port}/")) - await asyncio.sleep(0.25) - clock.advance(3600.0) - closer.reap() - await asyncio.sleep(0.05) - - assert held_client.is_closed is False, "closed a handler that was serving a request" - assert (await request).status_code == 200 - - clock.advance(3600.0) - closer.reap() - await asyncio.sleep(0.05) - - assert held_client.is_closed is True - assert handler.client.is_closed is False, "a held handler must self-heal after its evicted client is closed" - server.close() - - def test_the_pending_queue_cannot_grow_past_its_bound(): """A caller that churns the client cache must not be able to grow this queue.""" clock = FakeClock() diff --git a/tests/unit/caching/test_valkey_semantic_cache.py b/tests/unit/caching/test_valkey_semantic_cache.py index 749658784ac..c4c8827cd49 100644 --- a/tests/unit/caching/test_valkey_semantic_cache.py +++ b/tests/unit/caching/test_valkey_semantic_cache.py @@ -1,9 +1,6 @@ import hashlib import os import struct -import subprocess -import sys -import textwrap from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -508,29 +505,3 @@ async def test_index_info_uses_valkey_ft_info(): assert result == {"num_docs": 3} async_client.ft.assert_called_once_with("test_index") - - -def test_importing_caching_does_not_require_redis(): - # redis is an optional dependency (extra_proxy), so the base SDK can be - # installed without it. Selecting valkey-semantic needs redis, but merely - # importing litellm.caching.caching must not, or `import litellm` breaks for - # every base-SDK user. This runs in a subprocess with redis blocked so the - # check is not polluted by redis already being imported in this session. - code = textwrap.dedent(""" - import sys - for name in ("redis", "redis.asyncio", "redis.commands", - "redis.commands.search"): - sys.modules[name] = None - import litellm.caching.caching # must not import redis at module top - from litellm.types.caching import LiteLLMCacheType - assert LiteLLMCacheType.VALKEY_SEMANTIC == "valkey-semantic" - print("ok") - """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True, - env={**os.environ, "PYTHONPATH": _REPO_ROOT}, - ) - assert result.returncode == 0, result.stderr - assert "ok" in result.stdout diff --git a/tests/unit/enterprise/conftest.py b/tests/unit/enterprise/conftest.py index 4c95f967bc4..e815ba13edb 100644 --- a/tests/unit/enterprise/conftest.py +++ b/tests/unit/enterprise/conftest.py @@ -1,12 +1,17 @@ # conftest.py import asyncio -import importlib -import os +from typing import Final import pytest import litellm +import litellm.proxy.proxy_server +from tests._module_state import collect_flushable_caches, flush_caches, restore_mutable_state, snapshot_mutable_state + +_LITELLM_STATE: Final = snapshot_mutable_state(litellm) +_PROXY_SERVER_STATE: Final = snapshot_mutable_state(litellm.proxy.proxy_server) +_FLUSHABLE_CACHES: Final = collect_flushable_caches((litellm, litellm.proxy.proxy_server)) @pytest.fixture(scope="session") @@ -19,38 +24,17 @@ def event_loop(): loop.close() - - @pytest.fixture(scope="function", autouse=True) def setup_and_teardown(): - """ - This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. - """ - curr_dir = os.getcwd() # Get the current working directory - - from litellm import Router - - importlib.reload(litellm) - - try: - if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - importlib.reload(litellm.proxy.proxy_server) - except Exception as e: - print(f"Error reloading litellm.proxy.proxy_server: {e}") - - litellm.in_memory_llm_clients_cache.flush_cache() - - import asyncio + restore_mutable_state(litellm, _LITELLM_STATE) + restore_mutable_state(litellm.proxy.proxy_server, _PROXY_SERVER_STATE) + flush_caches(_FLUSHABLE_CACHES) loop = asyncio.get_event_loop_policy().new_event_loop() asyncio.set_event_loop(loop) - print(litellm) - # from litellm import Router, completion, aembedding, acompletion, embedding yield - - # Teardown code (executes after the yield point) - loop.close() # Close the loop created earlier - asyncio.set_event_loop(None) # Remove the reference to the loop + loop.close() + asyncio.set_event_loop(None) def pytest_collection_modifyitems(config, items): diff --git a/tests/unit/harness/sandbox/test_docker.py b/tests/unit/harness/sandbox/test_docker.py index 10cb8a092f3..40e7ce88f45 100644 --- a/tests/unit/harness/sandbox/test_docker.py +++ b/tests/unit/harness/sandbox/test_docker.py @@ -1,7 +1,5 @@ import asyncio -import hashlib import shutil -import subprocess from typing import Optional import pytest @@ -10,8 +8,6 @@ from litellm import sandbox from litellm.harness.errors import SandboxError from litellm.harness.sandbox import DockerSandbox, Sandbox from litellm.harness.sandbox.docker import parse_sha256sum - -DOCKER_IMAGE = "alpine:3.20" CID = "cid123" @@ -198,44 +194,3 @@ async def test_missing_docker_binary(monkeypatch): def test_parse_sha256sum_ignores_junk(): assert parse_sha256sum("garbage\n\n") == {} - - -def _docker_usable() -> bool: - if shutil.which("docker") is None: - return False - try: - return ( - subprocess.run( - ["docker", "info"], capture_output=True, timeout=20 - ).returncode - == 0 - ) - except (OSError, subprocess.SubprocessError): - return False - - -@pytest.mark.skipif(not _docker_usable(), reason="docker daemon not available") -async def test_real_docker_roundtrip(): - box = sandbox.docker(DOCKER_IMAGE, workdir="/workspace") - try: - result = await box.run(["echo", "hello"], timeout=120) - assert result.stdout.strip() == "hello" - assert result.exit_code == 0 - - await box.write("seed.txt", b"seed") - await box.write("sub/out.txt", b"from host") - assert await box.read("sub/out.txt") == b"from host" - await box.write("node_modules/skip.js", b"x") - - assert await box.which("sh") is not None - assert await box.which("definitely-not-a-binary-xyz") is None - tmp = await box.tempdir() - assert tmp.startswith("/") - - snap = await box.snapshot() - assert snap == { - "seed.txt": hashlib.sha256(b"seed").hexdigest(), - "sub/out.txt": hashlib.sha256(b"from host").hexdigest(), - } - finally: - await box.close() diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 375f3d47cd7..a8da1ad512d 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -4,12 +4,10 @@ import json import logging import os import pathlib -import subprocess import sys -import textwrap from collections.abc import Mapping from types import MappingProxyType -from typing import Final, List, Optional, Tuple, cast +from typing import Final, List, cast from unittest.mock import MagicMock, patch import pytest @@ -3501,54 +3499,6 @@ class TestConfiguredInjectionPointsSurviveClientMarks: assert sys2 == sys1 -class TestAnthropicPromptCachingEnvVars: - """Both settings are read from the environment at import, so an admin can enable - auto-caching without a config file. Each case re-imports litellm in a subprocess - so the env is read fresh without contaminating this process's module graph. - """ - - @staticmethod - def _import_litellm_with_env(env_override: dict) -> Tuple[bool, Optional[str]]: - env = os.environ.copy() - env.pop("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", None) - env.pop("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL", None) - env.update(env_override) - script = textwrap.dedent( - """ - import json, litellm - print(json.dumps([litellm.enable_anthropic_prompt_caching, litellm.anthropic_prompt_caching_ttl])) - """ - ) - result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, env=env, timeout=300) - assert result.returncode == 0, result.stderr - enabled, ttl = json.loads(result.stdout.strip().splitlines()[-1]) - return enabled, ttl - - def test_unset_env_leaves_auto_caching_off(self): - assert self._import_litellm_with_env({}) == (False, None) - - @pytest.mark.parametrize("value", ["true", "True", "TRUE"]) - def test_env_enables_auto_caching_case_insensitively(self, value): - enabled, _ = self._import_litellm_with_env({"LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING": value}) - assert enabled is True - - @pytest.mark.parametrize("value", ["false", "0", "yes", ""]) - def test_env_only_enables_on_true(self, value): - enabled, _ = self._import_litellm_with_env({"LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING": value}) - assert enabled is False - - @pytest.mark.parametrize("value", ["5m", "1h"]) - def test_ttl_env_is_applied(self, value): - _, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value}) - assert ttl == value - - @pytest.mark.parametrize("value", ["10m", "1H", "3600", "ephemeral"]) - def test_unsupported_ttl_env_falls_back_to_provider_default(self, value): - """An unparseable TTL must fall back to Anthropic's 5m default, never reach the provider verbatim.""" - _, ttl = self._import_litellm_with_env({"LITELLM_ANTHROPIC_PROMPT_CACHING_TTL": value}) - assert ttl is None - - def _contains_key(value, key) -> bool: if isinstance(value, dict): return key in value or any(_contains_key(v, key) for v in value.values()) diff --git a/tests/unit/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py b/tests/unit/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py index 90bf7ba2920..37606dd4a0b 100644 --- a/tests/unit/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py +++ b/tests/unit/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py @@ -215,36 +215,6 @@ async def test_hook_searches_through_the_injected_router_with_the_request_metada assert messages[0]["content"] == "Context:\n\ncontext from vs-router\n\n" -@pytest.mark.asyncio -@pytest.mark.parametrize("sdk_verbose", [False, True]) -async def test_hook_falls_back_to_the_sdk_when_the_runtime_has_no_router( - registry_with: RegisterStores, - warnings: list[logging.LogRecord], - monkeypatch: pytest.MonkeyPatch, - sdk_verbose: bool, -) -> None: - monkeypatch.setattr(litellm, "set_verbose", sdk_verbose) - registry_with("vs-sdk", custom_llm_provider="lit6752-not-a-provider") - - _, messages, _ = await _run_hook( - VectorStorePreCallHook(proxy_runtime=FakeProxyRuntime(router=None)), - ["vs-sdk"], - FakeLoggingObj({"user_api_key_team_id": "team-a"}), - ) - - assert messages == [{"role": "user", "content": "what is litellm?"}] - search_warnings: Final = tuple( - record.getMessage() - for record in warnings - if record.getMessage().startswith("Vector store search failed for vector_store_id=") - ) - assert len(search_warnings) == 1 - assert search_warnings[0].startswith( - "Vector store search failed for vector_store_id=vs-sdk, continuing without its context: " - ) - assert "is not a valid LlmProviders" in search_warnings[0] - - @pytest.mark.asyncio async def test_every_healthy_vector_store_contributes_its_own_context(registry_with: RegisterStores) -> None: """Regression (LIT-6752): each store appended its context to the original messages, so only the last one survived.""" diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 6ad72521d09..616e79ffac6 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -2,16 +2,13 @@ import asyncio import datetime import itertools import json -import os import queue -import subprocess import sys import threading import time from collections.abc import Mapping, Sequence from dataclasses import dataclass from itertools import chain -from pathlib import Path from types import CodeType, FrameType, MappingProxyType, TracebackType from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -2071,66 +2068,6 @@ async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logp ] -FINISH_ONLY_LOGPROBS_STREAM_SCRIPT: Final = """ -import json -import sys - -sys.path.insert(0, sys.argv[1]) - -import httpx -from openai import OpenAI - -import litellm - -upstream_choices = ( - {"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None}, - { - "index": 0, - "delta": {}, - "logprobs": {"content": [{"token": "<|return|>", "logprob": -0.5, "top_logprobs": []}]}, - "finish_reason": "stop", - }, -) -body = "".join( - "data: " - + json.dumps({"id": "c", "object": "chat.completion.chunk", "created": 1, "model": "m", "choices": [choice]}) - + "\\n\\n" - for choice in upstream_choices -) + "data: [DONE]\\n\\n" -client = OpenAI( - api_key="unused", - base_url="https://provider.example/v1", - http_client=httpx.Client( - transport=httpx.MockTransport( - lambda request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"}) - ) - ), -) -stream = litellm.completion( - model="openai/m", messages=[{"role": "user", "content": "hi"}], stream=True, logprobs=True, client=client -) -print(json.dumps([json.loads(chunk.model_dump_json(exclude_none=True)) for chunk in stream])) -""" - - -def test_stream_with_logprobs_only_on_finish_chunk_serializes_in_a_fresh_process() -> None: - result: Final = subprocess.run( - [sys.executable, "-I", "-c", FINISH_ONLY_LOGPROBS_STREAM_SCRIPT, str(Path(litellm.__file__).parent.parent)], - capture_output=True, - text=True, - timeout=120, - env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, - ) - - assert result.returncode == 0, result.stdout + result.stderr - wire_choices: Final = tuple( - itertools.chain.from_iterable(chunk["choices"] for chunk in json.loads(result.stdout.strip().splitlines()[-1])) - ) - finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")] - assert [choice["finish_reason"] for choice in finish_choices] == ["stop"] - assert [entry["token"] for entry in finish_choices[0]["logprobs"]["content"]] == ["<|return|>"] - - def _hosted_vllm_stream_wrapper(logging_obj: Logging, error_payload: dict) -> CustomStreamWrapper: """A CustomStreamWrapper over the real OpenAI-compatible line iterator, fed an HTTP 200 SSE body that carries an in-body error payload the way @@ -2398,7 +2335,6 @@ async def test_mid_stream_error_leaves_out_an_api_base_it_cannot_resolve(): assert "API Base:" not in debug_info - @pytest.mark.asyncio @pytest.mark.parametrize( "routed_api_base", @@ -7169,7 +7105,6 @@ def test_chunk_creator_skips_full_dump_of_model_response_stream_chunks( assert getattr(emitted_chunks[1], "custom_chunk_attribute") == "custom value" - def test_openai_chunk_creator_keeps_multiple_choice_logprobs_and_extras( logging_obj: Logging, ) -> None: @@ -9765,7 +9700,6 @@ async def test_logged_response_keeps_finish_reason_from_last_content_chunk( assert logged.choices[0].message.content == "Hello, this reply is cut off" - @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_no_synthetic_finish_reason_logged_when_provider_sent_none(sync_mode: bool, logging_obj: Logging): diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 4696203c914..b65f2fc383d 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -6,15 +6,12 @@ import copy import importlib import io import json -import os -import subprocess import sys import threading import time from collections.abc import Callable, Mapping from concurrent.futures import Future, ThreadPoolExecutor, wait from itertools import accumulate, chain -from pathlib import Path from queue import SimpleQueue from typing import Final, cast from unittest.mock import MagicMock, sentinel @@ -23,7 +20,7 @@ import anyio.to_thread import pytest import tiktoken from PIL import Image -from tokenizers import Regex, Tokenizer, models, pre_tokenizers +from tokenizers import Tokenizer from unittest.mock import AsyncMock, patch @@ -922,48 +919,6 @@ class NeedsToleranceUpdateError(Exception): # test_tokenizers() -def test_encoding_and_decoding(tmp_path: Path): - sample_text = "Hellö World, this is my input string!" - - # openai encoding + decoding - openai_tokens = encode(model="gpt-3.5-turbo", text=sample_text) - openai_text = decode(model="gpt-3.5-turbo", tokens=openai_tokens) - - assert openai_text == sample_text - - # claude encoding + decoding - claude_tokens = encode(model="claude-3-5-haiku-20241022", text=sample_text) - - claude_text = decode(model="claude-3-5-haiku-20241022", tokens=claude_tokens) - - assert claude_text == sample_text - - # cohere encoding + decoding - cohere_tokens = encode(model="command-nightly", text=sample_text) - cohere_text = decode(model="command-nightly", tokens=cohere_tokens) - - assert cohere_text == sample_text - - # llama2 encoding + decoding - words = sample_text.split() - result = _run_in_memory_hub( - HUB_ROUND_TRIP_SCRIPT, - { - "hf-internal-testing/llama-tokenizer": _word_level_tokenizer_json( - pre_tokenizers.WhitespaceSplit(), - vocab={"[UNK]": 0, **{word: i + 1 for i, word in enumerate(words)}}, - ) - }, - sample_text, - tmp_path, - ) - - assert result["decoded"] == sample_text - assert result["requested"] == ["hf-internal-testing/llama-tokenizer"] - assert len(result["tokens"]) == len(words) - assert len(result["tokens"]) != len(encode(model="gpt-3.5-turbo", text=sample_text)) - - # test_encoding_and_decoding() @@ -1162,7 +1117,6 @@ from litellm.utils import ( _HuggingFaceTokenizerLoadOutcomes, _return_huggingface_tokenizer, _select_tokenizer_helper, - claude_json_str, encoding, ) @@ -3564,129 +3518,6 @@ def test_image_estimate_bounds_header_decoding_for_large_inline_payloads() -> No assert len(decode_header.call_args.args[0]) <= 65_536 -HUB_SETUP_SCRIPT: Final = """ -import json -import sys -sys.path.insert(0, sys.argv[1]) -import httpx -import huggingface_hub -import litellm -served = json.loads(sys.argv[2]) -text = sys.argv[3] -requested = [] -def handle(request): - repo = request.url.path.lstrip("/").split("/resolve/")[0] - if repo not in served or not request.url.path.endswith("/tokenizer.json"): - return httpx.Response(404) - requested.append(repo) - payload = served[repo].encode() - headers = {"content-length": str(len(payload)), "etag": '"fixture"', "x-repo-commit": "a" * 40} - return httpx.Response(200, headers=headers, content=payload if request.method == "GET" else b"") -huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) -""" - -HUB_TOKENIZER_SCRIPT: Final = HUB_SETUP_SCRIPT + """ -litellm.cohere_models = {"command-r-v1"} -litellm.anthropic_models = {"claude-2"} -custom = litellm.create_pretrained_tokenizer("Xenova/llama-3-tokenizer") -print(json.dumps({ - "llama2": litellm.token_counter(model="meta-llama/Llama-2-7b-chat", text=text), - "llama3": litellm.token_counter(model="meta-llama/llama-3-70b-instruct", text=text), - "cohere": litellm.token_counter(model="command-r-v1", text=text), - "anthropic": litellm.token_counter(model="claude-2", text=text), - "custom": litellm.token_counter(custom_tokenizer=custom, text=text), - "requested": sorted(set(requested)), -})) -""" - -HUB_ROUND_TRIP_SCRIPT: Final = HUB_SETUP_SCRIPT + """ -tokens = litellm.encode(model="meta-llama/Llama-2-7b-chat", text=text) -print(json.dumps({"tokens": tokens, "decoded": litellm.decode(model="meta-llama/Llama-2-7b-chat", tokens=tokens), "requested": sorted(set(requested))})) -""" - -HUB_RETRY_DURING_EXIT_SCRIPT: Final = HUB_SETUP_SCRIPT + """ -import threading -import time -from litellm.utils import _HuggingFaceTokenizerLoadOutcomes, _return_huggingface_tokenizer -retry_reached_hub = threading.Event() -def answer_once_the_process_is_exiting(request): - retry_reached_hub.set() - while threading.main_thread().is_alive(): - time.sleep(0.01) - time.sleep(0.5) - return handle(request) -now = [0.0] -outcomes = _HuggingFaceTokenizerLoadOutcomes(clock=lambda: now[0]) -model = "meta-llama/llama-3-70b-instruct" -_return_huggingface_tokenizer(model, outcomes) -huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(answer_once_the_process_is_exiting))) -now[0] = 3600.0 -_return_huggingface_tokenizer(model, outcomes) -print(json.dumps({"retry_reached_hub": retry_reached_hub.wait(timeout=60)})) -""" - - -def _word_level_tokenizer_json( - pre_tokenizer: pre_tokenizers.PreTokenizer, vocab: Mapping[str, int] | None = None -) -> str: - tokenizer: Final = Tokenizer( - models.WordLevel(vocab=dict(vocab) if vocab is not None else {"[UNK]": 0}, unk_token="[UNK]") - ) - tokenizer.pre_tokenizer = pre_tokenizer - return tokenizer.to_str() - - -def _run_in_memory_hub(script: str, served: dict[str, str], text: str, tmp_path: Path) -> dict: - result: Final = subprocess.run( - [ - sys.executable, - "-I", - "-c", - script, - str(Path(litellm.__file__).parent.parent), - json.dumps(served), - text, - ], - capture_output=True, - text=True, - timeout=60, - env={ - **os.environ, - "HF_HOME": str(tmp_path / "home"), - "HF_HUB_CACHE": str(tmp_path / "cache"), - "HF_ENDPOINT": "http://127.0.0.1:9", - "HF_HUB_OFFLINE": "0", - "LITELLM_LOCAL_MODEL_COST_MAP": "True", - }, - ) - - assert result.returncode == 0, result.stdout + result.stderr - return json.loads(result.stdout.strip().splitlines()[-1]) - - -def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_tokenizer(tmp_path: Path) -> None: - sample: Final = "Tokenizers disagree: anthropic, tiktoken; llama-2 & llama-3!" - served: Final = { - "hf-internal-testing/llama-tokenizer": _word_level_tokenizer_json(pre_tokenizers.WhitespaceSplit()), - "Xenova/llama-3-tokenizer": _word_level_tokenizer_json(pre_tokenizers.Split(Regex("."), "isolated")), - "Xenova/c4ai-command-r-v01-tokenizer": _word_level_tokenizer_json(pre_tokenizers.Whitespace()), - } - expected: Final = {repo: len(Tokenizer.from_str(payload).encode(sample).ids) for repo, payload in served.items()} - anthropic_count: Final = len(Tokenizer.from_str(claude_json_str).encode(sample).ids) - tiktoken_count: Final = litellm.token_counter(model="gpt-3.5-turbo", text=sample) - assert len({*expected.values(), anthropic_count, tiktoken_count}) == len(expected) + 2 - - counts: Final = _run_in_memory_hub(HUB_TOKENIZER_SCRIPT, served, sample, tmp_path) - assert counts == { - "llama2": expected["hf-internal-testing/llama-tokenizer"], - "llama3": expected["Xenova/llama-3-tokenizer"], - "cohere": expected["Xenova/c4ai-command-r-v01-tokenizer"], - "anthropic": anthropic_count, - "custom": expected["Xenova/llama-3-tokenizer"], - "requested": sorted(served), - } - - def test_custom_openai_tokenizer_without_a_model_counts_with_the_default_encoding() -> None: sample: Final = "A custom tokenizer passed with no model still counts." @@ -3695,10 +3526,6 @@ def test_custom_openai_tokenizer_without_a_model_counts_with_the_default_encodin ) -def test_process_exits_cleanly_while_a_background_tokenizer_retry_is_downloading(tmp_path: Path) -> None: - assert _run_in_memory_hub(HUB_RETRY_DURING_EXIT_SCRIPT, {}, "", tmp_path) == {"retry_reached_hub": True} - - def test_empty_custom_tokenizer_uses_model_tokenizer() -> None: text_value: Final = "A tokenizer fallback should preserve the model encoding." custom_count: Final = _get_exact_count_function("gpt-3.5-turbo", {})(text_value) diff --git a/tests/unit/litellm_core_utils/test_tokenizer.py b/tests/unit/litellm_core_utils/test_tokenizer.py index 343b67699d3..fd605e5f2d1 100644 --- a/tests/unit/litellm_core_utils/test_tokenizer.py +++ b/tests/unit/litellm_core_utils/test_tokenizer.py @@ -1,7 +1,5 @@ import copy -import os import pickle -import subprocess import sys from pathlib import Path from typing import Final, Literal @@ -176,115 +174,6 @@ def test_public_tokenizer_objects_survive_pickle_and_deepcopy(tmp_path: Path) -> assert pickle.loads(pickle.dumps(litellm.encoding)).encode("hello") == litellm.encoding.encode("hello") -@pytest.mark.parametrize("offline", ("0", "1")) -def test_hub_loader_preserves_environment_auth_cache_and_offline(tmp_path: Path, offline: str) -> None: - script: Final = """ -import json -import sys -from pathlib import Path -sys.path.insert(0, sys.argv[1]) -import httpx -import huggingface_hub -from huggingface_hub.errors import LocalEntryNotFoundError -import litellm -payload = sys.argv[2].encode() -offline = sys.argv[3] == "1" -observed = [] -def handle(request): - assert not offline, "offline loading issued a request" - if request.url.path.endswith("/tokenizer.json"): - observed.append(request.headers.get("authorization")) - if request.headers.get("authorization") != "Bearer audit-fixture-token": - return httpx.Response(401) - return httpx.Response(200, headers={"content-length": str(len(payload)), "etag": '"fixture"', "x-repo-commit": "a" * 40}, content=payload if request.method == "GET" else b"") -if not offline: - huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) -try: - tokenizer = litellm.create_pretrained_tokenizer("test-fixture/tokenizer")["tokenizer"] -except LocalEntryNotFoundError: - assert offline - assert observed == [] -else: - assert not offline - assert "Bearer audit-fixture-token" in observed - assert tokenizer.decode(tokenizer.encode("Hello World").ids) == "Hello World" - assert tuple(Path(sys.argv[4]).rglob("tokenizer.json")) -print("compatible") -""" - result: Final = subprocess.run( - [ - sys.executable, - "-I", - "-c", - script, - str(Path(litellm.__file__).parent.parent), - TOKENIZER_JSON, - offline, - str(tmp_path / "cache"), - ], - capture_output=True, - text=True, - timeout=30, - env={ - **os.environ, - "HF_HOME": str(tmp_path / "home"), - "HF_HUB_CACHE": str(tmp_path / "cache"), - "HF_ENDPOINT": "http://127.0.0.1:9", - "HF_TOKEN": "audit-fixture-token", - "HF_HUB_OFFLINE": offline, - "HF_HUB_DISABLE_IMPLICIT_TOKEN": "0", - "LITELLM_LOCAL_MODEL_COST_MAP": "True", - }, - ) - assert result.returncode == 0, result.stdout + result.stderr - assert result.stdout.strip() == "compatible" - - -@pytest.mark.parametrize("rust", (None, "0", "1")) -def test_tokenization_stays_offline(tmp_path: Path, rust: str | None) -> None: - script: Final = """ -import sys -sys.path.insert(0, sys.argv[1]) -def reject_network(event, args): - if event == "socket.connect": - raise AssertionError("tokenizer attempted a network connection") -sys.addaudithook(reject_network) -import litellm -from litellm.rust_bridge.tokenizer import get_encoding -import tiktoken -from tokenizers import Tokenizer -assert isinstance(litellm.encoding, tiktoken.Encoding) -for name in ("cl100k_base", "o200k_base", "o200k_harmony", "p50k_base", "p50k_edit"): - encoding = get_encoding(name) - text = "offline café 漢字 🙂" + " " * 64 - assert encoding.decode(encoding.encode(text)) == text -ids = litellm.encode(text="hello world") -assert litellm.decode(tokens=ids) == "hello world" -assert litellm.token_counter(model=None, text="hello world") == len(ids) -custom = litellm.create_tokenizer(sys.argv[2]) -assert isinstance(custom["tokenizer"], Tokenizer) -custom["tokenizer"].enable_padding(pad_id=0, pad_token="[UNK]") -assert litellm.decode(tokens=litellm.encode(text="Hello World", custom_tokenizer=custom), custom_tokenizer=custom) == "Hello World" -print("compatible") -""" - result: Final = subprocess.run( - [sys.executable, "-I", "-c", script, str(Path(litellm.__file__).parent.parent), TOKENIZER_JSON], - capture_output=True, - text=True, - timeout=30, - cwd=tmp_path, - env={ - **{key: value for key, value in os.environ.items() if key != "LITELLM_RUST"}, - **({"LITELLM_RUST": rust} if rust is not None else {}), - "LITELLM_LOCAL_MODEL_COST_MAP": "True", - "TIKTOKEN_CACHE_DIR": str(tmp_path / "unused-tokenizer-cache"), - }, - ) - assert result.returncode == 0, result.stdout + result.stderr - assert result.stdout.strip() == "compatible" - assert not (tmp_path / "unused-tokenizer-cache").exists() - - @pytest.mark.parametrize("is_pretokenized", (False, True)) def test_huggingface_batch_sequence_containers_match_python(is_pretokenized: bool) -> None: reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index d105a7958b4..68e2260814d 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -14,7 +14,6 @@ import itertools import json import logging import os -import subprocess import sys import threading from collections.abc import Mapping @@ -4910,40 +4909,3 @@ def test_a_register_model_opt_out_for_one_claude_model_leaves_every_other_routed litellm.register_model({f"anthropic/{opted_out}": {"supports_prompt_caching": False}}) assert supports_anthropic_cache_control(untouched, "anthropic", {"id": "plain-id"}) is True - - -_MASKED_HOSTED_WEB_CALL_PROBE: Final = """ -import json - -from litellm.llms.anthropic.common_utils import write_request_hosted_web_call_arguments - -message = { - "role": "assistant", - "provider_specific_fields": { - "web_search_calls": [ - { - "id": "ws_srvtoolu_1", - "type": "web_search_call", - "status": "completed", - "action": {"type": "search", "query": "deploy token 12345", "queries": ["deploy token 12345"]}, - } - ] - }, -} -write_request_hosted_web_call_arguments(message, {"srvtoolu_1": json.dumps({"query": "deploy token [REDACTED]"})}) -print(json.dumps(message["provider_specific_fields"]["web_search_calls"])) -""" - - -def test_a_masked_replayed_hosted_web_call_serializes_before_its_openai_model_was_ever_built() -> None: - result: Final = subprocess.run( - (sys.executable, "-I", "-c", _MASKED_HOSTED_WEB_CALL_PROBE), - capture_output=True, - text=True, - check=True, - timeout=120, - ) - hosted: Final = json.loads(result.stdout.strip().splitlines()[-1]) - assert [call["action"] for call in hosted] == [ - {"type": "search", "query": "deploy token [REDACTED]", "queries": ["deploy token [REDACTED]"]} - ] diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index cafd00cb592..618aa15bd91 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -437,28 +437,6 @@ def test_initialize_honors_explicit_max_retries(setup_mocks, configured, expecte assert result["max_retries"] == expected -def test_default_max_retries_env_var_reaches_azure_sdk_client(): - import subprocess - import sys - - code = ( - "from litellm.llms.azure.common_utils import BaseAzureLLM\n" - "client = BaseAzureLLM().get_azure_openai_client(" - "api_key='test-api-key', api_base='https://test.openai.azure.com', api_version='2024-02-01'," - " client=None, _is_async=True, litellm_params={}, model='gpt-4')\n" - "print(client.max_retries)" - ) - completed = subprocess.run( - [sys.executable, "-c", code], - env={**os.environ, "DEFAULT_MAX_RETRIES": "0"}, - capture_output=True, - text=True, - check=True, - ) - - assert completed.stdout.strip() == "0" - - @pytest.mark.parametrize( "call_type", [ @@ -3268,4 +3246,3 @@ def test_non_chat_calls_on_a_foundry_project_v1_base_reach_azure( respx_mock.post().mock(return_value=httpx.Response(200, json=upstream_body)) assert call() == "ok" - diff --git a/tests/unit/llms/base_llm/auth/test_jwt_signing.py b/tests/unit/llms/base_llm/auth/test_jwt_signing.py index f1fbc698c82..62994fc0735 100644 --- a/tests/unit/llms/base_llm/auth/test_jwt_signing.py +++ b/tests/unit/llms/base_llm/auth/test_jwt_signing.py @@ -1,9 +1,7 @@ import base64 import hashlib import json -import subprocess import sys -import textwrap import time from typing import Final @@ -13,7 +11,6 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import ec, rsa from litellm.llms.base_llm.auth.jwt_signing import ( - MISSING_SIGNING_DEPENDENCIES_MESSAGE, build_jwk, build_jwks, jwks_document_json, @@ -183,33 +180,7 @@ class TestBaseSdkImport: and ``litellm/__init__`` reaches this module through the Anthropic provider, so a module-level import of either would break ``import litellm`` for every base SDK user.""" - def test_module_imports_with_pyjwt_and_cryptography_absent(self): - script: Final = textwrap.dedent( - """ - import sys - - class Blocker: - def find_spec(self, name, path=None, target=None): - if name.split(".")[0] in {"jwt", "cryptography"}: - raise ModuleNotFoundError(f"No module named {name!r}") - - sys.meta_path.insert(0, Blocker()) - import litellm - from litellm.llms.base_llm.auth.jwt_signing import jwks_document_json - try: - jwks_document_json("not a key") - except ImportError as e: - print(e) - """ - ) - result: Final = subprocess.run( - [sys.executable, "-I", "-c", script], capture_output=True, text=True, check=False - ) - assert result.returncode == 0, result.stderr[-2000:] - assert result.stdout.strip() == MISSING_SIGNING_DEPENDENCIES_MESSAGE - def test_signing_reports_the_missing_extra(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.setitem(sys.modules, "jwt", None) with pytest.raises(ImportError, match="litellm\\[proxy\\]"): sign_es256_jwt(pem_of(fixed_private_key()), {"sub": "x"}) - diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index 9698c4ac62f..2f7a15e1acf 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -1,19 +1,16 @@ import asyncio import gc -import importlib import json import logging import os import pathlib import ssl import threading -import time import weakref from collections.abc import AsyncIterator, Callable, Mapping from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager from contextvars import Context -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Final from unittest.mock import MagicMock, patch @@ -29,8 +26,6 @@ from yarl import URL import litellm from litellm.caching.llm_caching_handler import LLMClientCache from litellm.exceptions import Timeout as LitellmTimeout -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.litellm_core_utils.model_cost_table import publish from litellm.llms.base_llm.auth.token_exchange import JwtBearerTokenExchangeEngine, default_token_exchange_engine from litellm.llms.base_llm.auth.types import MintedToken, TokenExchangeSpec from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport @@ -46,7 +41,6 @@ from litellm.llms.custom_httpx.http_handler import ( realtime_ssl_for_url, ) from litellm.types.llms.custom_http import VerifyTypes -from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @pytest.mark.asyncio @@ -879,116 +873,6 @@ class TestDefaultCachedClientTimeoutHonorsRequestTimeout: assert client.timeout.read == 300.0 -async def _read_http_request(reader: asyncio.StreamReader) -> None: - raw = b"" - while b"\r\n\r\n" not in raw: - chunk = await reader.read(1024) - if not chunk: - return - raw += chunk - head, _, body = raw.partition(b"\r\n\r\n") - content_length = next( - (int(line.split(b":", 1)[1]) for line in head.split(b"\r\n") if line.lower().startswith(b"content-length")), - 0, - ) - while len(body) < content_length: - body += await reader.read(content_length - len(body)) - - -@pytest.mark.asyncio -async def test_init_held_async_handler_survives_external_client_close(): - handler = AsyncHTTPHandler(timeout=42.5) - held_client = handler.client - await held_client.aclose() - assert held_client.is_closed - - async def respond(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - await _read_http_request(reader) - writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") - await writer.drain() - writer.close() - - server = await asyncio.start_server(respond, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - try: - response = await handler.post(f"http://127.0.0.1:{port}/v1/compress", json={"messages": []}) - finally: - server.close() - await server.wait_closed() - - assert response.status_code == 200 - assert handler.client is not held_client - assert handler.client.timeout == httpx.Timeout(42.5) - await handler.close() - - -@pytest.mark.asyncio -async def test_init_held_async_handler_survives_evicted_client_close(): - from litellm.caching.evicted_client_closer import EvictedClientCloser - from litellm.caching.llm_caching_handler import LLMClientCache - - cache = LLMClientCache(evicted_client_closer=EvictedClientCloser(grace_seconds=0)) - handler = AsyncHTTPHandler(timeout=42.5) - held_client = handler.client - cache.set_cache("init-held-handler", handler, litellm_owned_client=True, ttl=0) - await asyncio.sleep(0.02) - assert cache.get_cache("init-held-handler") is None - await asyncio.sleep(0.05) - assert held_client.is_closed - - async def respond(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - await _read_http_request(reader) - writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") - await writer.drain() - writer.close() - - server = await asyncio.start_server(respond, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - try: - response = await handler.post(f"http://127.0.0.1:{port}/v1/compress", json={"messages": []}) - finally: - server.close() - await server.wait_closed() - - assert response.status_code == 200 - assert handler.client is not held_client - assert handler.client.timeout == httpx.Timeout(42.5) - await handler.close() - - -def test_init_held_sync_handler_recreates_closed_client(): - from http.server import BaseHTTPRequestHandler, HTTPServer - - class OkRequestHandler(BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - self.send_header("Content-Length", "2") - self.end_headers() - self.wfile.write(b"ok") - - def log_message(self, format, *args): - pass - - handler = HTTPHandler(timeout=7) - held_client = handler.client - held_client.close() - - server = HTTPServer(("127.0.0.1", 0), OkRequestHandler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - response = handler.get(f"http://127.0.0.1:{server.server_port}/") - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - assert response.status_code == 200 - assert handler.client is not held_client - assert handler.client.timeout == httpx.Timeout(7) - handler.close() - - def test_caller_supplied_sync_client_is_not_replaced_when_closed(): supplied = httpx.Client() handler = HTTPHandler(client=supplied) @@ -2245,66 +2129,6 @@ async def test_http2_disabled_by_default(monkeypatch: pytest.MonkeyPatch): assert AsyncHTTPHandler.should_use_aiohttp_transport() is True -@pytest.fixture() -def _vcr_outcome_gate(request, vcr): - install_live_call_probe(request, vcr) - yield - record_vcr_outcome(request, vcr) - - -@pytest.fixture(scope="function") -def isolate_litellm_state(): - """ - Per-function isolation fixture. - - Resets litellm globals to their true defaults before each test and - restores them afterward, so tests don't leak side effects. - Works safely under pytest-xdist parallel execution. - """ - original_state = {} - for attr in ( - "callbacks", - "success_callback", - "failure_callback", - "_async_success_callback", - "_async_failure_callback", - ): - if hasattr(litellm, attr): - val = getattr(litellm, attr) - original_state[attr] = val.copy() if val else [] - for attr in ("pre_call_rules", "post_call_rules"): - if hasattr(litellm, attr): - val = getattr(litellm, attr) - original_state[attr] = val.copy() if val else [] - for attr in _SCALAR_DEFAULTS: - if hasattr(litellm, attr): - original_state[attr] = getattr(litellm, attr) - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - for attr in ( - "callbacks", - "success_callback", - "failure_callback", - "_async_success_callback", - "_async_failure_callback", - "pre_call_rules", - "post_call_rules", - ): - if hasattr(litellm, attr): - setattr(litellm, attr, []) - for attr, default_val in _SCALAR_DEFAULTS.items(): - if hasattr(litellm, attr): - setattr(litellm, attr, default_val) - yield - asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue()) - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - for attr, original_value in original_state.items(): - if hasattr(litellm, attr): - setattr(litellm, attr, original_value) - publish(litellm.model_cost) - - _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), @@ -2326,73 +2150,6 @@ _SCALAR_DEFAULTS = { } -@pytest.fixture(scope="module") -def setup_and_teardown(): - """ - Module-scoped setup. Reloads litellm only in single-process mode - (skipped under xdist to avoid cross-worker interference). - """ - import litellm - - worker_id = os.environ.get("PYTEST_XDIST_WORKER", None) - if worker_id is None: - importlib.reload(litellm) - try: - if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - - importlib.reload(litellm.proxy.proxy_server) - except Exception: - pass - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - yield - - -_SERVER_DELAY_S = 5 - -_PER_REQUEST_TIMEOUT_S = 1.0 - -_CLIENT_DEFAULT_TIMEOUT_S = 60.0 - - -class _SlowHandler(BaseHTTPRequestHandler): - def do_POST(self): - time.sleep(_SERVER_DELAY_S) - try: - self.send_response(200) - self.end_headers() - self.wfile.write(b"{}") - except OSError: - pass - - def log_message(self, *args): - pass - - -@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") -def test_post_delay_exceeds_per_request_timeout_raises(): - server = ThreadingHTTPServer(("127.0.0.1", 0), _SlowHandler) - threading.Thread(target=server.serve_forever, daemon=True).start() - host, port = server.server_address - - handler = get_httpx_client(params={"timeout": _CLIENT_DEFAULT_TIMEOUT_S}) - try: - with pytest.raises(LitellmTimeout): - handler.post( - f"http://{host}:{port}/delay", - headers={"content-type": "application/json"}, - data=json.dumps({"model": "claude", "messages": []}), - timeout=_PER_REQUEST_TIMEOUT_S, - ) - except MaskedHTTPStatusError as e: - pytest.skip(f"httpbin.org unavailable: {e}") - finally: - handler.close() - server.shutdown() - server.server_close() - - def test_realtime_ssl_for_url_sends_no_tls_argument_for_a_plain_ws_endpoint() -> None: assert ( realtime_ssl_for_url("ws://127.0.0.1:8080/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent") diff --git a/tests/unit/llms/opencode/chat/test_transformation.py b/tests/unit/llms/opencode/chat/test_transformation.py index 576f2201130..a0a20daebf8 100644 --- a/tests/unit/llms/opencode/chat/test_transformation.py +++ b/tests/unit/llms/opencode/chat/test_transformation.py @@ -606,26 +606,6 @@ def cost_recorder(monkeypatch: pytest.MonkeyPatch) -> CostRecorder: return recorder -@pytest.mark.asyncio -@pytest.mark.parametrize("upstream", ["chat/completions", "messages", "responses", "gemini"]) -@pytest.mark.parametrize("ingress", ["chat", "responses"]) -async def test_reported_list_cost_is_billed_instead_of_rows( - go_catalog: None, cost_recorder: CostRecorder, upstream: str, ingress: str -) -> None: - model: Final = f"fixture-{upstream.split('/')[0]}" - with respx.mock(using="httpx") as transport: - transport.post().mock( - return_value=httpx.Response(200, json=go_priced_reply(upstream, model, GO_REPORTED_USAGE)) - ) - kwargs: Final = {"model": f"opencode_go/{model}", "api_key": "fake-go-key"} - if ingress == "chat": - response: Final = await litellm.acompletion(messages=[{"role": "user", "content": "hello"}], **kwargs) - else: - response = await litellm.aresponses(input="hello", **kwargs) - assert response._hidden_params["additional_headers"][RESPONSE_COST_HEADER] == pytest.approx(0.00412) - assert await logged_cost(cost_recorder) == pytest.approx(0.00412) - - @pytest.mark.asyncio @pytest.mark.parametrize("usage_extra", [{}, {"cost": 0}, {"opencode": {"billing_source": "go"}}]) async def test_rows_price_the_request_when_no_list_cost_is_reported( diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py index 233e5d888fa..26cb15cd27e 100644 --- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py +++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py @@ -1,11 +1,4 @@ -import asyncio import base64 -import uuid -from collections.abc import Generator -from contextlib import contextmanager -from dataclasses import dataclass, field -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from threading import Event, Thread from typing import Final, Literal from unittest.mock import patch @@ -298,131 +291,6 @@ async def test_openai_messages_keep_their_chat_conversion(custom_llm_provider: L _PNG_BYTES: Final = base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" ) -_LOOP_PROOF_TIMEOUT_SECONDS: Final = 5 - - -@dataclass(frozen=True, slots=True) -class _LoopProbe: - request_arrived: Event = field(default_factory=Event) - loop_ran: Event = field(default_factory=Event) - loop_ran_while_download_pending: Event = field(default_factory=Event) - - -def _png_handler_waiting_for_loop(probe: _LoopProbe) -> type[BaseHTTPRequestHandler]: - class _PngHandler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - probe.request_arrived.set() - if probe.loop_ran.wait(_LOOP_PROOF_TIMEOUT_SECONDS): - probe.loop_ran_while_download_pending.set() - self.send_response(200) - self.send_header("Content-Type", "image/png") - self.send_header("Content-Length", str(len(_PNG_BYTES))) - self.end_headers() - self.wfile.write(_PNG_BYTES) - - def log_message(self, format: str, *args: object) -> None: - return None - - return _PngHandler - - -@contextmanager -def _png_server_waiting_for_loop(probe: _LoopProbe) -> Generator[str, None, None]: - with ThreadingHTTPServer(("127.0.0.1", 0), _png_handler_waiting_for_loop(probe)) as server: - thread: Final = Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - yield f"http://127.0.0.1:{server.server_address[1]}" - finally: - server.shutdown() - - -async def _run_loop_once_download_starts(probe: _LoopProbe) -> None: - assert await asyncio.to_thread(probe.request_arrived.wait, _LOOP_PROOF_TIMEOUT_SECONDS) - probe.loop_ran.set() - - -@pytest.mark.parametrize("custom_llm_provider", ("vertex_ai", "gemini")) -async def test_remote_image_downloads_without_blocking_the_event_loop( - custom_llm_provider: Literal["vertex_ai", "gemini"], monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(litellm, "user_url_validation", False) - probe: Final = _LoopProbe() - with _png_server_waiting_for_loop(probe) as base_url: - image_url: Final = f"{base_url}/{uuid.uuid4().hex}.png" - _, contents = await asyncio.gather( - _run_loop_once_download_starts(probe), - gemini_count_contents( - ( - { - "role": "user", - "content": [ - {"type": "text", "text": "Describe it"}, - {"type": "image", "source": {"type": "url", "url": image_url}}, - ], - }, - ), - "anthropic", - "gemini-2.5-flash", - custom_llm_provider, - {}, - ), - ) - - assert contents == [ - { - "role": "user", - "parts": [ - {"text": "Describe it"}, - {"inline_data": {"data": base64.b64encode(_PNG_BYTES).decode(), "mime_type": "image/png"}}, - ], - } - ] - assert probe.loop_ran_while_download_pending.is_set() - - -class _MissingImageHandler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - self.send_response(404) - self.send_header("Content-Length", "0") - self.end_headers() - - def log_message(self, format: str, *args: object) -> None: - return None - - -@pytest.mark.parametrize("custom_llm_provider", ("vertex_ai", "gemini")) -async def test_image_that_cannot_be_fetched_returns_an_error_value( - custom_llm_provider: Literal["vertex_ai", "gemini"], monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(litellm, "user_url_validation", False) - with ThreadingHTTPServer(("127.0.0.1", 0), _MissingImageHandler) as server: - Thread(target=server.serve_forever, daemon=True).start() - try: - contents = await gemini_count_contents( - ( - { - "role": "user", - "content": [ - { - "type": "image", - "source": { - "type": "url", - "url": f"http://127.0.0.1:{server.server_address[1]}/{uuid.uuid4().hex}.png", - }, - } - ], - }, - ), - "anthropic", - "gemini-2.5-flash", - custom_llm_provider, - {}, - ) - finally: - server.shutdown() - - assert isinstance(contents, VertexGeminiCountContextError) async def test_gcs_media_type_lookup_uses_the_deployment_vertex_project() -> None: diff --git a/tests/unit/llms/vertex_ai/gemini/test_long_decode.py b/tests/unit/llms/vertex_ai/gemini/test_long_decode.py index 3fe9a5c7a92..91f099fae02 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_long_decode.py +++ b/tests/unit/llms/vertex_ai/gemini/test_long_decode.py @@ -1,16 +1,12 @@ import json import sys -import threading from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence from dataclasses import dataclass -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from itertools import chain, count, repeat +from itertools import chain, repeat from typing import Final import httpx import pytest -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -1185,158 +1181,6 @@ async def test_streaming_continuation_request_is_sent_with_the_callers_timeout(i assert read_timeouts == [7.5, 7.5] -@dataclass(frozen=True, slots=True) -class _TokenEndpoint: - credentials: str - down: threading.Event - - -def _service_account_json(token_uri: str) -> str: - private_key: Final = ( - rsa.generate_private_key(public_exponent=65537, key_size=2048) - .private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) - .decode() - ) - return json.dumps( - { - "type": "service_account", - "project_id": "p", - "private_key_id": "k", - "private_key": private_key, - "client_email": "sa@p.iam.gserviceaccount.com", - "token_uri": token_uri, - } - ) - - -@pytest.fixture -def token_endpoint() -> Iterator[_TokenEndpoint]: - down: Final = threading.Event() - minted: Final = count(1) - - class Handler(BaseHTTPRequestHandler): - def do_POST(self) -> None: - self.rfile.read(int(self.headers["Content-Length"])) - refused: Final = down.is_set() - payload: Final = json.dumps( - {"error": "invalid_grant"} if refused else {"access_token": f"token-{next(minted)}", "expires_in": 0} - ).encode() - self.send_response(400 if refused else 200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - def log_message(self, *_args: object) -> None: - return None - - server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - threading.Thread(target=server.serve_forever, daemon=True).start() - yield _TokenEndpoint(_service_account_json(f"http://127.0.0.1:{server.server_port}/token"), down) - server.shutdown() - server.server_close() - - -def _google_taking_each_access_token_once( - hops: Sequence[httpx.Response], used: list[str], on_answer: Callable[[], object] -) -> Callable[[httpx.Request], httpx.Response]: - def handler(request: httpx.Request) -> httpx.Response: - token: Final = request.headers["authorization"] - if token in used: - return httpx.Response(401, json={"error": {"code": 401, "status": "UNAUTHENTICATED"}}) - used.append(token) - on_answer() - return hops[len(used) - 1] - - return handler - - -async def _vertex_answer( - is_async: bool, - stream: bool, - handler: Callable[[httpx.Request], httpx.Response], - credentials: str, - extra_headers: Mapping[str, str] | None = None, -) -> tuple[str, list[str], list[object]]: - kwargs: Final = { - "model": "vertex_ai/gemini-3.8-flash", - "extra_headers": extra_headers, - "messages": list(MESSAGES), - "max_tokens": 100_000, - "stream": stream, - "vertex_project": "p", - "vertex_location": "global", - "vertex_credentials": credentials, - } - transport: Final = httpx.MockTransport(handler) - response: Final = ( - await litellm.acompletion(**kwargs, client=AsyncHTTPHandler(transport=transport)) - if is_async - else litellm.completion(**kwargs, client=HTTPHandler(client=httpx.Client(transport=transport))) - ) - if stream: - chunks: Final = [chunk async for chunk in response] if is_async else list(response) - return _stream_text(chunks), _stream_finish_reasons(chunks), _client_tokens(chunks) - choice: Final = response.choices[0] - fields: Final = choice.message.provider_specific_fields or {} - return choice.message.content, [choice.finish_reason], [fields["continuation_token"]] if fields else [] - - -def _two_hops(stream: bool) -> tuple[httpx.Response, httpx.Response]: - return _stream_hops() if stream else (httpx.Response(200, json=HOP_1), httpx.Response(200, json=HOP_2)) - - -@pytest.mark.parametrize("stream", [True, False]) -@pytest.mark.parametrize("is_async", [True, False]) -@pytest.mark.asyncio -async def test_vertex_long_decode_finishes_when_each_continuation_outlives_the_access_token_before_it( - is_async: bool, stream: bool, token_endpoint: _TokenEndpoint -): - used: Final[list[str]] = [] - google: Final = _google_taking_each_access_token_once(_two_hops(stream), used, lambda: None) - - answer: Final = await _vertex_answer(is_async, stream, google, token_endpoint.credentials) - - assert answer == ("hello world", ["stop"], []) - assert len(used) == 2 - - -@pytest.mark.parametrize("stream", [True, False]) -@pytest.mark.parametrize("is_async", [True, False]) -@pytest.mark.asyncio -async def test_vertex_long_decode_returns_the_answer_so_far_with_its_token_when_no_new_access_token_can_be_had( - is_async: bool, stream: bool, token_endpoint: _TokenEndpoint -): - used: Final[list[str]] = [] - google: Final = _google_taking_each_access_token_once(_two_hops(stream), used, token_endpoint.down.set) - - answer: Final = await _vertex_answer(is_async, stream, google, token_endpoint.credentials) - - assert answer == ("hello", ["length"], ["dG9rLTE="]) - assert len(used) == 1 - - -@pytest.mark.parametrize("stream", [True, False]) -@pytest.mark.parametrize("is_async", [True, False]) -@pytest.mark.asyncio -async def test_vertex_long_decode_keeps_the_callers_own_authorization_header_on_every_continuation( - is_async: bool, stream: bool, token_endpoint: _TokenEndpoint -): - hops: Final = _two_hops(stream) - seen: Final[list[str]] = [] - - def google(request: httpx.Request) -> httpx.Response: - seen.append(request.headers["authorization"]) - return hops[len(seen) - 1] - - answer: Final = await _vertex_answer( - is_async, stream, google, token_endpoint.credentials, {"Authorization": "Bearer mine"} - ) - - assert answer == ("hello world", ["stop"], []) - assert seen == ["Bearer mine", "Bearer mine"] - - BUDGET_ENV: Final = "LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS" ARGON: Final = "gemini/gemini-4-argon" UNMAPPED_GEMINI: Final = "gemini/gemini-99-unreleased" @@ -1555,46 +1399,6 @@ async def test_env_budget_leaves_a_snake_case_generation_config_from_extra_body_ assert _output_limits(sent[0]) == {} -def _gemini_chat_models_answering_no_text() -> tuple[str, ...]: - rows: Final = ( - (name, row) - for name, row in litellm.model_cost.items() - if row.get("mode") == "chat" and "text" not in row.get("supported_output_modalities", ("text",)) - ) - return tuple( - name if name.startswith("gemini/") else f"vertex_ai/{name}" - for name, row in rows - if row.get("litellm_provider") in ("gemini", "vertex_ai-language-models") - ) - - -@pytest.mark.asyncio -async def test_env_budget_skips_every_chat_model_the_cost_map_lists_as_answering_no_text( - monkeypatch: pytest.MonkeyPatch, token_endpoint: _TokenEndpoint -): - monkeypatch.setenv(BUDGET_ENV, "50") - models: Final = _gemini_chat_models_answering_no_text() - sent: Final[list[dict[str, object]]] = [] - replies: Final = tuple(httpx.Response(200, json=HOP_2) for _ in models) - client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_replay(replies, sent))) - - for model in models: - await litellm.acompletion( - model=model, - messages=list(MESSAGES), - api_key="fake-gemini-key", - vertex_project="p", - vertex_location="us-central1", - vertex_credentials=token_endpoint.credentials, - client=client, - ) - - assert models - assert {model: _output_limits(request) for model, request in zip(models, sent, strict=True)} == dict.fromkeys( - models, {} - ) - - @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.asyncio async def test_env_budget_leaves_a_continuation_token_resume_with_no_limit_alone( @@ -1619,28 +1423,6 @@ async def test_env_budget_leaves_a_continuation_token_resume_with_no_limit_alone assert _output_limits(sent[0]) == {} -@pytest.mark.parametrize("model", ["gemini/gemini-2.5-flash", "vertex_ai/gemini-2.5-flash", ARGON]) -@pytest.mark.asyncio -async def test_env_budget_above_a_models_output_limit_sends_that_limit_instead( - monkeypatch: pytest.MonkeyPatch, token_endpoint: _TokenEndpoint, model: str -): - budget: Final = 2 * _cost_map_max_output_tokens(model) - monkeypatch.setenv(BUDGET_ENV, str(budget)) - sent: Final[list[dict[str, object]]] = [] - - await litellm.acompletion( - model=model, - messages=list(MESSAGES), - api_key="fake-gemini-key", - vertex_project="p", - vertex_location="us-central1", - vertex_credentials=token_endpoint.credentials, - client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), - ) - - assert _output_limits(sent[0]) == {"max_output_tokens": _cost_map_max_output_tokens(model)} - - @pytest.mark.asyncio async def test_env_budget_never_goes_above_the_cost_map_limit_when_a_deployment_overrides_it( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/mcp_client/test_mcp_client.py b/tests/unit/mcp_client/test_mcp_client.py index 866a29b70e9..c657e17695c 100644 --- a/tests/unit/mcp_client/test_mcp_client.py +++ b/tests/unit/mcp_client/test_mcp_client.py @@ -15,7 +15,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import anyio import httpx2 import pytest -from mcp import ClientSession, MCPError +from mcp import MCPError from mcp.client.session import ClientRequestContext, ElicitationFnT from mcp.client.streamable_http import streamable_http_client from mcp.server.mcpserver import Context @@ -47,7 +47,7 @@ from starlette.applications import Starlette from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint from starlette.requests import Request from starlette.responses import Response -from starlette.types import ASGIApp, Message, Receive, Scope, Send +from starlette.types import ASGIApp # Add the parent directory to the path so we can import litellm import litellm.mcp_client.client as mcp_client_module @@ -55,7 +55,6 @@ from litellm.mcp_client.client import ( MCPClient, MCPSigV4Auth, PersistentMCPSession, - UpstreamSessionClosedError, _first_non_cancelled_cause, _TransportContext, _TransportStreams, @@ -1511,31 +1510,6 @@ def test_mcp_extra_matches_proxy_extra_and_supports_streamable_http(): assert specifier.contains(mcp_versions[0]) -@pytest.mark.parametrize("module", ["mcp", "mcp_types", "httpx2", "httpcore2"]) -def test_base_sdk_guard_rejects_mcp_dependencies(tmp_path: Path, module: str) -> None: - import subprocess - import sys - - (tmp_path / f"{module}.py").write_text("") - checker = Path(__file__).parents[2] / "base_sdk_tests" / "check_base_sdk_install.py" - result = subprocess.run( - [ - sys.executable, - "-S", - "-c", - "import runpy, sys; sys.path.insert(0, sys.argv[2]); " - "runpy.run_path(sys.argv[1])['check_environment_is_base_only']()", - str(checker), - str(tmp_path), - ], - capture_output=True, - text=True, - check=False, - ) - assert result.returncode != 0, f"base-only guard accepted installed {module}" - assert f"{module} installed" in result.stderr - - @pytest.mark.parametrize( "auth_type, default_header", [ @@ -2094,220 +2068,6 @@ async def test_invalid_tool_list_schema_is_identified_as_an_upstream_response() assert "secret" not in message -class _DiagnosticSSEStream(httpx2.AsyncByteStream): - def __init__(self, messages: asyncio.Queue[bytes | Exception | None]) -> None: - self.messages = messages - - async def __aiter__(self) -> AsyncIterator[bytes]: - yield b"event: endpoint\ndata: /messages\n\n" - while True: - message: Final = await self.messages.get() - if message is None: - return - if isinstance(message, Exception): - raise message - yield b"event: message\ndata: " + message + b"\n\n" - - -_DIAGNOSTIC_STDIO_SERVER: Final = """ -import json, sys -mode, failure_method = sys.argv[1:] -for line in sys.stdin: - request = json.loads(line) - if "method" not in request or "id" not in request: - continue - if request["method"] == failure_method: - if mode == "bad-json": - print("secret-invalid-json", flush=True) - continue - if mode == "closed": - sys.exit(0) - if mode == "silent": - print(json.dumps({"jsonrpc": "2.0", "method": "notifications/message", "params": {"level": "info", "data": "Waiting"}}), flush=True) - continue - if request["method"] == "initialize": - result = {"protocolVersion": request["params"]["protocolVersion"], "capabilities": {"tools": {}, "logging": {}}, "serverInfo": {"name": "diagnostic", "version": "1"}} - elif request["method"] == "tools/list": - print(json.dumps({"jsonrpc": "2.0", "method": "notifications/message", "params": {"level": "info", "data": "Listing tools"}}), flush=True) - print(json.dumps({"jsonrpc": "2.0", "id": "unmatched", "result": {}}), flush=True) - print(json.dumps({"jsonrpc": "2.0", "id": "server-ping", "method": "ping"}), flush=True) - result = {"tools": [{"name": "ping", "inputSchema": {"type": "object"}}]} - else: - result = {"content": [{"type": "text", "text": "pong"}], "isError": False} - print(json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": result}), flush=True) -""" - - -def _diagnostic_transport(transport: MCPTransport, mode: str, failure_method: str) -> _TransportContext: - from mcp import StdioServerParameters - from mcp.client.sse import sse_client - from mcp.client.stdio import stdio_client - - if transport == MCPTransport.stdio: - return stdio_client( - StdioServerParameters( - command=sys.executable, args=["-u", "-c", _DIAGNOSTIC_STDIO_SERVER, mode, failure_method] - ) - ) - messages: Final[asyncio.Queue[bytes | Exception | None]] = asyncio.Queue() - - async def respond(request: httpx2.Request) -> httpx2.Response: - if request.method == "GET": - return httpx2.Response( - 200, headers={"Content-Type": "text/event-stream"}, stream=_DiagnosticSSEStream(messages) - ) - payload: Final = json.loads(request.content) - if "method" not in payload or "id" not in payload: - return httpx2.Response(202) - if payload["method"] == failure_method and mode != "ok": - if mode == "bad-json": - await messages.put(b"secret-invalid-json") - elif mode == "io-error": - await messages.put(httpx2.ReadError("secret-read-error")) - elif mode == "closed": - await messages.put(None) - elif mode == "silent": - await messages.put( - b'{"jsonrpc":"2.0","method":"notifications/message","params":{"level":"info","data":"Waiting"}}' - ) - return httpx2.Response(202) - if payload["method"] == "tools/list": - for message in ( - { - "jsonrpc": "2.0", - "method": "notifications/message", - "params": {"level": "info", "data": "Listing tools"}, - }, - {"jsonrpc": "2.0", "id": "unmatched", "result": {}}, - {"jsonrpc": "2.0", "id": "server-ping", "method": "ping"}, - ): - await messages.put(json.dumps(message).encode()) - result: Final = ( - { - "protocolVersion": payload["params"]["protocolVersion"], - "capabilities": {"tools": {}, "logging": {}}, - "serverInfo": {"name": "diagnostic", "version": "1"}, - } - if payload["method"] == "initialize" - else {"tools": [{"name": "ping", "inputSchema": {"type": "object"}}]} - if payload["method"] == "tools/list" - else {"content": [{"type": "text", "text": "pong"}], "isError": False} - ) - await messages.put(json.dumps({"jsonrpc": "2.0", "id": payload["id"], "result": result}).encode()) - return httpx2.Response(202) - - def factory( - headers: dict[str, str] | None = None, - timeout: httpx2.Timeout | None = None, - auth: httpx2.Auth | None = None, - ) -> httpx2.AsyncClient: - return httpx2.AsyncClient(transport=httpx2.MockTransport(respond), headers=headers, timeout=timeout, auth=auth) - - return sse_client("https://example.com/sse", httpx_client_factory=factory) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("transport", [MCPTransport.sse, MCPTransport.stdio]) -@pytest.mark.parametrize("failure_method", ["initialize", "tools/list"]) -async def test_transport_parsing_failure_is_preserved(transport: MCPTransport, failure_method: str) -> None: - client: Final = MCPClient(server_url="https://example.com/sse", transport_type=transport, timeout=0.2) - with pytest.raises(ValidationError): - await asyncio.wait_for( - client._execute_session_operation( - _diagnostic_transport(transport, "bad-json", failure_method), lambda session: session.list_tools() - ), - timeout=3, - ) - - -def test_sse_read_failure_is_preserved() -> None: - loop: Final = AutojumpClockLoop() - - async def run() -> None: - client: Final = MCPClient(server_url="https://example.com/sse", transport_type=MCPTransport.sse, timeout=0.2) - with pytest.raises(httpx2.ReadError, match="secret-read-error"): - await asyncio.wait_for( - client._execute_session_operation( - _diagnostic_transport(MCPTransport.sse, "io-error", "tools/list"), - lambda session: session.list_tools(), - ), - timeout=3, - ) - - try: - loop.run_until_complete(run()) - finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.run_until_complete(loop.shutdown_default_executor()) - loop.close() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("protocol_version", ["auto", "2025-06-18"]) -@pytest.mark.parametrize("transport", [MCPTransport.sse, MCPTransport.stdio]) -@pytest.mark.parametrize("mode", ["ok", "closed", "silent"]) -async def test_transport_completion_and_normal_messages( - transport: MCPTransport, mode: str, protocol_version: str -) -> None: - from mcp import ClientSession - - from litellm.proxy.mcp_server.rest_endpoints import _connection_error_message - - logging_callback: Final = AsyncMock() - read_timeout: Final = 0.2 if mode == "silent" else 30 - client: Final = MCPClient( - server_url="https://example.com/sse", - transport_type=transport, - timeout=read_timeout, - logging_callback=logging_callback, - protocol_version=protocol_version, - ) - - async def operation(session: ClientSession) -> CallToolResult: - tools: Final = await session.list_tools() - assert [tool.name for tool in tools.tools] == ["ping"] - return await session.call_tool("ping", {}) - - pending: Final = client._execute_session_operation(_diagnostic_transport(transport, mode, "tools/list"), operation) - if mode == "ok": - result: Final = await asyncio.wait_for(pending, timeout=3) - assert result.is_error is False - assert result.content[0].text == "pong" - logging_callback.assert_awaited_once_with(LoggingMessageNotificationParams(level="info", data="Listing tools")) - else: - with pytest.raises(MCPError) as caught: - await asyncio.wait_for(pending, timeout=3) - if mode == "closed": - assert "connection was closed" in _connection_error_message(caught.value, client.server_url, read_timeout) - else: - assert isinstance(as_mcp_read_timeout(caught.value), TimeoutError) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("transport", [MCPTransport.sse, MCPTransport.stdio]) -async def test_transport_cancellation_cleans_up_a_pending_request(transport: MCPTransport) -> None: - ready: Final = asyncio.Event() - - async def on_log(message: LoggingMessageNotificationParams) -> None: - if message.data == "Waiting": - ready.set() - - client: Final = MCPClient( - server_url="https://example.com/sse", transport_type=transport, timeout=30, logging_callback=on_log - ) - task: Final = asyncio.create_task( - client._execute_session_operation( - _diagnostic_transport(transport, "silent", "tools/list"), lambda session: session.list_tools() - ) - ) - try: - await asyncio.wait_for(ready.wait(), timeout=3) - finally: - task.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(task, timeout=3) - - class _InterruptedHTTPBody(httpx2.AsyncByteStream): async def __aiter__(self) -> AsyncIterator[bytes]: yield b'{"jsonrpc":' @@ -2670,125 +2430,6 @@ async def test_optional_discovery_collects_all_pages(method: str, session_id: st assert sum(call.args[0].method == "DELETE" for call in responder.call_args_list) == (1 if session_id else 0) -@pytest.mark.parametrize("method", ("prompts/list", "resources/list", "resources/templates/list")) -@pytest.mark.parametrize( - "failure", ("repeat", "cycle", "cap", "method_not_found", "internal_error", "unauthorized", "deadline") -) -@pytest.mark.parametrize("strict", (False, True)) -def test_optional_discovery_rejects_incomplete_walks( - method: str, failure: str, strict: bool, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture -) -> None: - monkeypatch.setattr( - mcp_client_module, - "MCP_TOOL_LISTING_MAX_PAGES", - 3 if failure in ("cycle", "repeat") else 2, - raising=False, - ) - monkeypatch.setattr(mcp_client_module, "MCP_TOOL_LISTING_TIMEOUT", 0.05) - field: Final = { - "prompts/list": "prompts", - "resources/list": "resources", - "resources/templates/list": "resourceTemplates", - }[method] - entry: Final = { - "prompts/list": {"name": "first"}, - "resources/list": {"name": "first", "uri": "test://first"}, - "resources/templates/list": {"name": "first", "uriTemplate": "test://{name}"}, - }[method] - loop: Final = ManualClockLoop() - - async def run() -> None: - cancelled: Final = asyncio.Event() - - async def respond(request: httpx2.Request) -> httpx2.Response: - payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) - if not isinstance(payload, JSONRPCRequest): - return httpx2.Response(202) - if payload.method == "initialize": - return httpx2.Response( - 200, - json={ - "jsonrpc": "2.0", - "id": payload.id, - "result": { - "protocolVersion": (payload.params or {})["protocolVersion"], - "capabilities": {"prompts": {}, "resources": {}}, - "serverInfo": {"name": "interrupted", "version": "1"}, - }, - }, - ) - assert payload.method == method - cursor: Final = (payload.params or {}).get("cursor") - if cursor is not None: - if failure == "deadline": - loop.advance(0.15) - try: - for _ in range(1_000): - await asyncio.sleep(0) - except asyncio.CancelledError: - cancelled.set() - raise - return httpx2.Response(500) - if failure == "unauthorized": - return httpx2.Response(401) - if failure in ("method_not_found", "internal_error"): - return httpx2.Response( - 200, - json={ - "jsonrpc": "2.0", - "id": payload.id, - "error": { - "code": -32601 if failure == "method_not_found" else -32603, - "message": "Later page unavailable", - }, - }, - ) - if failure == "deadline" and cursor is None: - loop.advance(0.1) - next_cursor: Final = ( - "private-cursor-2" if cursor == "private-cursor-1" and failure != "repeat" else "private-cursor-1" - ) - return httpx2.Response( - 200, json={"jsonrpc": "2.0", "id": payload.id, "result": {field: [entry], "nextCursor": next_cursor}} - ) - - responder: Final = AsyncMock(side_effect=respond) - client: Final = _MockTransportClient(responder, server_url="https://example.com/mcp", timeout=0.2) - operation: Final = { - "prompts/list": client.list_prompts, - "resources/list": client.list_resources, - "resources/templates/list": client.list_resource_templates, - }[method] - if strict: - error_type: Final = { - "internal_error": MCPError, - "unauthorized": httpx2.HTTPStatusError, - "deadline": TimeoutError, - }.get(failure, RuntimeError) - with pytest.raises(error_type): - await operation(raise_on_error=True) - else: - assert await operation() == [] - assert len( - tuple( - payload - for call in responder.call_args_list - if isinstance(payload := _JSONRPC_MESSAGE_ADAPTER.validate_json(call.args[0].content), JSONRPCRequest) - and payload.method == method - ) - ) == (3 if failure == "cycle" else 2) - assert "private-cursor" not in caplog.text - if failure == "deadline": - assert cancelled.is_set() - - try: - loop.run_until_complete(run()) - finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.run_until_complete(loop.shutdown_default_executor()) - loop.close() - - @pytest.mark.asyncio @pytest.mark.parametrize("method", ("prompts/list", "resources/list", "resources/templates/list")) async def test_optional_discovery_allows_exhaustion_at_page_cap(method: str, monkeypatch: pytest.MonkeyPatch) -> None: @@ -2867,24 +2508,6 @@ def test_standalone_client_request_does_not_require_proxy_dependencies(): assert result.stdout.strip() == "standalone request succeeded" -def test_client_import_before_proxy_credentials_succeeds_in_fresh_process(): - import subprocess - - result = subprocess.run( - [ - sys.executable, - "-c", - "import litellm.mcp_client.client; from litellm.proxy.mcp_server.mcp_server_manager import MCPServerManager; print(MCPServerManager.__name__)", - ], - capture_output=True, - text=True, - timeout=60, - check=False, - ) - assert result.returncode == 0, result.stderr - assert result.stdout.strip() == "MCPServerManager" - - @pytest.mark.asyncio @pytest.mark.parametrize("resolved", (False, True)) async def test_discovery_auth_fingerprint_tracks_effective_credentials(resolved: bool) -> None: @@ -3227,162 +2850,6 @@ async def test_http_close_cancellation_cannot_turn_into_success(original_error: assert task.cancelled() -@pytest.mark.asyncio -@pytest.mark.parametrize("protocol_version", ("auto", "2025-06-18")) -@pytest.mark.parametrize("cancel_mode", ("scope", "task", "wait_for", "read_timeout")) -@pytest.mark.parametrize("concurrency", (1, 5)) -@pytest.mark.parametrize("termination", ("ok", "hang", "hang_body")) -@pytest.mark.parametrize("raise_on_error", (False, True)) -async def test_cancellation_delivers_termination_over_tcp( - cancel_mode: str, concurrency: int, termination: str, raise_on_error: bool, protocol_version: str -) -> None: - started: Final = asyncio.Event() - scope_ready: Final[asyncio.Future[anyio.CancelScope]] = asyncio.get_running_loop().create_future() - terminations: Final[list[bytes]] = [] - starts: Final[list[bytes]] = [] - stop: Final = asyncio.Event() - connections: Final[list[asyncio.Task[None]]] = [] - - async def handle_connection(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - connection: Final = asyncio.current_task() - assert connection is not None - connections.append(connection) - try: - request_line: Final = await reader.readline() - if not request_line: - return - method: Final = request_line.split()[0] - headers: Final = await reader.readuntil(b"\r\n\r\n") - length: Final = next( - ( - int(line.split(b":", 1)[1]) - for line in headers.splitlines() - if line.lower().startswith(b"content-length:") - ), - 0, - ) - try: - body: Final = await reader.readexactly(length) - except asyncio.IncompleteReadError: - return - if method == b"DELETE": - terminations.append(body) - if termination != "ok": - await stop.wait() - return - writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") - elif method == b"GET": - writer.write(b"HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") - else: - payload: Final = json.loads(body) - if payload["method"] == "tools/call": - if termination == "hang_body": - writer.write( - b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" - b"Content-Length: 200\r\nConnection: close\r\n\r\n" - ) - await writer.drain() - starts.append(body) - if len(starts) == concurrency: - started.set() - await stop.wait() - return - if payload["method"] == "initialize": - if cancel_mode != "read_timeout": - await asyncio.sleep(0.75) - response: Final = json.dumps( - { - "jsonrpc": "2.0", - "id": payload["id"], - "result": { - "protocolVersion": "2025-06-18", - "capabilities": {"tools": {}}, - "serverInfo": {"name": "tcp-peer", "version": "1"}, - }, - } - ).encode() - writer.write( - b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nMcp-Session-Id: tcp-session\r\n" - + f"Content-Length: {len(response)}\r\nConnection: close\r\n\r\n".encode() - + response - ) - else: - writer.write(b"HTTP/1.1 202 Accepted\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") - await writer.drain() - finally: - writer.close() - await writer.wait_closed() - - listener: Final = await asyncio.start_server(handle_connection, "127.0.0.1", 0) - port: Final = listener.sockets[0].getsockname()[1] - client_timeout: Final = 2 if cancel_mode == "read_timeout" else 30 - client: Final = MCPClient( - server_url=f"http://127.0.0.1:{port}/mcp", - protocol_version=protocol_version, - timeout=client_timeout, - ) - - async def calls(): - results: Final = await asyncio.gather( - *( - client.call_tool(CallToolRequestParams(name="slow", arguments={}), raise_on_error=raise_on_error) - for _ in range(concurrency) - ), - return_exceptions=cancel_mode == "read_timeout", - ) - if cancel_mode == "read_timeout": - if raise_on_error: - assert all(isinstance(result, TimeoutError) for result in results) - else: - assert all(isinstance(result, CallToolResult) and result.is_error for result in results) - return results - - async def invoke(): - if cancel_mode == "scope": - with anyio.fail_after(None) as scope: - scope_ready.set_result(scope) - return await calls() - return await calls() - - try: - task: Final = asyncio.create_task(invoke()) - await asyncio.wait_for(started.wait(), 30) - if cancel_mode == "scope": - (await scope_ready).deadline = anyio.current_time() + 0.2 - if cancel_mode == "task": - task.cancel() - expected_error: Final = ( - TimeoutError - if cancel_mode == "read_timeout" - else asyncio.CancelledError - if cancel_mode == "task" - else TimeoutError - ) - if cancel_mode == "read_timeout": - done, _ = await asyncio.wait((task,), timeout=8) - assert task in done, "Read timeout and bounded cleanup must complete without external cancellation" - await task - elif cancel_mode == "wait_for": - with pytest.raises(expected_error): - await asyncio.wait_for(task, 0.2) - else: - with pytest.raises(expected_error): - await task - assert len(starts) == concurrency - assert len(terminations) == concurrency, "Each cancelled call must send DELETE over a fresh TCP connection" - finally: - stop.set() - if not task.done(): - task.cancel() - await asyncio.wait((task,), timeout=8) - listener.close() - for connection in connections: - connection.cancel() - closed: Final = await asyncio.wait_for(asyncio.gather(*connections, return_exceptions=True), 2) - assert all(result is None or isinstance(result, asyncio.CancelledError) for result in closed), closed - await asyncio.wait_for(listener.wait_closed(), 2) - - @pytest.mark.asyncio @pytest.mark.parametrize("revision", ["2024-11-05", "2025-03-26", "2025-06-18", "2025-11-25", "auto"]) @pytest.mark.parametrize("accepted", [True, False]) @@ -3710,80 +3177,6 @@ async def test_modern_call_requires_schema_from_bounded_listing( assert observed.count("tools/call") == int(schema_available) -def test_modern_call_uses_discovery_deadline_for_multiple_pages() -> None: - from mcp.types import ToolsCapability - - loop: Final = AutojumpClockLoop() - - async def respond(request: httpx2.Request) -> httpx2.Response: - payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) - assert isinstance(payload, JSONRPCRequest) - if payload.method == "server/discover": - return httpx2.Response( - 200, - json={ - "jsonrpc": "2.0", - "id": payload.id, - "result": DiscoverResult( - supported_versions=["2026-07-28"], capabilities=ServerCapabilities(tools=ToolsCapability()) - ).model_dump(by_alias=True, exclude_none=True), - }, - ) - if payload.method == "tools/list": - await asyncio.sleep(0.06) - assert payload.params is not None - page: Final = ( - {"tools": [], "nextCursor": "second"} - if "cursor" not in payload.params - else { - "tools": [ - { - "name": "quote", - "inputSchema": { - "type": "object", - "properties": {"workspace": {"type": "string", "x-mcp-header": "Workspace"}}, - }, - } - ] - } - ) - return httpx2.Response( - 200, - json={ - "jsonrpc": "2.0", - "id": payload.id, - "result": {"resultType": "complete", "cacheScope": "private", "ttlMs": 0, **page}, - }, - ) - assert payload.method == "tools/call" - assert request.headers["mcp-param-workspace"] == "engineering" - return httpx2.Response( - 200, - json={ - "jsonrpc": "2.0", - "id": payload.id, - "result": {"resultType": "complete", "content": [{"type": "text", "text": "quoted"}], "isError": False}, - }, - ) - - async def run() -> None: - client: Final = _MockTransportClient( - respond, server_url="https://example.com/mcp", protocol_version="2026-07-28", timeout=0.1 - ) - assert [tool.name for tool in await client.list_tools(raise_on_error=True)] == ["quote"] - result: Final = await client.call_tool( - CallToolRequestParams(name="quote", arguments={"workspace": "engineering"}), raise_on_error=True - ) - assert result.is_error is False - assert result.content[0].text == "quoted" - - try: - loop.run_until_complete(run()) - finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() - - @pytest.mark.asyncio async def test_cancelled_modern_catalog_load_prevents_tool_execution() -> None: from queue import SimpleQueue @@ -4138,45 +3531,6 @@ class _StatefulUpstreamClient(MCPClient): return streamable_http_client(self.server_url, http_client=http_client), http_client -class _CleanupHoldingASGITransport(httpx2.ASGITransport): - def __init__(self, app: ASGIApp, cleanup_started: asyncio.Event, release_cleanup: asyncio.Event) -> None: - super().__init__(app=app) - self._cleanup_started: Final = cleanup_started - self._release_cleanup: Final = release_cleanup - - async def aclose(self) -> None: - self._cleanup_started.set() - await self._release_cleanup.wait() - await super().aclose() - - -class _CleanupHoldingStatefulUpstreamClient(_StatefulUpstreamClient): - def __init__( - self, - app: ASGIApp, - *, - server_url: str, - transport_type: MCPTransport, - cleanup_started: asyncio.Event, - release_cleanup: asyncio.Event, - ) -> None: - super().__init__(app, server_url=server_url, transport_type=transport_type) - self._cleanup_started: Final = cleanup_started - self._release_cleanup: Final = release_cleanup - - def _create_transport_context(self) -> tuple[_TransportContext, httpx2.AsyncClient]: - transport: Final = _CleanupHoldingASGITransport( - self._app, - self._cleanup_started, - self._release_cleanup, - ) - http_client: Final = self._create_httpx_client_factory(transport=transport)( - headers=self._get_auth_headers(), - timeout=httpx2.Timeout(self.timeout), - ) - return streamable_http_client(self.server_url, http_client=http_client), http_client - - class _InProcessUpstreamClient(MCPClient): def __init__(self, server: UpstreamServer, *, elicitation_callback: ElicitationFnT | None = None) -> None: super().__init__( @@ -4258,32 +3612,6 @@ def _upstream_with_status_for_method(method: str, status_code: int) -> Starlette return app -def _upstream_with_malformed_response_for_method(method: str) -> Starlette: - service: Final = UpstreamServer("malformed response upstream") - - @service.tool() - def malformed_tool() -> str: - return "unreachable" - - app: Final = service.streamable_http_app( - stateless_http=False, - json_response=True, - transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False), - ) - - async def malformed_method(request: Request, call_next: RequestResponseEndpoint) -> Response: - if request.method == "GET": - return Response(content="data: {}\n\n", media_type="text/event-stream") - if request.method == "POST": - payload: Final = await request.json() - if isinstance(payload, dict) and payload.get("method") == method: - return Response(content="not a JSON-RPC message", media_type="text/event-stream") - return await call_next(request) - - app.add_middleware(BaseHTTPMiddleware, dispatch=malformed_method) - return app - - @pytest.mark.asyncio async def test_persistent_session_keeps_upstream_state_across_tool_calls() -> None: app: Final = _stateful_upstream() @@ -4319,48 +3647,6 @@ def _client_with_session( return client, client.open_persistent_session(admission) -@pytest.mark.asyncio -async def test_cancelling_one_waiter_for_session_readiness_does_not_cancel_other_waiters() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - initialize_started: Final = asyncio.Event() - release_initialize: Final = asyncio.Event() - - async def gated_server(scope: Scope, receive: Receive, send: Send) -> None: - async def gated_send(message: Message) -> None: - if message["type"] == "http.response.body" and not initialize_started.is_set(): - initialize_started.set() - await release_initialize.wait() - await send(message) - - await app(scope, receive, gated_send) - - gated_app: Final[ASGIApp] = gated_server - client: Final = _StatefulUpstreamClient( - gated_app, server_url="http://upstream/mcp", transport_type=MCPTransport.http - ) - session: Final = client.open_persistent_session() - - async def operation(_: ClientSession) -> str: - return "ready" - - cancelled_waiter: Final = asyncio.create_task(session.run(operation)) - remaining_waiter: Final = asyncio.create_task(session.run(operation)) - try: - await asyncio.wait_for(initialize_started.wait(), 5) - await asyncio.sleep(0) - cancelled_waiter.cancel() - with pytest.raises(asyncio.CancelledError): - await cancelled_waiter - release_initialize.set() - assert await asyncio.wait_for(remaining_waiter, 5) == "ready" - assert not session.closed - finally: - release_initialize.set() - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - - @pytest.mark.asyncio async def test_persistent_session_survives_a_caller_timeout_on_one_operation() -> None: app: Final = _stateful_upstream() @@ -4388,99 +3674,6 @@ async def test_persistent_session_survives_a_caller_timeout_on_one_operation() - await asyncio.wait_for(session.wait_closed(), 5) -@pytest.mark.asyncio -async def test_persistent_session_survives_recoverable_mcp_errors_and_closes_on_connection_closed() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - client, session = _client_with_session(app) - - async def invalid_operation(_: ClientSession) -> str: - raise MCPError(code=-32602, message="invalid params") - - try: - with pytest.raises(MCPError): - await session.run(invalid_operation) - assert not session.closed - selected: Final = await client.call_tool( - CallToolRequestParams(name="select_project", arguments={"name": "recoverable"}), - persistent_session=session, - ) - created: Final = await client.call_tool( - CallToolRequestParams(name="create_feature", arguments={"title": "still-open"}), - persistent_session=session, - ) - finally: - session.close() - assert selected.is_error is False - assert created.content[0].text == "recoverable/still-open" - await asyncio.wait_for(session.wait_closed(), 5) - - connection_app: Final = _stateful_upstream() - async with connection_app.router.lifespan_context(connection_app): - _, session = _client_with_session(connection_app) - started: Final = asyncio.Event() - release: Final = asyncio.Event() - - async def disconnected_operation(_: ClientSession) -> str: - started.set() - await release.wait() - raise MCPError(code=CONNECTION_CLOSED, message="upstream connection closed") - - async def pending_operation(_: ClientSession) -> str: - return "unused" - - active: Final = asyncio.create_task(session.run(disconnected_operation)) - await asyncio.wait_for(started.wait(), 5) - pending: Final = asyncio.create_task(session.run(pending_operation)) - await asyncio.sleep(0) - release.set() - with pytest.raises(MCPError) as caught: - await asyncio.wait_for(active, 5) - assert caught.value.error.code == CONNECTION_CLOSED - with pytest.raises((MCPError, UpstreamSessionClosedError)): - await asyncio.wait_for(pending, 5) - await asyncio.wait_for(session.wait_closed(), 5) - assert session.closed - - -@pytest.mark.asyncio -async def test_ended_persistent_session_is_not_reusable_during_transport_cleanup() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - cleanup_started: Final = asyncio.Event() - release_cleanup: Final = asyncio.Event() - client: Final = _CleanupHoldingStatefulUpstreamClient( - app, - server_url="http://upstream/mcp", - transport_type=MCPTransport.http, - cleanup_started=cleanup_started, - release_cleanup=release_cleanup, - ) - session: Final = client.open_persistent_session() - - async def disconnected_operation(_: ClientSession) -> str: - raise MCPError(code=CONNECTION_CLOSED, message="upstream connection closed") - - try: - with pytest.raises(MCPError) as caught: - await asyncio.wait_for(session.run(disconnected_operation), 5) - assert caught.value.error.code == CONNECTION_CLOSED - await asyncio.wait_for(cleanup_started.wait(), 5) - - assert not session.reusable - assert not session.closed - session.retire() - - release_cleanup.set() - await asyncio.wait_for(session.wait_closed(), 5) - assert session.closed - finally: - release_cleanup.set() - if not session.closed: - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - - @pytest.mark.asyncio async def test_persistent_tool_call_preserves_upstream_http_status_error() -> None: app: Final = _upstream_with_status_for_method("tools/call", 401) @@ -4534,104 +3727,6 @@ async def test_persistent_tool_call_classifies_initialized_notification_status(s await asyncio.wait_for(session.wait_closed(), 5) -@pytest.mark.asyncio -async def test_persistent_session_remaps_stream_error_over_mcp_error() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - client: Final = _StatefulUpstreamClient( - app, - server_url="http://upstream/mcp", - transport_type=MCPTransport.http, - ) - session: Final = client.open_persistent_session() - request: Final = httpx2.Request("POST", "http://upstream/mcp") - response: Final = httpx2.Response(401, request=request, headers={"WWW-Authenticate": "Bearer"}) - http_error: Final = httpx2.HTTPStatusError("401 Unauthorized", request=request, response=response) - - async def operation(_: ClientSession) -> str: - stream_error: Final[asyncio.Future[Exception] | None] = session._stream_error - assert stream_error is not None - stream_error.set_result(http_error) - raise MCPError(code=-32603, message="response stream closed") - - try: - with pytest.raises(httpx2.HTTPStatusError) as caught: - await session.run(operation) - assert caught.value is http_error - assert not session.reusable - finally: - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - - -@pytest.mark.asyncio -async def test_persistent_session_ends_when_a_malformed_response_is_remapped() -> None: - app: Final = _upstream_with_malformed_response_for_method("tools/call") - async with app.router.lifespan_context(app): - client: Final = _StatefulUpstreamClient( - app, - server_url="http://upstream/mcp", - transport_type=MCPTransport.http, - ) - session: Final = client.open_persistent_session() - try: - - async def ready(_: ClientSession) -> str: - return "ready" - - assert await asyncio.wait_for(session.run(ready), 5) == "ready" - stream_error: Final[asyncio.Future[Exception] | None] = session._stream_error - assert stream_error is not None - recorded_stream_error: Final = await asyncio.wait_for(asyncio.shield(stream_error), 5) - assert isinstance(recorded_stream_error, ValidationError) - - with pytest.raises(ValidationError) as caught: - await client.call_tool( - CallToolRequestParams(name="malformed_tool", arguments={}), - raise_on_error=True, - persistent_session=session, - ) - assert caught.value is recorded_stream_error - assert not session.reusable - await asyncio.wait_for(session.wait_closed(), 5) - assert session.closed - finally: - if not session.closed: - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - - -@pytest.mark.asyncio -async def test_failed_persistent_initialize_is_not_reusable_during_transport_cleanup() -> None: - app: Final = _upstream_with_status_for_method("initialize", 500) - async with app.router.lifespan_context(app): - cleanup_started: Final = asyncio.Event() - release_cleanup: Final = asyncio.Event() - client: Final = _CleanupHoldingStatefulUpstreamClient( - app, - server_url="http://upstream/mcp", - transport_type=MCPTransport.http, - cleanup_started=cleanup_started, - release_cleanup=release_cleanup, - ) - session: Final = client.open_persistent_session() - pending: Final = asyncio.create_task(session.run(lambda _: "ready")) - try: - await asyncio.wait_for(cleanup_started.wait(), 5) - assert not session.reusable - assert not session.closed - release_cleanup.set() - with pytest.raises(httpx2.HTTPStatusError): - await asyncio.wait_for(pending, 5) - await asyncio.wait_for(session.wait_closed(), 5) - assert session.closed - finally: - release_cleanup.set() - if not session.closed: - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - - @pytest.mark.asyncio async def test_persistent_session_routes_elicitation_to_the_current_clients_callback() -> None: service: Final = UpstreamServer("elicitation") @@ -4687,406 +3782,6 @@ async def test_persistent_session_without_elicitation_callback_uses_sdk_default( assert "elicitation callback is unavailable" not in str(persistent.model_dump(mode="json")) -@pytest.mark.asyncio -async def test_cancelled_queued_operation_does_not_run_and_session_accepts_a_later_operation() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - _, session = _client_with_session(app) - first_started: Final = asyncio.Event() - release_first: Final = asyncio.Event() - cancelled_operation_ran: Final = asyncio.Event() - - async def first_operation(_: ClientSession) -> str: - first_started.set() - await release_first.wait() - return "first" - - async def cancelled_operation(_: ClientSession) -> str: - cancelled_operation_ran.set() - return "cancelled" - - async def third_operation(_: ClientSession) -> str: - return "third" - - first_task: Final = asyncio.create_task(session.run(first_operation)) - try: - await asyncio.wait_for(first_started.wait(), 5) - second_task: Final = asyncio.create_task(session.run(cancelled_operation)) - await asyncio.sleep(0) - assert session._queue.qsize() == 1 - second_task.cancel() - with pytest.raises(asyncio.CancelledError): - await second_task - third_task: Final = asyncio.create_task(session.run(third_operation)) - await asyncio.sleep(0) - assert session._queue.qsize() == 2 - release_first.set() - assert await asyncio.wait_for(first_task, 5) == "first" - assert await asyncio.wait_for(third_task, 5) == "third" - assert not cancelled_operation_ran.is_set() - assert not session.closed - finally: - release_first.set() - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - - -@pytest.mark.asyncio -async def test_retiring_persistent_session_drains_in_flight_and_queued_operations() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - _, session = _client_with_session(app) - first_started: Final = asyncio.Event() - release_first: Final = asyncio.Event() - - async def first_operation(_: ClientSession) -> str: - first_started.set() - await release_first.wait() - return "first" - - async def queued_operation(_: ClientSession) -> str: - return "queued" - - first_waiter: Final = asyncio.create_task(session.run(first_operation)) - try: - await asyncio.wait_for(first_started.wait(), 5) - queued_waiter: Final = asyncio.create_task(session.run(queued_operation)) - await asyncio.sleep(0) - assert session._queue.qsize() == 1 - session.retire() - release_first.set() - - results: Final = await asyncio.wait_for(asyncio.gather(first_waiter, queued_waiter), 5) - assert results == ["first", "queued"] - await asyncio.wait_for(session.wait_closed(), 5) - assert session.closed - finally: - release_first.set() - if not session.closed: - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - - -@pytest.mark.asyncio -async def test_timed_out_persistent_operation_waiting_for_admission_is_skipped() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - admission_slots: Final = asyncio.Semaphore(1) - admission_requested: Final = asyncio.Event() - - def admission() -> AbstractAsyncContextManager[None]: - admission_requested.set() - return admission_slots - - _, session = _client_with_session(app, admission) - operation_ran: Final = asyncio.Event() - - async def warmup(_: ClientSession) -> str: - return "ready" - - async def timed_out_operation(_: ClientSession) -> str: - operation_ran.set() - return "expired" - - async def later_operation(_: ClientSession) -> str: - return "later" - - try: - assert await session.run(warmup) == "ready" - admission_requested.clear() - async with admission_slots: - timed_out_call: Final = asyncio.create_task(session.run(timed_out_operation)) - await asyncio.wait_for(admission_requested.wait(), 1) - with pytest.raises(asyncio.TimeoutError): - await asyncio.wait_for(timed_out_call, 0.05) - assert not operation_ran.is_set() - - later_result: Final = await asyncio.wait_for(session.run(later_operation), 1) - - assert later_result == "later" - assert session.reusable - assert not operation_ran.is_set() - finally: - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - - -@pytest.mark.asyncio -async def test_persistent_session_holds_admission_until_initialize_completes() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - initialize_seen: Final = asyncio.Event() - admission_slots: Final = asyncio.Semaphore(1) - - async def observing_app(scope: Scope, receive: Receive, send: Send) -> None: - async def observe_send(message: Message) -> None: - if message["type"] == "http.response.body": - initialize_seen.set() - await send(message) - - await app(scope, receive, observe_send) - - client: Final = _StatefulUpstreamClient( - observing_app, - server_url="http://upstream/mcp", - transport_type=MCPTransport.http, - ) - session: Final = client.open_persistent_session(lambda: admission_slots) - - async def operation(_: ClientSession) -> str: - return "ready" - - run_task: Final = asyncio.create_task(session.run(operation)) - try: - async with admission_slots: - with pytest.raises(asyncio.TimeoutError): - await asyncio.wait_for(initialize_seen.wait(), 0.05) - assert not session._ready.done() - assert not run_task.done() - - result: Final = await asyncio.wait_for(run_task, 5) - assert result == "ready" - assert initialize_seen.is_set() - finally: - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - await asyncio.gather(run_task, return_exceptions=True) - - -@pytest.mark.asyncio -async def test_persistent_session_releases_admission_when_connect_fails() -> None: - async def failing_app(_: Scope, __: Receive, ___: Send) -> None: - raise httpx2.ConnectError("connect failed") - - client: Final = _StatefulUpstreamClient( - failing_app, - server_url="http://upstream/mcp", - transport_type=MCPTransport.http, - ) - admission_slots: Final = asyncio.Semaphore(1) - session: Final = client.open_persistent_session(lambda: admission_slots) - - async def operation(_: ClientSession) -> str: - return "unreachable" - - try: - with pytest.raises(httpx2.ConnectError, match="connect failed"): - await asyncio.wait_for(session.run(operation), 1) - await asyncio.wait_for(session.wait_closed(), 1) - - await asyncio.wait_for(admission_slots.acquire(), 0.1) - admission_slots.release() - finally: - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - - -@pytest.mark.asyncio -async def test_failed_connect_is_not_reusable_during_admission_teardown() -> None: - connect_failed: Final = asyncio.Event() - admission_exit_started: Final = asyncio.Event() - release_admission_exit: Final = asyncio.Event() - - async def failing_app(_: Scope, __: Receive, ___: Send) -> None: - connect_failed.set() - raise httpx2.ConnectError("connect failed") - - @asynccontextmanager - async def blocking_admission() -> AsyncIterator[None]: - try: - yield - finally: - admission_exit_started.set() - await release_admission_exit.wait() - - client: Final = _StatefulUpstreamClient( - failing_app, - server_url="http://upstream/mcp", - transport_type=MCPTransport.http, - ) - session: Final = client.open_persistent_session(blocking_admission) - - async def operation(_: ClientSession) -> str: - return "unreachable" - - run_task: Final = asyncio.create_task(session.run(operation)) - try: - await asyncio.wait_for(connect_failed.wait(), 1) - await asyncio.wait_for(admission_exit_started.wait(), 1) - - assert not run_task.done() - assert not session.reusable - assert not session.closed - - release_admission_exit.set() - with pytest.raises(httpx2.ConnectError, match="connect failed"): - await asyncio.wait_for(run_task, 1) - await asyncio.wait_for(session.wait_closed(), 1) - assert session.closed - finally: - release_admission_exit.set() - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - await asyncio.gather(run_task, return_exceptions=True) - - -@pytest.mark.asyncio -async def test_persistent_session_admission_is_acquired_only_when_operation_runs() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - admission_slots: Final = asyncio.Semaphore(2) - _, session = _client_with_session(app, lambda: admission_slots) - first_started: Final = asyncio.Event() - release_first: Final = asyncio.Event() - operation_order: Final[list[str]] = [] - - async def first_operation(_: ClientSession) -> str: - operation_order.append("first") - first_started.set() - await release_first.wait() - return "first" - - async def second_operation(_: ClientSession) -> str: - operation_order.append("second") - return "second" - - async def third_operation(_: ClientSession) -> str: - operation_order.append("third") - return "third" - - first_task: Final = asyncio.create_task(session.run(first_operation)) - second_task: Final = asyncio.create_task(session.run(second_operation)) - third_task: Final = asyncio.create_task(session.run(third_operation)) - try: - await asyncio.wait_for(first_started.wait(), 5) - await asyncio.sleep(0) - assert session._queue.qsize() == 2 - await asyncio.wait_for(admission_slots.acquire(), 0.1) - try: - assert admission_slots.locked() - finally: - admission_slots.release() - assert operation_order == ["first"] - - release_first.set() - results: Final = await asyncio.wait_for( - asyncio.gather(first_task, second_task, third_task), - 5, - ) - - assert results == ["first", "second", "third"] - assert operation_order == ["first", "second", "third"] - finally: - release_first.set() - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - await asyncio.gather(first_task, second_task, third_task, return_exceptions=True) - - -@pytest.mark.asyncio -async def test_retiring_idle_persistent_session_closes_and_rejects_new_operations() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - _, session = _client_with_session(app) - - async def operation(_: ClientSession) -> str: - return "ready" - - try: - assert await session.run(operation) == "ready" - session.retire() - await asyncio.wait_for(session.wait_closed(), 1) - - assert session.closed - with pytest.raises(UpstreamSessionClosedError): - await session.run(operation) - finally: - if not session.closed: - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - - -@pytest.mark.asyncio -async def test_closing_persistent_session_mid_operation_fails_the_waiter_instead_of_hanging() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - _, session = _client_with_session(app) - started: Final = asyncio.Event() - - async def slow_operation(_: object) -> str: - started.set() - await asyncio.sleep(30) - return "never" - - waiter: Final = asyncio.ensure_future(session.run(slow_operation)) - await asyncio.wait_for(started.wait(), 5) - session.close() - with pytest.raises(RuntimeError, match="upstream MCP session closed"): - await asyncio.wait_for(waiter, 5) - await asyncio.wait_for(session.wait_closed(), 5) - - -@pytest.mark.asyncio -async def test_persistent_session_reports_an_upstream_cancellation_as_a_runtime_error_not_a_cancelled_caller() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - client, session = _client_with_session(app) - - async def cancelled_upstream(_: object) -> str: - raise asyncio.CancelledError() - - try: - with pytest.raises(RuntimeError, match="cancelled"): - await asyncio.wait_for(session.run(cancelled_upstream), 5) - selected: Final = await client.call_tool( - CallToolRequestParams(name="select_project", arguments={"name": "e"}), persistent_session=session - ) - finally: - session.close() - assert selected.is_error is False, "an upstream cancellation must not be mistaken for a cancelled caller" - await asyncio.wait_for(session.wait_closed(), 5) - - -@pytest.mark.asyncio -async def test_closing_persistent_session_fails_a_caller_blocked_on_a_full_queue_instead_of_hanging() -> None: - from litellm.mcp_client.client import _MAX_PENDING_OPERATIONS - - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - _, session = _client_with_session(app) - started: Final = asyncio.Event() - - async def slow_operation(_: object) -> str: - started.set() - await asyncio.sleep(30) - return "never" - - waiters: Final = tuple( - asyncio.ensure_future(session.run(slow_operation)) for _ in range(_MAX_PENDING_OPERATIONS + 2) - ) - await asyncio.wait_for(started.wait(), 5) - await asyncio.sleep(0) - session.close() - outcomes: Final = await asyncio.wait_for(asyncio.gather(*waiters, return_exceptions=True), 5) - assert all(isinstance(outcome, RuntimeError) for outcome in outcomes), outcomes - await asyncio.wait_for(session.wait_closed(), 5) - - -@pytest.mark.asyncio -async def test_persistent_session_ends_after_a_broken_stream_so_the_next_call_gets_a_fresh_one() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - _, session = _client_with_session(app) - - async def broken_stream(_: object) -> str: - raise anyio.BrokenResourceError() - - with pytest.raises(anyio.BrokenResourceError): - await asyncio.wait_for(session.run(broken_stream), 5) - await asyncio.wait_for(session.wait_closed(), 5) - assert session.closed, "a dead transport must end the session instead of being reused" - - @pytest.mark.asyncio @pytest.mark.parametrize("session_started_in_request", [True, False]) async def test_persistent_session_releases_the_current_requests_scope_response(session_started_in_request) -> None: @@ -5134,76 +3829,6 @@ async def test_persistent_session_releases_the_current_requests_scope_response(s session.close() await asyncio.wait_for(session.wait_closed(), 5) -@pytest.mark.asyncio -@pytest.mark.parametrize("legacy_callback", [False, True]) -async def test_tool_progress_preserves_message_and_legacy_callback(legacy_callback: bool) -> None: - received = AsyncMock() - - async def legacy(progress: float, total: float | None) -> None: - await received(progress, total) - - async def modern(progress: float, total: float | None, message: str | None) -> None: - await received(progress, total, message) - - async def call_tool(**kwargs): - await kwargs["progress_callback"](1.0, 2.0, "halfway") - return CallToolResult(content=[]) - - session = AsyncMock(spec=ClientSession, call_tool=AsyncMock(side_effect=call_tool)) - - async def run(operation, **kwargs): - return await operation(session) - - persistent = MagicMock(spec=PersistentMCPSession, run=AsyncMock(side_effect=run)) - client = MCPClient(server_url="https://example.com/mcp") - result = await client.call_tool( - CallToolRequestParams(name="slow"), - host_progress_callback=legacy if legacy_callback else modern, - persistent_session=persistent, - ) - assert not result.is_error - if legacy_callback: - received.assert_awaited_once_with(1.0, 2.0) - else: - received.assert_awaited_once_with(1.0, 2.0, "halfway") - - -@pytest.mark.asyncio -async def test_cancelling_persistent_operation_stops_work_and_keeps_session_usable() -> None: - app: Final = _stateful_upstream() - async with app.router.lifespan_context(app): - client, session = _client_with_session(app) - started: Final = asyncio.Event() - stopped: Final = asyncio.Event() - - async def slow_operation(_: ClientSession) -> None: - started.set() - try: - await asyncio.Event().wait() - finally: - stopped.set() - - waiter: Final = asyncio.create_task(session.run(slow_operation)) - try: - await asyncio.wait_for(started.wait(), 5) - waiter.cancel() - with pytest.raises(asyncio.CancelledError): - await waiter - await asyncio.sleep(0.1) - assert stopped.is_set(), "Caller cancellation must stop the running upstream operation" - result: Final = await asyncio.wait_for( - client.call_tool( - CallToolRequestParams(name="select_project", arguments={"name": "after-cancellation"}), - persistent_session=session, - ), - 5, - ) - assert not result.is_error - finally: - session.close() - await asyncio.wait_for(session.wait_closed(), 5) - await asyncio.gather(waiter, return_exceptions=True) - @pytest.mark.asyncio @pytest.mark.parametrize("header_name", ["Authorization", "x-upstream-token"]) diff --git a/tests/unit/proxy/auth/test_master_key_boot_check.py b/tests/unit/proxy/auth/test_master_key_boot_check.py index dc490068c4d..50075a97c6c 100644 --- a/tests/unit/proxy/auth/test_master_key_boot_check.py +++ b/tests/unit/proxy/auth/test_master_key_boot_check.py @@ -1,10 +1,6 @@ import asyncio import re -import shutil -import subprocess -import sys from collections.abc import Mapping -from pathlib import Path from typing import Final import pytest @@ -26,7 +22,6 @@ from litellm.proxy.auth.master_key_boot_check import ( UnsafeMasterKeyError, UnsafeMasterKeyReason, UnsafeMasterKeyRefused, - announce_on_stderr_at_exit, enforce_master_key_boot_verdict, master_key_boot_verdict, render_refusal, @@ -361,24 +356,6 @@ def test_migration_with_an_exported_key_replaces_it_in_place_and_numbers_every_s assert text.index("os.environ/") < text.index(MIGRATE_FROM_MASTER_KEY_ENV_VAR + "=") < text.index("Start the proxy") -@pytest.mark.skipif(shutil.which("openssl") is None, reason="the printed command shells out to openssl") -@pytest.mark.parametrize("from_master_key", [PUBLICLY_KNOWN_KEY, "", " " + PUBLICLY_KNOWN_KEY]) -def test_printed_migration_commands_save_both_keys_to_the_env_file(tmp_path: Path, from_master_key: str): - from dotenv import dotenv_values - - text = render_refusal( - _refusal(migration=StoredSecretsMigration(from_master_key=from_master_key, encrypted_value_count=1)) - ) - commands = [line.strip() for line in text.splitlines() if line.strip().startswith("echo ")] - - subprocess.run(["bash", "-c", "\n".join(commands)], cwd=tmp_path, capture_output=True, text=True, check=True) - - saved = dotenv_values(tmp_path / ".env") - assert saved[MIGRATE_FROM_MASTER_KEY_ENV_VAR] == from_master_key - assert _verdict(saved[MASTER_KEY_ENV_VAR]) == SafeMasterKey() - assert len(commands) == 2 - - @pytest.mark.parametrize("migration", [_MIGRATION, None]) def test_override_hint_is_the_last_paragraph(migration: StoredSecretsMigration | None): text = render_refusal(_refusal(migration=migration)) @@ -388,30 +365,6 @@ def test_override_hint_is_the_last_paragraph(migration: StoredSecretsMigration | assert f"general_settings.{WEAK_OR_UNSET_MASTER_KEY_OVERRIDE_SETTING}" in last_paragraph -@pytest.mark.skipif(shutil.which("openssl") is None, reason="the printed command shells out to openssl") -def test_printed_command_saves_a_key_the_boot_check_accepts(tmp_path: Path): - completed = subprocess.run( - ["bash", "-c", GENERATE_MASTER_KEY_COMMAND], cwd=tmp_path, capture_output=True, text=True, check=True - ) - - saved = (tmp_path / ".env").read_text() - match = re.fullmatch(rf"{MASTER_KEY_ENV_VAR}=(sk-[0-9a-f]{{64}})\n", saved) - assert match is not None - assert completed.stdout == saved - assert _verdict(match.group(1)) == SafeMasterKey() - - -@pytest.mark.skipif(shutil.which("openssl") is None, reason="the printed command shells out to openssl") -def test_rotation_command_prints_a_key_the_boot_check_accepts_and_saves_nothing(tmp_path: Path): - completed = subprocess.run( - ["bash", "-c", PRINT_NEW_MASTER_KEY_COMMAND], cwd=tmp_path, capture_output=True, text=True, check=True - ) - - assert re.fullmatch(r"sk-[0-9a-f]{64}\n", completed.stdout) is not None - assert _verdict(completed.stdout.strip()) == SafeMasterKey() - assert list(tmp_path.iterdir()) == [] - - def test_refusal_announces_the_fix_and_aborts_the_boot(): announced: list[str] = [] refusal = _refusal(source=ConfigFileSource(config_file_path="/app/config.yaml")) @@ -429,19 +382,3 @@ def test_safe_and_overridden_keys_boot_without_announcing(verdict: MasterKeyBoot enforce_master_key_boot_verdict(verdict, announce=announced.append) assert announced == [] - - -def test_announced_fix_is_the_last_thing_a_crashing_process_prints(): - crash_after_announcing = ( - "from litellm.proxy.auth.master_key_boot_check import announce_on_stderr_at_exit\n" - "announce_on_stderr_at_exit('THE FIX')\n" - "print('buffered stdout')\n" - "raise RuntimeError('lifespan failed')\n" - ) - - completed = subprocess.run([sys.executable, "-c", crash_after_announcing], capture_output=True, text=True) - - assert completed.returncode != 0 - assert "RuntimeError: lifespan failed" in completed.stderr - assert completed.stderr.endswith("THE FIX") - assert completed.stdout == "buffered stdout\n" diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index b5dc6042474..ba0f436f3dd 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -3,16 +3,12 @@ import contextvars import json import logging import os -import subprocess -import sys import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from contextlib import contextmanager, nullcontext from datetime import datetime, timedelta, timezone from functools import partial from io import BytesIO -from pathlib import Path -from textwrap import dedent from types import MappingProxyType, SimpleNamespace from typing import Final, Literal from unittest.mock import ANY, AsyncMock, MagicMock, patch @@ -12170,129 +12166,6 @@ def test_the_request_span_parents_datastore_spans_of_the_request(monkeypatch, al assert contextvars.copy_context().run(ambient_parent_after_auth) is request_span -_RECORDING_DDTRACE = dedent( - """ - import functools - import inspect - - - class _Span: - def __enter__(self): - return self - - def __exit__(self, *exc): - return None - - - class _Tracer: - def __init__(self): - self.spans = [] - - def wrap(self, name=None, **kwargs): - def decorator(f): - span_name = name or f"{f.__module__}.{f.__name__}" - if inspect.iscoroutinefunction(f): - - @functools.wraps(f) - async def async_wrapped(*args, **kw): - self.spans.append(span_name) - return await f(*args, **kw) - - return async_wrapped - - @functools.wraps(f) - def wrapped(*args, **kw): - self.spans.append(span_name) - return f(*args, **kw) - - return wrapped - - return decorator - - def trace(self, name, **kwargs): - return _Span() - - def current_span(self): - return None - - def current_root_span(self): - return None - - - tracer = _Tracer() - """ -) - -_DDTRACE_AUTH_PROBE = dedent( - """ - import asyncio - import json - - import ddtrace - from fastapi import Request - from starlette.datastructures import URL - - import litellm.proxy.proxy_server as proxy_server - from litellm.proxy._types import ProxyException - from litellm.proxy.auth.user_api_key_auth import user_api_key_auth - - proxy_server.master_key = "sk-probe" - - - async def auth(api_key): - request = Request(scope={"type": "http", "headers": [], "method": "POST", "path": "/chat/completions"}) - request._url = URL(url="/chat/completions") - try: - await user_api_key_auth( - request=request, - api_key=api_key, - azure_api_key_header="", - anthropic_api_key_header=None, - google_ai_studio_api_key_header=None, - azure_apim_header=None, - custom_litellm_key_header=None, - ) - return "accepted" - except ProxyException: - return "rejected" - - - async def main(): - outcomes = [await auth("Bearer sk-probe"), await auth("Bearer sk-wrong")] - print(json.dumps({"outcomes": outcomes, "spans": ddtrace.tracer.spans})) - - - asyncio.run(main()) - """ -) - - -def test_user_api_key_auth_opens_a_datadog_span_for_accepted_and_rejected_keys(tmp_path: Path): - stub_root = tmp_path / "site" - (stub_root / "ddtrace").mkdir(parents=True) - (stub_root / "ddtrace" / "__init__.py").write_text(_RECORDING_DDTRACE) - probe = tmp_path / "probe.py" - probe.write_text(_DDTRACE_AUTH_PROBE) - repo_root = Path(litellm.__file__).resolve().parent.parent - env = { - **os.environ, - "USE_DDTRACE": "true", - "PYTHONPATH": os.pathsep.join( - [str(stub_root), str(repo_root)] + [p for p in (os.environ.get("PYTHONPATH"),) if p] - ), - } - - result = subprocess.run( - [sys.executable, str(probe)], env=env, cwd=repo_root, capture_output=True, text=True, check=False - ) - - assert result.returncode == 0, result.stderr[-4000:] - report = json.loads(result.stdout.strip().splitlines()[-1]) - assert report["outcomes"] == ["accepted", "rejected"] - auth_span = "litellm.proxy.auth.user_api_key_auth.user_api_key_auth" - assert [span for span in report["spans"] if span == auth_span] == [auth_span, auth_span] - - @pytest.mark.asyncio @pytest.mark.parametrize("is_proxy_admin", [False, True], ids=["standard-return", "proxy-admin-return"]) async def test_jwt_builder_returns_every_team_grant_the_key_path_gets(is_proxy_admin): diff --git a/tests/unit/proxy/client/cli/test_claude_settings.py b/tests/unit/proxy/client/cli/test_claude_settings.py index 80aa86c5058..655d70d4ef9 100644 --- a/tests/unit/proxy/client/cli/test_claude_settings.py +++ b/tests/unit/proxy/client/cli/test_claude_settings.py @@ -1,9 +1,6 @@ import json import os -import pathlib -import shlex import stat -import subprocess import sys import time from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError @@ -56,7 +53,6 @@ def _owners(*backup_paths): CLAUDE_SETTINGS_MODULE = "litellm.proxy.client.cli.commands.claude_settings" -AUTH_MODULE = "litellm.proxy.client.cli.commands.auth" @pytest.fixture def paths(tmp_path): return tmp_path / "claude" / "settings.json", tmp_path / "backup.json" @@ -174,35 +170,6 @@ class TestConfigureClaudeSettings: class TestLiteApiKeyHelper: - @pytest.mark.skipif(sys.platform.startswith("win"), reason="runs the command through sh, as Claude Code does on POSIX") - def test_sh_runs_the_lite_on_path_against_this_proxy(self, tmp_path): - lite = tmp_path / "my tools" / "lite" - lite.parent.mkdir() - lite.write_text('#!/bin/sh\nprintf "%s\\n" "$0" "$@"\n') - lite.chmod(0o700) - - helper = lite_api_key_helper("https://proxy.example.com/", platform="linux", which=lambda name: str(lite)) - - assert helper is not None - ran = subprocess.run(["sh", "-c", helper.command], capture_output=True, text=True, check=True) - assert ran.stdout.splitlines() == [str(lite), "--base-url", "https://proxy.example.com", "auth", "print-token"] - - @pytest.mark.skipif(sys.platform.startswith("win"), reason="runs the command through sh, as Claude Code does on POSIX") - def test_a_lite_found_through_a_relative_path_entry_keeps_running_from_any_directory(self, tmp_path, monkeypatch): - for repo, name in ((tmp_path / "login-dir", "mine"), (tmp_path / "other-repo", "planted")): - lite = repo / "bin" / "lite" - lite.parent.mkdir(parents=True) - lite.write_text(f"#!/bin/sh\necho {name}\n") - lite.chmod(0o700) - monkeypatch.chdir(tmp_path / "login-dir") - - helper = lite_api_key_helper("https://proxy.example.com", platform="linux", which=lambda name: "bin/lite") - - assert helper is not None - ran = subprocess.run( - ["sh", "-c", helper.command], cwd=tmp_path / "other-repo", capture_output=True, text=True, check=True - ) - assert ran.stdout == "mine\n" def test_quotes_every_token_for_cmd_on_windows(self): helper = lite_api_key_helper( @@ -262,23 +229,6 @@ class TestInstalledExecutable: path.write_text(f"#!/bin/sh\n{body}\n") path.chmod(0o700) - def test_the_helper_runs_the_lite_installed_beside_this_python_over_a_shim_earlier_on_path( - self, tmp_path, monkeypatch - ): - self._script(tmp_path / "shims" / "lite", "echo 'pyenv: lite: command not found' >&2; exit 127") - self._script(tmp_path / "venv" / "bin" / "lite", "echo real") - monkeypatch.setenv("PATH", f"{tmp_path / 'shims'}{os.pathsep}{os.environ['PATH']}") - - helper = lite_api_key_helper( - "https://proxy.example.com", - platform="linux", - which=lambda name: installed_executable(name, scripts_dir=str(tmp_path / "venv" / "bin")), - ) - - assert helper is not None - ran = subprocess.run(["sh", "-c", helper.command], capture_output=True, text=True) - assert (ran.returncode, ran.stdout) == (0, "real\n") - def test_falls_back_to_path_when_this_python_has_no_lite(self, tmp_path, monkeypatch): self._script(tmp_path / "pipx" / "lite", "echo pipx") (tmp_path / "venv" / "bin").mkdir(parents=True) @@ -931,18 +881,6 @@ class TestStatusLine: merged = merge_claude_settings({}, PROXY, StaticToken("tok"), status_line=self.COMMAND) assert merged["statusLine"] == {"type": "command", "command": self.COMMAND} - def test_the_installed_script_is_the_bundled_one_and_the_command_runs_this_interpreter(self, tmp_path): - from litellm.proxy.client.cli.commands import statusline_script - - script = tmp_path / "lite" / "statusline.py" - command = install_statusline_script(script) - assert script.read_bytes().split(b"\n", 1)[1] == pathlib.Path(statusline_script.__file__).read_bytes() - assert shlex.split(command) == [sys.executable, str(script)] - assert command == statusline_command(script) - assert stat.S_IMODE(script.stat().st_mode) == 0o600 - assert stat.S_IMODE(script.parent.stat().st_mode) == 0o700 - assert install_statusline_script(script) == command - def test_a_reinstall_replaces_the_script_in_one_step_and_a_refused_one_leaves_the_old_script_whole(self, tmp_path): # Claude Code may be running the script at the moment `lite` reinstalls it; the file it has open # must stay complete, and a reinstall that cannot land must not leave a truncated script behind. diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index 1aec7831b8e..05463bd3555 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -2,15 +2,12 @@ import _hashlib import asyncio -import copy import hashlib import importlib -import inspect import logging import os import sys import tempfile -import warnings from collections.abc import Awaitable, Callable, Iterator from contextlib import contextmanager from types import ModuleType @@ -36,6 +33,7 @@ from litellm.integrations.otel.model.payloads import ServiceSpanData from litellm.integrations.otel.model.spans import service_span_name from tests.unit.litellm_core_utils.fake_secret_vault import FakeSecretVault from tests._master_key import MASTER_KEY +from tests._module_state import collect_flushable_caches, flush_caches, restore_mutable_state, snapshot_mutable_state class StubClientNotConnectedError(ClientNotConnectedError): @@ -56,104 +54,9 @@ def disconnected_prisma() -> DisconnectedPrisma: return DisconnectedPrisma() -# Top-level assignments of these types are the ones importlib.reload(litellm) -# would have effectively reset. We snapshot them at conftest import time and -# deep-copy the snapshot back before every test. -_SNAPSHOT_TYPES = (list, dict, set, tuple, str, int, float, bool, bytes) - - -def _snapshot_mutable_state(module): - """Capture a per-module snapshot of primitive and collection attributes.""" - snapshot = {} - for attr in list(vars(module)): - if attr.startswith("_"): - continue - try: - value = getattr(module, attr) - except Exception as exc: - warnings.warn( - f"conftest: could not read {module.__name__}.{attr} during snapshot: {exc}", - stacklevel=2, - ) - continue - if value is None or isinstance(value, _SNAPSHOT_TYPES): - try: - snapshot[attr] = _restored_value(value) - except Exception as exc: - warnings.warn( - f"conftest: could not snapshot {module.__name__}.{attr}: {exc}", - stacklevel=2, - ) - return snapshot - - -_MUTABLE_CONTAINERS = (list, dict, set, bytearray) - - -def _holds_mutable_container(value) -> bool: - if isinstance(value, _MUTABLE_CONTAINERS): - return True - if isinstance(value, tuple): - return any(_holds_mutable_container(element) for element in value) - return False - - -def _restored_value(value): - return copy.deepcopy(value) if _holds_mutable_container(value) else value - - -def _restore_mutable_state(module, snapshot): - for attr, default in snapshot.items(): - try: - setattr(module, attr, _restored_value(default)) - except Exception as exc: - warnings.warn( - f"conftest: could not restore {module.__name__}.{attr}: {exc}", - stacklevel=2, - ) - - -def _collect_flushable_caches(): - """Return (module, attr) pairs whose values expose flush_cache().""" - targets = [] - for module in (litellm, litellm.proxy.proxy_server): - for attr in list(vars(module)): - if attr.startswith("_"): - continue - try: - value = getattr(module, attr) - except Exception: - continue - # Only instances — a class reference has an unbound flush_cache - # that can't be called without a self argument. - if inspect.isclass(value) or inspect.ismodule(value): - continue - if callable(getattr(value, "flush_cache", None)): - targets.append((module, attr)) - return targets - - -def _flush_caches(targets): - for module, attr in targets: - try: - value = getattr(module, attr) - except Exception: - continue - flush = getattr(value, "flush_cache", None) - if callable(flush): - try: - flush() - except Exception as exc: - warnings.warn( - f"conftest: flush_cache failed on {module.__name__}.{attr}: {exc}", - stacklevel=2, - ) - - -# Snapshot once at conftest import — these are the "clean" module states. -_LITELLM_STATE = _snapshot_mutable_state(litellm) -_PROXY_SERVER_STATE = _snapshot_mutable_state(litellm.proxy.proxy_server) -_FLUSHABLE_CACHES = _collect_flushable_caches() +_LITELLM_STATE: Final = snapshot_mutable_state(litellm) +_PROXY_SERVER_STATE: Final = snapshot_mutable_state(litellm.proxy.proxy_server) +_FLUSHABLE_CACHES: Final = collect_flushable_caches((litellm, litellm.proxy.proxy_server)) _PROXY_APP_LIFESPAN: Final = litellm.proxy.proxy_server.app.router.lifespan_context @@ -182,9 +85,9 @@ def setup_and_teardown(): Use pytest's monkeypatch.setattr() or a local fixture for those cases — don't rely on this autouse fixture to undo them. """ - _restore_mutable_state(litellm, _LITELLM_STATE) - _restore_mutable_state(litellm.proxy.proxy_server, _PROXY_SERVER_STATE) - _flush_caches(_FLUSHABLE_CACHES) + restore_mutable_state(litellm, _LITELLM_STATE) + restore_mutable_state(litellm.proxy.proxy_server, _PROXY_SERVER_STATE) + flush_caches(_FLUSHABLE_CACHES) loop = asyncio.get_event_loop_policy().new_event_loop() asyncio.set_event_loop(loop) diff --git a/tests/unit/proxy/hooks/litellm_skills/test_main.py b/tests/unit/proxy/hooks/litellm_skills/test_main.py index dc7ebf6e2ec..f716a8533d8 100644 --- a/tests/unit/proxy/hooks/litellm_skills/test_main.py +++ b/tests/unit/proxy/hooks/litellm_skills/test_main.py @@ -1,7 +1,3 @@ -import os -import subprocess -import sys -from typing import Final from unittest.mock import AsyncMock, patch import pytest @@ -69,19 +65,3 @@ async def test_execute_code_loop_dispatches_litellm_skill_tool(): mock_exec.assert_awaited_once() assert mock_exec.await_args.args[0] == SKILL_TOOL_NAME assert result is final_response - - -def test_importing_proxy_hooks_does_not_mutate_litellm_callbacks(): - script: Final = ( - "import litellm; " - "litellm.callbacks = []; " - "import litellm.proxy.hooks; " - "assert len(litellm.callbacks) == 0, f'mutated: {litellm.callbacks}'" - ) - result: Final = subprocess.run( - [sys.executable, "-c", script], - capture_output=True, - text=True, - env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, - ) - assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}" diff --git a/tests/unit/proxy/mcp_server/conftest.py b/tests/unit/proxy/mcp_server/conftest.py index da5b7d8d9a3..605d910e0b7 100644 --- a/tests/unit/proxy/mcp_server/conftest.py +++ b/tests/unit/proxy/mcp_server/conftest.py @@ -1,11 +1,15 @@ import asyncio -import importlib import os +from typing import Final import pytest import litellm from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from tests._module_state import collect_flushable_caches, flush_caches, restore_mutable_state, snapshot_mutable_state + +_LITELLM_STATE: Final = snapshot_mutable_state(litellm) +_FLUSHABLE_CACHES: Final = collect_flushable_caches((litellm,)) @pytest.fixture(scope="session") @@ -20,11 +24,8 @@ def event_loop(): @pytest.fixture(scope="function", autouse=True) def setup_and_teardown(): - """ - This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. - """ - importlib.reload(litellm) - import asyncio + restore_mutable_state(litellm, _LITELLM_STATE) + flush_caches(_FLUSHABLE_CACHES) loop = asyncio.get_event_loop_policy().new_event_loop() asyncio.set_event_loop(loop) diff --git a/tests/unit/proxy/mcp_server/test_openapi_to_mcp_generator.py b/tests/unit/proxy/mcp_server/test_openapi_to_mcp_generator.py index adb185e4242..a658433b00c 100644 --- a/tests/unit/proxy/mcp_server/test_openapi_to_mcp_generator.py +++ b/tests/unit/proxy/mcp_server/test_openapi_to_mcp_generator.py @@ -1636,21 +1636,3 @@ class TestBoundedOpenAPISpecLoading: else: assert await load_openapi_spec_async("https://93.184.216.34/spec.json", max_bytes=100) == {"paths": {}} assert destination.call_count == 1 - - -def test_openapi_generator_import_does_not_require_mcp_sdk() -> None: - import subprocess - import sys - - script = """ -import builtins -original_import = builtins.__import__ -def without_mcp(name, *args, **kwargs): - if name == 'mcp' or name.startswith('mcp.'): - raise ModuleNotFoundError('MCP SDK unavailable') - return original_import(name, *args, **kwargs) -builtins.__import__ = without_mcp -import litellm.proxy.mcp_server.openapi_to_mcp_generator -""" - result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) - assert result.returncode == 0, result.stderr diff --git a/tests/unit/proxy/rust_forward/supervisor/test_runtime.py b/tests/unit/proxy/rust_forward/supervisor/test_runtime.py index c5d1bbc7a36..1dda28b7e1f 100644 --- a/tests/unit/proxy/rust_forward/supervisor/test_runtime.py +++ b/tests/unit/proxy/rust_forward/supervisor/test_runtime.py @@ -1,5 +1,4 @@ import signal -import subprocess import sys from dataclasses import dataclass, field from pathlib import Path @@ -9,9 +8,7 @@ from typing import Final import anyio import pytest -from litellm.proxy.rust_forward.platform import ChildProcess, Platform, current_platform -from litellm.proxy.rust_forward.platform.windows import WindowsPlatform -from litellm.proxy.rust_forward.supervisor import supervise +from litellm.proxy.rust_forward.platform import ChildProcess, current_platform from litellm.proxy.rust_forward.supervisor.children import ChildSpec from litellm.proxy.rust_forward.supervisor.lifecycle import ( Action, @@ -24,42 +21,6 @@ from litellm.proxy.rust_forward.supervisor.lifecycle import ( Spawn, ) from litellm.proxy.rust_forward.supervisor.runtime import SpawnFailed, drive, exit_like, open_children, spawn_subprocess -from tests.unit.proxy.rust_forward.supervisor._support import python_server - - -PYTHON_CHILD: Final = """ -import json, os, pathlib, socket, time -time.sleep(0.5) -pathlib.Path(os.environ["MARKERS"], "listening").touch() -private = json.loads(os.environ["_LITELLM_GATEWAY_SUPERVISED"])["listener"] -listener = socket.socket(fileno=private["fd"]) if "fd" in private else socket.create_server(("127.0.0.1", private["port"])) -listener.listen() -time.sleep(10) -""" -RUST_CHILD: Final = """ -import os, pathlib -markers = pathlib.Path(os.environ["MARKERS"]) -(markers / "rust_saw_python_listening").write_text(str((markers / "listening").exists())) -""" - - -def _fake_gateway(scripts: Path) -> Platform: - if sys.platform != "win32": - gateway: Final = scripts / "litellm-gateway" - gateway.write_text(f"#!{sys.executable} -I\n{RUST_CHILD}") - gateway.chmod(0o755) - return current_platform() - script: Final = scripts / "rust_child.py" - script.write_text(RUST_CHILD) - (scripts / "litellm-gateway.cmd").write_text(f'@"{sys.executable}" -I "{script}"\r\n') - return WindowsPlatform(gateway_name="litellm-gateway.cmd") - - -def test_rust_opens_the_public_port_only_once_python_listens(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("MARKERS", str(tmp_path)) - command: Final = (sys.executable, "-I", "-c", PYTHON_CHILD) - supervise(python_server(), command, scripts_dir=str(tmp_path), platform=_fake_gateway(tmp_path)) - assert (tmp_path / "rust_saw_python_listening").read_text() == "True" TERM: Final = signal.SIGTERM @@ -190,15 +151,6 @@ async def test_a_cancelled_supervisor_still_waits_for_the_child_it_stops() -> No assert (while_child_runs, once_child_exits) == (((child,), 1, False), True) -async def test_exiting_children_are_reported_as_events() -> None: - async with open_children( - current_platform(), spawn_subprocess, _children((sys.executable, "-c", "exit(7)")), 1 - ) as children: - assert await children.spawn(Child.RUST) is None - with anyio.fail_after(5): - assert await children.next_event() == ChildExited(Child.RUST, 7) - - async def test_a_child_that_cannot_start_is_returned_as_a_failure(tmp_path: Path) -> None: async with open_children( current_platform(), spawn_subprocess, _children((str(tmp_path / "missing"),)), 1 @@ -223,13 +175,3 @@ def _dropped(_signum: signal.Signals) -> None: ) def test_a_supervisor_the_kernel_will_not_signal_still_exits_as_python_alone_would(status: int, exits: int) -> None: assert exit_like(status, STOPS, die_by=_dropped) == exits - - -@POSIX_ONLY -@pytest.mark.parametrize("signum", [signal.SIGTERM, signal.SIGKILL]) -def test_the_supervisor_dies_by_the_signal_that_killed_python(signum: signal.Signals) -> None: - script: Final = ( - "from litellm.proxy.rust_forward.supervisor.runtime import exit_like\n" - f"raise SystemExit(exit_like({-signum}, frozenset()))" - ) - assert subprocess.run((sys.executable, "-I", "-c", script), check=False, timeout=60).returncode == -signum diff --git a/tests/unit/proxy/test_prometheus_metrics_server.py b/tests/unit/proxy/test_prometheus_metrics_server.py index e52110e3b75..b080493a63b 100644 --- a/tests/unit/proxy/test_prometheus_metrics_server.py +++ b/tests/unit/proxy/test_prometheus_metrics_server.py @@ -151,37 +151,6 @@ def test_main_rejects_missing_multiproc_dir(monkeypatch: pytest.MonkeyPatch): run.assert_not_called() -def test_start_metrics_server_process_returns_only_once_child_serves(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): - monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", str(tmp_path)) - _write_worker_sample(pid=3001, value=4) - port: Final = _free_port() - with patch("atexit.register") as register: - process: Final = start_metrics_server_process(host="127.0.0.1", port=port, multiproc_dir=str(tmp_path)) - try: - register.assert_called_once_with(process.terminate) - assert process.poll() is None - startup_metrics: Final = httpx.get(f"http://127.0.0.1:{port}/metrics", follow_redirects=True, timeout=5.0) - assert startup_metrics.status_code == 200 - assert startup_metrics.headers[PID_HEADER] == str(process.pid) - metrics: Final = httpx.get(f"http://127.0.0.1:{port}/metrics", follow_redirects=True, timeout=10.0) - assert 'litellm_requests_metric_total{model="gpt-5"} 4.0' in metrics.text - finally: - process.kill() - process.wait(timeout=10) - - -def test_start_metrics_server_process_fails_when_port_is_taken(tmp_path: Path): - with socket.socket() as occupied: - occupied.bind(("127.0.0.1", 0)) - occupied.listen() - port: Final = occupied.getsockname()[1] - with ( - patch("atexit.register"), - pytest.raises(MetricsServerStartupError), - ): - start_metrics_server_process(host="127.0.0.1", port=port, multiproc_dir=str(tmp_path)) - - class _ImpostorMetrics(BaseHTTPRequestHandler): """An unrelated service already on the port that answers /metrics with 200 and plausible metrics.""" @@ -196,22 +165,6 @@ class _ImpostorMetrics(BaseHTTPRequestHandler): return -def test_start_metrics_server_process_rejects_metrics_from_another_service_on_the_port(tmp_path: Path): - with ThreadingHTTPServer(("127.0.0.1", 0), _ImpostorMetrics) as impostor: - threading.Thread(target=impostor.serve_forever, daemon=True).start() - port: Final = impostor.server_address[1] - impostor_response: Final = httpx.get(f"http://127.0.0.1:{port}/metrics") - assert impostor_response.status_code == 200 - assert "# HELP impostor_metric" in impostor_response.text - assert PID_HEADER not in impostor_response.headers - with ( - patch("atexit.register"), - pytest.raises(MetricsServerStartupError), - ): - start_metrics_server_process(host="127.0.0.1", port=port, multiproc_dir=str(tmp_path)) - impostor.shutdown() - - def test_metrics_server_process_serves_and_exits_with_parent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", str(tmp_path)) _write_worker_sample(pid=2001, value=7) diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 7fb034022d0..fdc38e3434a 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -3201,33 +3201,6 @@ async def test_user_config_router_enforces_its_limit_across_proxy_requests( ) -@pytest.mark.parametrize("arrives_with_logging_obj", [True, False]) -@pytest.mark.asyncio -async def test_concurrent_user_config_requests_each_count_once_against_the_tpm_limit( - fake_redis_port: int, arrives_with_logging_obj: bool -) -> None: - tpm_for_three_requests: Final = await _tpm_for_three_requests() - - def request() -> Coroutine[None, None, str]: - return _unsettled_user_config_request_outcome( - fake_redis_port, - {"routing_strategy": "usage-based-routing-v2"}, - {"tpm": tpm_for_three_requests}, - _proxy_logging_obj() if arrives_with_logging_obj else None, - ) - - async def settled(*requests: Coroutine[None, None, str]) -> list[str]: - outcomes: Final = await asyncio.gather(*requests) - await _settle_router_accounting() - return outcomes - - await _wait_out_a_rate_limit_minute_rollover() - concurrent_outcomes: Final = await settled(request(), request()) - later_outcomes: Final = [*(await settled(request())), *(await settled(request()))] - - assert [*concurrent_outcomes, *later_outcomes] == ["200", "200", "200", "429"] - - @pytest.mark.asyncio async def test_overlapping_user_config_requests_share_one_rpm_limit(fake_redis_port: int) -> None: def request() -> Coroutine[None, None, str]: diff --git a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py index 627f00b96b9..9835e0e2017 100644 --- a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -1,8 +1,6 @@ import asyncio import importlib -import subprocess import sys -import textwrap import types from types import MappingProxyType from collections.abc import Mapping, Sequence @@ -1032,52 +1030,6 @@ async def test_execute_tool_calls_propagates_request_tags_to_function_setup(monk assert captured["metadata"]["tags"] == ["team-a", "prod"] -def test_completion_with_function_tools_works_without_fastapi_installed(): - script = textwrap.dedent( - """ - import sys - - class _FastapiBlocker: - def find_spec(self, fullname, path=None, target=None): - if fullname == "fastapi" or fullname.startswith("fastapi."): - raise ModuleNotFoundError("No module named 'fastapi'") - return None - - sys.meta_path.insert(0, _FastapiBlocker()) - - import litellm - - response = litellm.completion( - model="openai/gpt-5.5", - messages=[{"role": "user", "content": "What is the weather in SF?"}], - tools=[ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": {"location": {"type": "string"}}, - "required": ["location"], - }, - }, - } - ], - mock_response="sunny", - ) - assert response.choices[0].message.content == "sunny" - """ - ) - result = subprocess.run( - [sys.executable, "-c", script], - capture_output=True, - text=True, - timeout=120, - ) - assert result.returncode == 0, result.stderr - - def test_extract_tool_call_details_reads_anthropic_tool_use_input(): """ Regression test (LIT-4517): an Anthropic tool_use block carries its arguments diff --git a/tests/unit/rust_bridge/test_configuration.py b/tests/unit/rust_bridge/test_configuration.py index 70223421471..714718b9afc 100644 --- a/tests/unit/rust_bridge/test_configuration.py +++ b/tests/unit/rust_bridge/test_configuration.py @@ -1,8 +1,4 @@ from __future__ import annotations - -import os -import subprocess -import sys from collections.abc import Generator from concurrent.futures import ThreadPoolExecutor from typing import Final @@ -132,21 +128,3 @@ def test_process_override_and_reset_apply_to_existing_threads() -> None: assert executor.submit(configuration.rust_enabled).result() is True configuration.reset_rust_configuration() assert executor.submit(configuration.rust_enabled).result() is False - - -@pytest.mark.parametrize(("value", "expected"), (("1", "True"), ("0", "False"))) -def test_environment_controls_startup(value: str, expected: str) -> None: - environment: Final = {**os.environ, "LITELLM_RUST": value} - result: Final = subprocess.run( - ( - sys.executable, - "-c", - "from litellm.rust_bridge.configuration import rust_enabled; print(rust_enabled())", - ), - check=True, - capture_output=True, - text=True, - env=environment, - ) - - assert result.stdout.strip() == expected diff --git a/tests/unit/test_lazy_imports.py b/tests/unit/test_lazy_imports.py index 1f39bdede5a..850eac03011 100644 --- a/tests/unit/test_lazy_imports.py +++ b/tests/unit/test_lazy_imports.py @@ -1,7 +1,4 @@ import importlib -import os -import subprocess -import sys from typing import Final import pytest @@ -12,22 +9,6 @@ from litellm._lazy_imports_registry import LITELLM_ATTRIBUTES, UTILS_ATTRIBUTES from tests.test_litellm_rust.support.child_interpreter import run_child_interpreter -def test_import_litellm_does_not_load_fastapi_or_bpe_table(): - result: Final = subprocess.run( - [ - sys.executable, - "-c", - "import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.proxy.proxy_cli','litellm.litellm_core_utils.default_encoding','openai.types.realtime') if m in sys.modules))", - ], - check=True, - capture_output=True, - text=True, - env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, - ) - - assert result.stdout.strip() == "" - - def _declared(location: tuple[str, str]) -> object: module_path, attr_name = location return getattr(importlib.import_module(module_path, package="litellm"), attr_name) diff --git a/tests/unit/test_litellm_cli.py b/tests/unit/test_litellm_cli.py index 6015bd1458b..fcd3ca6bf05 100644 --- a/tests/unit/test_litellm_cli.py +++ b/tests/unit/test_litellm_cli.py @@ -1,8 +1,4 @@ -import json -import os -import subprocess import sys -from collections.abc import Mapping from pathlib import Path from typing import Final @@ -16,373 +12,8 @@ if sys.version_info >= (3, 11): else: import tomli as tomllib -_HELP_ARGV: Final = ( - ("--help",), - ("logout", "--help"), - ("login", "--help"), - ("models", "list", "--help"), - ("claude", "--help"), - ("opencode", "--help"), - ("codex", "--help"), - ("pi", "--help"), - ("keys", "list", "--help"), -) - -_COMMAND_ARGV: Final = (("logout",), ("whoami",)) - -_RUN_EACH_ARGV: Final = """ -import contextlib, io, json, sys -from {module} import cli -runs = {{}} -for argv in json.loads(sys.argv[1]): - sys.argv = ["lite", *argv] - output = io.StringIO() - with contextlib.redirect_stdout(output): - try: - cli() - except SystemExit as stopped: - code = stopped.code - else: - code = 0 - runs[" ".join(argv)] = {{"exit": code, "output": output.getvalue()}} -print(json.dumps({{"runs": runs, "sdk_initialized": "completion" in vars(sys.modules["litellm"])}})) -""" - -_USE_SDK_AFTER_HELP: Final = """ -import builtins, importlib, json, sys -interpreter_imports = (builtins.__import__, importlib.import_module) -from {module} import cli -sys.argv = ["lite", "--help"] -try: - cli() -except SystemExit: - pass -before = "completion" in vars(sys.modules["litellm"]) -{first_use} -from litellm.router import Router -import litellm -import litellm.responses.main -from litellm import acompletion -print(json.dumps({{ - "before": before, - "router": Router.__name__, - "completion": callable(litellm.completion), - "acompletion": acompletion is litellm.acompletion, - "responses_is_function": callable(litellm.responses) and not isinstance(litellm.responses, type(litellm)), - "namespace": sorted(vars(litellm)), - "module_type": type(litellm).__name__, - "imports_restored": (builtins.__import__, importlib.import_module) == interpreter_imports, -}})) -""" - -_FIRST_USE_FROM_THREADS: Final = """ -import json, sys, threading -from litellm_cli.main import cli -sys.argv = ["lite", "--help"] -try: - cli() -except SystemExit: - pass -import litellm -start = threading.Barrier(4) -seen = [] -def use_sdk(): - start.wait() - try: - seen.append(callable(litellm.completion)) - except AttributeError as error: - seen.append(str(error)) -threads = [threading.Thread(target=use_sdk) for _ in range(4)] -for thread in threads: - thread.start() -for thread in threads: - thread.join() -print(json.dumps({"seen": seen})) -""" - -_STAR_IMPORT_AFTER_HELP: Final = """ -import json, sys -from litellm_cli.main import cli -sys.argv = ["lite", "--help"] -try: - cli() -except SystemExit: - pass -before = "completion" in vars(sys.modules["litellm"]) -from litellm import * -print(json.dumps({"before": before, "completion": callable(globals().get("completion")), "acompletion": callable(globals().get("acompletion"))})) -""" - -_SDK_WHOSE_SUBMODULE_READS_IT_WHILE_LOADING: Final = { - "sdk_sync.py": """ -import threading -initializing = threading.Event() -submodule_loading = threading.Event() -""", - "litellm/__init__.py": """ -import sdk_sync -sdk_sync.initializing.set() -sdk_sync.submodule_loading.wait(2) -from litellm import submodule -ready = True -""", - "litellm/submodule.py": """ -import litellm -import sdk_sync -sdk_sync.submodule_loading.set() -ready_when_loaded = getattr(litellm, "ready", False) -""", - "litellm/_process_defaults.py": "def apply_process_defaults() -> None: ...\n", - "litellm/proxy/__init__.py": "", - "litellm/proxy/client/__init__.py": "", - "litellm/proxy/client/cli/__init__.py": "", - "litellm/proxy/client/cli/main.py": """ -import json, threading -import litellm -import sdk_sync - -def cli(): - def initialize_sdk(): - litellm.ready - - def import_submodule_once_initialization_starts(): - sdk_sync.initializing.wait() - import litellm.submodule - - threads = [threading.Thread(target=initialize_sdk, daemon=True), - threading.Thread(target=import_submodule_once_initialization_starts, daemon=True)] - for thread in threads: - thread.start() - for thread in threads: - thread.join(10) - print(json.dumps({"finished": [not thread.is_alive() for thread in threads]})) - -litellm_proxy_cli = cli -""", -} - -_SDK_WHOSE_INIT_FAILS: Final = { - "litellm/__init__.py": 'raise ImportError("a dependency of the SDK is broken")\n', - "litellm/utils.py": "", - "litellm/_process_defaults.py": "def apply_process_defaults() -> None: ...\n", - "litellm/proxy/__init__.py": "", - "litellm/proxy/client/__init__.py": "", - "litellm/proxy/client/cli/__init__.py": "", - "litellm/proxy/client/cli/main.py": """ -import json -import litellm - -def error_from(use): - try: - use() - except Exception as error: - return f"{type(error).__name__}: {error}" - return None - -def import_sdk_code(): - import litellm.utils - -def cli(): - print(json.dumps([error_from(lambda: litellm.completion), error_from(lambda: litellm.completion), error_from(import_sdk_code)])) - -litellm_proxy_cli = cli -""", -} - -_RUN_LITE_AGAINST_SDK_AT: Final = """ -import sys -sys.path.insert(0, sys.argv[1]) -from litellm_cli.main import cli -cli() -""" - _PYPROJECT: Final = Path(__file__).resolve().parents[2] / "pyproject.toml" -_RUN_ENTRY_POINT_HELP: Final = """ -import contextlib, io, json, sys -from {module} import {function} -imported_sdk = "litellm" in sys.modules -sys.argv = ["litellm", "--help"] -output = io.StringIO() -with contextlib.redirect_stdout(output): - try: - {function}() - except SystemExit as stopped: - code = stopped.code - else: - code = 0 -print(json.dumps({{"imported_sdk": imported_sdk, "exit": code, "help": output.getvalue()}})) -""" - -_WORKER_LOGGERS_UNDER_A_DEFAULT_LOG_CONFIG: Final = """ -import json, logging, os -os.environ["LITELLM_WORKER_LOG_CONFIG"] = json.dumps({{"version": 1, "root": {{"level": "INFO"}}}}) -from litellm_cli.main import proxy_worker_app -proxy_worker_app() -print(json.dumps({{name: logging.getLogger(name).disabled for name in {names!r}}})) -""" - -_COMMAND_MODULES_LOADED_BY: Final = """ -import json, sys -from litellm_cli.main import cli -sys.argv = ["lite", *sys.argv[1:]] -try: - cli() -except SystemExit: - pass -prefix = "litellm.proxy.client.cli.commands." -print(json.dumps({"modules": sorted(name[len(prefix):] for name in sys.modules if name.startswith(prefix))})) -""" - - -def _run(source: str, argv: tuple[str, ...], home: Path) -> dict[str, object]: - checkout: Final = str(_PYPROJECT.parent) - bootstrap: Final = ( - "import pathlib,sys\n" - f"sys.path.insert(0, {checkout!r})\n" - f"sys.path.insert(0, str(pathlib.Path({checkout!r}) / '.github/packaging'))\n" - "from cli_package import editable_support\n" - f"exec(editable_support(pathlib.Path({checkout!r}) / 'litellm/cli')[0][1])\n" - ) - result: Final = subprocess.run( - [sys.executable, "-I", "-c", bootstrap + source, *argv], - capture_output=True, - text=True, - timeout=120, - env={**os.environ, "HOME": str(home), "LITELLM_LOCAL_MODEL_COST_MAP": "True"}, - ) - assert result.returncode == 0, result.stderr - return json.loads(result.stdout.rstrip("\n").rpartition("\n")[2]) - - -@pytest.fixture(scope="module") -def cli_runs(tmp_path_factory: pytest.TempPathFactory) -> Mapping[str, dict[str, object]]: - argv: Final = (json.dumps(_HELP_ARGV + _COMMAND_ARGV),) - return { - "thin": _run(_RUN_EACH_ARGV.format(module="litellm_cli.main"), argv, tmp_path_factory.mktemp("thin")), - "full": _run( - _RUN_EACH_ARGV.format(module="litellm.proxy.client.cli.main"), argv, tmp_path_factory.mktemp("full") - ), - } - - -def test_litellm_command_imports_the_proxy_only_when_it_runs(tmp_path: Path): - module, _, function = tomllib.loads(_PYPROJECT.read_text())["project"]["scripts"]["litellm"].partition(":") - run: Final = _run(_RUN_ENTRY_POINT_HELP.format(module=module, function=function), (), tmp_path) - assert run["imported_sdk"] is False - assert run["exit"] == 0 - assert "--num_workers" in str(run["help"]) - - -def test_a_worker_log_config_silences_the_sdk_loggers_but_not_the_ones_the_proxy_app_creates(tmp_path: Path): - disabled: Final = _run( - _WORKER_LOGGERS_UNDER_A_DEFAULT_LOG_CONFIG.format(names=("LiteLLM Proxy", "litellm.proxy.health_check")), - (), - tmp_path, - ) - assert disabled == {"LiteLLM Proxy": True, "litellm.proxy.health_check": False} - - -def test_lite_runs_commands_without_initializing_the_sdk(cli_runs: Mapping[str, dict[str, object]]): - assert cli_runs["thin"]["sdk_initialized"] is False - assert cli_runs["full"]["sdk_initialized"] is True - - -@pytest.mark.parametrize("argv", _HELP_ARGV + _COMMAND_ARGV, ids=" ".join) -def test_lite_prints_what_the_full_sdk_cli_prints(argv: tuple[str, ...], cli_runs: Mapping[str, dict[str, object]]): - thin: Final = cli_runs["thin"]["runs"] - full: Final = cli_runs["full"]["runs"] - assert isinstance(thin, dict) and isinstance(full, dict) - - assert thin[" ".join(argv)] == full[" ".join(argv)] - assert thin[" ".join(argv)]["output"] - - -@pytest.mark.parametrize( - ("argv", "loaded", "not_loaded"), - [ - (("logout", "--help"), {"auth"}, {"agents", "autoroute.commands", "models", "keys"}), - (("models", "list", "--help"), {"models"}, {"agents", "autoroute.commands", "keys"}), - (("claude", "--help"), {"agents"}, {"autoroute.commands", "models", "keys"}), - ], -) -def test_lite_imports_only_the_command_it_runs( - argv: tuple[str, ...], loaded: set[str], not_loaded: set[str], tmp_path: Path -): - modules: Final = _run(_COMMAND_MODULES_LOADED_BY, argv, tmp_path)["modules"] - assert isinstance(modules, list) - - assert loaded <= set(modules) - assert not not_loaded & set(modules) - - -@pytest.fixture(scope="module") -def sdk_used_after_full_cli(tmp_path_factory: pytest.TempPathFactory) -> dict[str, object]: - return _run( - _USE_SDK_AFTER_HELP.format(module="litellm.proxy.client.cli.main", first_use="pass"), - (), - tmp_path_factory.mktemp("full"), - ) - - -@pytest.mark.parametrize( - "first_use", - [ - "import litellm; litellm.completion", - "import litellm; litellm.types.utils.ModelResponse", - "from litellm import acompletion", - "from litellm.router import Router", - "from litellm.types import utils", - "import litellm.responses.main", - 'importlib.import_module("litellm.router")', - 'importlib.import_module(".utils", "litellm")', - 'exec("from .types import utils", {"__name__": "litellm.lazy_import", "__package__": "litellm"})', - ], -) -def test_sdk_initializes_in_place_on_first_use_after_lite_imports( - first_use: str, sdk_used_after_full_cli: dict[str, object], tmp_path: Path -): - thin: Final = _run(_USE_SDK_AFTER_HELP.format(module="litellm_cli.main", first_use=first_use), (), tmp_path) - - assert thin["before"] is False - assert sdk_used_after_full_cli["before"] is True - assert {**thin, "before": True} == sdk_used_after_full_cli - assert (thin["router"], thin["completion"], thin["acompletion"], thin["responses_is_function"]) == ( - "Router", - True, - True, - True, - ) - - -def test_star_import_after_lite_imports_gets_the_whole_sdk(tmp_path: Path): - assert _run(_STAR_IMPORT_AFTER_HELP, (), tmp_path) == {"before": False, "completion": True, "acompletion": True} - - -def test_threads_that_use_the_sdk_at_once_all_wait_for_one_initialization(tmp_path: Path): - used: Final = _run(_FIRST_USE_FROM_THREADS, (), tmp_path) - - assert used == {"seen": [True, True, True, True]} - - -def _write_sdk(root: Path, files: Mapping[str, str]) -> Path: - for relative_path, source in files.items(): - (root / relative_path).parent.mkdir(parents=True, exist_ok=True) - (root / relative_path).write_text(source) - return root - - -def test_importing_sdk_code_while_another_thread_initializes_the_sdk_does_not_hang(tmp_path: Path): - sdk_root: Final = _write_sdk(tmp_path / "sdk", _SDK_WHOSE_SUBMODULE_READS_IT_WHILE_LOADING) - - assert _run(_RUN_LITE_AGAINST_SDK_AT, (str(sdk_root),), tmp_path) == {"finished": [True, True]} - - -def test_every_use_after_a_failed_sdk_init_raises_the_init_error(tmp_path: Path): - sdk_root: Final = _write_sdk(tmp_path / "sdk", _SDK_WHOSE_INIT_FAILS) - - assert _run(_RUN_LITE_AGAINST_SDK_AT, (str(sdk_root),), tmp_path) == ["ImportError: a dependency of the SDK is broken"] * 3 - def test_lite_keeps_an_sdk_that_was_already_imported( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] diff --git a/tests/unit/test_router/test_router_fuzz_regressions_rust_only.py b/tests/unit/test_router/test_router_fuzz_regressions_rust_only.py index 8551e6ea884..3c55c2a2c15 100644 --- a/tests/unit/test_router/test_router_fuzz_regressions_rust_only.py +++ b/tests/unit/test_router/test_router_fuzz_regressions_rust_only.py @@ -3,15 +3,12 @@ Router behavior the router-code differential fuzz run found changed once the Rus only core: count settings below zero, an allowed_fails_policy write that is no policy, a model response object as a deployment's mock_response, and endpoints read or patched on the Router class. """ - -import asyncio import gc import json import socket import threading import weakref from collections.abc import Iterator, Mapping -from datetime import datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Final from unittest import mock @@ -245,95 +242,6 @@ def upstream() -> Iterator[str]: thread.join() -def _served(name: str, deployment_id: str, model: str, api_base: str) -> dict[str, object]: - return { - "model_name": name, - "litellm_params": {"model": model, "api_key": "k", "api_base": api_base}, - "model_info": {"id": deployment_id}, - } - - -def _request_logging(call_type: str, *, stream: bool) -> object: - from litellm.litellm_core_utils.litellm_logging import Logging - - return Logging( - model="g", - messages=MESSAGES, - stream=stream, - call_type=call_type, - start_time=datetime.now(), # noqa: DTZ005 # litellm's own naive start time - litellm_call_id="call-1", - function_id="f", - ) - - -async def _recorded() -> None: - """Let the background recordings of a stream's outcome run.""" - for _ in range(50): - await asyncio.sleep(0.02) - - -@pytest.mark.asyncio -async def test_a_responses_stream_cut_after_it_opened_counts_one_fail_with_a_shared_logging_object( - upstream: str, -) -> None: - router: Final = Router( - model_list=[_served("g", "d1", "openai/gpt-4o", f"{upstream}/cut/v1")], num_retries=0, disable_cooldowns=True - ) - - stream: Final = await router.aresponses( - model="g", input="hi", stream=True, litellm_logging_obj=_request_logging("aresponses", stream=True) - ) - with pytest.raises(Exception): # noqa: B017, PT011 # the transport's read error, whatever class maps it - async for _ in stream: # pyright: ignore[reportGeneralTypeIssues] # a stream when stream=True - pass - await _recorded() - - assert router.health().store.get_cache(key="d1:fails") == 1 - - -@pytest.mark.asyncio -async def test_a_blocked_response_raised_for_a_fallback_counts_no_success_with_a_shared_logging_object( - upstream: str, -) -> None: - router: Final = Router( - model_list=[ - _served("a", "a1", "openai/gpt-4o", f"{upstream}/filtered/v1"), - _served("safe", "s1", "openai/gpt-4o", f"{upstream}/ok/v1"), - ], - content_policy_fallbacks=[{"a": ["safe"]}], - num_retries=0, - disable_cooldowns=True, - ) - - response: Final = await router.acompletion( - model="a", messages=MESSAGES, litellm_logging_obj=_request_logging("acompletion", stream=False) - ) - await _recorded() - - assert response.choices[0].message.content == "ok" - assert router.health().store.get_cache(key="a1:successes") is None - assert router.health().store.get_cache(key="s1:successes") == 1 - - -@pytest.mark.asyncio -async def test_a_text_completion_stream_that_fails_before_its_first_chunk_counts_nothing(upstream: str) -> None: - router: Final = Router( - model_list=[_served("g", "d1", "openai/gpt-3.5-turbo-instruct", f"{upstream}/down/v1")], - num_retries=0, - disable_cooldowns=True, - ) - - stream: Final = await router.atext_completion(model="g", prompt="hi", stream=True) - with pytest.raises(litellm.BadGatewayError): - async for _ in stream: # pyright: ignore[reportGeneralTypeIssues] # a stream when stream=True - pass - await _recorded() - - assert router.health().store.get_cache(key="d1:successes") is None - assert router.health().store.get_cache(key="d1:fails") is None - - @pytest.mark.asyncio async def test_a_text_completion_without_a_prompt_raises_type_error_before_routing() -> None: router: Final = Router(model_list=[_deployment("g", "ok")], num_retries=0) @@ -389,20 +297,6 @@ async def test_a_refused_pick_by_deployment_id_names_the_deployments_group() -> await router.async_get_available_deployment(model="p2", messages=MESSAGES, request_kwargs={}) -@pytest.mark.asyncio -async def test_a_request_refused_before_the_provider_counts_against_no_deployment(upstream: str) -> None: - router: Final = Router( - model_list=[_served("g", "d1", "openai/gpt-4o", f"{upstream}/ok/v1")], num_retries=0, disable_cooldowns=True - ) - - with pytest.raises(litellm.UnsupportedParamsError): - await router.acompletion(model="g", messages=MESSAGES, reasoning_effort="low") - await _recorded() - - assert router.health().store.get_cache(key="d1:fails") is None - assert not [key for key in router.health().store.in_memory_cache.cache_dict if ":rpm" in key] - - @pytest.mark.asyncio async def test_the_async_pick_read_on_the_class_is_a_function_taking_the_router() -> None: router: Final = Router(model_list=[_deployment("g", "ok")]) diff --git a/tests/unit/test_router/test_router_host.py b/tests/unit/test_router/test_router_host.py index 1da96e71d7b..7be9d6b3767 100644 --- a/tests/unit/test_router/test_router_host.py +++ b/tests/unit/test_router/test_router_host.py @@ -1,4 +1,3 @@ -import subprocess import sys import pytest @@ -7,19 +6,6 @@ import litellm.router_host as router_host_module from litellm.router_host import RouterHost, router_host -def test_router_uses_the_proxy_host_without_importing_the_proxy_first(): - # A fresh interpreter, so no earlier test has imported litellm.proxy. - script = ( - "import sys\n" - "from litellm.router import Router\n" - "from litellm.router_host import router_host\n" - "assert 'litellm.proxy' not in sys.modules, 'importing the router imported the proxy'\n" - "print(type(router_host()).__name__)\n" - ) - result = subprocess.run([sys.executable, "-P", "-c", script], capture_output=True, text=True, check=True) - assert result.stdout.strip() == "ProxyRouterHost" - - @pytest.fixture(autouse=True) def nothing_installed(): saved = router_host_module._installed[0] diff --git a/tests/windows_tests/AGENTS.md b/tests/windows_tests/AGENTS.md index 565dba91ce6..36e16a99b5e 100644 --- a/tests/windows_tests/AGENTS.md +++ b/tests/windows_tests/AGENTS.md @@ -1,6 +1,6 @@ # Windows tests -The `windows-sdk` job in `.github/workflows/test-merge-smoke.yml` is the only place these run, on a Windows runner with no proxy dependencies installed. They prove the SDK works on Windows without them, so a test here must not import uvicorn, fastapi, psutil or anything else outside the `dev` group, and must not need the `litellm-gateway.exe` binary +The `windows-sdk` job in `.github/workflows/test-merge-smoke.yml` is the only place these run, on pushes to main and manual runs but not on pull requests, on a Windows runner with no proxy dependencies installed. They prove the SDK works on Windows without them, so a test here must not import uvicorn, fastapi, psutil or anything else outside the `dev` group, and must not need the `litellm-gateway.exe` binary A test that cannot run off Windows carries a `skipif` on `sys.platform`, so collecting this folder on another OS never fails From a1ea22e9966c73addc0a8db75cbdb9444ca2092e Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Sat, 14 Nov 2026 15:45:11 -0800 Subject: [PATCH 07/57] feat(otel v2): proxy-level default langfuse_span_scope for key and team destinations (#51726) * feat(otel v2): proxy-level default langfuse_span_scope for key and team destinations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): regenerate dashboard API documentation types Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): preserve tenant destinations with malformed otel settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): cover explicit narrow scope and non-Langfuse defaults Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * Read only the prefixed Langfuse destination scope environment variable Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/otel/model/config.py | 47 +++++- .../integrations/otel/presets/destinations.py | 9 +- litellm/proxy/litellm_pre_call_utils.py | 10 +- .../team_callback_endpoints.py | 5 +- .../components/team/LoggingSettings.test.tsx | 5 + .../src/components/team/LoggingSettings.tsx | 3 + .../src/i18n/locales/en.json | 1 + .../src/lib/http/schema.d.ts | 5 +- .../test_langfuse_destination_span_scope.py | 138 +++++++++++++++++ .../otel/test_otel_v2_destinations.py | 141 ++++++++++++++++++ 10 files changed, 353 insertions(+), 11 deletions(-) create mode 100644 tests/integration/observability/test_langfuse_destination_span_scope.py diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index ea570bc19c9..3d4de310753 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from enum import Enum -from functools import lru_cache +from functools import cache, lru_cache from typing import Annotated, Final from pydantic import AliasChoices, ConfigDict, Field, TypeAdapter, ValidationError, field_validator, model_validator @@ -218,7 +218,8 @@ class OpenTelemetryV2Config(BaseSettings): description=( "``llm_only`` keeps just the model-call spans on the operator's own Langfuse " "exporter (the spec whose owner is ``langfuse_otel``). Other exporters and " - "key/team destinations are not affected." + "key/team destinations are not affected. Destinations default to " + "``langfuse_destination_span_scope`` instead." ), ) excluded_services: Annotated[frozenset[str], NoDecode] = Field( @@ -317,9 +318,7 @@ class OpenTelemetryV2Config(BaseSettings): @field_validator("langfuse_span_scope", mode="before") @classmethod def _normalize_langfuse_span_scope(cls, value: object) -> object: - if isinstance(value, str): - return value.strip().lower() - return value + return _normalized_span_scope(value) @field_validator( "baggage_promoted_keys", @@ -417,6 +416,44 @@ def capture_message_content_from_env() -> str: return _CaptureMessageContentSettings().capture_message_content +def _normalized_span_scope(value: object) -> object: + return value.strip().lower() if isinstance(value, str) else value + + +class _LangfuseDestinationSpanScopeSettings(BaseSettings): + model_config = SettingsConfigDict(extra="ignore", frozen=True) + + langfuse_destination_span_scope: OtelSpanScope = Field( + default="full", + validation_alias=AliasChoices("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE"), + ) + + @field_validator("langfuse_destination_span_scope", mode="before") + @classmethod + def _normalize_scope(cls, value: object) -> object: + return _normalized_span_scope(value) + + +def langfuse_destination_span_scope(configured: object = None) -> OtelSpanScope: + return _cached_langfuse_destination_span_scope(None if configured is None else str(configured)) + + +@cache +def _cached_langfuse_destination_span_scope(configured: str | None) -> OtelSpanScope: + try: + return _LangfuseDestinationSpanScopeSettings.model_validate( + {"LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE": configured} if configured is not None else {} + ).langfuse_destination_span_scope + except ValidationError as error: + verbose_logger.warning( + "Invalid LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE or " + "callback_settings.otel.langfuse_destination_span_scope. " + "Allowed values are 'full' and 'llm_only'. Falling back to 'full'. %s", + error, + ) + return "full" + + _EXCLUDED_SERVICES_INPUT: Final[TypeAdapter[str | tuple[object, ...]]] = TypeAdapter(str | tuple[object, ...]) diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index a14d1b69b5a..f14e92686d8 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -161,10 +161,12 @@ def _resource_defaults(callback_name: str) -> Mapping[str, str]: return backend() if backend is not None else _NO_ATTRS -def _span_scope(callback_name: str, params: StandardCallbackDynamicParams) -> OtelSpanScope: +def _span_scope( + callback_name: str, params: StandardCallbackDynamicParams, langfuse_destination_span_scope: OtelSpanScope +) -> OtelSpanScope: if callback_name != "langfuse_otel": return "full" - return params.get("langfuse_span_scope") or "full" + return params.get("langfuse_span_scope") or langfuse_destination_span_scope def destination_capable_backends() -> frozenset[str]: @@ -199,6 +201,7 @@ def destination_for( capture_message_content: str | None = None, excluded_db_systems: frozenset[str] | None = None, metadata_keys: tuple[str, ...] | None = None, + langfuse_destination_span_scope: OtelSpanScope = "full", ) -> OtelDestination | None: """The destination ``params`` names for ``callback_name``, or ``None``. @@ -228,7 +231,7 @@ def destination_for( resource_defaults=_resource_defaults(callback_name), callback_name=callback_name, protocol=protocol, - span_scope=_span_scope(callback_name, params), + span_scope=_span_scope(callback_name, params, langfuse_destination_span_scope), capture_message_content=capture_message_content, excluded_db_systems=excluded_db_systems, metadata_keys=metadata_keys or (), diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 86f16411724..4fc2a1419c8 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1595,7 +1595,11 @@ def _tenant_destination( name: str, callbacks: Sequence[AddTeamCallback], service_name: str | None ) -> "OtelDestination | None": from litellm.integrations.otel.model.baggage import parse_destination_metadata_keys - from litellm.integrations.otel.model.config import parse_capture_message_content, parse_excluded_services + from litellm.integrations.otel.model.config import ( + langfuse_destination_span_scope, + parse_capture_message_content, + parse_excluded_services, + ) from litellm.integrations.otel.presets.destinations import destination_for callback_vars: Final = MappingProxyType( @@ -1605,6 +1609,7 @@ def _tenant_destination( ) ) ) + otel_settings: Final = as_str_mapping((litellm.callback_settings or {}).get("otel")) or {} return destination_for( name, _tenant_otel_params(callback_vars), @@ -1612,6 +1617,9 @@ def _tenant_destination( parse_capture_message_content(callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR)), parse_excluded_services(callback_vars.get(EXCLUDED_SERVICES_VAR)), parse_destination_metadata_keys(callback_vars.get(BAGGAGE_METADATA_KEYS_VAR)), + langfuse_destination_span_scope=langfuse_destination_span_scope( + otel_settings.get("langfuse_destination_span_scope") + ), ) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 24675406567..ffe530cb5db 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -301,7 +301,10 @@ async def add_team_callbacks( - langfuse_secret: The secret for the Langfuse callback - langfuse_host: The host for the Langfuse callback - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT) - - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans + - langfuse_span_scope: For langfuse_otel, "full" sends the whole request trace, + "llm_only" sends only model-call spans. + Omitted follows LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE (default "full"), also configurable as + callback_settings.otel.langfuse_destination_span_scope. - capture_message_content: For OTel v2 callbacks, "no_content" or "span_only". An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting. "span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out diff --git a/litellm/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx b/litellm/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx index 670bd710d38..d418b0b1466 100644 --- a/litellm/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx @@ -230,6 +230,11 @@ describe("LoggingSettings", () => { renderWithProviders(); expect(screen.queryByPlaceholderText("os.environ/LANGFUSE_SPAN_SCOPE")).not.toBeInTheDocument(); + expect( + screen.getByText( + "Leave empty to use the proxy default (LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE, full if unset).", + ), + ).toBeInTheDocument(); await user.click(screen.getByRole("combobox", { name: "langfuse span scope" })); expect((await screen.findAllByRole("option")).map((option) => option.textContent)).toEqual(["full", "llm_only"]); await user.click(screen.getByRole("option", { name: "llm_only" })); diff --git a/litellm/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx b/litellm/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx index f091bbbb929..77bb2a792f4 100644 --- a/litellm/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx +++ b/litellm/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx @@ -250,6 +250,9 @@ const LoggingSettings: React.FC = ({ {paramType === "number" && ( {t("teams.loggingSettings.numberHint")} )} + {paramName === "langfuse_span_scope" && ( +

{t("teams.loggingSettings.langfuseSpanScopeHint")}

+ )} {renderParamControl(config, configIndex, paramName, { type: paramType, options: paramType === "select" ? paramOptions[paramName] || [] : [], diff --git a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json index b52a8f6e4a8..5f9d04881e8 100644 --- a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json @@ -17339,6 +17339,7 @@ "number": "Number" }, "numberHint": "Value must be between 0 and 1", + "langfuseSpanScopeHint": "Leave empty to use the proxy default (LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE, full if unset).", "disabledCallbacks": { "title": "Disabled Callbacks", "tooltip": "Select callbacks to disable for this key. Disabled callbacks will not receive any logging data.", diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts index 311b7297f4d..855aac16f03 100644 --- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -18844,7 +18844,10 @@ export interface paths { * - langfuse_secret: The secret for the Langfuse callback * - langfuse_host: The host for the Langfuse callback * - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT) - * - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans + * - langfuse_span_scope: For langfuse_otel, "full" sends the whole request trace, + * "llm_only" sends only model-call spans. + * Omitted follows LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE (default "full"), also configurable as + * callback_settings.otel.langfuse_destination_span_scope. * - capture_message_content: For OTel v2 callbacks, "no_content" or "span_only". * An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting. * "span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out diff --git a/tests/integration/observability/test_langfuse_destination_span_scope.py b/tests/integration/observability/test_langfuse_destination_span_scope.py new file mode 100644 index 00000000000..e393828fd19 --- /dev/null +++ b/tests/integration/observability/test_langfuse_destination_span_scope.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import uuid +from collections.abc import Iterator +from typing import Final, Literal + +import pytest +from integration._support.client import Gateway, gateway_from_environment, held +from integration._support.otlp_sink import ( + AuditConfigWriter, + SpanSinks, + langfuse_callback_variables, + recorded_spans, + spans_for_trace, +) +from integration._support.process import owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + + +def _chat(request: Request) -> Reply: + assert request.target.endswith("/chat/completions"), request.target + assert b'"messages"' in request.body, request.body + return Reply( + body=( + b'{"id":"chatcmpl-scope","object":"chat.completion","created":1,"model":"gpt-4o-mini",' + b'"choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],' + b'"usage":{"prompt_tokens":7,"completion_tokens":2,"total_tokens":9}}' + ) + ) + + +@pytest.fixture(scope="module") +def provider() -> Iterator[Wire]: + with wire_server(_chat) as wire: + yield wire + + +@pytest.fixture(scope="module", params=["llm_only", None, "full"], ids=["proxy-llm-only", "proxy-unset", "proxy-full"]) +def proxy( + request: pytest.FixtureRequest, + audit_sinks: SpanSinks, + otel_audit_config: AuditConfigWriter, + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[tuple[Gateway, bool]]: + directory: Final = tmp_path_factory.mktemp("langfuse-destination-scope") + narrowed: Final = request.param == "llm_only" + default: Final = "llm_only" if narrowed else "full" + with ( + gateway_from_environment() as gateway, + owned_proxy_process( + gateway, + directory, + { + "LITELLM_OTEL_V2": "true", + "ARIZE_HTTP_ENDPOINT": audit_sinks.arize + "/v1/traces", + **({"LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE": default} if request.param is not None else {}), + }, + config=otel_audit_config(directory, {}), + remove_environment=( + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LANGFUSE_HOST", + "LITELLM_OTEL_LANGFUSE_SPAN_SCOPE", + *(("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE",) if request.param is None else ()), + ), + workers=2, + ) as owned, + ): + yield owned.gateway, narrowed + + +@pytest.mark.parametrize( + ("level", "scope", "backend"), + [ + ("key", None, "langfuse_otel"), + ("key", "full", "langfuse_otel"), + ("team", None, "langfuse_otel"), + ("key", "llm_only", "langfuse_otel"), + ("key", None, "arize"), + ], + ids=["key-default", "key-full", "team-default", "key-llm-only", "arize-default"], +) +def test_proxy_default_limits_only_destinations_without_an_explicit_scope( + level: Literal["key", "team"], + scope: str | None, + backend: Literal["langfuse_otel", "arize"], + proxy: tuple[Gateway, bool], + provider: Wire, + audit_sinks: SpanSinks, +) -> None: + gateway, narrowed = proxy + logging: Final[list[JsonValue]] = [ + { + "callback_name": backend, + "callback_type": "success", + "callback_vars": { + **( + langfuse_callback_variables(audit_sinks) + if backend == "langfuse_otel" + else {"arize_api_key": "arize-test-key", "arize_space_id": "arize-test-space"} + ), + **({"langfuse_span_scope": scope} if scope else {}), + }, + } + ] + sink: Final = audit_sinks.tenant if backend == "langfuse_otel" else audit_sinks.arize + with gateway.scenario() as scenario: + model: Final = scenario.model(api_base=provider.url + "/v1") + key: Final = ( + scenario.key(metadata={"logging": logging}) + if level == "key" + else scenario.key(team_id=scenario.team(metadata={"logging": logging})) + ) + since: Final = recorded_spans(sink)[0] + trace_id: Final = uuid.uuid4().hex + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "destination span scope"}]}, + key=key, + headers={"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"}, + ) + assert response.status_code == 200, response.text + spans: Final = held( + lambda: spans_for_trace(recorded_spans(sink, since)[1], trace_id), + lambda found: any("gen_ai.operation.name" in span["attributes"] for span in found), + holding=3, + seconds=40, + ) + if backend == "langfuse_otel" and (scope == "llm_only" or (narrowed and scope is None)): + assert len(spans) == 1, [(span["name"], span["kind"]) for span in spans] + assert "gen_ai.operation.name" in spans[0]["attributes"], spans + assert not any(span["kind"] == 2 or span["attributes"].get("db.system") == "postgresql" for span in spans) + return + assert any(span["kind"] == 2 for span in spans), [(span["name"], span["kind"]) for span in spans] + assert any("gen_ai.operation.name" in span["attributes"] for span in spans), spans + assert any(span["attributes"].get("db.system") == "postgresql" for span in spans), spans diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 60635c5ce45..5dff8f6a048 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -41,7 +41,9 @@ from litellm.integrations.otel.model.config import ( ExporterOwner, ExporterSpec, OpenTelemetryV2Config, + _cached_langfuse_destination_span_scope, is_otel_v2_enabled, + langfuse_destination_span_scope, parse_excluded_services, ) from litellm.integrations.otel.model.destination import OtelDestination @@ -115,6 +117,7 @@ def isolate_published_provider(monkeypatch): def forget_otel_v2_flag_after_each_test(): yield is_otel_v2_enabled.cache_clear() + _cached_langfuse_destination_span_scope.cache_clear() def in_fresh_context(fn, *args): @@ -3035,6 +3038,144 @@ class TestSpanScope: assert destination_for("langfuse_otel", creds).span_scope == "full" + @pytest.mark.parametrize("level", ["key", "team"]) + @pytest.mark.parametrize("source", ["env", "callback_settings"]) + @pytest.mark.parametrize( + ("default", "explicit", "expected"), + [("llm_only", None, "llm_only"), ("llm_only", "full", "full"), (None, None, "full")], + ) + def test_proxy_destination_scope_falls_back_without_overriding_tenant_choices( + self, + monkeypatch: pytest.MonkeyPatch, + allow_test_hosts: None, + level: str, + default: str | None, + explicit: str | None, + expected: str, + source: str, + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_SPAN_SCOPE", "llm_only") + monkeypatch.setattr(litellm, "callback_settings", {}) + monkeypatch.delenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", raising=False) + if default is not None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", default if source == "env" else "full") + if source == "callback_settings": + monkeypatch.setattr( + litellm, "callback_settings", {"otel": {"langfuse_destination_span_scope": default}} + ) + is_otel_v2_enabled.cache_clear() + metadata: Final = { + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "langfuse_host": "http://x", + **({"langfuse_span_scope": explicit} if explicit is not None else {}), + }, + } + ] + } + auth: Final = UserAPIKeyAuth(**({"metadata": metadata} if level == "key" else {"team_metadata": metadata})) + destinations: Final = resolve_tenant_otel_destinations(auth) + assert [destination.span_scope for destination in destinations] == [expected] + + @pytest.mark.parametrize("otel_settings", [None, "invalid", [], 42]) + @pytest.mark.parametrize("default", [None, "llm_only"]) + def test_non_mapping_otel_settings_preserve_key_destination( + self, + monkeypatch: pytest.MonkeyPatch, + allow_test_hosts: None, + otel_settings: object, + default: str | None, + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setattr(litellm, "callback_settings", {"otel": otel_settings}) + monkeypatch.delenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", raising=False) + if default is not None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", default) + is_otel_v2_enabled.cache_clear() + auth: Final = UserAPIKeyAuth( + metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "langfuse_host": "http://x", + }, + } + ] + } + ) + + assert [destination.span_scope for destination in resolve_tenant_otel_destinations(auth)] == [default or "full"] + + def test_callback_settings_destination_scope_wins_over_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", "full") + assert langfuse_destination_span_scope(" LLM_ONLY\n") == "llm_only" + + @pytest.mark.parametrize( + ("bare", "prefixed", "configured", "expected"), + [ + ("llm_only", None, None, "full"), + ("full", "llm_only", None, "llm_only"), + ("full", "full", "llm_only", "llm_only"), + ("llm_only", "llm_only", "full", "full"), + ], + ) + def test_only_prefixed_environment_sets_destination_scope( + self, + monkeypatch: pytest.MonkeyPatch, + bare: str, + prefixed: str | None, + configured: str | None, + expected: str, + ) -> None: + monkeypatch.setenv("langfuse_destination_span_scope", bare) + if prefixed is None: + monkeypatch.delenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", raising=False) + else: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", prefixed) + + assert langfuse_destination_span_scope(configured) == expected + + @pytest.mark.parametrize("spelling", ["LLM_ONLY", " llm_only\n", "full"]) + def test_destination_scope_config_normalizes_like_operator_scope( + self, + monkeypatch: pytest.MonkeyPatch, + spelling: str, + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", spelling) + assert langfuse_destination_span_scope() == spelling.strip().lower() + + @pytest.mark.parametrize("invalid", ["", "everything"]) + @pytest.mark.parametrize("source", ["env", "callback_settings"]) + def test_invalid_destination_scope_falls_back_and_warns_once( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, invalid: str, source: str + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", invalid if source == "env" else "full") + with caplog.at_level("WARNING", logger="LiteLLM"): + assert langfuse_destination_span_scope(invalid if source == "callback_settings" else None) == "full" + assert langfuse_destination_span_scope(invalid if source == "callback_settings" else None) == "full" + warnings: Final = [record.message for record in caplog.records if "DESTINATION_SPAN_SCOPE" in record.message] + assert len(warnings) == 1 + assert repr(invalid) in warnings[0] + assert "'full'" in warnings[0] and "'llm_only'" in warnings[0] + + def test_destination_scope_environment_is_a_process_snapshot(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", "llm_only") + assert langfuse_destination_span_scope() == "llm_only" + monkeypatch.setenv("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE", "full") + assert langfuse_destination_span_scope() == "llm_only" + assert langfuse_destination_span_scope("full") == "full" + assert langfuse_destination_span_scope("llm_only") == "llm_only" + def test_only_langfuse_honours_the_scope_var(self): arize = destination_for( "arize", {"arize_api_key": "k", "arize_space_id": "s", "langfuse_span_scope": "llm_only"} From 4e97503afa2c733052dff35049da346127cd1398 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 14 Nov 2026 23:57:30 +0000 Subject: [PATCH 08/57] test(router): hold the concurrent upsert where its build reads the list back (#51878) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_router_concurrent_deployment_delete_sdk.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py b/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py index 4cc6aca5a03..f27e05c781e 100644 --- a/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py +++ b/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py @@ -32,6 +32,12 @@ _CHURN_UPSERTS: Final = 120 _UNREPORTED_MODEL_ID: Final = frozenset({"aresponses-stream", "aanthropic-messages"}) _LIST_REFRESH: Final = HostedModelList.refresh.__code__ _HAND_CHANGES: Final = HostRegistry._hand_changes.__code__ +_REREADS_AFTER_A_WRITE: Final = frozenset( + { + (_LIST_REFRESH, _HAND_CHANGES), + (HostedModelList.end_build.__code__, HostRegistry._run_build.__code__), + } +) def _router_source_files() -> frozenset[str]: @@ -104,9 +110,7 @@ def _read_during_held_write(read: Callable[[], T], write: Callable[[], object]) def hold_at_reindex(frame: FrameType, event: str, _arg: object) -> None: caller: Final = frame.f_back - refreshes_after_a_write: Final = ( - frame.f_code is _LIST_REFRESH and caller is not None and caller.f_code is _HAND_CHANGES - ) + refreshes_after_a_write: Final = caller is not None and (frame.f_code, caller.f_code) in _REREADS_AFTER_A_WRITE if event == "call" and refreshes_after_a_write and not held.is_set(): held.set() released_in_time.append(release.wait(timeout=_SYNC_SECONDS)) From 21215cebac633da4b2d4d0d29ef5557a240bf141 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 15 Nov 2026 00:28:42 +0000 Subject: [PATCH 09/57] chore(cost-map): add fal_ai seedream v5 pro US edit and text-to-image rows priced per image from their fal.ai pages (#51881) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 26 +++++++++++++++++++ model_prices_and_context_window.json | 26 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d5a26bd3ae9..bca7e61d873 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -101932,5 +101932,31 @@ "supports_parallel_function_calling": true, "supports_reasoning": true, "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/pro/us/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per output image by total pixel area at the default image_size auto_2K with one input image; outputs up to and including 1536x1536 pixels are $0.0675 and outputs above that up to 2048x2048 are $0.135; the first input image is free and each additional input image adds $0.0045 per output, which is not modeled; fal calls this pricing tentative"}, + "mode": "image_generation", + "output_cost_per_image": 0.135, + "output_cost_per_image_1K": 0.0675, + "output_cost_per_image_2K": 0.135, + "source": "https://fal.ai/models/bytedance/seedream/v5/pro/us/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/pro/us/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per output image by total pixel area at the default image_size auto_2K; outputs up to and including 1536x1536 pixels are $0.0675 and outputs above that up to 2048x2048 are $0.135, so 1.5K has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.135, + "output_cost_per_image_1K": 0.0675, + "output_cost_per_image_2K": 0.135, + "source": "https://fal.ai/models/bytedance/seedream/v5/pro/us/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d5a26bd3ae9..bca7e61d873 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -101932,5 +101932,31 @@ "supports_parallel_function_calling": true, "supports_reasoning": true, "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/pro/us/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per output image by total pixel area at the default image_size auto_2K with one input image; outputs up to and including 1536x1536 pixels are $0.0675 and outputs above that up to 2048x2048 are $0.135; the first input image is free and each additional input image adds $0.0045 per output, which is not modeled; fal calls this pricing tentative"}, + "mode": "image_generation", + "output_cost_per_image": 0.135, + "output_cost_per_image_1K": 0.0675, + "output_cost_per_image_2K": 0.135, + "source": "https://fal.ai/models/bytedance/seedream/v5/pro/us/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5/pro/us/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per output image by total pixel area at the default image_size auto_2K; outputs up to and including 1536x1536 pixels are $0.0675 and outputs above that up to 2048x2048 are $0.135, so 1.5K has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.135, + "output_cost_per_image_1K": 0.0675, + "output_cost_per_image_2K": 0.135, + "source": "https://fal.ai/models/bytedance/seedream/v5/pro/us/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] } } From fbc7fd30b2ac4000a1e58519d6764da2ab095355 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 14 Nov 2026 16:42:35 -0800 Subject: [PATCH 10/57] bump: litellm-enterprise 0.1.85 -> 0.1.86, litellm-proxy-extras 0.4.116 -> 0.4.117 (#51876) --- litellm/enterprise/pyproject.toml | 4 ++-- litellm/proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 4 ++-- uv.lock | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/litellm/enterprise/pyproject.toml b/litellm/enterprise/pyproject.toml index 352aaa5d4dc..56098cb1028 100644 --- a/litellm/enterprise/pyproject.toml +++ b/litellm/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.85" +version = "0.1.86" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.85" +version = "0.1.86" version_files = [ "pyproject.toml:^version", "../../pyproject.toml:litellm-enterprise==", diff --git a/litellm/proxy-extras/pyproject.toml b/litellm/proxy-extras/pyproject.toml index 1d5f3038ac5..567aecc8f69 100644 --- a/litellm/proxy-extras/pyproject.toml +++ b/litellm/proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.116" +version = "0.4.117" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -30,7 +30,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.116" +version = "0.4.117" version_files = [ "pyproject.toml:^version", "../../pyproject.toml:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index e490d2f14b0..7ae10633fe7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,8 +87,8 @@ proxy = [ "mcp>=2.3.0,<3", "httpx2>=2.10.0,<3", "pydantic>=2.12.0,<3", - "litellm-proxy-extras==0.4.116", - "litellm-enterprise==0.1.85", + "litellm-proxy-extras==0.4.117", + "litellm-enterprise==0.1.86", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", diff --git a/uv.lock b/uv.lock index 9f7ec3ea812..e9640c98052 100644 --- a/uv.lock +++ b/uv.lock @@ -5583,7 +5583,7 @@ provides-extras = ["test"] [[package]] name = "litellm-enterprise" -version = "0.1.85" +version = "0.1.86" source = { editable = "litellm/enterprise" } [[package]] @@ -5593,7 +5593,7 @@ source = { editable = "litellm/packaging/litellm-proxy" } [[package]] name = "litellm-proxy-extras" -version = "0.4.116" +version = "0.4.117" source = { editable = "litellm/proxy-extras" } dependencies = [ { name = "psycopg" }, From 4a8fa7c60af70500a069a309b5306c229c83d753 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 14 Nov 2026 16:55:01 -0800 Subject: [PATCH 11/57] test(e2e): run the batch lifecycle with a key that only reaches its own deployment (#51857) test_batch_lifecycle[azure-provider_fallback] has failed in most scheduled e2e runs since the suite moved from 8 to 12 xdist workers. Batch create returned 404 Resource not found from Azure after a successful upload. A batch call that names a provider and no model takes its credentials from the first deployment of that provider the key may use. The test used an all-models key, so on the shared proxy that was whichever Azure deployment got registered first. With 12 workers the realtime suite starts in the same instant as the batches suite, and its session-long azure-realtime deployment carries api_version 2025-08-28. When it registers before azure-batch, the batch create goes out with that api_version and Azure answers 404. The upload still works because the file-create route overrides the borrowed credentials with files_settings. The file delete on cleanup does not, so the model_param and provider_fallback cleanups were also sent with the wrong api_version and their 404 was read as already deleted. The lifecycle key now belongs to a team that may only use the batch deployment under test, so provider-only calls resolve that deployment in any registration order. --- tests/e2e/batches/COVERAGE.md | 10 +++++++++- tests/e2e/batches/test_batches_e2e.py | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index ac6f9cd08e4..00247ec550f 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -49,7 +49,7 @@ and batch id carry the shape that scenario must produce (`matches_id_shape`): | `encoded` | upload with `?model=` -> model-encoded file id -> create with just that id | model-encoded | model-encoded | | `unified` | upload with `target_model_names=` -> unified managed file id -> create with that id | managed | managed | | `model_param` | raw file (provider-fallback upload) -> create with `model` in the body | raw | model-encoded | -| `provider_fallback` | raw file -> `POST /{provider}/v1/batches`, env creds, no model | raw | raw (native provider shape) | +| `provider_fallback` | raw file -> `POST /{provider}/v1/batches`, no model, credentials of the key's deployment for that provider | raw | raw (native provider shape) | "managed" ids base64-decode to a `litellm_proxy` marker; "model-encoded" ids keep the provider prefix and base64-encode `litellm:;model,`; "raw" ids are the @@ -59,6 +59,14 @@ provider also fails create (the file id / model do not belong there), and the `provider_fallback` raw batch id is additionally checked against the provider's native shape (`raw_id_matches_provider`). +The `model_param` file calls and every `provider_fallback` call name a provider and no +model. For those the proxy uses the credentials of the first deployment of that provider +the key may use, and falls back to env credentials only when the key can reach none. So +the lifecycle key belongs to a team that may only use the batch deployment under test +(`key_in_team_limited_to`). An all-models key would resolve whichever deployment of that +provider another suite registered first on the shared proxy, and the call would go out +with that deployment's `api_base` and `api_version`. + ## Key model restriction `test_batch_key_model_access_denied` mints a key restricted to one model diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index f28a8361aeb..c0933a7d8e0 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -71,7 +71,7 @@ from e2e_http import ( unwrap, ) from lifecycle import ResourceManager -from models import KeyGenerateBody, KeyMetadata, LiteLLMParamsBody, SpendLogRow +from models import KeyGenerateBody, KeyMetadata, LiteLLMParamsBody, SpendLogRow, TeamNewBody from pydantic import BaseModel, Field pytestmark = pytest.mark.e2e @@ -248,6 +248,16 @@ def assert_batch_object(batch: BatchObject) -> None: ), "batch.created_at missing" +def key_in_team_limited_to(client: BatchClient, resources: ResourceManager, model: str) -> str: + team_id: Final = client.proxy.create_team( + TeamNewBody(team_alias=f"e2e-batch-lifecycle-{unique_marker()}", models=[model]) + ) + resources.defer(lambda: client.proxy.delete_team(team_id)) + key: Final = client.proxy.generate_key(KeyGenerateBody(team_id=team_id, user_id="e2e-test-user")) + resources.defer(lambda: client.proxy.delete_key(key)) + return key + + @pytest.mark.parametrize( "cap", [ @@ -265,7 +275,7 @@ def test_batch_lifecycle( resources: ResourceManager, batch_deployments: None, ) -> None: - key = resources.key() + key = key_in_team_limited_to(client, resources, cap.model) provider = op_provider(cap) file = unwrap(upload_for_scenario(client, cap, render_jsonl(cap.jsonl_model), key)) From e38187324ad3b49e8111c6823a8cfb3c7d6c2f7d Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Sat, 14 Nov 2026 17:47:36 -0800 Subject: [PATCH 12/57] fix(spend): support PostgreSQL 12 delivery migrations (#51868) * fix(spend): support PostgreSQL 12 delivery migrations The SpendLogDelivery migration defaulted delivery_id to gen_random_uuid(), which PostgreSQL 12 lacks without pgcrypto, so migrate deploy failed there. The writer always supplies delivery_id, so the column no longer needs a database default. Databases that already applied the original migration drop the default in a follow-up migration without rewriting rows. * test(spend): apply the ledger enum migration before the delivery migration tests --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../migration.sql | 2 +- .../migration.sql | 2 + .../litellm_proxy_extras/schema.prisma | 2 +- litellm/proxy/schema.prisma | 2 +- litellm/schema.prisma | 2 +- .../test_drop_accounting_event_tables.py | 58 ++++++++++++++++++- .../database/test_spendlogs_ledger_parity.py | 41 +++++++------ 7 files changed, 86 insertions(+), 23 deletions(-) create mode 100644 litellm/proxy-extras/litellm_proxy_extras/migrations/20261104020000_spendlogs_delivery_id_drop_default/migration.sql diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql index b00cf892b68..ce7171637a2 100644 --- a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql @@ -1,6 +1,6 @@ SET LOCAL lock_timeout = '10s'; CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogDelivery" ( - "delivery_id" TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text, + "delivery_id" TEXT PRIMARY KEY, "request_id" TEXT NOT NULL, "state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending', "attempts" INTEGER NOT NULL DEFAULT 0, diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261104020000_spendlogs_delivery_id_drop_default/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261104020000_spendlogs_delivery_id_drop_default/migration.sql new file mode 100644 index 00000000000..ee5c8652f6b --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261104020000_spendlogs_delivery_id_drop_default/migration.sql @@ -0,0 +1,2 @@ +SET LOCAL lock_timeout = '10s'; +ALTER TABLE "LiteLLM_SpendLogDelivery" ALTER COLUMN "delivery_id" DROP DEFAULT; diff --git a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma index db88b24dcb0..3ca6db43106 100644 --- a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma @@ -2308,7 +2308,7 @@ enum AccountingDeliveryState { } model LiteLLM_SpendLogDelivery { - delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + delivery_id String @id @default(uuid()) request_id String state AccountingDeliveryState @default(pending) accounting_payload Json diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index db88b24dcb0..3ca6db43106 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2308,7 +2308,7 @@ enum AccountingDeliveryState { } model LiteLLM_SpendLogDelivery { - delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + delivery_id String @id @default(uuid()) request_id String state AccountingDeliveryState @default(pending) accounting_payload Json diff --git a/litellm/schema.prisma b/litellm/schema.prisma index db88b24dcb0..3ca6db43106 100644 --- a/litellm/schema.prisma +++ b/litellm/schema.prisma @@ -2308,7 +2308,7 @@ enum AccountingDeliveryState { } model LiteLLM_SpendLogDelivery { - delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + delivery_id String @id @default(uuid()) request_id String state AccountingDeliveryState @default(pending) accounting_payload Json diff --git a/tests/integration/database/migrations/test_drop_accounting_event_tables.py b/tests/integration/database/migrations/test_drop_accounting_event_tables.py index 0894cf05c4e..72049395cbf 100644 --- a/tests/integration/database/migrations/test_drop_accounting_event_tables.py +++ b/tests/integration/database/migrations/test_drop_accounting_event_tables.py @@ -9,6 +9,7 @@ _MIGRATIONS: Final = Path(__file__).parents[4] / "litellm/proxy-extras/litellm_p _OLD_LEDGER: Final = _MIGRATIONS / "20261027000000_add_accounting_ledger_tables/migration.sql" _SPEND_LOG_DELIVERY: Final = _MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql" _DROP: Final = _MIGRATIONS / "20261104010000_drop_accounting_event_tables/migration.sql" +_DROP_DELIVERY_ID_DEFAULT: Final = _MIGRATIONS / "20261104020000_spendlogs_delivery_id_drop_default/migration.sql" _DROPPED: Final = ( "AccountingDeliveryTarget", "AccountingEventKind", @@ -51,8 +52,8 @@ def test_drop_removes_the_old_ledger_and_keeps_spend_log_deliveries(old_states: (f"event-{index}", state), ) connection.execute( - 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, state, accounting_payload, is_batch) ' - "VALUES ('request', 'pending', '{}', false)" + 'INSERT INTO "LiteLLM_SpendLogDelivery" (delivery_id, request_id, state, accounting_payload, is_batch) ' + "VALUES ('request', 'request', 'pending', '{}', false)" ) _apply(connection, _DROP) @@ -62,3 +63,56 @@ def test_drop_removes_the_old_ledger_and_keeps_spend_log_deliveries(old_states: assert connection.execute('SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [ ("request", "pending") ] + + +def _deliveries(connection: psycopg.Connection[tuple[object, ...]]) -> list[tuple[object, ...]]: + return connection.execute( + 'SELECT delivery_id, request_id FROM "LiteLLM_SpendLogDelivery" ORDER BY request_id' + ).fetchall() + + +def _insert_delivery(connection: psycopg.Connection[tuple[object, ...]], request_id: str) -> None: + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (delivery_id, request_id, accounting_payload, is_batch) ' + "VALUES (%s, %s, '{}', false)", + (f"delivery-{request_id}", request_id), + ) + + +def test_delivery_table_stores_application_ids_without_database_uuid_generation() -> None: + with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True) as connection: + _apply(connection, _OLD_LEDGER) + _apply(connection, _SPEND_LOG_DELIVERY) + + _insert_delivery(connection, "request") + with pytest.raises(psycopg.errors.NotNullViolation): + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch) ' + "VALUES ('no-id', '{}', false)" + ) + + assert _deliveries(connection) == [("delivery-request", "request")] + + +def test_drop_default_keeps_deliveries_written_under_the_database_generated_id() -> None: + with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True) as connection: + _apply(connection, _OLD_LEDGER) + _apply(connection, _SPEND_LOG_DELIVERY) + connection.execute( + 'ALTER TABLE "LiteLLM_SpendLogDelivery" ALTER COLUMN "delivery_id" SET DEFAULT gen_random_uuid()::text' + ) + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch) ' + "VALUES ('existing', '{}', false)" + ) + existing: Final = _deliveries(connection) + + _apply(connection, _DROP_DELIVERY_ID_DEFAULT) + _insert_delivery(connection, "new") + with pytest.raises(psycopg.errors.NotNullViolation): + connection.execute( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch) ' + "VALUES ('no-id', '{}', false)" + ) + + assert _deliveries(connection) == [*existing, ("delivery-new", "new")] diff --git a/tests/integration/database/test_spendlogs_ledger_parity.py b/tests/integration/database/test_spendlogs_ledger_parity.py index c33391ea6e4..bf58f22f82e 100644 --- a/tests/integration/database/test_spendlogs_ledger_parity.py +++ b/tests/integration/database/test_spendlogs_ledger_parity.py @@ -114,9 +114,10 @@ async def test_worker_removes_more_expired_receipts_than_one_cleanup_batch_in_on client = PrismaClient(database_url=parity_database, proxy_logging_obj=proxy_server.proxy_logging_obj) with psycopg.connect(parity_database, autocommit=True) as connection: connection.execute( - 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch, state, applied_at) ' - "SELECT 'expired-' || n, '{}'::jsonb, false, 'applied', CURRENT_TIMESTAMP - INTERVAL '2 days' " - "FROM generate_series(1, 501) AS n" + 'INSERT INTO "LiteLLM_SpendLogDelivery" ' + "(delivery_id, request_id, accounting_payload, is_batch, state, applied_at) " + "SELECT 'expired-' || n, 'expired-' || n, '{}'::jsonb, false, 'applied', " + "CURRENT_TIMESTAMP - INTERVAL '2 days' FROM generate_series(1, 501) AS n" ) await client.db.connect() try: @@ -810,7 +811,8 @@ async def test_window_seed_excludes_pending_and_parked_but_preserves_legacy_cuto ) for request_id, state in (("applied", "applied"), ("pending", "pending"), ("parked", "parked")): await client.writer_db.execute_raw( - 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id,state,accounting_payload,is_batch) VALUES ($1,$2::"AccountingDeliveryState",$$ {} $$::jsonb,false)', + 'INSERT INTO "LiteLLM_SpendLogDelivery" (delivery_id,request_id,state,accounting_payload,is_batch) ' + 'VALUES ($1,$1,$2::"AccountingDeliveryState",$$ {} $$::jsonb,false)', request_id, state, ) @@ -944,8 +946,9 @@ async def test_worker_exposes_invalid_accounting_as_delayed_then_parked_work( try: with psycopg.connect(parity_database, autocommit=True) as connection: connection.execute( - 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, created_at, accounting_payload, is_batch) ' - "VALUES ('invalid-accounting', CURRENT_TIMESTAMP - INTERVAL '1 hour', " + 'INSERT INTO "LiteLLM_SpendLogDelivery" ' + "(delivery_id, request_id, created_at, accounting_payload, is_batch) " + "VALUES ('invalid-accounting', 'invalid-accounting', CURRENT_TIMESTAMP - INTERVAL '1 hour', " '$$ {"metadata": {"accounting_windows": [{}]}} $$::jsonb, false)' ) await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) @@ -1001,8 +1004,9 @@ async def test_worker_delivers_under_another_leaders_lock_then_cleans_reports_an assert reports_before is not None with psycopg.connect(parity_database, autocommit=True) as connection: connection.execute( - 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch, attempts) ' - "VALUES ('fresh', %s::jsonb, false, 0), ('retry', %s::jsonb, false, 1)", + 'INSERT INTO "LiteLLM_SpendLogDelivery" ' + "(delivery_id, request_id, accounting_payload, is_batch, attempts) " + "VALUES ('fresh', 'fresh', %s::jsonb, false, 0), ('retry', 'retry', %s::jsonb, false, 1)", ( json.dumps({"spend": 0.25, "metadata": {"accounting_entities": {"user": "user"}}}), json.dumps({"spend": 0.5, "metadata": {"accounting_entities": {"user": "user"}}}), @@ -1010,8 +1014,8 @@ async def test_worker_delivers_under_another_leaders_lock_then_cleans_reports_an ) connection.execute( 'INSERT INTO "LiteLLM_SpendLogDelivery" ' - '(request_id, accounting_payload, is_batch, state, applied_at) ' - "VALUES ('expired', '{}'::jsonb, false, 'applied', CURRENT_TIMESTAMP - INTERVAL '2 days')" + '(delivery_id, request_id, accounting_payload, is_batch, state, applied_at) ' + "VALUES ('expired', 'expired', '{}'::jsonb, false, 'applied', CURRENT_TIMESTAMP - INTERVAL '2 days')" ) assert await other_leader.acquire_lock(cronjob_id="accounting_delivery", allow_reentrant=False) await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) @@ -1442,8 +1446,9 @@ async def test_retained_batch_storage_threshold_alerts_without_deleting_receipts try: with psycopg.connect(parity_database, autocommit=True) as connection: connection.execute( - 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, state, applied_at, accounting_payload, is_batch) ' - "VALUES ('old-batch', 'applied', CURRENT_TIMESTAMP - INTERVAL '400 days', '{}', true)" + 'INSERT INTO "LiteLLM_SpendLogDelivery" ' + "(delivery_id, request_id, state, applied_at, accounting_payload, is_batch) " + "VALUES ('old-batch', 'old-batch', 'applied', CURRENT_TIMESTAMP - INTERVAL '400 days', '{}', true)" ) await flush_accounting_deliveries(DBSpendUpdateWriter(), client, alerts) with psycopg.connect(parity_database) as connection: @@ -1482,15 +1487,17 @@ async def test_worker_drains_recovered_and_expired_work_at_its_configured_capaci with psycopg.connect(parity_database, autocommit=True) as connection: if backlog == "retries": connection.execute( - 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch, attempts) ' - "SELECT 'retry-' || i, accounting_payload, false, 1 " + 'INSERT INTO "LiteLLM_SpendLogDelivery" ' + "(delivery_id, request_id, accounting_payload, is_batch, attempts) " + "SELECT 'retry-' || i || '-' || delivery_id, 'retry-' || i, accounting_payload, false, 1 " 'FROM "LiteLLM_SpendLogDelivery" CROSS JOIN generate_series(1, 3) i' ) else: connection.execute( - 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id, accounting_payload, is_batch, state, applied_at) ' - "SELECT 'expired-' || i, '{}', false, 'applied', CURRENT_TIMESTAMP - INTERVAL '2 days' " - 'FROM generate_series(1, 1100) i' + 'INSERT INTO "LiteLLM_SpendLogDelivery" ' + "(delivery_id, request_id, accounting_payload, is_batch, state, applied_at) " + "SELECT 'expired-' || i, 'expired-' || i, '{}', false, 'applied', " + "CURRENT_TIMESTAMP - INTERVAL '2 days' FROM generate_series(1, 1100) i" ) await flush_accounting_deliveries(writer, client, proxy_server.proxy_logging_obj) with psycopg.connect(parity_database) as connection: From 745886e5ff4fb3036727b5b735d7d1822fcd83e0 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Sun, 15 Nov 2026 11:16:47 +0900 Subject: [PATCH 13/57] fix(packaging): keep the litellm-proxy package version in step with litellm (#51807) The litellm-proxy manifest still said 1.109.0 after litellm moved to 1.110.0 and 1.111.0, because version bumps only rewrote the root pyproject.toml. The Docker image builds that manifest unchanged, so ghcr.io/berriai/litellm:v1.111.0-dev.2 reports litellm-proxy 1.109.0, a version that was never published. Set the manifest and uv.lock to 1.111.0, add the manifest to the commitizen version_files so cz bump rewrites it, and add a test that fails when the two versions or the lock entry disagree. --- litellm/packaging/litellm-proxy/pyproject.toml | 2 +- pyproject.toml | 1 + tests/unit/test_package_metadata.py | 7 +++++++ uv.lock | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/litellm/packaging/litellm-proxy/pyproject.toml b/litellm/packaging/litellm-proxy/pyproject.toml index 8efdca8db74..915b0e3708d 100644 --- a/litellm/packaging/litellm-proxy/pyproject.toml +++ b/litellm/packaging/litellm-proxy/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy" -version = "1.109.0" +version = "1.111.0" description = "Opt-in Rust HTTP forwarding server for the LiteLLM Python proxy" requires-python = ">=3.10, <3.16" license = "MIT" diff --git a/pyproject.toml b/pyproject.toml index 7ae10633fe7..baffe10a43e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -445,6 +445,7 @@ members = ["litellm/enterprise", "litellm/proxy-extras", "litellm/packaging/lite version = "1.111.0" version_files = [ "pyproject.toml:^version", + "litellm/packaging/litellm-proxy/pyproject.toml:^version", ] [tool.pytest.ini_options] diff --git a/tests/unit/test_package_metadata.py b/tests/unit/test_package_metadata.py index d7dce33bd74..6ff5b62ff7a 100644 --- a/tests/unit/test_package_metadata.py +++ b/tests/unit/test_package_metadata.py @@ -76,6 +76,13 @@ def test_importing_litellm_reports_the_declared_release_version() -> None: assert litellm._version.version == PYPROJECT["project"]["version"] +def test_gateway_binary_package_ships_at_the_litellm_release_version() -> None: + manifest: Final = tomllib.loads((PROJECT_ROOT / "litellm/packaging/litellm-proxy/pyproject.toml").read_text()) + + assert manifest["project"]["version"] == PYPROJECT["project"]["version"] + assert _locked_package("litellm-proxy")["version"] == PYPROJECT["project"]["version"] + + def test_every_declared_extra_is_locked_with_its_requirements() -> None: declared: Final = { canonicalize_name(extra): _requirement_names(requirements) diff --git a/uv.lock b/uv.lock index e9640c98052..3dd89c01d44 100644 --- a/uv.lock +++ b/uv.lock @@ -5588,7 +5588,7 @@ source = { editable = "litellm/enterprise" } [[package]] name = "litellm-proxy" -version = "1.109.0" +version = "1.111.0" source = { editable = "litellm/packaging/litellm-proxy" } [[package]] From 61483d8f04cfd667a3c2b5d6f826715cfc2dda53 Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Sun, 15 Nov 2026 10:23:35 +0800 Subject: [PATCH 14/57] fix(ui): rename Customer Portal to Customer Management (#51891) * fix(ui): rename Customer Portal to Customer Management Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(ui): polish Customer Management product copy Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * style(ui): format Customer Management test assertions Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- litellm/proxy/auth/customer_hierarchy_auth.py | 4 ++-- .../management_endpoints/management_v1/customers.py | 11 ++++++----- .../CreateCustomerDialog.integration.test.tsx | 4 ++-- .../CreateCustomerHierarchyDialog.test.tsx | 6 ++++-- .../CustomerHierarchySettings.integration.test.tsx | 2 +- .../customers/limits.integration.test.tsx | 4 ++-- .../Navbar/ViewSwitcher.integration.test.tsx | 8 ++++---- .../src/components/leftnav.test.tsx | 11 +++++++---- .../ui/litellm-dashboard/src/components/leftnav.tsx | 6 +++--- .../ui/litellm-dashboard/src/i18n/locales/en.json | 12 ++++++------ .../ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 ++-- .../management_v1/test_customers.py | 10 +++++----- 12 files changed, 44 insertions(+), 38 deletions(-) diff --git a/litellm/proxy/auth/customer_hierarchy_auth.py b/litellm/proxy/auth/customer_hierarchy_auth.py index 11b6ae2c488..24dd06e64d4 100644 --- a/litellm/proxy/auth/customer_hierarchy_auth.py +++ b/litellm/proxy/auth/customer_hierarchy_auth.py @@ -553,7 +553,7 @@ async def _register_organization( if isinstance(registered, Failure): if registered.code is ErrorCode.FREE_TIER_LIMIT: verbose_proxy_logger.warning( - "CUSTOMER TRACKING OFF for organization %s because the free customer portal limit was reached. " + "CUSTOMER TRACKING OFF for organization %s because the Customer Management free tier limit was reached. " "Upgrade to Enterprise to remove the limits: https://www.litellm.ai/enterprise", customer_org_alias, ) @@ -620,7 +620,7 @@ async def _register_member( if isinstance(registered, Failure): if registered.code is ErrorCode.FREE_TIER_LIMIT: verbose_proxy_logger.warning( - "CUSTOMER TRACKING OFF for member %s because the free customer portal limit was reached. " + "CUSTOMER TRACKING OFF for member %s because the Customer Management free tier limit was reached. " "Upgrade to Enterprise to remove the limits: https://www.litellm.ai/enterprise", customer_member_id, ) diff --git a/litellm/proxy/management_endpoints/management_v1/customers.py b/litellm/proxy/management_endpoints/management_v1/customers.py index a678a535385..f451344f8cc 100644 --- a/litellm/proxy/management_endpoints/management_v1/customers.py +++ b/litellm/proxy/management_endpoints/management_v1/customers.py @@ -54,13 +54,13 @@ _ERROR_DETAILS: Final[Mapping[ErrorCode, tuple[int, str, str]]] = MappingProxyTy { ErrorCode.CAPACITY_BUSY: ( 503, - "Customer portal busy", - "Customer portal capacity is being updated. Retry the request.", + "Customer Management busy", + "Customer Management capacity is being updated. Retry the request.", ), ErrorCode.FREE_TIER_LIMIT: ( 403, - "Free customer portal limit reached", - "Free customer portal limit reached: up to 5 customer organizations, 5 teams per organization and " + "Customer Management free tier limit reached", + "Customer Management free tier limit reached: up to 5 customer organizations, 5 teams per organization and " "5 members per team without a LiteLLM Enterprise license. Upgrade to Enterprise to remove the limits: " "https://www.litellm.ai/enterprise", ), @@ -213,7 +213,8 @@ AuthCache: TypeAlias = Annotated[UserApiKeyCache, Depends(get_user_api_key_cache @router.get( "/customer_portal/limits", response_model=CustomerPortalLimits, - description="Get customer portal free-tier limits and current capacity. Proxy admins only.", + summary="Get Customer Management Limits", + description="Get Customer Management free-tier limits and current capacity. Proxy admins only.", ) @management_endpoint_wrapper async def get_customer_portal_limits( diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.integration.test.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.integration.test.tsx index ebc3ede926f..b8426673b83 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerDialog.integration.test.tsx @@ -40,7 +40,7 @@ beforeEach(() => { creations.push(await request.json()); if (freeTierReached) return Response.json( - { detail: "Free customer portal limit reached. Upgrade to Enterprise." }, + { detail: "Customer Management free tier limit reached. Upgrade to Enterprise." }, { status: 403 }, ); return rejectCreation @@ -65,7 +65,7 @@ describe("CreateCustomerDialog", () => { fireEvent.click(screen.getByRole("button", { name: "Create customer" })); await waitFor(() => expect(toast.error).toHaveBeenCalledWith( - expect.stringContaining("Free customer portal limit reached. Upgrade to Enterprise."), + expect.stringContaining("Customer Management free tier limit reached. Upgrade to Enterprise."), ), ); }); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerHierarchyDialog.test.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerHierarchyDialog.test.tsx index 5db40e0ce5a..c08b6bc7359 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerHierarchyDialog.test.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CreateCustomerHierarchyDialog.test.tsx @@ -164,7 +164,9 @@ describe("CreateCustomerHierarchyDialog", () => { }); it.each(["team", "member"] as const)("surfaces the free-tier 403 detail when creating a %s", async (kind) => { - fetchMock.mockResolvedValue(json({ detail: "Free customer portal limit reached. Upgrade to Enterprise." }, 403)); + fetchMock.mockResolvedValue( + json({ detail: "Customer Management free tier limit reached. Upgrade to Enterprise." }, 403), + ); render( { fireEvent.change(screen.getByRole("textbox", { name: "Display name" }), { target: { value: "Example" } }); fireEvent.click(screen.getByRole("button", { name: "Create" })); expect(await screen.findByRole("alert")).toHaveTextContent( - "Free customer portal limit reached. Upgrade to Enterprise.", + "Customer Management free tier limit reached. Upgrade to Enterprise.", ); expect(onOpenChange).not.toHaveBeenCalled(); expect(onCreated).not.toHaveBeenCalled(); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchySettings.integration.test.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchySettings.integration.test.tsx index 555b32f3450..48c8787d2f2 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchySettings.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchySettings.integration.test.tsx @@ -385,7 +385,7 @@ describe("CustomerHierarchySettings", () => { /> , ); - expect(await screen.findByRole("alert")).toHaveTextContent("Free customer portal limit reached"); + expect(await screen.findByRole("alert")).toHaveTextContent("Customer Management free tier limit reached"); fireEvent.click(screen.getByRole("button", { name: "Delete customer member" })); const dialog = await screen.findByRole("dialog", { name: "Delete customer member" }); fireEvent.change(within(dialog).getByPlaceholderText("member-1"), { target: { value: "member-1" } }); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/limits.integration.test.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/limits.integration.test.tsx index 7b8f6cdf7ce..fc79778727a 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/limits.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/limits.integration.test.tsx @@ -8,7 +8,7 @@ afterEach(() => { vi.unstubAllGlobals(); }); -describe("Customer portal limits", () => { +describe("Customer Management limits", () => { it.each([ { limitsApply: true, reached: false }, { limitsApply: true, reached: true }, @@ -50,7 +50,7 @@ describe("Customer portal limits", () => { expect(screen.queryByText(/Free tier:/)).not.toBeInTheDocument(); } if (limitsApply && reached) { - expect(await screen.findByRole("alert")).toHaveTextContent("Free customer portal limit reached"); + expect(await screen.findByRole("alert")).toHaveTextContent("Customer Management free tier limit reached"); expect(screen.getByRole("alert")).toHaveTextContent( "Requests from new customers or members still go through but are not tracked", ); diff --git a/litellm/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.integration.test.tsx index a0f6e59d029..65f2f977267 100644 --- a/litellm/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.integration.test.tsx @@ -106,23 +106,23 @@ describe("ViewSwitcher", () => { expect(assignSpy).not.toHaveBeenCalled(); }); - it("should open the Customer Portal at the customers page and come back to the gateway", async () => { + it("should open Customer Management at the customers page and come back to the gateway", async () => { const user = userEvent.setup(); renderSwitcher(); await user.click(screen.getByRole("button", { name: "Switch workspace, current: AI Gateway" })); - await user.click(await screen.findByRole("menuitem", { name: /Customer Portal/ })); + await user.click(await screen.findByRole("menuitem", { name: /Customer Management/ })); expect(pushMock).toHaveBeenCalledWith("/ui/customers"); expect(assignSpy).not.toHaveBeenCalled(); expect(localStorage.getItem("litellm_plugin_mode")).toBeNull(); }); - it("should route back to the gateway from Customer Portal mode instead of only flipping the mode", async () => { + it("should route back to the gateway from Customer Management mode instead of only flipping the mode", async () => { localStorage.setItem("litellm_plugin_mode", "customer-portal"); const user = userEvent.setup(); renderSwitcher(); - await user.click(await screen.findByRole("button", { name: "Switch workspace, current: Customer Portal" })); + await user.click(await screen.findByRole("button", { name: "Switch workspace, current: Customer Management" })); await user.click(await screen.findByRole("menuitem", { name: /^AI Gateway/ })); expect(pushMock).toHaveBeenCalledWith("/ui/api-keys"); }); diff --git a/litellm/ui/litellm-dashboard/src/components/leftnav.test.tsx b/litellm/ui/litellm-dashboard/src/components/leftnav.test.tsx index 416671dac1f..19cbcf27d87 100644 --- a/litellm/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -178,7 +178,7 @@ describe("Sidebar (leftnav)", () => { expect(screen.getByRole("link", { name: /litellm home/i })).toHaveAttribute("href", "/ui"); }); - it("should make customer organizations reachable from the Customer Portal sidebar", () => { + it("should make customer organizations reachable from the Customer Management sidebar", () => { localStorage.setItem("litellm_plugin_mode", "customer-portal"); navState.pathname = "/ui/customers/manage"; renderWithProviders( @@ -765,14 +765,17 @@ describe("Sidebar (leftnav)", () => { describe("getBreadcrumb", () => { it("should identify the customer organizations list and its hierarchy routes", () => { expect(getBreadcrumb("/ui/customers/usage")).toEqual({ - section: "Customer Portal", + section: "Customer Management", title: "Customer organizations", }); expect(getBreadcrumb("/ui/customers/manage")).toEqual({ - section: "Customer Portal", + section: "Customer Management", + title: "Customer organizations", + }); + expect(getBreadcrumb("/ui/customers/")).toEqual({ + section: "Customer Management", title: "Customer organizations", }); - expect(getBreadcrumb("/ui/customers/")).toEqual({ section: "Customer Portal", title: "Customer organizations" }); }); it("resolves a top-level route to its section + title", () => { expect(getBreadcrumb("/ui/api-keys")).toEqual({ section: "AI Gateway", title: "Virtual Keys" }); diff --git a/litellm/ui/litellm-dashboard/src/components/leftnav.tsx b/litellm/ui/litellm-dashboard/src/components/leftnav.tsx index 897eda4263e..1bd9a472d28 100644 --- a/litellm/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/litellm/ui/litellm-dashboard/src/components/leftnav.tsx @@ -198,7 +198,7 @@ export const visibleMenuGroups = (groups: readonly MenuGroup[], context: MenuVis // icons changed to lucide as part of the sidebar redesign. const customerPortalGroups: MenuGroup[] = [ { - groupLabel: "CUSTOMER PORTAL", + groupLabel: "CUSTOMER MANAGEMENT", items: [ { key: "customers", @@ -531,7 +531,7 @@ const findMenuItemKey = (route: string): string => { const GROUP_LABELS: Record = { "AI GATEWAY": "shell.nav.groups.aiGateway", - "CUSTOMER PORTAL": "shell.nav.groups.customerPortal", + "CUSTOMER MANAGEMENT": "shell.nav.groups.customerPortal", OBSERVABILITY: "shell.nav.groups.observability", "ACCESS CONTROL": "shell.nav.groups.accessControl", "DEVELOPER TOOLS": "shell.nav.groups.developerTools", @@ -543,7 +543,7 @@ export const groupText = (groupLabel: string): string => const SECTION_DISPLAY: Record = { "AI GATEWAY": "shell.nav.sections.aiGateway", - "CUSTOMER PORTAL": "shell.nav.sections.customerPortal", + "CUSTOMER MANAGEMENT": "shell.nav.sections.customerPortal", OBSERVABILITY: "shell.nav.sections.observability", "ACCESS CONTROL": "shell.nav.sections.accessControl", "DEVELOPER TOOLS": "shell.nav.sections.developerTools", diff --git a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json index 5f9d04881e8..e8c979032ac 100644 --- a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json @@ -8164,7 +8164,7 @@ "accessControl": "ACCESS CONTROL", "developerTools": "DEVELOPER TOOLS", "settings": "SETTINGS", - "customerPortal": "CUSTOMER PORTAL" + "customerPortal": "CUSTOMER MANAGEMENT" }, "sections": { "aiGateway": "AI Gateway", @@ -8172,7 +8172,7 @@ "accessControl": "Access Control", "developerTools": "Developer Tools", "settings": "Settings", - "customerPortal": "Customer Portal" + "customerPortal": "Customer Management" }, "items": { "home": "Home", @@ -20672,15 +20672,15 @@ }, "limits": { "byline": "Free tier: up to 5 customers, 5 teams per customer, 5 members per team. LiteLLM Enterprise removes these limits.", - "reachedTitle": "Free customer portal limit reached", - "reachedDescription": "New customers, teams and members can't be created. Requests from new customers or members still go through but are not tracked in the customer portal.", + "reachedTitle": "Customer Management free tier limit reached", + "reachedDescription": "New customers, teams and members can't be created. Requests from new customers or members still go through but are not tracked in Customer Management.", "upgrade": "Upgrade to Enterprise" }, "hierarchy": { "customerNavigation": "Customer navigation", "usage": "Usage", "manage": "Manage", - "title": "Customer management", + "title": "Customer Management", "subtitle": "Manage customer teams, customer members, and settings for this customer organization.", "teamSubtitle": "Manage customer members and settings for this customer team.", "memberSubtitle": "Manage settings and budget for this customer member.", @@ -20778,7 +20778,7 @@ "deleteInformationTitle": "{{kind}} details" }, "switcher": { - "label": "Customer Portal", + "label": "Customer Management", "description": "Customer organizations and usage" }, "status": { diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts index 855aac16f03..a115de0fdd0 100644 --- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9934,8 +9934,8 @@ export interface paths { cookie?: never; }; /** - * Get Customer Portal Limits - * @description Get customer portal free-tier limits and current capacity. Proxy admins only. + * Get Customer Management Limits + * @description Get Customer Management free-tier limits and current capacity. Proxy admins only. */ get: operations["get_customer_portal_limits_management_v1_customer_portal_limits_get"]; put?: never; diff --git a/tests/unit/proxy/management_endpoints/management_v1/test_customers.py b/tests/unit/proxy/management_endpoints/management_v1/test_customers.py index 7a862d99422..64126c6df52 100644 --- a/tests/unit/proxy/management_endpoints/management_v1/test_customers.py +++ b/tests/unit/proxy/management_endpoints/management_v1/test_customers.py @@ -78,7 +78,7 @@ def test_customer_portal_limits_and_free_tier_problem(premium: bool) -> None: assert sixth.status_code == (201 if premium else 403), sixth.text if not premium: assert sixth.json()["detail"] == ( - "Free customer portal limit reached: up to 5 customer organizations, 5 teams per organization and " + "Customer Management free tier limit reached: up to 5 customer organizations, 5 teams per organization and " "5 members per team without a LiteLLM Enterprise license. Upgrade to Enterprise to remove the limits: " "https://www.litellm.ai/enterprise" ) @@ -161,9 +161,9 @@ def test_capacity_contention_is_retryable_without_hiding_other_database_errors(e assert response.status_code == 503, response.text assert response.json() == { "type": "urn:litellm:error:customer-hierarchy-capacity-busy", - "title": "Customer portal busy", + "title": "Customer Management busy", "status": 503, - "detail": "Customer portal capacity is being updated. Retry the request.", + "detail": "Customer Management capacity is being updated. Retry the request.", } assert http.get("/management/v1/customer_members/blocked").status_code == 404 @@ -197,9 +197,9 @@ def test_team_model_write_contention_is_retryable_and_rolls_back(error: DataErro assert response.status_code == 503, response.text assert response.json() == { "type": "urn:litellm:error:customer-hierarchy-capacity-busy", - "title": "Customer portal busy", + "title": "Customer Management busy", "status": 503, - "detail": "Customer portal capacity is being updated. Retry the request.", + "detail": "Customer Management capacity is being updated. Retry the request.", } assert http.get("/management/v1/customer_teams/blocked").status_code == 404 assert not database.litellm_budgettable.rows From aeb9ed07aa4b304fd343f21a47d95d06c6798087 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:09:22 +0000 Subject: [PATCH 15/57] test(database): expect no native pool in proxy lifespan cleanup (#51911) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/database/test_proxy_database_lifespan.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/database/test_proxy_database_lifespan.py b/tests/integration/database/test_proxy_database_lifespan.py index c79c0ba4b35..e24f565c437 100644 --- a/tests/integration/database/test_proxy_database_lifespan.py +++ b/tests/integration/database/test_proxy_database_lifespan.py @@ -10,7 +10,6 @@ from typing import Final, Literal, TypeAlias import psutil import pytest -from litellm.rust_bridge._native import NativeDatabase from tests.integration._support.client import eventually from tests.integration._support.database import read_rows, scratch_database @@ -80,7 +79,7 @@ async def _exercise_lifespans(mode: Mode) -> None: async with proxy_server.proxy_startup_event(proxy_server.app) as state: database: Final = state["database"] assert database is not None - assert isinstance(database.native, NativeDatabase) + assert database.native is None assert await database.prisma.writer_db.query_raw("SELECT 42 AS answer") == [{"answer": 42}] assert await database.prisma.read_db.query_raw("SELECT 43 AS answer") == [{"answer": 43}] assert psutil.Process().children(recursive=True) From bc78cbe9a2b87be48b32f1c1fc00f13fea3d1bac Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sun, 15 Nov 2026 11:09:35 +0800 Subject: [PATCH 16/57] fix(ui): clarify customer usage periods and preserve date ranges (#51899) * fix(ui): clarify customer usage periods and preserve date ranges * fix(ui): preserve fixed customer dates with stale preset labels --- .../_components/CustomerBudgetSummary.tsx | 8 +- .../_components/CustomerHierarchyManager.tsx | 13 ++-- .../CustomerHierarchyNavigation.tsx | 32 ++++++-- .../_components/CustomerUsageHeader.tsx | 9 ++- .../_components/customerUsageRange.test.ts | 46 ++++++++++- .../_components/customerUsageRange.ts | 76 ++++++++++++++----- .../_components/useCustomerHierarchyUsage.ts | 32 ++++---- .../manage/page.integration.test.tsx | 16 ++++ .../customers/page.integration.test.tsx | 49 ++++++------ .../customers/usage/page.integration.test.tsx | 43 +++++++++-- .../app/(dashboard)/customers/usage/page.tsx | 6 +- 11 files changed, 242 insertions(+), 88 deletions(-) diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerBudgetSummary.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerBudgetSummary.tsx index 69c98d614ec..6384f14d5ab 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerBudgetSummary.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerBudgetSummary.tsx @@ -1,6 +1,6 @@ import { useTranslation } from "react-i18next"; import type { CustomerUsageEntity } from "./customerUsageApi"; -import CustomerBudgetSpend, { CustomerBudgetWindow } from "./CustomerBudgetSpend"; +import { CustomerBudgetWindow } from "./CustomerBudgetSpend"; import { CUSTOMER_BUDGET_LABELS, customerBudgetPeriod, customerBudgetUsage } from "./customerBudgetUsage"; export default function CustomerBudgetSummary({ @@ -35,10 +35,10 @@ export default function CustomerBudgetSummary({ ); })} - {budgets.length === 0 && ( - + {budgets.length === 0 &&

{t("customerPortal.analytics.noBudget")}

} + {budgets.length > 0 && ( +

{t("customerPortal.analytics.budgetUsageHint")}

)} -

{t("customerPortal.analytics.budgetUsageHint")}

); } diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchyManager.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchyManager.tsx index ad75a9b62f5..f3d33f82db1 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchyManager.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerHierarchyManager.tsx @@ -9,8 +9,7 @@ import { ErrorState } from "@/components/shared/ErrorState"; import { Button } from "@/components/ui/button"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { extractProxyErrorMessage } from "@/lib/http/client"; -import { uiHref } from "@/utils/uiHref"; -import CustomerHierarchyNavigation, { customerHierarchyHref } from "./CustomerHierarchyNavigation"; +import CustomerHierarchyNavigation, { useCustomerHierarchyLinks } from "./CustomerHierarchyNavigation"; import CustomerHierarchySettings from "./CustomerHierarchySettings"; import CustomerBudgetGroups from "./CustomerBudgetGroups"; import CustomerBudgetSummary from "./CustomerBudgetSummary"; @@ -39,6 +38,7 @@ const DESCRIPTION_KEYS = { function HierarchyHeader({ hierarchy, selected }: Pick) { const { t } = useTranslation(); + const { customerHierarchyHref, customersHref } = useCustomerHierarchyLinks(); const entity = selected.entity; const team = selected.team; const label = customerEntityLabel(entity); @@ -70,7 +70,7 @@ function HierarchyHeader({ hierarchy, selected }: Pick - + {t("customerPortal.entities.organizations")}