From 9b5562f89b34601295fa21f3877c85046929318b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 2 Oct 2026 14:32:17 -0700 Subject: [PATCH] test: repair stale and polluting tests red on scheduled main CI (#44229) * test(proxy): stop the proxy_server app fixture leaking LITELLM_LOG The session app fixture set LITELLM_LOG=ERROR with os.environ.setdefault and never removed it, so later tests on the same xdist worker inherited it. test_drop_params_env_var spawns a subprocess with os.environ and lost the warning it asserts on. Scope the variable to the import with a MonkeyPatch context * test(secret-detection): give the hand-built redaction request an ASGI path Since #43975 _read_request_body checks the route path via request.scope, and a scope without path raised KeyError that was swallowed into an empty body, so chat_completion failed with a missing messages parameter. Real ASGI scopes always carry path * test(integration): isolate litellm callback lists per sdk test usage-based-routing-v2 Routers register their selector in litellm.callbacks and nothing removes it, not even Router.reset(). The counter TTL and Redis service metrics tests left their selectors behind, and the next usage routing test ran their pre-call checks against its own rpm=1 deployments, raising "Deployment over defined rpm limit". An autouse fixture now gives each sdk test copies of the callback lists and restores the originals afterwards * test(integration): keep the owner-lookup fault proxy off the shared read replica The owned proxy points DATABASE_URL at a scratch database but inherited DATABASE_URL_READ_REPLICA from the replica job, so auth read the shared database and rejected the freshly created key with token_not_found_in_db. Drop the replica variable like the other scratch-database owned proxies * test(integration): request every seeded key in the team owner breakdown The aggregated team activity endpoint now caps breakdown.api_keys at the top 100 keys by default (#43398), so the 300 seeded keys came back as 100 rows. The test guarantees each key is reported with its own owner, so ask for an api_key_limit that covers all seeded keys * test(integration): give every owned Redis its own port in the redis-cache container On CircleCI every owned Redis ran on the fixed port 16379 inside the shared redis-cache container. When an earlier server still held that port, the new one failed to bind, readiness pinged the old server, the pidfile read failed and cleanup then reported "Owned Redis still serves after shutdown" Reserve an ephemeral port for the docker-exec path the same way the local binary path already does, and refuse to start when something already serves the chosen port so the failure names the real cause * test(e2e): skip the Vertex Mistral partner case the e2e project cannot reach The e2e Vertex project gets a 404 publisher model not found for vertex_ai/mistral-small-2503, so the case can only fail * test(e2e): skip the Vertex gpt-oss partner case the e2e project never serves vertex_ai/openai/gpt-oss-120b-maas has hit a 60s read timeout with no response headers on every run in the e2e Vertex project since the case was ported, and no other Vertex partner chat model passes there to switch to * test(e2e): check only stored message content for a leaked card number The Presidio spend-log check ran the card-number pattern over the whole serialized response, so a Luhn-valid usage.cost float (0.0003466000000000001) failed the streaming /v1/messages case although the stored content was . The check now reads the content and text strings of the stored response, which is where a raw card would land, and still requires the placeholder there * test(e2e): assert the proxy decodes token-array embeddings for titan The port in #44120 carried over a legacy SDK-direct test that expected Bedrock to reject token ids with a 400. Through the proxy, /embeddings decodes token arrays to text for providers that cannot embed tokens, so titan answers 200. The test now sends a token array and its decoded sentence and requires the two vectors to match, which fails if the proxy stops decoding or decodes with the wrong tokenizer * test(e2e): run the Bedrock extended-thinking round trip on a model that honors enabled thinking us.anthropic.claude-sonnet-5-5 is adaptive-only, so litellm sends thinking.type=enabled with a 1024 budget as adaptive with low effort, and Bedrock returned no reasoning blocks on 5 of 5 identical Converse calls (boto3 direct agreed). us.anthropic.claude-sonnet-4-6 accepts the legacy shape verbatim and returned reasoning on 5 of 5. The non-thinking Bedrock case stays on sonnet-5-5 * test(proxy): stop unit modules forcing DEBUG logging into the event-loop lag tests Five tests/unit modules set verbose_proxy_logger to DEBUG at import, so every xdist worker that collected them logged the 2.4MB pass-through response from a worker thread, and secret redaction of that line held the GIL for ~0.8s+ inside the timed window. The lag tests now pin the LiteLLM loggers to WARNING and freeze gc while timing, and the module-level DEBUG overrides are removed * test(e2e): cite the tokenizer and date behind the titan token-array fixture * test(e2e): let migration seed replicas finish their request-log indexes before cloning Since #43948 a serving proxy builds the two LiteLLM_SpendLogs indexes on a background thread after it reports ready. The seed fixtures stopped the replica at readiness, so every cloned legacy database lacked an index no real deployment would be missing, and the v2 baseline diff refused it. Seeds now wait until both indexes exist and are valid in the database's schema * test(passthrough): give the pass-through MockRequest an httpx URL and ASGI scope #43626 made get_request_route read request.scope during pass-through kwarg setup; the MockRequest in tests/unit/passthrough had neither a scope nor a URL object, so both stream-param tests raised before reaching the code they check. Mirrors the repair #43626 made to the tests/pass_through_unit_tests fake * test(integration): ignore foreign allow_all_keys MCP servers in the access matrix tool list test_toolset_gateway_url_serves_a_team_granted_toolset_to_a_key_without_its_own_grant (#43908) registers an allow_all_keys server on the shared gateway, and allow_all_keys servers are listed to every key by design, so a matrix case running on another xdist worker at the same time saw its tools. The matrix now drops tools of allow_all_keys servers it did not create, read from LiteLLM_MCPServerTable before and after listing, and still compares everything else exactly --- .../guardrails/test_presidio_masking_e2e.py | 22 +++++-- .../test_chat_completions_regression_e2e.py | 13 +++- .../test_chat_tool_round_trip_e2e.py | 9 +-- .../test_embeddings_endpoint_e2e.py | 62 ++++++++++--------- tests/e2e/migrations/conftest.py | 4 +- tests/e2e/migrations/containers.py | 13 ++++ tests/e2e/migrations/test_legacy.py | 4 +- tests/integration/_support/redis_process.py | 24 +++---- .../management/test_tool_policy_user.py | 8 ++- .../integration/mcp/test_mcp_access_matrix.py | 15 ++++- tests/integration/sdk/conftest.py | 20 ++++++ .../test_daily_activity_key_owner_faults.py | 4 +- .../local_testing/test_secret_detect_hook.py | 1 + .../test_project_endpoints_prisma.py | 3 - .../unit/litellm_core_utils/event_loop_lag.py | 7 ++- .../unit/passthrough/test_passthrough_main.py | 4 +- .../test_e2e_pod_lock_manager.py | 3 - .../test_key_generate_prisma.py | 1 - .../test_audit_logs_proxy.py | 3 - .../test_streaming_handler.py | 9 ++- tests/unit/proxy/proxy_server/conftest.py | 5 +- tests/unit/proxy/test_proxy_token_counter.py | 3 - 22 files changed, 159 insertions(+), 78 deletions(-) create mode 100644 tests/integration/sdk/conftest.py diff --git a/tests/e2e/guardrails/test_presidio_masking_e2e.py b/tests/e2e/guardrails/test_presidio_masking_e2e.py index 1fbc4a68bbd..35eb4884ddf 100644 --- a/tests/e2e/guardrails/test_presidio_masking_e2e.py +++ b/tests/e2e/guardrails/test_presidio_masking_e2e.py @@ -29,15 +29,14 @@ this suite deliberately requires the detected-entity details to remain visible. from __future__ import annotations -import json import os import re import time -from collections.abc import Callable +from collections.abc import Callable, Iterator from typing import Final, Literal import pytest -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, JsonValue, TypeAdapter from e2e_config import unique_marker from e2e_http import Result, StreamingResponse, Success @@ -74,6 +73,7 @@ FAKE_PHONE = "+1 415-555-0134" FAKE_VISA_TEST_CARD = "4111 1111 1111 1111" _CARD_DIGIT_RUN: Final = re.compile(r"(?:\d[ -]?){13,19}") +_CONTENT_KEYS: Final = frozenset({"content", "text"}) def _presidio_bases() -> tuple[str, str]: @@ -516,7 +516,21 @@ def _spend_log_response_text(client: GuardrailsClient, key: str, call_id: str) - ) row = next((row for row in rows if row.litellm_call_id == call_id), None) assert row is not None, f"no spend log row ever appeared for x-litellm-call-id {call_id}" - return json.dumps(row.response) + return "\n".join(_stored_content(row.response)) + + +def _stored_content(node: JsonValue, key: str | None = None) -> Iterator[str]: + match node: + case str() if key in _CONTENT_KEYS: + yield node + case dict(): + for child_key, child in node.items(): + yield from _stored_content(child, child_key) + case list(): + for item in node: + yield from _stored_content(item, key) + case _: + return class TestPresidioSpendLogStoresMaskedOutput: diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index b6b0ef3ebbe..90ac18c16ac 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -53,7 +53,18 @@ OPENAI_BACKEND = "openai/gpt-5.6" ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5-20251001" BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" BEDROCK_NOVA_BACKEND: Final = "bedrock/us.amazon.nova-2-lite-v1:0" -VERTEX_PARTNER_BACKENDS: Final = ("vertex_ai/mistral-small-2503", "vertex_ai/openai/gpt-oss-120b-maas") +VERTEX_PARTNER_BACKENDS: Final = ( + pytest.param( + "vertex_ai/mistral-small-2503", + marks=pytest.mark.skip( + reason="the e2e Vertex project has no access to mistral-small-2503 (404 publisher model not found)" + ), + ), + pytest.param( + "vertex_ai/openai/gpt-oss-120b-maas", + marks=pytest.mark.skip(reason="never served by the e2e Vertex project (60s read timeout, no headers)"), + ), +) PDF_DOCUMENT_URL: Final = ( "https://cdn.jsdelivr.net/gh/BerriAI/litellm" "@d769e81c90d453240c61fc572cdb27fae06a89d0" diff --git a/tests/e2e/llm_translation/test_chat_tool_round_trip_e2e.py b/tests/e2e/llm_translation/test_chat_tool_round_trip_e2e.py index c6e9f813099..75ba5c23ff8 100644 --- a/tests/e2e/llm_translation/test_chat_tool_round_trip_e2e.py +++ b/tests/e2e/llm_translation/test_chat_tool_round_trip_e2e.py @@ -28,6 +28,7 @@ GEMINI_BACKEND: Final = "gemini/gemini-3.5-flash-lite" MISTRAL_BACKEND: Final = "mistral/mistral-medium-3.5" ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5" BEDROCK_CONVERSE_BACKEND: Final = "bedrock/converse/us.anthropic.claude-sonnet-5-5" +BEDROCK_LEGACY_THINKING_BACKEND: Final = "bedrock/converse/us.anthropic.claude-sonnet-4-6" PROMPT: Final = "What is the weather in Paris and in Tokyo? Use the get_weather tool for each city." CITY_TEMPERATURES: Final = MappingProxyType({"paris": "22", "tokyo": "31"}) @@ -54,9 +55,9 @@ def _api_key_params(backend: str, env: str) -> LiteLLMParamsBody: return LiteLLMParamsBody(model=backend, api_key=f"os.environ/{env}") -def _bedrock_params() -> LiteLLMParamsBody: +def _bedrock_params(backend: str) -> LiteLLMParamsBody: return LiteLLMParamsBody( - model=BEDROCK_CONVERSE_BACKEND, + model=backend, aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", aws_region_name="os.environ/AWS_REGION", @@ -146,7 +147,7 @@ class TestChatToolResultRoundTrip: _assert_tool_results_reach_the_model(client, key, model, thinking=None, tool_choice="required") def test_bedrock_converse(self, client: PassthroughClient, resources: ResourceManager) -> None: - model, key = _register(client, resources, _bedrock_params()) + model, key = _register(client, resources, _bedrock_params(BEDROCK_CONVERSE_BACKEND)) _assert_tool_results_reach_the_model(client, key, model, thinking=None, tool_choice="required") def test_anthropic_with_extended_thinking(self, client: PassthroughClient, resources: ResourceManager) -> None: @@ -156,5 +157,5 @@ class TestChatToolResultRoundTrip: def test_bedrock_converse_with_extended_thinking( self, client: PassthroughClient, resources: ResourceManager ) -> None: - model, key = _register(client, resources, _bedrock_params()) + model, key = _register(client, resources, _bedrock_params(BEDROCK_LEGACY_THINKING_BACKEND)) _assert_tool_results_reach_the_model(client, key, model, thinking=THINKING, tool_choice=None) diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index 42fe6537d4f..0e5bac556cc 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -10,6 +10,7 @@ SDK refuses to build stay on the shared transport. from __future__ import annotations +import math from typing import Final import pytest @@ -25,6 +26,10 @@ pytestmark = pytest.mark.e2e VERTEX_TEXT_EMBEDDING: Final = "vertex_ai/text-embedding-005" VERTEX_MULTIMODAL_EMBEDDING: Final = "vertex_ai/multimodalembedding@001" +TOKENS_TEXT: Final = "The quick brown fox jumps over the lazy dog" +# tiktoken 0.12.0 cl100k_base encoding of TOKENS_TEXT (checked 2026-10-02), the vocabulary behind the proxy's +# litellm.decode(model="gpt-3.5-turbo") token-array decode +TOKENS: Final = (791, 4062, 14198, 39935, 35308, 927, 279, 16053, 5679) class _OptionalEmbeddingsBody(BaseModel): @@ -32,9 +37,19 @@ class _OptionalEmbeddingsBody(BaseModel): input: str | list[str] | None = None -class _TokenEmbeddingsBody(BaseModel): - model: str - input: list[list[int]] +def _cosine(left: list[float], right: list[float]) -> float: + dot: Final = sum(a * b for a, b in zip(left, right, strict=True)) + norms: Final = math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right)) + return dot / norms + + +def _titan_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model="bedrock/amazon.titan-embed-text-v2:0", + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ) def _vertex_params(model: str) -> LiteLLMParamsBody: @@ -94,12 +109,7 @@ class TestEmbeddingsEndpoint: resources, sdk, "e2e-embeddings-bedrock", - LiteLLMParamsBody( - model="bedrock/amazon.titan-embed-text-v2:0", - aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", - aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", - aws_region_name="os.environ/AWS_REGION", - ), + _titan_params(), ) @pytest.mark.covers("llm.embeddings.cohere.basic.nonstream.works") @@ -171,28 +181,20 @@ class TestEmbeddingsEndpoint: cost = response_header(raw.headers, "x-litellm-response-cost") assert cost is not None and float(cost) > 0, f"multimodal embedding was not costed: {cost!r}" - def test_bedrock_titan_rejects_token_array_input_as_bad_request( - self, proxy: ProxyClient, resources: ResourceManager + def test_bedrock_titan_embeds_token_array_input_as_its_decoded_text( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model, key = _register( - proxy, - resources, - "e2e-embeddings-titan-tokens", - LiteLLMParamsBody( - model="bedrock/amazon.titan-embed-text-v2:0", - aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", - aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", - aws_region_name="os.environ/AWS_REGION", - ), - ) - result = proxy.transport.send( - "/embeddings", - headers=proxy.transport.bearer(key), - json=_TokenEmbeddingsBody(model=model, input=[[1]]), - ) - assert result.status_code == 400, ( - f"titan cannot embed token arrays, so the caller must get a 400, got {result.status_code}: " - f"{result.body[:300]}" + model, key = _register(proxy, resources, "e2e-embeddings-titan-tokens", _titan_params()) + client: Final = sdk.openai(key) + + from_tokens: Final = client.embeddings.create(model=model, input=[TOKENS], extra_body=NO_PROXY_CACHE) + from_text: Final = client.embeddings.create(model=model, input=TOKENS_TEXT, extra_body=NO_PROXY_CACHE) + + assert len(from_tokens.data) == 1, f"one token array must yield one vector: {from_tokens!r}" + similarity: Final = _cosine(from_tokens.data[0].embedding, from_text.data[0].embedding) + assert similarity > 0.99, ( + f"titan cannot embed token ids, so the proxy must decode them to {TOKENS_TEXT!r} first; " + f"the token-array vector only has cosine {similarity:.4f} with that text's vector" ) @pytest.mark.replayable diff --git a/tests/e2e/migrations/conftest.py b/tests/e2e/migrations/conftest.py index b7604a4fdda..ac3618a4542 100644 --- a/tests/e2e/migrations/conftest.py +++ b/tests/e2e/migrations/conftest.py @@ -8,7 +8,7 @@ from urllib.parse import urlsplit import pytest from _pytest.fixtures import SubRequest -from .containers import Containers, docker, ready +from .containers import Containers, docker, ready, seeded from .database import Database, Databases @@ -44,7 +44,7 @@ def migrated_template( output: Final = Path(os.environ.get("MIGRATION_TEST_OUTPUT", str(tmp_path_factory.getbasetemp()))) / "seed" with databases.create() as database: with Containers(migration_image, output).start(database) as replica: - ready((replica,), database) + seeded(replica, database) yield database diff --git a/tests/e2e/migrations/containers.py b/tests/e2e/migrations/containers.py index dd126b994d3..30374dedcf3 100644 --- a/tests/e2e/migrations/containers.py +++ b/tests/e2e/migrations/containers.py @@ -97,6 +97,19 @@ def ready(replicas: tuple[Replica, ...], database: Database) -> None: replica.usable(database) +def seeded(seed: Replica, database: Database) -> None: + ready((seed,), database) + until("the seed replica to finish its request-log indexes", lambda: request_log_indexes_built(database)) + + +def request_log_indexes_built(database: Database) -> bool: + return database.query( + "SELECT count(*) FROM pg_index x JOIN pg_class i ON i.oid = x.indexrelid " + "JOIN pg_namespace n ON n.oid = i.relnamespace WHERE n.nspname = current_schema() AND x.indisvalid " + "AND i.relname IN ('LiteLLM_SpendLogs_api_key_startTime_idx', 'LiteLLM_SpendLogs_litellm_call_id_idx')" + ) == ((2,),) + + def failed(replicas: tuple[Replica, ...], marker: str) -> None: def all_stopped() -> bool: observations: Final = tuple(replica.observe() for replica in replicas) diff --git a/tests/e2e/migrations/test_legacy.py b/tests/e2e/migrations/test_legacy.py index ba5e77a3070..4cad808d3ff 100644 --- a/tests/e2e/migrations/test_legacy.py +++ b/tests/e2e/migrations/test_legacy.py @@ -5,7 +5,7 @@ from typing import Final, Literal import pytest from .checks import COMPLETE, assert_completed, confirmed_history, assert_original_proof, start_replicas -from .containers import Containers, failed, ready +from .containers import Containers, failed, ready, seeded from .database import Database, Databases pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup] @@ -69,7 +69,7 @@ class TestLegacyMigrations: ) -> None: with databases.create(schema="migration tenant") as database: with containers.start(database) as seed: - ready((seed,), database) + seeded(seed, database) match scenario: case "upgrade": with ExitStack() as stack: diff --git a/tests/integration/_support/redis_process.py b/tests/integration/_support/redis_process.py index 86abcbe024e..3a5bf8bec23 100644 --- a/tests/integration/_support/redis_process.py +++ b/tests/integration/_support/redis_process.py @@ -25,8 +25,16 @@ class OwnedRedis: process: subprocess.Popen | None = None server_pid: int | None = None + def serves(self) -> bool: + with Redis(host=self.host, port=self.port, socket_connect_timeout=0.2, socket_timeout=0.2) as client: + try: + return bool(client.ping()) + except RedisConnectionError: + return False + def start(self) -> None: assert self.process is None + assert not self.serves(), f"Another Redis already serves {self.host}:{self.port} before the owned one starts" self.process = subprocess.Popen(self.command, stdout=self.log, stderr=subprocess.STDOUT, start_new_session=True) deadline: Final = time.monotonic() + 8 with Redis(host=self.host, port=self.port, socket_connect_timeout=0.2, socket_timeout=0.2) as client: @@ -68,14 +76,7 @@ class OwnedRedis: self.process.wait(timeout=3) self.process = None self.server_pid = None - with Redis(host=self.host, port=self.port, socket_connect_timeout=0.2, socket_timeout=0.2) as client: - try: - client.ping() - except RedisConnectionError: - stopped = True - else: - stopped = False - assert stopped, "Owned Redis still serves after shutdown" + assert not self.serves(), "Owned Redis still serves after shutdown" assert failure is None and not forced, f"Owned Redis required shutdown recovery: {failure!r}" def signal(self, action: signal.Signals) -> None: @@ -92,16 +93,15 @@ class OwnedRedis: @contextmanager def owned_redis(directory: Path) -> Iterator[OwnedRedis]: binary: Final = shutil.which("redis-server") + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + port: Final = reservation.getsockname()[1] if binary: - with socket.socket() as reservation: - reservation.bind(("127.0.0.1", 0)) - port = reservation.getsockname()[1] host = "127.0.0.1" prefix = (binary,) else: host = subprocess.check_output(["docker", "inspect", "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", "redis-cache"], text=True).strip() assert host, "CircleCI owned Redis container has no address" - port = 16379 prefix = ("docker", "exec", "redis-cache", "redis-server") output: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(directory))) output.mkdir(parents=True, exist_ok=True) diff --git a/tests/integration/management/test_tool_policy_user.py b/tests/integration/management/test_tool_policy_user.py index 00b4605ee2e..b93fe190e28 100644 --- a/tests/integration/management/test_tool_policy_user.py +++ b/tests/integration/management/test_tool_policy_user.py @@ -462,7 +462,13 @@ def test_owner_lookup_failure_keeps_tools_listed_without_a_user(gateway: Gateway config: Final = _proxy_config(tmp_path, model, gateway.upstream_url, {}) with ( scratch_database() as database_url, - owned_proxy(gateway, tmp_path, {"DATABASE_URL": database_url}, config=config) as candidate, + owned_proxy( + gateway, + tmp_path, + {"DATABASE_URL": database_url}, + config=config, + remove_environment=("DATABASE_URL_READ_REPLICA",), + ) as candidate, ): alias: Final = "fault-owner-" + uuid.uuid4().hex user: Final = string_value( diff --git a/tests/integration/mcp/test_mcp_access_matrix.py b/tests/integration/mcp/test_mcp_access_matrix.py index 13759ce245c..e458911cc68 100644 --- a/tests/integration/mcp/test_mcp_access_matrix.py +++ b/tests/integration/mcp/test_mcp_access_matrix.py @@ -3,6 +3,7 @@ from typing import Final import pytest from integration._support.client import Gateway +from integration._support.database import read_rows from integration._support.mcp import ( ENTRY_POINTS, EntryPoint, @@ -27,6 +28,16 @@ def _name(entry: EntryPoint, alias: str, tool: str) -> str: return tool if entry == "rest" else f"{alias}-{tool}" +def _open_aliases() -> frozenset[str]: + rows: Final = read_rows('SELECT alias FROM "LiteLLM_MCPServerTable" WHERE allow_all_keys', ()) + return frozenset(str(row["alias"]) for row in rows) + + +def _without_foreign_open_servers(tools: tuple[str, ...], open_aliases: frozenset[str]) -> set[str]: + prefixes: Final = tuple(f"{alias}-" for alias in open_aliases) + return {tool for tool in tools if not tool.startswith(prefixes)} + + def _assert_denied(caller: McpCaller, peer: McpPeer, name: str, identity: str, entry: EntryPoint) -> None: peer.drain() outcome: Final = caller.call(name, CALLABLE["add"], _server_scoped(entry, identity)) @@ -51,14 +62,16 @@ def test_subject_grant_lists_only_reachable_tools_and_denies_the_rest( scenario, subject, (granted,), (granted, denied), access_group=group, allowed_tools={granted: ("add",)} ) reach: Final = McpCaller(gateway, caller.key, entry, granted_alias, caller.headers) + open_before: Final = _open_aliases() listed: Final = reach.list_tools(_server_scoped(entry, granted)) assert listed.ok, listed.raw + open_aliases: Final = open_before | _open_aliases() expected: Final = ( {_name(entry, granted_alias, "add")} if subject in ("toolset", "allowed_tools") else {_name(entry, granted_alias, tool) for tool in ("add", "multiply", "fail")} ) - assert set(listed.tools) == expected, listed.tools + assert _without_foreign_open_servers(listed.tools, open_aliases) == expected, listed.tools for tool, arguments in CALLABLE.items(): name: Final = _name(entry, granted_alias, tool) if name not in listed.tools: diff --git a/tests/integration/sdk/conftest.py b/tests/integration/sdk/conftest.py new file mode 100644 index 00000000000..066cca6250a --- /dev/null +++ b/tests/integration/sdk/conftest.py @@ -0,0 +1,20 @@ +from typing import Final + +import litellm +import pytest + +CALLBACK_LISTS: Final = ( + "callbacks", + "success_callback", + "failure_callback", + "input_callback", + "_async_success_callback", + "_async_failure_callback", + "_async_input_callback", +) + + +@pytest.fixture(autouse=True) +def isolate_litellm_callback_lists(monkeypatch: pytest.MonkeyPatch) -> None: + for name in CALLBACK_LISTS: + monkeypatch.setattr(litellm, name, list(getattr(litellm, name))) diff --git a/tests/integration/spend/test_daily_activity_key_owner_faults.py b/tests/integration/spend/test_daily_activity_key_owner_faults.py index 998cd2396ae..2ab56622ec4 100644 --- a/tests/integration/spend/test_daily_activity_key_owner_faults.py +++ b/tests/integration/spend/test_daily_activity_key_owner_faults.py @@ -161,7 +161,9 @@ def test_every_key_of_a_team_is_reported_with_its_own_user(gateway: Gateway) -> ) with daily_rows(rows): response: Final = gateway.request( - "GET", AGGREGATED_TEAM_ACTIVITY, params={"start_date": DAY, "end_date": DAY, "team_ids": team} + "GET", + AGGREGATED_TEAM_ACTIVITY, + params={"start_date": DAY, "end_date": DAY, "team_ids": team, "api_key_limit": KEYS_OF_ONE_TEAM}, ) assert response.status_code == 200, response.text body: Final = object_value(response.json()) diff --git a/tests/local_testing/test_secret_detect_hook.py b/tests/local_testing/test_secret_detect_hook.py index 0ee0f596177..c560637b785 100644 --- a/tests/local_testing/test_secret_detect_hook.py +++ b/tests/local_testing/test_secret_detect_hook.py @@ -272,6 +272,7 @@ async def test_chat_completion_request_with_redaction(): scope={ "type": "http", "method": "POST", + "path": "/chat/completions", "headers": [(b"content-type", b"application/json")], "query_string": query_params.encode(), } diff --git a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index 36878fa698c..226755e7b8e 100644 --- a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -9,12 +9,10 @@ from fastapi import HTTPException, Request load_dotenv() import time -import logging import pytest import litellm -from litellm._logging import verbose_proxy_logger from litellm.proxy.management_endpoints.team_endpoints import ( new_team, ) @@ -30,7 +28,6 @@ from litellm.proxy.proxy_server import ( from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.utils import PrismaClient, ProxyLogging -verbose_proxy_logger.setLevel(level=logging.DEBUG) from litellm.caching.caching import DualCache diff --git a/tests/unit/litellm_core_utils/event_loop_lag.py b/tests/unit/litellm_core_utils/event_loop_lag.py index 1cac0365547..5697b3203f5 100644 --- a/tests/unit/litellm_core_utils/event_loop_lag.py +++ b/tests/unit/litellm_core_utils/event_loop_lag.py @@ -1,4 +1,5 @@ import asyncio +import gc import time from collections.abc import Awaitable, Callable from typing import Final, TypeVar @@ -32,7 +33,11 @@ async def timed_with_loop_lags(run: Callable[[], Awaitable[T]]) -> tuple[T, floa finally: finished.set() - (result, took), lags = await asyncio.gather(timed(), loop_wake_lags(finished)) + gc.freeze() + try: + (result, took), lags = await asyncio.gather(timed(), loop_wake_lags(finished)) + finally: + gc.unfreeze() return result, took, lags diff --git a/tests/unit/passthrough/test_passthrough_main.py b/tests/unit/passthrough/test_passthrough_main.py index 82825ec2802..729b03b7df4 100644 --- a/tests/unit/passthrough/test_passthrough_main.py +++ b/tests/unit/passthrough/test_passthrough_main.py @@ -205,8 +205,8 @@ def mock_request(): self.query_params = QueryParams() self.method = method self.request_body = request_body or {} - # Add url attribute that the actual code expects - self.url = "http://localhost:8000/test" + self.url = httpx.URL("http://localhost:8000/test") + self.scope = {"type": "http", "method": method, "path": "/test"} async def body(self) -> bytes: return bytes(json.dumps(self.request_body), "utf-8") diff --git a/tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py b/tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py index 6fac731a60d..e90184ce45b 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py @@ -19,12 +19,10 @@ import fakeredis # this file is to test litellm/proxy import asyncio -import logging import pytest from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager import litellm -from litellm._logging import verbose_proxy_logger from litellm.proxy.management_endpoints.internal_user_endpoints import ( new_user, user_info, @@ -66,7 +64,6 @@ from litellm.proxy.spend_tracking.spend_management_endpoints import ( ) from litellm.proxy.utils import PrismaClient, ProxyLogging, hash_token, update_spend -verbose_proxy_logger.setLevel(level=logging.DEBUG) from starlette.datastructures import URL diff --git a/tests/unit/proxy/management_endpoints/test_key_generate_prisma.py b/tests/unit/proxy/management_endpoints/test_key_generate_prisma.py index fb5e84c8294..348924b4064 100644 --- a/tests/unit/proxy/management_endpoints/test_key_generate_prisma.py +++ b/tests/unit/proxy/management_endpoints/test_key_generate_prisma.py @@ -87,7 +87,6 @@ from litellm.proxy.spend_tracking.spend_management_endpoints import ( ) from litellm.proxy.utils import PrismaClient, ProxyLogging, hash_token, update_spend -verbose_proxy_logger.setLevel(level=logging.DEBUG) from starlette.datastructures import URL diff --git a/tests/unit/proxy/management_helpers/test_audit_logs_proxy.py b/tests/unit/proxy/management_helpers/test_audit_logs_proxy.py index 878e19f5b6f..98922801296 100644 --- a/tests/unit/proxy/management_helpers/test_audit_logs_proxy.py +++ b/tests/unit/proxy/management_helpers/test_audit_logs_proxy.py @@ -14,13 +14,11 @@ import time # this file is to test litellm/proxy import asyncio -import logging load_dotenv() import pytest import litellm -from litellm._logging import verbose_proxy_logger from litellm.proxy.proxy_server import ( LitellmUserRoles, @@ -35,7 +33,6 @@ from litellm.proxy.proxy_server import ( from litellm.proxy.utils import PrismaClient, ProxyLogging, hash_token, update_spend -verbose_proxy_logger.setLevel(level=logging.DEBUG) from starlette.datastructures import URL diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py index 9d4532df49a..fdd21afbc67 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Iterator from datetime import datetime from unittest.mock import MagicMock @@ -158,7 +159,7 @@ def _interrupted_anthropic_stream(model: str, output_text: str) -> list[bytes]: @pytest.mark.asyncio -async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event_loop(): +async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event_loop(caplog): from unittest.mock import AsyncMock from tests.large_text import text @@ -168,6 +169,8 @@ async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event warm_tokenizer, ) + caplog.set_level(logging.WARNING, logger="LiteLLM") + caplog.set_level(logging.WARNING, logger="LiteLLM Proxy") model = "claude-fable-5" warm_tokenizer(model) logging_obj = _logging_obj() @@ -197,7 +200,7 @@ async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event @pytest.mark.asyncio -async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop(): +async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop(caplog): from unittest.mock import AsyncMock from tests.large_text import text @@ -207,6 +210,8 @@ async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop( warm_tokenizer, ) + caplog.set_level(logging.WARNING, logger="LiteLLM") + caplog.set_level(logging.WARNING, logger="LiteLLM Proxy") model = "claude-fable-5" warm_tokenizer(model) logging_obj = _logging_obj() diff --git a/tests/unit/proxy/proxy_server/conftest.py b/tests/unit/proxy/proxy_server/conftest.py index 0d7ec4812ce..9baf3206fc4 100644 --- a/tests/unit/proxy/proxy_server/conftest.py +++ b/tests/unit/proxy/proxy_server/conftest.py @@ -73,8 +73,9 @@ def app(): so the startup event (DB connect, Router init, OTEL setup) never fires. Module import still runs once; module-level globals are harmless. """ - os.environ.setdefault("LITELLM_LOG", "ERROR") - from litellm.proxy.proxy_server import app as _app + with pytest.MonkeyPatch.context() as environment: + environment.setenv("LITELLM_LOG", os.environ.get("LITELLM_LOG", "ERROR")) + from litellm.proxy.proxy_server import app as _app return _app diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 8590e959961..fcd72e3777c 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2,7 +2,6 @@ # 1. Generate a Key, and use it to make a call -import logging from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -18,7 +17,6 @@ from fastapi import HTTPException, Request import litellm from litellm import Router -from litellm._logging import verbose_proxy_logger from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler @@ -29,7 +27,6 @@ from litellm.proxy.anthropic_endpoints.endpoints import ( from litellm.proxy.proxy_server import token_counter from litellm.types.utils import TokenCountResponse -verbose_proxy_logger.setLevel(level=logging.DEBUG) @pytest.mark.asyncio