scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-02 02:11:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

test(proxy): delete the legacy proxy test tree and shard tests/unit/proxy by glob (#44018)

Browse source 
* test(proxy): delete the legacy proxy test tree and serve the redirect test from loopback

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): exercise the shard check directly for unit_selection-owned children

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): serve the redirect test from respx instead of a socket

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): credit shard ownership only to unit flags wired in gha

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): implement the wired-flag shard crediting the tests assert

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): split the root proxy test files into their own unit shard

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): point the rate-limit skip reason at the usage-based-routing-v2 RPM tests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yuneng <yuneng@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-10-01 11:40:45 -07:00 • committed by GitHub
parent a76b59db9f
commit 2cfa5ec126
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 146 additions and 486 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.circleci/scripts/unit_selection.sh
View file

@ -116,7 +116,7 @@ legacy_paths() {
echo tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
echo tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
echo tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
echo tests/unit/proxy/response_polling/test_response_polling_handler.py
echo tests/unit/proxy/response_polling
echo tests/unit/proxy/test_custom_tokenizer_bug.py
echo tests/unit/proxy/test_get_favicon.py
echo tests/unit/proxy/test_get_image.py
@ -151,6 +151,7 @@ legacy_paths() {
proxy-extras) echo tests/unit/litellm_proxy_extras ;;
proxy-infra)
echo tests/unit/gateway
echo tests/unit/proxy/management
echo tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
echo tests/unit/proxy/roi_calculator ;;
responses-caching-types)

48
.github/scripts/assert_ci_coverage.py vendored
View file

@ -9,6 +9,7 @@ import sys
import warnings
from collections.abc import Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
import yaml
@ -35,7 +36,7 @@ GLOB_CHARS = frozenset("*?")
# itself decomposed one level deeper and is checked through its own entry.
SHARDED_ROOTS: tuple[str, ...] = (
"tests/test_litellm",
"tests/test_litellm/proxy",
"tests/unit/proxy",
)
@ -119,11 +120,48 @@ def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
)
def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]:
SELECTION_ARM_RE = re.compile(r"(?ms)^\s*([A-Za-z0-9_|*-]+)\)\s*(.*?);;")
def _unit_selection_arms(repo_root: pathlib.Path = REPO_ROOT) -> Mapping[str, frozenset[str]]:
script: Final = repo_root / ".circleci/scripts/unit_selection.sh"
if not script.is_file():
return frozenset()
return frozenset(match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(_uncommented(script.read_text())))
return MappingProxyType({})
text: Final = _uncommented(script.read_text())
return MappingProxyType(
{
label: frozenset(
match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body)
)
for label, body in SELECTION_ARM_RE.findall(text)
}
)
def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]:
return frozenset(
token for tokens in _unit_selection_arms(repo_root).values() for token in tokens
)
def _wired_unit_flags(scalars: Iterable[Scalar]) -> frozenset[str]:
return frozenset(
scalar.value
for scalar in scalars
if scalar.key == "unit-flag" and "${{" not in scalar.value
)
def _shard_tokens(
scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]]
) -> frozenset[str]:
wired: Final = _wired_unit_flags(scalars)
return _invoked_test_tokens(scalars) | frozenset(
token
for label, tokens in arms.items()
if label in wired
for token in tokens
)
def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
@ -480,7 +518,7 @@ def _check_slices() -> int:
def _check_shards() -> int:
findings = _unassigned_shard_children(_invoked_test_tokens(_all_scalars()))
findings = _unassigned_shard_children(_shard_tokens(_all_scalars(), _unit_selection_arms()))
if findings:
_report(
"test directories and files that no shard claims",

116
.github/workflows/test-unit.yml vendored
View file

@ -204,7 +204,6 @@ jobs:
tests/unit/proxy/spend_tracking
--ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py
tests/unit/proxy/pass_through_endpoints
tests/test_litellm/proxy/pass_through_endpoints
tests/unit/proxy/_experimental
--ignore=tests/unit/proxy/_experimental/mcp_server
tests/unit/proxy/experimental
@ -217,93 +216,46 @@ jobs:
tests/unit/proxy/enterprise_billing
tests/unit/proxy/types_utils
tests/unit/proxy/logging_endpoints
tests/unit/proxy/test__types.py
tests/unit/proxy/test_aiohttp_cleanup_closed.py
tests/unit/proxy/test_aiohttp_session_recovery.py
tests/unit/proxy/test_api_key_masking_in_errors.py
tests/unit/proxy/test_audio_speech_prometheus_hooks.py
tests/unit/proxy/test_batch_expiry.py
tests/unit/proxy/test_batch_metadata_none_fix.py
tests/unit/proxy/test_batch_retrieve_bedrock.py
tests/unit/proxy/test_batch_x_litellm_model_encoding.py
tests/unit/proxy/test_blocked_response_usage.py
tests/unit/proxy/test_body_snapshot_callback_params.py
tests/unit/proxy/test_budget_reservation.py
tests/unit/proxy/test_bug_report_config.py
tests/unit/proxy/test_caching_routes.py
tests/unit/proxy/test_chat_completion_metadata.py
tests/unit/proxy/test_claude_code_marketplace.py
tests/unit/proxy/test_collector.py
tests/unit/proxy/test_common_request_processing.py
tests/unit/proxy/test_component_allowlists.py
tests/unit/proxy/test_conftest.py
tests/unit/proxy/test_cors_config.py
tests/unit/proxy/test_custom_proxy.py
tests/unit/proxy/test_dynamic_mcp_route.py
tests/unit/proxy/test_empty_model_list.py
tests/unit/proxy/test_enforce_user_param.py
tests/unit/proxy/test_fallback_management_endpoints.py
tests/unit/proxy/test_fastapi_offline_routes.py
tests/unit/proxy/test_filter_models_by_team_access_group.py
tests/unit/proxy/test_health_check_functions.py
tests/unit/proxy/test_health_check_max_tokens.py
tests/unit/proxy/test_init_litellm_callbacks.py
tests/unit/proxy/test_langfuse_passthrough_security.py
tests/unit/proxy/test_lazy_openapi_snapshot.py
tests/unit/proxy/test_litellm_pre_call_utils.py
tests/unit/proxy/test_max_budget_env_var.py
tests/unit/proxy/test_mcp_asgi_response.py
tests/unit/proxy/test_model_based_routing_files_batches.py
tests/unit/proxy/test_model_deprecations_endpoint.py
tests/unit/proxy/test_model_dump_with_preserved_fields.py
tests/unit/proxy/test_model_id_header_propagation.py
tests/unit/proxy/test_model_info_default_limits.py
tests/unit/proxy/test_model_level_guardrails.py
tests/unit/proxy/test_model_list_aliases.py
tests/unit/proxy/test_model_list_callback_filter.py
tests/unit/proxy/test_model_list_discoverable.py
tests/unit/proxy/test_model_list_healthy_only.py
tests/unit/proxy/test_modify_response_streaming_passthrough.py
tests/unit/proxy/test_native_compaction.py
tests/unit/proxy/test_openai_ws_passthrough_routes.py
tests/unit/proxy/test_openapi_schema_validation.py
tests/unit/proxy/test_plugin_routes.py
tests/unit/proxy/test_pointfive_dashboard_config.py
tests/unit/proxy/test_pointfive_ui_callback.py
tests/unit/proxy/test_pricing_field_strip.py
tests/unit/proxy/test_prisma_engine_watchdog.py
tests/unit/proxy/test_prisma_migration.py
tests/unit/proxy/test_prometheus_cleanup.py
tests/unit/proxy/test_prometheus_metrics_server.py
tests/unit/proxy/test_provider_url_destination_guard.py
tests/unit/proxy/test_proxy_cli.py
tests/unit/proxy/test_proxy_logging_hook_detection.py
tests/unit/proxy/test_proxy_types.py
tests/unit/proxy/test_pyroscope.py
tests/unit/proxy/test_read_model_list.py
tests/unit/proxy/test_redis_auth_cache_flag.py
tests/unit/proxy/test_response_model_sanitization.py
tests/unit/proxy/test_route_a2a_models.py
tests/unit/proxy/test_route_llm_request.py
tests/unit/proxy/test_route_priority.py
tests/unit/proxy/test_sensitive_route_auth.py
tests/unit/proxy/test_shared_health_check.py
tests/unit/proxy/test_spend_log_cleanup.py
tests/unit/proxy/test_swagger_chat_completions.py
tests/unit/proxy/test_team_member_update.py
tests/unit/proxy/test_team_org_move.py
tests/unit/proxy/test_tools_allowlist_enforcement.py
tests/unit/proxy/test_tracing_endpoints.py
tests/unit/proxy/test_update_llm_router_resilience.py
tests/unit/proxy/test_zerobus_dashboard_config.py
tests/unit/proxy/test_proxy_server_endpoints_and_startup.py
tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py
unit-flag: proxy-infra
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-infra-root
artifact-name: proxy-infra-root
test-path: >-
tests/unit/proxy/test_*.py
--ignore=tests/unit/proxy/test_aproxy_startup.py
--ignore=tests/unit/proxy/test_credential_slot_registry.py
--ignore=tests/unit/proxy/test_custom_callback_input.py
--ignore=tests/unit/proxy/test_custom_logger_s3_gcs.py
--ignore=tests/unit/proxy/test_custom_tokenizer_bug.py
--ignore=tests/unit/proxy/test_db_schema_changes.py
--ignore=tests/unit/proxy/test_deprecated_key_grace_period.py
--ignore=tests/unit/proxy/test_get_favicon.py
--ignore=tests/unit/proxy/test_get_image.py
--ignore=tests/unit/proxy/test_prisma_client_backoff_retry.py
--ignore=tests/unit/proxy/test_prompt_test_endpoint.py
--ignore=tests/unit/proxy/test_proxy_config_unit_test.py
--ignore=tests/unit/proxy/test_proxy_custom_auth.py
--ignore=tests/unit/proxy/test_proxy_reject_logging.py
--ignore=tests/unit/proxy/test_proxy_server.py
--ignore=tests/unit/proxy/test_proxy_setting_guardrails.py
--ignore=tests/unit/proxy/test_proxy_token_counter.py
--ignore=tests/unit/proxy/test_proxy_utils.py
--ignore=tests/unit/proxy/test_reducto_ocr_route.py
--ignore=tests/unit/proxy/test_response_polling_pre_call_checks.py
--ignore=tests/unit/proxy/test_server_root_path.py
--ignore=tests/unit/proxy/test_ui_path_detection.py
--ignore=tests/unit/proxy/test_unit_test_proxy_hooks.py
--ignore=tests/unit/proxy/test_update_spend.py
--ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: caching-local
artifact-name: caching-local
test-path: ""

89
Makefile
View file

@ -1,7 +1,7 @@
# LiteLLM Makefile
# Simple Makefile for running tests and basic development tasks
.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc test-unit-proxy-root \
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
test-rust-extension rust-sqlx-prepare \
@ -47,6 +47,7 @@ help:
@echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
@echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
@echo " make test-unit-proxy-misc - Run proxy misc tests (~77 files)"
@echo " make test-unit-proxy-root - Run proxy root-file tests (tests/unit/proxy/test_*.py)"
@echo " make test-unit-integrations - Run integration tests (~60 files)"
@echo " make test-unit-core-utils - Run core utils tests (~32 files)"
@echo " make test-unit-other - Run other tests (caching, responses, etc., ~69 files)"
@ -326,89 +327,11 @@ test-unit-proxy-guardrails: install-test-deps
test-unit-proxy-core: install-test-deps
$(UV_RUN) pytest tests/unit/proxy/auth tests/unit/proxy/client tests/unit/proxy/db tests/unit/proxy/hooks tests/unit/proxy/policy_engine --ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py --ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py --tb=short -vv -n 4 --durations=20
PROXY_INFRA_ROOT_TESTS := \
tests/unit/proxy/test__types.py \
tests/unit/proxy/test_aiohttp_cleanup_closed.py \
tests/unit/proxy/test_aiohttp_session_recovery.py \
tests/unit/proxy/test_api_key_masking_in_errors.py \
tests/unit/proxy/test_audio_speech_prometheus_hooks.py \
tests/unit/proxy/test_batch_expiry.py \
tests/unit/proxy/test_batch_metadata_none_fix.py \
tests/unit/proxy/test_batch_retrieve_bedrock.py \
tests/unit/proxy/test_batch_x_litellm_model_encoding.py \
tests/unit/proxy/test_blocked_response_usage.py \
tests/unit/proxy/test_body_snapshot_callback_params.py \
tests/unit/proxy/test_budget_reservation.py \
tests/unit/proxy/test_bug_report_config.py \
tests/unit/proxy/test_caching_routes.py \
tests/unit/proxy/test_chat_completion_metadata.py \
tests/unit/proxy/test_claude_code_marketplace.py \
tests/unit/proxy/test_collector.py \
tests/unit/proxy/test_common_request_processing.py \
tests/unit/proxy/test_component_allowlists.py \
tests/unit/proxy/test_conftest.py \
tests/unit/proxy/test_cors_config.py \
tests/unit/proxy/test_custom_proxy.py \
tests/unit/proxy/test_dynamic_mcp_route.py \
tests/unit/proxy/test_empty_model_list.py \
tests/unit/proxy/test_enforce_user_param.py \
tests/unit/proxy/test_fallback_management_endpoints.py \
tests/unit/proxy/test_fastapi_offline_routes.py \
tests/unit/proxy/test_filter_models_by_team_access_group.py \
tests/unit/proxy/test_health_check_functions.py \
tests/unit/proxy/test_health_check_max_tokens.py \
tests/unit/proxy/test_init_litellm_callbacks.py \
tests/unit/proxy/test_langfuse_passthrough_security.py \
tests/unit/proxy/test_lazy_openapi_snapshot.py \
tests/unit/proxy/test_litellm_pre_call_utils.py \
tests/unit/proxy/test_max_budget_env_var.py \
tests/unit/proxy/test_mcp_asgi_response.py \
tests/unit/proxy/test_model_based_routing_files_batches.py \
tests/unit/proxy/test_model_deprecations_endpoint.py \
tests/unit/proxy/test_model_dump_with_preserved_fields.py \
tests/unit/proxy/test_model_id_header_propagation.py \
tests/unit/proxy/test_model_info_default_limits.py \
tests/unit/proxy/test_model_level_guardrails.py \
tests/unit/proxy/test_model_list_aliases.py \
tests/unit/proxy/test_model_list_callback_filter.py \
tests/unit/proxy/test_model_list_discoverable.py \
tests/unit/proxy/test_model_list_healthy_only.py \
tests/unit/proxy/test_modify_response_streaming_passthrough.py \
tests/unit/proxy/test_native_compaction.py \
tests/unit/proxy/test_openai_ws_passthrough_routes.py \
tests/unit/proxy/test_openapi_schema_validation.py \
tests/unit/proxy/test_plugin_routes.py \
tests/unit/proxy/test_pointfive_dashboard_config.py \
tests/unit/proxy/test_pointfive_ui_callback.py \
tests/unit/proxy/test_pricing_field_strip.py \
tests/unit/proxy/test_prisma_engine_watchdog.py \
tests/unit/proxy/test_prisma_migration.py \
tests/unit/proxy/test_prometheus_cleanup.py \
tests/unit/proxy/test_prometheus_metrics_server.py \
tests/unit/proxy/test_provider_url_destination_guard.py \
tests/unit/proxy/test_proxy_cli.py \
tests/unit/proxy/test_proxy_logging_hook_detection.py \
tests/unit/proxy/test_proxy_types.py \
tests/unit/proxy/test_pyroscope.py \
tests/unit/proxy/test_read_model_list.py \
tests/unit/proxy/test_redis_auth_cache_flag.py \
tests/unit/proxy/test_response_model_sanitization.py \
tests/unit/proxy/test_route_a2a_models.py \
tests/unit/proxy/test_route_llm_request.py \
tests/unit/proxy/test_route_priority.py \
tests/unit/proxy/test_sensitive_route_auth.py \
tests/unit/proxy/test_shared_health_check.py \
tests/unit/proxy/test_spend_log_cleanup.py \
tests/unit/proxy/test_swagger_chat_completions.py \
tests/unit/proxy/test_team_member_update.py \
tests/unit/proxy/test_team_org_move.py \
tests/unit/proxy/test_tools_allowlist_enforcement.py \
tests/unit/proxy/test_tracing_endpoints.py \
tests/unit/proxy/test_update_llm_router_resilience.py \
tests/unit/proxy/test_zerobus_dashboard_config.py
test-unit-proxy-misc: install-test-deps
$(UV_RUN) pytest tests/unit/proxy/agent_endpoints tests/unit/proxy/anthropic_endpoints tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py tests/unit/proxy/discovery_endpoints tests/unit/proxy/experimental tests/unit/proxy/google_endpoints tests/unit/proxy/health_endpoints tests/unit/proxy/image_endpoints tests/unit/proxy/middleware --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/openai_files_endpoint tests/unit/proxy/pass_through_endpoints tests/test_litellm/proxy/pass_through_endpoints tests/unit/proxy/prompts tests/unit/proxy/public_endpoints tests/unit/proxy/response_api_endpoints tests/unit/proxy/shutdown tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/ui_crud_endpoints tests/unit/proxy/vector_store_endpoints $(PROXY_INFRA_ROOT_TESTS) tests/unit/proxy/test_proxy_server_endpoints_and_startup.py tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py --tb=short -vv -n 4 --durations=20
$(UV_RUN) pytest tests/unit/proxy/agent_endpoints tests/unit/proxy/anthropic_endpoints tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py tests/unit/proxy/discovery_endpoints tests/unit/proxy/experimental tests/unit/proxy/google_endpoints tests/unit/proxy/health_endpoints tests/unit/proxy/image_endpoints tests/unit/proxy/middleware --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/openai_files_endpoint tests/unit/proxy/pass_through_endpoints tests/unit/proxy/prompts tests/unit/proxy/public_endpoints tests/unit/proxy/response_api_endpoints tests/unit/proxy/shutdown tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/ui_crud_endpoints tests/unit/proxy/vector_store_endpoints tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py --tb=short -vv -n 4 --durations=20
test-unit-proxy-root: install-test-deps
$(UV_RUN) pytest tests/unit/proxy/test_*.py --ignore=tests/unit/proxy/test_aproxy_startup.py --ignore=tests/unit/proxy/test_credential_slot_registry.py --ignore=tests/unit/proxy/test_custom_callback_input.py --ignore=tests/unit/proxy/test_custom_logger_s3_gcs.py --ignore=tests/unit/proxy/test_custom_tokenizer_bug.py --ignore=tests/unit/proxy/test_db_schema_changes.py --ignore=tests/unit/proxy/test_deprecated_key_grace_period.py --ignore=tests/unit/proxy/test_get_favicon.py --ignore=tests/unit/proxy/test_get_image.py --ignore=tests/unit/proxy/test_prisma_client_backoff_retry.py --ignore=tests/unit/proxy/test_prompt_test_endpoint.py --ignore=tests/unit/proxy/test_proxy_config_unit_test.py --ignore=tests/unit/proxy/test_proxy_custom_auth.py --ignore=tests/unit/proxy/test_proxy_reject_logging.py --ignore=tests/unit/proxy/test_proxy_server.py --ignore=tests/unit/proxy/test_proxy_setting_guardrails.py --ignore=tests/unit/proxy/test_proxy_token_counter.py --ignore=tests/unit/proxy/test_proxy_utils.py --ignore=tests/unit/proxy/test_reducto_ocr_route.py --ignore=tests/unit/proxy/test_response_polling_pre_call_checks.py --ignore=tests/unit/proxy/test_server_root_path.py --ignore=tests/unit/proxy/test_ui_path_detection.py --ignore=tests/unit/proxy/test_unit_test_proxy_hooks.py --ignore=tests/unit/proxy/test_update_spend.py --ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py --tb=short -vv -n 4 --durations=20
test-unit-integrations: install-test-deps
$(UV_RUN) pytest tests/unit/integrations --tb=short -vv -n 4 --durations=20

2
tests/e2e/batches/COVERAGE.md
View file

@ -185,6 +185,6 @@ never landed.
Unified (managed) batch cost is owned by the hourly `CheckBatchCost` poller, and a
terminal DB status short-circuits retrieve for those ids, so the terminal-state cell
uses the encoded path; poller timing does not fit an e2e gate and belongs in a
DI-stubbed proxy integration test under `tests/test_litellm/proxy/`. Gemini
DI-stubbed proxy integration test under `tests/unit/proxy/`. Gemini
(non-Vertex) file content raises `NotImplementedError` upstream and is not a
coverage cell.

1
tests/test_litellm/proxy/__init__.py
View file

@ -1 +0,0 @@
# This file makes the tests/test_litellm/proxy directory a Python package

284
tests/test_litellm/proxy/conftest.py
View file

@ -1,284 +0,0 @@
"""
Shared fixtures and helpers for proxy tests.
This module provides reusable utilities for creating proxy test clients
with database and Redis cache configuration.
"""
import asyncio
import os
import tempfile
from typing import Dict, Optional
import pytest
import yaml
from fastapi.testclient import TestClient
from prisma.errors import ClientNotConnectedError
_PROXY_MODULE_GLOBALS_TO_ISOLATE = (
"master_key",
"prisma_client",
"llm_router",
)
class StubClientNotConnectedError(ClientNotConnectedError):
pass
class DisconnectedPrisma:
"""Mimics prisma-client-py after disconnect(): ``is_connected()`` is False
and the ``_engine`` property raises ``ClientNotConnectedError``."""
def is_connected(self) -> bool:
return False
@property
def _engine(self) -> None:
raise StubClientNotConnectedError()
@pytest.fixture
def disconnected_prisma() -> DisconnectedPrisma:
"""A stand-in for a Prisma client wedged in the disconnected state."""
return DisconnectedPrisma()
_MODULE_GLOBAL_MISSING = object()
_proxy_module_globals_snapshot = pytest.StashKey[Dict[str, object]]()
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_setup(item):
"""
Snapshot module-level globals on litellm.proxy.proxy_server before any
fixture runs, and restore them in pytest_runtest_teardown after every
fixture finalizer has run.
Without this, a leaked value (e.g. master_key set by a sibling test)
flips the auth short-circuit in user_api_key_auth and causes unrelated
tests in the same xdist worker to return 401 instead of 200. A leaked
llm_router does the same to anything that reads the running router out
of sys.modules, such as the PTU rollup's deployment scan, which then
counts a sibling test's deployments as if the proxy owned them.
This must be a hook pair, not an autouse fixture: an autouse fixture in
the root conftest requests monkeypatch, so monkeypatch's undo stack
unwinds after every other fixture finalizer. A test that monkeypatches a
global while a fixture has it patched records the fixture's mock as the
"original", and monkeypatch.undo re-plants that mock after all restores
have run, poisoning the global for the rest of the xdist worker.
"""
from litellm.proxy import proxy_server
item.stash[_proxy_module_globals_snapshot] = {
name: getattr(proxy_server, name, _MODULE_GLOBAL_MISSING)
for name in _PROXY_MODULE_GLOBALS_TO_ISOLATE
}
yield
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_teardown(item, nextitem):
yield
snapshot = item.stash.get(_proxy_module_globals_snapshot, None)
if snapshot is None:
return
from litellm.proxy import proxy_server
for name, value in snapshot.items():
if value is _MODULE_GLOBAL_MISSING:
if hasattr(proxy_server, name):
delattr(proxy_server, name)
else:
setattr(proxy_server, name, value)
@pytest.fixture(autouse=True)
def _reset_graceful_shutdown_state():
"""Graceful shutdown state is process-scoped; keep it from leaking between tests."""
from litellm.proxy.shutdown.graceful_shutdown_manager import (
GracefulShutdownManager,
)
GracefulShutdownManager.reset()
yield
GracefulShutdownManager.reset()
def build_cache_config(enable_cache: bool = True) -> Optional[Dict]:
"""
Build Redis cache configuration from environment variables.
Args:
enable_cache: Whether to enable cache (default: True)
Returns:
dict: Cache configuration dict with 'cache' and 'cache_params' keys, or None
"""
if not enable_cache:
return None
redis_host = os.getenv("REDIS_HOST")
if not redis_host:
return None
redis_port = os.getenv("REDIS_PORT", "6379")
cache_params = {
"type": "redis",
"host": redis_host,
"port": int(redis_port) if redis_port.isdigit() else redis_port,
}
redis_password = os.getenv("REDIS_PASSWORD")
if redis_password:
cache_params["password"] = redis_password
return {"cache": True, "cache_params": cache_params}
def build_minimal_proxy_config(
database_url: Optional[str] = None, **init_options
) -> Dict:
"""
Build a minimal proxy configuration YAML.
Args:
database_url: Optional database URL (falls back to DATABASE_URL env var)
**init_options: Additional configuration options:
- master_key: API key for authentication (default: "sk-1234")
- enable_cache: Whether to enable Redis cache (default: True)
- success_callback: Callback function for success events
Returns:
dict: Configuration dictionary ready to be written as YAML
"""
config = {
"general_settings": {"master_key": init_options.get("master_key", "sk-1234")},
"litellm_settings": {},
}
# Configure database
db_url = database_url or os.getenv("DATABASE_URL")
if db_url:
config["general_settings"]["database_url"] = db_url
# Configure cache if Redis is available
enable_cache = init_options.get("enable_cache", True)
cache_config = build_cache_config(enable_cache=enable_cache)
if cache_config:
config["litellm_settings"].update(cache_config)
# Add success_callback if provided (for realistic readiness endpoint)
if init_options.get("success_callback") is not None:
config["litellm_settings"]["success_callback"] = init_options[
"success_callback"
]
# Add any other litellm_settings from init_options
excluded_keys = {
"master_key",
"debug",
"success_callback",
"database_url",
"enable_cache",
}
for key, value in init_options.items():
if key not in excluded_keys and key not in config["litellm_settings"]:
config["litellm_settings"][key] = value
return config
def set_proxy_environment_variables(
monkeypatch, database_url: Optional[str] = None
) -> None:
"""
Set environment variables for database and Redis.
Args:
monkeypatch: pytest monkeypatch fixture
database_url: Optional database URL (falls back to DATABASE_URL env var)
"""
# Set database URL
db_url = database_url or os.getenv("DATABASE_URL")
if db_url:
monkeypatch.setenv("DATABASE_URL", db_url)
# Set Redis environment variables if available
redis_host = os.getenv("REDIS_HOST")
if redis_host:
monkeypatch.setenv("REDIS_HOST", redis_host)
monkeypatch.setenv("REDIS_PORT", os.getenv("REDIS_PORT", "6379"))
redis_password = os.getenv("REDIS_PASSWORD")
if redis_password:
monkeypatch.setenv("REDIS_PASSWORD", redis_password)
def create_proxy_test_client(
monkeypatch, database_url: Optional[str] = None, **init_options
) -> TestClient:
"""
Create a proxy TestClient with optional database and Redis cache configuration.
Args:
monkeypatch: pytest monkeypatch fixture
database_url: Optional database URL (falls back to DATABASE_URL env var)
**init_options: Additional configuration options:
- master_key: API key for authentication (default: "sk-1234")
- enable_cache: Whether to enable Redis cache (default: True)
- success_callback: Callback function for success events
- debug: Enable debug mode
Returns:
TestClient: FastAPI test client for the proxy server
"""
from litellm.proxy.proxy_server import (
cleanup_router_config_variables,
initialize,
app,
)
cleanup_router_config_variables()
# Get config file path
filepath = os.path.dirname(os.path.abspath(__file__))
default_config_fp = os.path.join(
filepath, "test_configs", "test_config_no_auth.yaml"
)
# Check if we need to create a minimal config with Redis/database
enable_cache = init_options.get("enable_cache", True)
needs_redis = enable_cache and os.getenv("REDIS_HOST") is not None
needs_db = (database_url or os.getenv("DATABASE_URL")) is not None
# Create minimal config if:
# 1. Default config file doesn't exist, OR
# 2. We need Redis/database config that might not be in the default config
if not os.path.exists(default_config_fp) or needs_redis or needs_db:
minimal_config = build_minimal_proxy_config(
database_url=database_url, **init_options
)
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
yaml.dump(minimal_config, f)
config_fp = f.name
else:
config_fp = default_config_fp
# Set environment variables
set_proxy_environment_variables(monkeypatch, database_url=database_url)
monkeypatch.setenv("LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY", "true")
# Initialize proxy
asyncio.run(initialize(config=config_fp, debug=init_options.get("debug", False)))
return TestClient(app)
@pytest.fixture
def fresh_agent_read_through(monkeypatch):
from litellm.proxy.common_utils import registry_read_through
read_through = registry_read_through.RegistryReadThrough(resync=registry_read_through._resync_agents)
monkeypatch.setattr(registry_read_through, "agent_registry_read_through", read_through)
return read_through

4
tests/test_ratelimit.py
View file

@ -135,8 +135,8 @@ def test_async_rate_limit(
if num_try_send > num_allowed_send:
pytest.skip(
"RPM tracking via background thread is racy; "
"rate-limit enforcement is tested in "
"tests/test_litellm/proxy/test_router_rate_limit.py"
"RPM over-limit rejection is tested for usage-based-routing-v2 in "
"tests/unit/router_strategy/test_router_routing_groups.py"
)
list_of_messages = generate_list_of_messages(max(num_try_send, num_allowed_send))

43
tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py → tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -15,6 +15,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
from fastapi import HTTPException, Request, Response, UploadFile
from fastapi.responses import StreamingResponse
from pydantic import TypeAdapter, ValidationError
@ -2711,10 +2712,10 @@ async def test_pass_through_request_merge_query_params_rewrites_managed_ids_on_t
@pytest.mark.asyncio
async def test_pass_through_with_httpbin_redirect():
async def test_pass_through_request_follows_redirect_to_final_response(httpx_transport):
"""
Integration test using httpbin.org redirect endpoint to test real redirect handling.
This tests the actual redirect handling capability end-to-end using the full pass_through_request function.
The proxy must follow the upstream redirect and return the final response,
not the 302.
"""
from unittest.mock import MagicMock
@ -2725,44 +2726,40 @@ async def test_pass_through_with_httpbin_redirect():
pass_through_request,
)
# Create mock request
mock_request = MagicMock(spec=Request)
mock_request.method = "GET"
mock_request.headers = Headers({})
mock_request.query_params = QueryParams("")
# Mock the body method to return empty bytes for GET request
async def mock_body():
return b""
mock_request.body = mock_body
# Mock user API key dict
mock_user_api_key_dict = MagicMock()
try:
# Test with httpbin.org redirect endpoint
# This will redirect to httpbin.org/get
with respx.mock(assert_all_called=True) as upstream:
upstream.get("https://upstream.test/redirect/1").respond(
302, headers={"Location": "/get"}
)
upstream.get("https://upstream.test/get").respond(
200, json={"url": "https://upstream.test/get"}
)
response = await pass_through_request(
request=mock_request,
target="https://httpbin.org/redirect/1",
target="https://upstream.test/redirect/1",
custom_headers={},
user_api_key_dict=mock_user_api_key_dict,
)
requested_urls: Final = [str(call.request.url) for call in upstream.calls]
# Should get the final response (200) from /get endpoint, not the redirect (302)
assert response.status_code == 200
# The response should be from the /get endpoint
response_content = bytes(response.body).decode("utf-8")
# httpbin.org/get returns JSON with info about the request
assert '"url": "https://httpbin.org/get"' in response_content
except Exception as e:
# If httpbin.org is not accessible, skip the test
import pytest
pytest.skip(f"Could not reach httpbin.org for integration test: {e}")
assert response.status_code == 200
assert json.loads(bytes(response.body))["url"] == "https://upstream.test/get"
assert requested_urls == [
"https://upstream.test/redirect/1",
"https://upstream.test/get",
]
@pytest.mark.asyncio

2
tests/unit/proxy/utils/prisma_and_spend/conftest.py
View file

@ -1,4 +1,4 @@
"""Shared fixtures for tests/test_litellm/proxy/utils/prisma_and_spend/.
"""Shared fixtures for tests/unit/proxy/utils/prisma_and_spend/.
All fixtures used by PR2 test files live here. Do NOT add fixtures inside
individual test files; if a fixture is missing, add it here and update the

2
tests/unit/proxy/utils/proxy_logging/conftest.py
View file

@ -1,4 +1,4 @@
"""Shared fixtures for tests/test_litellm/proxy/utils/proxy_logging/.
"""Shared fixtures for tests/unit/proxy/utils/proxy_logging/.
All fixtures used by PR1 of the proxy/utils.py behavior-pinning project
live here. Tests should not declare fixtures inline.

38
tests/unit/test_assert_ci_coverage.py
View file

@ -92,7 +92,7 @@ def test_a_glob_names_only_what_it_matches_not_what_sits_below_it():
glob = "tests/test_litellm/test_*.py"
assert coverage._token_names(glob, "tests/test_litellm/test_router.py") is True
assert coverage._token_names(glob, "tests/test_litellm/test_router.py/nested.py") is False
assert coverage._token_names(glob, "tests/test_litellm/proxy/test_router.py") is False
assert coverage._token_names(glob, "tests/test_litellm/nested/test_router.py") is False
def test_a_glob_still_covers_the_subtree_for_the_census():
@ -169,10 +169,44 @@ def test_every_sharded_root_named_in_the_script_exists_on_disk():
def test_the_repo_as_it_stands_has_every_shard_child_assigned():
findings = coverage._unassigned_shard_children(coverage._invoked_test_tokens(coverage._all_scalars()))
findings = coverage._unassigned_shard_children(
coverage._shard_tokens(coverage._all_scalars(), coverage._unit_selection_arms())
)
assert [f.subject for f in findings] == []
def test_shard_tokens_credits_only_wired_unit_flags(tmp_path):
root = tmp_path / "tests" / "tree"
(root / "wired").mkdir(parents=True)
(root / "wired" / "test_a.py").write_text("def test_a(): assert True\n")
(root / "unwired").mkdir(parents=True)
(root / "unwired" / "test_b.py").write_text("def test_b(): assert True\n")
script = tmp_path / ".circleci" / "scripts" / "unit_selection.sh"
script.parent.mkdir(parents=True)
script.write_text(
"legacy_paths() {\n"
" case \"$1\" in\n"
" wired-flag) echo tests/tree/wired ;;\n"
" unwired-flag)\n"
" echo tests/tree/unwired ;;\n"
" esac\n"
"}\n"
)
scalars: Final = (coverage.Scalar(key="unit-flag", value="wired-flag"),)
findings = coverage._unassigned_shard_children(
coverage._shard_tokens(scalars, coverage._unit_selection_arms(tmp_path)),
roots=("tests/tree",),
repo_root=tmp_path,
)
assert tuple(f.subject for f in findings) == ("tests/tree/unwired",)
def test_check_shards_passes_on_the_repo_as_it_stands(capsys):
assert coverage._check_shards() == 0
# --------------------------------------------------------------------------- #
# Slice guard: a job can glob a file and its -k can then throw the file out
# --------------------------------------------------------------------------- #