From 2cfa5ec1262024185af0a2a2f5297aef8c794002 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:40:45 -0700 Subject: [PATCH] test(proxy): delete the legacy proxy test tree and shard tests/unit/proxy by glob (#44018) * test(proxy): delete the legacy proxy test tree and serve the redirect test from loopback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): exercise the shard check directly for unit_selection-owned children Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): serve the redirect test from respx instead of a socket Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): credit shard ownership only to unit flags wired in gha Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): implement the wired-flag shard crediting the tests assert Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): split the root proxy test files into their own unit shard Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): point the rate-limit skip reason at the usage-based-routing-v2 RPM tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yuneng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .circleci/scripts/unit_selection.sh | 3 +- .github/scripts/assert_ci_coverage.py | 48 ++- .github/workflows/test-unit.yml | 116 +++---- Makefile | 89 +----- tests/e2e/batches/COVERAGE.md | 2 +- tests/test_litellm/proxy/__init__.py | 1 - tests/test_litellm/proxy/conftest.py | 284 ------------------ tests/test_ratelimit.py | 4 +- .../test_pass_through_endpoints.py | 43 ++- .../proxy/utils/prisma_and_spend/conftest.py | 2 +- .../proxy/utils/proxy_logging/conftest.py | 2 +- tests/unit/test_assert_ci_coverage.py | 38 ++- 12 files changed, 146 insertions(+), 486 deletions(-) delete mode 100644 tests/test_litellm/proxy/__init__.py delete mode 100644 tests/test_litellm/proxy/conftest.py rename tests/{test_litellm => unit}/proxy/pass_through_endpoints/test_pass_through_endpoints.py (99%) diff --git a/.circleci/scripts/unit_selection.sh b/.circleci/scripts/unit_selection.sh index dc6c1f2ffaa..bfaa27c3ef3 100755 --- a/.circleci/scripts/unit_selection.sh +++ b/.circleci/scripts/unit_selection.sh @@ -116,7 +116,7 @@ legacy_paths() { echo tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py echo tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py echo tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py - echo tests/unit/proxy/response_polling/test_response_polling_handler.py + echo tests/unit/proxy/response_polling echo tests/unit/proxy/test_custom_tokenizer_bug.py echo tests/unit/proxy/test_get_favicon.py echo tests/unit/proxy/test_get_image.py @@ -151,6 +151,7 @@ legacy_paths() { proxy-extras) echo tests/unit/litellm_proxy_extras ;; proxy-infra) echo tests/unit/gateway + echo tests/unit/proxy/management echo tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py echo tests/unit/proxy/roi_calculator ;; responses-caching-types) diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index a483dcec9d7..3022f94a599 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -9,6 +9,7 @@ import sys import warnings from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Final import yaml @@ -35,7 +36,7 @@ GLOB_CHARS = frozenset("*?") # itself decomposed one level deeper and is checked through its own entry. SHARDED_ROOTS: tuple[str, ...] = ( "tests/test_litellm", - "tests/test_litellm/proxy", + "tests/unit/proxy", ) @@ -119,11 +120,48 @@ def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]: ) -def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]: +SELECTION_ARM_RE = re.compile(r"(?ms)^\s*([A-Za-z0-9_|*-]+)\)\s*(.*?);;") + + +def _unit_selection_arms(repo_root: pathlib.Path = REPO_ROOT) -> Mapping[str, frozenset[str]]: script: Final = repo_root / ".circleci/scripts/unit_selection.sh" if not script.is_file(): - return frozenset() - return frozenset(match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(_uncommented(script.read_text()))) + return MappingProxyType({}) + text: Final = _uncommented(script.read_text()) + return MappingProxyType( + { + label: frozenset( + match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body) + ) + for label, body in SELECTION_ARM_RE.findall(text) + } + ) + + +def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]: + return frozenset( + token for tokens in _unit_selection_arms(repo_root).values() for token in tokens + ) + + +def _wired_unit_flags(scalars: Iterable[Scalar]) -> frozenset[str]: + return frozenset( + scalar.value + for scalar in scalars + if scalar.key == "unit-flag" and "${{" not in scalar.value + ) + + +def _shard_tokens( + scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]] +) -> frozenset[str]: + wired: Final = _wired_unit_flags(scalars) + return _invoked_test_tokens(scalars) | frozenset( + token + for label, tokens in arms.items() + if label in wired + for token in tokens + ) def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]: @@ -480,7 +518,7 @@ def _check_slices() -> int: def _check_shards() -> int: - findings = _unassigned_shard_children(_invoked_test_tokens(_all_scalars())) + findings = _unassigned_shard_children(_shard_tokens(_all_scalars(), _unit_selection_arms())) if findings: _report( "test directories and files that no shard claims", diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index c79614c049e..20096a0e373 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -204,7 +204,6 @@ jobs: tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/pass_through_endpoints - tests/test_litellm/proxy/pass_through_endpoints tests/unit/proxy/_experimental --ignore=tests/unit/proxy/_experimental/mcp_server tests/unit/proxy/experimental @@ -217,93 +216,46 @@ jobs: tests/unit/proxy/enterprise_billing tests/unit/proxy/types_utils tests/unit/proxy/logging_endpoints - tests/unit/proxy/test__types.py - tests/unit/proxy/test_aiohttp_cleanup_closed.py - tests/unit/proxy/test_aiohttp_session_recovery.py - tests/unit/proxy/test_api_key_masking_in_errors.py - tests/unit/proxy/test_audio_speech_prometheus_hooks.py - tests/unit/proxy/test_batch_expiry.py - tests/unit/proxy/test_batch_metadata_none_fix.py - tests/unit/proxy/test_batch_retrieve_bedrock.py - tests/unit/proxy/test_batch_x_litellm_model_encoding.py - tests/unit/proxy/test_blocked_response_usage.py - tests/unit/proxy/test_body_snapshot_callback_params.py - tests/unit/proxy/test_budget_reservation.py - tests/unit/proxy/test_bug_report_config.py - tests/unit/proxy/test_caching_routes.py - tests/unit/proxy/test_chat_completion_metadata.py - tests/unit/proxy/test_claude_code_marketplace.py - tests/unit/proxy/test_collector.py - tests/unit/proxy/test_common_request_processing.py - tests/unit/proxy/test_component_allowlists.py - tests/unit/proxy/test_conftest.py - tests/unit/proxy/test_cors_config.py - tests/unit/proxy/test_custom_proxy.py - tests/unit/proxy/test_dynamic_mcp_route.py - tests/unit/proxy/test_empty_model_list.py - tests/unit/proxy/test_enforce_user_param.py - tests/unit/proxy/test_fallback_management_endpoints.py - tests/unit/proxy/test_fastapi_offline_routes.py - tests/unit/proxy/test_filter_models_by_team_access_group.py - tests/unit/proxy/test_health_check_functions.py - tests/unit/proxy/test_health_check_max_tokens.py - tests/unit/proxy/test_init_litellm_callbacks.py - tests/unit/proxy/test_langfuse_passthrough_security.py - tests/unit/proxy/test_lazy_openapi_snapshot.py - tests/unit/proxy/test_litellm_pre_call_utils.py - tests/unit/proxy/test_max_budget_env_var.py - tests/unit/proxy/test_mcp_asgi_response.py - tests/unit/proxy/test_model_based_routing_files_batches.py - tests/unit/proxy/test_model_deprecations_endpoint.py - tests/unit/proxy/test_model_dump_with_preserved_fields.py - tests/unit/proxy/test_model_id_header_propagation.py - tests/unit/proxy/test_model_info_default_limits.py - tests/unit/proxy/test_model_level_guardrails.py - tests/unit/proxy/test_model_list_aliases.py - tests/unit/proxy/test_model_list_callback_filter.py - tests/unit/proxy/test_model_list_discoverable.py - tests/unit/proxy/test_model_list_healthy_only.py - tests/unit/proxy/test_modify_response_streaming_passthrough.py - tests/unit/proxy/test_native_compaction.py - tests/unit/proxy/test_openai_ws_passthrough_routes.py - tests/unit/proxy/test_openapi_schema_validation.py - tests/unit/proxy/test_plugin_routes.py - tests/unit/proxy/test_pointfive_dashboard_config.py - tests/unit/proxy/test_pointfive_ui_callback.py - tests/unit/proxy/test_pricing_field_strip.py - tests/unit/proxy/test_prisma_engine_watchdog.py - tests/unit/proxy/test_prisma_migration.py - tests/unit/proxy/test_prometheus_cleanup.py - tests/unit/proxy/test_prometheus_metrics_server.py - tests/unit/proxy/test_provider_url_destination_guard.py - tests/unit/proxy/test_proxy_cli.py - tests/unit/proxy/test_proxy_logging_hook_detection.py - tests/unit/proxy/test_proxy_types.py - tests/unit/proxy/test_pyroscope.py - tests/unit/proxy/test_read_model_list.py - tests/unit/proxy/test_redis_auth_cache_flag.py - tests/unit/proxy/test_response_model_sanitization.py - tests/unit/proxy/test_route_a2a_models.py - tests/unit/proxy/test_route_llm_request.py - tests/unit/proxy/test_route_priority.py - tests/unit/proxy/test_sensitive_route_auth.py - tests/unit/proxy/test_shared_health_check.py - tests/unit/proxy/test_spend_log_cleanup.py - tests/unit/proxy/test_swagger_chat_completions.py - tests/unit/proxy/test_team_member_update.py - tests/unit/proxy/test_team_org_move.py - tests/unit/proxy/test_tools_allowlist_enforcement.py - tests/unit/proxy/test_tracing_endpoints.py - tests/unit/proxy/test_update_llm_router_resilience.py - tests/unit/proxy/test_zerobus_dashboard_config.py - tests/unit/proxy/test_proxy_server_endpoints_and_startup.py - tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py unit-flag: proxy-infra workers: 4 reruns: 2 timeout-minutes: 20 job-timeout-minutes: 60 + - shard: proxy-infra-root + artifact-name: proxy-infra-root + test-path: >- + tests/unit/proxy/test_*.py + --ignore=tests/unit/proxy/test_aproxy_startup.py + --ignore=tests/unit/proxy/test_credential_slot_registry.py + --ignore=tests/unit/proxy/test_custom_callback_input.py + --ignore=tests/unit/proxy/test_custom_logger_s3_gcs.py + --ignore=tests/unit/proxy/test_custom_tokenizer_bug.py + --ignore=tests/unit/proxy/test_db_schema_changes.py + --ignore=tests/unit/proxy/test_deprecated_key_grace_period.py + --ignore=tests/unit/proxy/test_get_favicon.py + --ignore=tests/unit/proxy/test_get_image.py + --ignore=tests/unit/proxy/test_prisma_client_backoff_retry.py + --ignore=tests/unit/proxy/test_prompt_test_endpoint.py + --ignore=tests/unit/proxy/test_proxy_config_unit_test.py + --ignore=tests/unit/proxy/test_proxy_custom_auth.py + --ignore=tests/unit/proxy/test_proxy_reject_logging.py + --ignore=tests/unit/proxy/test_proxy_server.py + --ignore=tests/unit/proxy/test_proxy_setting_guardrails.py + --ignore=tests/unit/proxy/test_proxy_token_counter.py + --ignore=tests/unit/proxy/test_proxy_utils.py + --ignore=tests/unit/proxy/test_reducto_ocr_route.py + --ignore=tests/unit/proxy/test_response_polling_pre_call_checks.py + --ignore=tests/unit/proxy/test_server_root_path.py + --ignore=tests/unit/proxy/test_ui_path_detection.py + --ignore=tests/unit/proxy/test_unit_test_proxy_hooks.py + --ignore=tests/unit/proxy/test_update_spend.py + --ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py + workers: 4 + reruns: 2 + timeout-minutes: 20 + job-timeout-minutes: 60 + - shard: caching-local artifact-name: caching-local test-path: "" diff --git a/Makefile b/Makefile index def7c57a324..e512960949c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # LiteLLM Makefile # Simple Makefile for running tests and basic development tasks -.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \ +.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc test-unit-proxy-root \ test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \ test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ test-rust-extension rust-sqlx-prepare \ @@ -47,6 +47,7 @@ help: @echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)" @echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)" @echo " make test-unit-proxy-misc - Run proxy misc tests (~77 files)" + @echo " make test-unit-proxy-root - Run proxy root-file tests (tests/unit/proxy/test_*.py)" @echo " make test-unit-integrations - Run integration tests (~60 files)" @echo " make test-unit-core-utils - Run core utils tests (~32 files)" @echo " make test-unit-other - Run other tests (caching, responses, etc., ~69 files)" @@ -326,89 +327,11 @@ test-unit-proxy-guardrails: install-test-deps test-unit-proxy-core: install-test-deps $(UV_RUN) pytest tests/unit/proxy/auth tests/unit/proxy/client tests/unit/proxy/db tests/unit/proxy/hooks tests/unit/proxy/policy_engine --ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py --ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py --tb=short -vv -n 4 --durations=20 -PROXY_INFRA_ROOT_TESTS := \ - tests/unit/proxy/test__types.py \ - tests/unit/proxy/test_aiohttp_cleanup_closed.py \ - tests/unit/proxy/test_aiohttp_session_recovery.py \ - tests/unit/proxy/test_api_key_masking_in_errors.py \ - tests/unit/proxy/test_audio_speech_prometheus_hooks.py \ - tests/unit/proxy/test_batch_expiry.py \ - tests/unit/proxy/test_batch_metadata_none_fix.py \ - tests/unit/proxy/test_batch_retrieve_bedrock.py \ - tests/unit/proxy/test_batch_x_litellm_model_encoding.py \ - tests/unit/proxy/test_blocked_response_usage.py \ - tests/unit/proxy/test_body_snapshot_callback_params.py \ - tests/unit/proxy/test_budget_reservation.py \ - tests/unit/proxy/test_bug_report_config.py \ - tests/unit/proxy/test_caching_routes.py \ - tests/unit/proxy/test_chat_completion_metadata.py \ - tests/unit/proxy/test_claude_code_marketplace.py \ - tests/unit/proxy/test_collector.py \ - tests/unit/proxy/test_common_request_processing.py \ - tests/unit/proxy/test_component_allowlists.py \ - tests/unit/proxy/test_conftest.py \ - tests/unit/proxy/test_cors_config.py \ - tests/unit/proxy/test_custom_proxy.py \ - tests/unit/proxy/test_dynamic_mcp_route.py \ - tests/unit/proxy/test_empty_model_list.py \ - tests/unit/proxy/test_enforce_user_param.py \ - tests/unit/proxy/test_fallback_management_endpoints.py \ - tests/unit/proxy/test_fastapi_offline_routes.py \ - tests/unit/proxy/test_filter_models_by_team_access_group.py \ - tests/unit/proxy/test_health_check_functions.py \ - tests/unit/proxy/test_health_check_max_tokens.py \ - tests/unit/proxy/test_init_litellm_callbacks.py \ - tests/unit/proxy/test_langfuse_passthrough_security.py \ - tests/unit/proxy/test_lazy_openapi_snapshot.py \ - tests/unit/proxy/test_litellm_pre_call_utils.py \ - tests/unit/proxy/test_max_budget_env_var.py \ - tests/unit/proxy/test_mcp_asgi_response.py \ - tests/unit/proxy/test_model_based_routing_files_batches.py \ - tests/unit/proxy/test_model_deprecations_endpoint.py \ - tests/unit/proxy/test_model_dump_with_preserved_fields.py \ - tests/unit/proxy/test_model_id_header_propagation.py \ - tests/unit/proxy/test_model_info_default_limits.py \ - tests/unit/proxy/test_model_level_guardrails.py \ - tests/unit/proxy/test_model_list_aliases.py \ - tests/unit/proxy/test_model_list_callback_filter.py \ - tests/unit/proxy/test_model_list_discoverable.py \ - tests/unit/proxy/test_model_list_healthy_only.py \ - tests/unit/proxy/test_modify_response_streaming_passthrough.py \ - tests/unit/proxy/test_native_compaction.py \ - tests/unit/proxy/test_openai_ws_passthrough_routes.py \ - tests/unit/proxy/test_openapi_schema_validation.py \ - tests/unit/proxy/test_plugin_routes.py \ - tests/unit/proxy/test_pointfive_dashboard_config.py \ - tests/unit/proxy/test_pointfive_ui_callback.py \ - tests/unit/proxy/test_pricing_field_strip.py \ - tests/unit/proxy/test_prisma_engine_watchdog.py \ - tests/unit/proxy/test_prisma_migration.py \ - tests/unit/proxy/test_prometheus_cleanup.py \ - tests/unit/proxy/test_prometheus_metrics_server.py \ - tests/unit/proxy/test_provider_url_destination_guard.py \ - tests/unit/proxy/test_proxy_cli.py \ - tests/unit/proxy/test_proxy_logging_hook_detection.py \ - tests/unit/proxy/test_proxy_types.py \ - tests/unit/proxy/test_pyroscope.py \ - tests/unit/proxy/test_read_model_list.py \ - tests/unit/proxy/test_redis_auth_cache_flag.py \ - tests/unit/proxy/test_response_model_sanitization.py \ - tests/unit/proxy/test_route_a2a_models.py \ - tests/unit/proxy/test_route_llm_request.py \ - tests/unit/proxy/test_route_priority.py \ - tests/unit/proxy/test_sensitive_route_auth.py \ - tests/unit/proxy/test_shared_health_check.py \ - tests/unit/proxy/test_spend_log_cleanup.py \ - tests/unit/proxy/test_swagger_chat_completions.py \ - tests/unit/proxy/test_team_member_update.py \ - tests/unit/proxy/test_team_org_move.py \ - tests/unit/proxy/test_tools_allowlist_enforcement.py \ - tests/unit/proxy/test_tracing_endpoints.py \ - tests/unit/proxy/test_update_llm_router_resilience.py \ - tests/unit/proxy/test_zerobus_dashboard_config.py - test-unit-proxy-misc: install-test-deps - $(UV_RUN) pytest tests/unit/proxy/agent_endpoints tests/unit/proxy/anthropic_endpoints tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py tests/unit/proxy/discovery_endpoints tests/unit/proxy/experimental tests/unit/proxy/google_endpoints tests/unit/proxy/health_endpoints tests/unit/proxy/image_endpoints tests/unit/proxy/middleware --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/openai_files_endpoint tests/unit/proxy/pass_through_endpoints tests/test_litellm/proxy/pass_through_endpoints tests/unit/proxy/prompts tests/unit/proxy/public_endpoints tests/unit/proxy/response_api_endpoints tests/unit/proxy/shutdown tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/ui_crud_endpoints tests/unit/proxy/vector_store_endpoints $(PROXY_INFRA_ROOT_TESTS) tests/unit/proxy/test_proxy_server_endpoints_and_startup.py tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/unit/proxy/agent_endpoints tests/unit/proxy/anthropic_endpoints tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py tests/unit/proxy/discovery_endpoints tests/unit/proxy/experimental tests/unit/proxy/google_endpoints tests/unit/proxy/health_endpoints tests/unit/proxy/image_endpoints tests/unit/proxy/middleware --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/openai_files_endpoint tests/unit/proxy/pass_through_endpoints tests/unit/proxy/prompts tests/unit/proxy/public_endpoints tests/unit/proxy/response_api_endpoints tests/unit/proxy/shutdown tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/ui_crud_endpoints tests/unit/proxy/vector_store_endpoints tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-root: install-test-deps + $(UV_RUN) pytest tests/unit/proxy/test_*.py --ignore=tests/unit/proxy/test_aproxy_startup.py --ignore=tests/unit/proxy/test_credential_slot_registry.py --ignore=tests/unit/proxy/test_custom_callback_input.py --ignore=tests/unit/proxy/test_custom_logger_s3_gcs.py --ignore=tests/unit/proxy/test_custom_tokenizer_bug.py --ignore=tests/unit/proxy/test_db_schema_changes.py --ignore=tests/unit/proxy/test_deprecated_key_grace_period.py --ignore=tests/unit/proxy/test_get_favicon.py --ignore=tests/unit/proxy/test_get_image.py --ignore=tests/unit/proxy/test_prisma_client_backoff_retry.py --ignore=tests/unit/proxy/test_prompt_test_endpoint.py --ignore=tests/unit/proxy/test_proxy_config_unit_test.py --ignore=tests/unit/proxy/test_proxy_custom_auth.py --ignore=tests/unit/proxy/test_proxy_reject_logging.py --ignore=tests/unit/proxy/test_proxy_server.py --ignore=tests/unit/proxy/test_proxy_setting_guardrails.py --ignore=tests/unit/proxy/test_proxy_token_counter.py --ignore=tests/unit/proxy/test_proxy_utils.py --ignore=tests/unit/proxy/test_reducto_ocr_route.py --ignore=tests/unit/proxy/test_response_polling_pre_call_checks.py --ignore=tests/unit/proxy/test_server_root_path.py --ignore=tests/unit/proxy/test_ui_path_detection.py --ignore=tests/unit/proxy/test_unit_test_proxy_hooks.py --ignore=tests/unit/proxy/test_update_spend.py --ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py --tb=short -vv -n 4 --durations=20 test-unit-integrations: install-test-deps $(UV_RUN) pytest tests/unit/integrations --tb=short -vv -n 4 --durations=20 diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index 2ba49a492ff..1ede9c89b97 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -185,6 +185,6 @@ never landed. Unified (managed) batch cost is owned by the hourly `CheckBatchCost` poller, and a terminal DB status short-circuits retrieve for those ids, so the terminal-state cell uses the encoded path; poller timing does not fit an e2e gate and belongs in a -DI-stubbed proxy integration test under `tests/test_litellm/proxy/`. Gemini +DI-stubbed proxy integration test under `tests/unit/proxy/`. Gemini (non-Vertex) file content raises `NotImplementedError` upstream and is not a coverage cell. diff --git a/tests/test_litellm/proxy/__init__.py b/tests/test_litellm/proxy/__init__.py deleted file mode 100644 index 1fb5d377d15..00000000000 --- a/tests/test_litellm/proxy/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# This file makes the tests/test_litellm/proxy directory a Python package diff --git a/tests/test_litellm/proxy/conftest.py b/tests/test_litellm/proxy/conftest.py deleted file mode 100644 index 49dc8d02bdb..00000000000 --- a/tests/test_litellm/proxy/conftest.py +++ /dev/null @@ -1,284 +0,0 @@ -""" -Shared fixtures and helpers for proxy tests. - -This module provides reusable utilities for creating proxy test clients -with database and Redis cache configuration. -""" - -import asyncio -import os -import tempfile -from typing import Dict, Optional - -import pytest -import yaml -from fastapi.testclient import TestClient -from prisma.errors import ClientNotConnectedError - -_PROXY_MODULE_GLOBALS_TO_ISOLATE = ( - "master_key", - "prisma_client", - "llm_router", -) - - -class StubClientNotConnectedError(ClientNotConnectedError): - pass - - -class DisconnectedPrisma: - """Mimics prisma-client-py after disconnect(): ``is_connected()`` is False - and the ``_engine`` property raises ``ClientNotConnectedError``.""" - - def is_connected(self) -> bool: - return False - - @property - def _engine(self) -> None: - raise StubClientNotConnectedError() - - -@pytest.fixture -def disconnected_prisma() -> DisconnectedPrisma: - """A stand-in for a Prisma client wedged in the disconnected state.""" - return DisconnectedPrisma() - - -_MODULE_GLOBAL_MISSING = object() -_proxy_module_globals_snapshot = pytest.StashKey[Dict[str, object]]() - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_setup(item): - """ - Snapshot module-level globals on litellm.proxy.proxy_server before any - fixture runs, and restore them in pytest_runtest_teardown after every - fixture finalizer has run. - - Without this, a leaked value (e.g. master_key set by a sibling test) - flips the auth short-circuit in user_api_key_auth and causes unrelated - tests in the same xdist worker to return 401 instead of 200. A leaked - llm_router does the same to anything that reads the running router out - of sys.modules, such as the PTU rollup's deployment scan, which then - counts a sibling test's deployments as if the proxy owned them. - - This must be a hook pair, not an autouse fixture: an autouse fixture in - the root conftest requests monkeypatch, so monkeypatch's undo stack - unwinds after every other fixture finalizer. A test that monkeypatches a - global while a fixture has it patched records the fixture's mock as the - "original", and monkeypatch.undo re-plants that mock after all restores - have run, poisoning the global for the rest of the xdist worker. - """ - from litellm.proxy import proxy_server - - item.stash[_proxy_module_globals_snapshot] = { - name: getattr(proxy_server, name, _MODULE_GLOBAL_MISSING) - for name in _PROXY_MODULE_GLOBALS_TO_ISOLATE - } - yield - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_teardown(item, nextitem): - yield - snapshot = item.stash.get(_proxy_module_globals_snapshot, None) - if snapshot is None: - return - from litellm.proxy import proxy_server - - for name, value in snapshot.items(): - if value is _MODULE_GLOBAL_MISSING: - if hasattr(proxy_server, name): - delattr(proxy_server, name) - else: - setattr(proxy_server, name, value) - - -@pytest.fixture(autouse=True) -def _reset_graceful_shutdown_state(): - """Graceful shutdown state is process-scoped; keep it from leaking between tests.""" - from litellm.proxy.shutdown.graceful_shutdown_manager import ( - GracefulShutdownManager, - ) - - GracefulShutdownManager.reset() - yield - GracefulShutdownManager.reset() - - -def build_cache_config(enable_cache: bool = True) -> Optional[Dict]: - """ - Build Redis cache configuration from environment variables. - - Args: - enable_cache: Whether to enable cache (default: True) - - Returns: - dict: Cache configuration dict with 'cache' and 'cache_params' keys, or None - """ - if not enable_cache: - return None - - redis_host = os.getenv("REDIS_HOST") - if not redis_host: - return None - - redis_port = os.getenv("REDIS_PORT", "6379") - cache_params = { - "type": "redis", - "host": redis_host, - "port": int(redis_port) if redis_port.isdigit() else redis_port, - } - - redis_password = os.getenv("REDIS_PASSWORD") - if redis_password: - cache_params["password"] = redis_password - - return {"cache": True, "cache_params": cache_params} - - -def build_minimal_proxy_config( - database_url: Optional[str] = None, **init_options -) -> Dict: - """ - Build a minimal proxy configuration YAML. - - Args: - database_url: Optional database URL (falls back to DATABASE_URL env var) - **init_options: Additional configuration options: - - master_key: API key for authentication (default: "sk-1234") - - enable_cache: Whether to enable Redis cache (default: True) - - success_callback: Callback function for success events - - Returns: - dict: Configuration dictionary ready to be written as YAML - """ - config = { - "general_settings": {"master_key": init_options.get("master_key", "sk-1234")}, - "litellm_settings": {}, - } - - # Configure database - db_url = database_url or os.getenv("DATABASE_URL") - if db_url: - config["general_settings"]["database_url"] = db_url - - # Configure cache if Redis is available - enable_cache = init_options.get("enable_cache", True) - cache_config = build_cache_config(enable_cache=enable_cache) - if cache_config: - config["litellm_settings"].update(cache_config) - - # Add success_callback if provided (for realistic readiness endpoint) - if init_options.get("success_callback") is not None: - config["litellm_settings"]["success_callback"] = init_options[ - "success_callback" - ] - - # Add any other litellm_settings from init_options - excluded_keys = { - "master_key", - "debug", - "success_callback", - "database_url", - "enable_cache", - } - for key, value in init_options.items(): - if key not in excluded_keys and key not in config["litellm_settings"]: - config["litellm_settings"][key] = value - - return config - - -def set_proxy_environment_variables( - monkeypatch, database_url: Optional[str] = None -) -> None: - """ - Set environment variables for database and Redis. - - Args: - monkeypatch: pytest monkeypatch fixture - database_url: Optional database URL (falls back to DATABASE_URL env var) - """ - # Set database URL - db_url = database_url or os.getenv("DATABASE_URL") - if db_url: - monkeypatch.setenv("DATABASE_URL", db_url) - - # Set Redis environment variables if available - redis_host = os.getenv("REDIS_HOST") - if redis_host: - monkeypatch.setenv("REDIS_HOST", redis_host) - monkeypatch.setenv("REDIS_PORT", os.getenv("REDIS_PORT", "6379")) - redis_password = os.getenv("REDIS_PASSWORD") - if redis_password: - monkeypatch.setenv("REDIS_PASSWORD", redis_password) - - -def create_proxy_test_client( - monkeypatch, database_url: Optional[str] = None, **init_options -) -> TestClient: - """ - Create a proxy TestClient with optional database and Redis cache configuration. - - Args: - monkeypatch: pytest monkeypatch fixture - database_url: Optional database URL (falls back to DATABASE_URL env var) - **init_options: Additional configuration options: - - master_key: API key for authentication (default: "sk-1234") - - enable_cache: Whether to enable Redis cache (default: True) - - success_callback: Callback function for success events - - debug: Enable debug mode - - Returns: - TestClient: FastAPI test client for the proxy server - """ - from litellm.proxy.proxy_server import ( - cleanup_router_config_variables, - initialize, - app, - ) - - cleanup_router_config_variables() - - # Get config file path - filepath = os.path.dirname(os.path.abspath(__file__)) - default_config_fp = os.path.join( - filepath, "test_configs", "test_config_no_auth.yaml" - ) - - # Check if we need to create a minimal config with Redis/database - enable_cache = init_options.get("enable_cache", True) - needs_redis = enable_cache and os.getenv("REDIS_HOST") is not None - needs_db = (database_url or os.getenv("DATABASE_URL")) is not None - - # Create minimal config if: - # 1. Default config file doesn't exist, OR - # 2. We need Redis/database config that might not be in the default config - if not os.path.exists(default_config_fp) or needs_redis or needs_db: - minimal_config = build_minimal_proxy_config( - database_url=database_url, **init_options - ) - - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: - yaml.dump(minimal_config, f) - config_fp = f.name - else: - config_fp = default_config_fp - - # Set environment variables - set_proxy_environment_variables(monkeypatch, database_url=database_url) - monkeypatch.setenv("LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY", "true") - - # Initialize proxy - asyncio.run(initialize(config=config_fp, debug=init_options.get("debug", False))) - return TestClient(app) - - -@pytest.fixture -def fresh_agent_read_through(monkeypatch): - from litellm.proxy.common_utils import registry_read_through - - read_through = registry_read_through.RegistryReadThrough(resync=registry_read_through._resync_agents) - monkeypatch.setattr(registry_read_through, "agent_registry_read_through", read_through) - return read_through diff --git a/tests/test_ratelimit.py b/tests/test_ratelimit.py index 7959f182a3a..94d48f0accf 100644 --- a/tests/test_ratelimit.py +++ b/tests/test_ratelimit.py @@ -135,8 +135,8 @@ def test_async_rate_limit( if num_try_send > num_allowed_send: pytest.skip( "RPM tracking via background thread is racy; " - "rate-limit enforcement is tested in " - "tests/test_litellm/proxy/test_router_rate_limit.py" + "RPM over-limit rejection is tested for usage-based-routing-v2 in " + "tests/unit/router_strategy/test_router_routing_groups.py" ) list_of_messages = generate_list_of_messages(max(num_try_send, num_allowed_send)) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py similarity index 99% rename from tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py rename to tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 793db970dd5..52acdf93f35 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -15,6 +15,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import respx from fastapi import HTTPException, Request, Response, UploadFile from fastapi.responses import StreamingResponse from pydantic import TypeAdapter, ValidationError @@ -2711,10 +2712,10 @@ async def test_pass_through_request_merge_query_params_rewrites_managed_ids_on_t @pytest.mark.asyncio -async def test_pass_through_with_httpbin_redirect(): +async def test_pass_through_request_follows_redirect_to_final_response(httpx_transport): """ - Integration test using httpbin.org redirect endpoint to test real redirect handling. - This tests the actual redirect handling capability end-to-end using the full pass_through_request function. + The proxy must follow the upstream redirect and return the final response, + not the 302. """ from unittest.mock import MagicMock @@ -2725,44 +2726,40 @@ async def test_pass_through_with_httpbin_redirect(): pass_through_request, ) - # Create mock request mock_request = MagicMock(spec=Request) mock_request.method = "GET" mock_request.headers = Headers({}) mock_request.query_params = QueryParams("") - # Mock the body method to return empty bytes for GET request async def mock_body(): return b"" mock_request.body = mock_body - # Mock user API key dict mock_user_api_key_dict = MagicMock() - try: - # Test with httpbin.org redirect endpoint - # This will redirect to httpbin.org/get + with respx.mock(assert_all_called=True) as upstream: + upstream.get("https://upstream.test/redirect/1").respond( + 302, headers={"Location": "/get"} + ) + upstream.get("https://upstream.test/get").respond( + 200, json={"url": "https://upstream.test/get"} + ) + response = await pass_through_request( request=mock_request, - target="https://httpbin.org/redirect/1", + target="https://upstream.test/redirect/1", custom_headers={}, user_api_key_dict=mock_user_api_key_dict, ) + requested_urls: Final = [str(call.request.url) for call in upstream.calls] - # Should get the final response (200) from /get endpoint, not the redirect (302) - assert response.status_code == 200 - - # The response should be from the /get endpoint - response_content = bytes(response.body).decode("utf-8") - - # httpbin.org/get returns JSON with info about the request - assert '"url": "https://httpbin.org/get"' in response_content - except Exception as e: - # If httpbin.org is not accessible, skip the test - import pytest - - pytest.skip(f"Could not reach httpbin.org for integration test: {e}") + assert response.status_code == 200 + assert json.loads(bytes(response.body))["url"] == "https://upstream.test/get" + assert requested_urls == [ + "https://upstream.test/redirect/1", + "https://upstream.test/get", + ] @pytest.mark.asyncio diff --git a/tests/unit/proxy/utils/prisma_and_spend/conftest.py b/tests/unit/proxy/utils/prisma_and_spend/conftest.py index c502fe4800e..455eb423ddc 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/conftest.py +++ b/tests/unit/proxy/utils/prisma_and_spend/conftest.py @@ -1,4 +1,4 @@ -"""Shared fixtures for tests/test_litellm/proxy/utils/prisma_and_spend/. +"""Shared fixtures for tests/unit/proxy/utils/prisma_and_spend/. All fixtures used by PR2 test files live here. Do NOT add fixtures inside individual test files; if a fixture is missing, add it here and update the diff --git a/tests/unit/proxy/utils/proxy_logging/conftest.py b/tests/unit/proxy/utils/proxy_logging/conftest.py index 74508a74e3b..17c8caabc52 100644 --- a/tests/unit/proxy/utils/proxy_logging/conftest.py +++ b/tests/unit/proxy/utils/proxy_logging/conftest.py @@ -1,4 +1,4 @@ -"""Shared fixtures for tests/test_litellm/proxy/utils/proxy_logging/. +"""Shared fixtures for tests/unit/proxy/utils/proxy_logging/. All fixtures used by PR1 of the proxy/utils.py behavior-pinning project live here. Tests should not declare fixtures inline. diff --git a/tests/unit/test_assert_ci_coverage.py b/tests/unit/test_assert_ci_coverage.py index 8524a905745..cc25627c651 100644 --- a/tests/unit/test_assert_ci_coverage.py +++ b/tests/unit/test_assert_ci_coverage.py @@ -92,7 +92,7 @@ def test_a_glob_names_only_what_it_matches_not_what_sits_below_it(): glob = "tests/test_litellm/test_*.py" assert coverage._token_names(glob, "tests/test_litellm/test_router.py") is True assert coverage._token_names(glob, "tests/test_litellm/test_router.py/nested.py") is False - assert coverage._token_names(glob, "tests/test_litellm/proxy/test_router.py") is False + assert coverage._token_names(glob, "tests/test_litellm/nested/test_router.py") is False def test_a_glob_still_covers_the_subtree_for_the_census(): @@ -169,10 +169,44 @@ def test_every_sharded_root_named_in_the_script_exists_on_disk(): def test_the_repo_as_it_stands_has_every_shard_child_assigned(): - findings = coverage._unassigned_shard_children(coverage._invoked_test_tokens(coverage._all_scalars())) + findings = coverage._unassigned_shard_children( + coverage._shard_tokens(coverage._all_scalars(), coverage._unit_selection_arms()) + ) assert [f.subject for f in findings] == [] +def test_shard_tokens_credits_only_wired_unit_flags(tmp_path): + root = tmp_path / "tests" / "tree" + (root / "wired").mkdir(parents=True) + (root / "wired" / "test_a.py").write_text("def test_a(): assert True\n") + (root / "unwired").mkdir(parents=True) + (root / "unwired" / "test_b.py").write_text("def test_b(): assert True\n") + script = tmp_path / ".circleci" / "scripts" / "unit_selection.sh" + script.parent.mkdir(parents=True) + script.write_text( + "legacy_paths() {\n" + " case \"$1\" in\n" + " wired-flag) echo tests/tree/wired ;;\n" + " unwired-flag)\n" + " echo tests/tree/unwired ;;\n" + " esac\n" + "}\n" + ) + + scalars: Final = (coverage.Scalar(key="unit-flag", value="wired-flag"),) + findings = coverage._unassigned_shard_children( + coverage._shard_tokens(scalars, coverage._unit_selection_arms(tmp_path)), + roots=("tests/tree",), + repo_root=tmp_path, + ) + + assert tuple(f.subject for f in findings) == ("tests/tree/unwired",) + + +def test_check_shards_passes_on_the_repo_as_it_stands(capsys): + assert coverage._check_shards() == 0 + + # --------------------------------------------------------------------------- # # Slice guard: a job can glob a file and its -k can then throw the file out # --------------------------------------------------------------------------- #