Merge remote-tracking branch 'origin/main' into litellm_agent365_fail_open_default

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.circleci/scripts/unit_selection.sh

@@ -7,6 +7,7 @@ legacy_flags=(
   caching-local
   enterprise-package
   enterprise-routing
+  integrations
   llm-other-providers
   llm-vertex-ai
   mcp-integration
@@ -52,6 +53,7 @@ legacy_paths() {
       echo tests/unit/enterprise/proxy/test_file_deletion_blocking.py
       echo tests/unit/enterprise/proxy/test_managed_files_access_check.py
       echo tests/unit/enterprise/proxy/test_managed_files_hook.py ;;
+    integrations) echo tests/unit/integrations ;;
     llm-other-providers) find tests/unit/llms -name 'test_*.py' -not -path 'tests/unit/llms/vertex_ai/*' ;;
     llm-vertex-ai) echo tests/unit/llms/vertex_ai ;;
     mcp-integration)
@@ -75,6 +77,7 @@ legacy_paths() {
       echo tests/unit/messages
       echo tests/unit/rag
       echo tests/unit/rerank_api
+      echo tests/unit/secret_managers
       echo tests/unit/vector_stores
       echo tests/unit/videos ;;
     proxy-db-auth-checks)

.circleci/tests.yml

@@ -369,6 +369,13 @@ workflows:
           reruns: 2
           base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
           pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
+      - unit:
+          name: unit-integrations
+          flag: integrations
+          shards: 2
+          reruns: 3
+          base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
+          pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
       - unit:
           name: unit-misc
           flag: misc

.github/workflows/test-unit.yml

@@ -80,7 +80,8 @@ jobs:
 
           - shard: integrations
             artifact-name: integrations
-            test-path: "tests/test_litellm/integrations"
+            test-path: ""
+            unit-flag: integrations
             workers: 2
             reruns: 3
             timeout-minutes: 20
@@ -107,7 +108,6 @@ jobs:
           - shard: misc
             artifact-name: misc
             test-path: >-
-              tests/test_litellm/secret_managers
               tests/test_litellm/interactions
               tests/test_litellm/ocr
               tests/test_litellm/passthrough

Makefile

@@ -326,13 +326,13 @@ test-unit-proxy-misc: install-test-deps
 	$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
 
 test-unit-integrations: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/integrations --tb=short -vv -n 4 --durations=20
 
 test-unit-core-utils: install-test-deps
 	$(UV_RUN) pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
 
 test-unit-other: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/test_litellm/anthropic_interface tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/test_litellm/anthropic_interface tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20
 
 test-unit-root: install-test-deps
 	$(UV_RUN) pytest tests/unit/test_*.py tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md

@@ -2,7 +2,7 @@
 
 Every `litellm_*` metric family the proxy can expose on `/metrics` (136 families across 97 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
 
-Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
+Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/unit/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
 
 The first eleven rows need only `callbacks: ["prometheus"]`. The last three rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected
 

litellm-rust/crates/secrets-hashicorp/tests/secret_manager/support.rs

@@ -118,7 +118,7 @@ pub(super) struct ParityCase {
 pub(super) fn parity_cases() -> Vec<ParityCase> {
     serde_json::from_str(include_str!(concat!(
         env!("CARGO_MANIFEST_DIR"),
-        "/../../../tests/test_litellm/secret_managers/hashicorp_vault_parity.json"
+        "/../../../tests/unit/secret_managers/hashicorp_vault_parity.json"
     )))
     .unwrap()
 }

litellm-rust/crates/secrets/PARITY.md

@@ -32,7 +32,7 @@ The rollout decision comes from [catalog.py](../../../litellm/rust_bridge/catalo
 
 `_SecretManagerRuntime` is a private implementation detail, not a replacement SDK class. Its async methods return Futures; public `async def` methods retain lazy coroutine creation and `asyncio.create_task` support. Passing the same names and arguments is insufficient to claim parity until the remaining return-value, error, cache and configuration differences above are closed
 
-## [tests/test_litellm/secret_managers/test_aws_secret_manager_replication.py](../../../tests/test_litellm/secret_managers/test_aws_secret_manager_replication.py)
+## [tests/unit/secret_managers/test_aws_secret_manager_replication.py](../../../tests/unit/secret_managers/test_aws_secret_manager_replication.py)
 
 | Python test | Rust coverage or boundary |
 | --- | --- |
@@ -48,7 +48,7 @@ The rollout decision comes from [catalog.py](../../../litellm/rust_bridge/catalo
 | `test_replicate_secret_http_error_raises` | [direct_replication_returns_response_or_service_error](../secrets-aws/tests/secret_manager/writes.rs) |
 | `test_replicate_secret_timeout_raises` | [write_and_replication_timeouts_remain_errors](../secrets-aws/tests/secret_manager/writes.rs) |
 
-## [tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py](../../../tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py)
+## [tests/unit/secret_managers/test_aws_secret_manager_rotation.py](../../../tests/unit/secret_managers/test_aws_secret_manager_rotation.py)
 
 | Python test | Rust coverage or boundary |
 | --- | --- |
@@ -59,7 +59,7 @@ The rollout decision comes from [catalog.py](../../../litellm/rust_bridge/catalo
 | `test_write_secret_to_name_inside_recovery_window_restores_and_stores_new_value` | [recovery_window_alias_is_restored_updated_and_tagged](../secrets-aws/tests/secret_manager/writes.rs) |
 | `test_write_secret_to_live_existing_name_still_fails_without_overwriting` | [create_failure_does_not_overwrite_an_alias_without_a_deletion_date](../secrets-aws/tests/secret_manager/writes.rs) |
 
-## [tests/test_litellm/secret_managers/test_aws_secret_manager_v2.py](../../../tests/test_litellm/secret_managers/test_aws_secret_manager_v2.py)
+## [tests/unit/secret_managers/test_aws_secret_manager_v2.py](../../../tests/unit/secret_managers/test_aws_secret_manager_v2.py)
 
 | Python test | Rust coverage or boundary |
 | --- | --- |
@@ -70,14 +70,14 @@ The rollout decision comes from [catalog.py](../../../litellm/rust_bridge/catalo
 | `test_prepare_request_explicit_bedrock_runtime_endpoint_param_still_wins` | [endpoint_overrides_replace_the_service_and_override_the_region](../secrets-aws/tests/secret_manager/configuration.rs) |
 | `test_prepare_request_env_bedrock_runtime_endpoint_still_wins` | [endpoint_overrides_replace_the_service_and_override_the_region](../secrets-aws/tests/secret_manager/configuration.rs) |
 
-## [tests/test_litellm/secret_managers/test_base_secret_manager.py](../../../tests/test_litellm/secret_managers/test_base_secret_manager.py)
+## [tests/unit/secret_managers/test_base_secret_manager.py](../../../tests/unit/secret_managers/test_base_secret_manager.py)
 
 | Python test | Rust coverage or boundary |
 | --- | --- |
 | `test_raise_if_unsafe_secret_name_rejects_traversal_and_line_breaks` | [names_reject_path_traversal_and_control_characters](../secrets-types/tests/rotation.rs) |
 | `test_raise_if_unsafe_secret_name_allows_legitimate_aliases` | [names_allow_safe_values](../secrets-types/tests/rotation.rs) |
 
-## [tests/test_litellm/secret_managers/test_custom_secret_manager.py](../../../tests/test_litellm/secret_managers/test_custom_secret_manager.py)
+## [tests/unit/secret_managers/test_custom_secret_manager.py](../../../tests/unit/secret_managers/test_custom_secret_manager.py)
 
 | Python test | Rust coverage or boundary |
 | --- | --- |
@@ -89,7 +89,7 @@ The rollout decision comes from [catalog.py](../../../litellm/rust_bridge/catalo
 | `test_custom_secret_manager_integration_with_litellm` | [manager_strings_are_coerced_like_literal_eval](../secrets/tests/resolution.rs) |
 | `test_minimal_custom_secret_manager` | Exercises the Python example subclass itself or Python default methods. Caller-authored Python implementations remain Python callbacks; resolver integration is covered by `manager_strings_are_coerced_like_literal_eval` |
 
-## [tests/test_litellm/secret_managers/test_cyberark_secret_manager.py](../../../tests/test_litellm/secret_managers/test_cyberark_secret_manager.py)
+## [tests/unit/secret_managers/test_cyberark_secret_manager.py](../../../tests/unit/secret_managers/test_cyberark_secret_manager.py)
 
 | Python test | Rust coverage or boundary |
 | --- | --- |
@@ -97,7 +97,7 @@ The rollout decision comes from [catalog.py](../../../litellm/rust_bridge/catalo
 | `test_async_write_matches_parity_fixture` | [writes_match_python_parity_fixture](../secrets-cyberark/tests/secret_manager/writes.rs) |
 | `test_missing_credentials_raise_value_error` | [new_validates_credentials_before_license_and_configuration](../secrets-cyberark/tests/secret_manager/configuration.rs) |
 
-## [tests/test_litellm/secret_managers/test_get_azure_ad_token_provider.py](../../../tests/test_litellm/secret_managers/test_get_azure_ad_token_provider.py)
+## [tests/unit/secret_managers/test_get_azure_ad_token_provider.py](../../../tests/unit/secret_managers/test_get_azure_ad_token_provider.py)
 
 | Python test | Rust coverage or boundary |
 | --- | --- |
@@ -115,7 +115,7 @@ The rollout decision comes from [catalog.py](../../../litellm/rust_bridge/catalo
 | `test_get_azure_ad_token_provider_prefers_workload_identity_over_managed_identity` | Credential-selection contract belongs to `litellm-auth-azure`, not a secrets crate |
 | `test_get_azure_ad_token_provider_defaults_to_default_azure_credential` | Credential-selection contract belongs to `litellm-auth-azure`, not a secrets crate |
 
-## [tests/test_litellm/secret_managers/test_hashicorp_secret_manager.py](../../../tests/test_litellm/secret_managers/test_hashicorp_secret_manager.py)
+## [tests/unit/secret_managers/test_hashicorp_secret_manager.py](../../../tests/unit/secret_managers/test_hashicorp_secret_manager.py)
 
 | Python test | Rust coverage or boundary |
 | --- | --- |
@@ -130,13 +130,13 @@ The rollout decision comes from [catalog.py](../../../litellm/rust_bridge/catalo
 | `test_tls_login_uses_login_namespace` | [tls_login_posts_the_role_and_uses_the_client_identity](../secrets-hashicorp/tests/secret_manager/configuration.rs) |
 | `test_configuration_matches_native_parity_fixture` | [configuration_matches_python_parity_fixture](../secrets-hashicorp/tests/secret_manager/configuration.rs) |
 
-## [tests/test_litellm/secret_managers/test_secret_manager_handler.py](../../../tests/test_litellm/secret_managers/test_secret_manager_handler.py)
+## [tests/unit/secret_managers/test_secret_manager_handler.py](../../../tests/unit/secret_managers/test_secret_manager_handler.py)
 
 | Python test | Rust coverage or boundary |
 | --- | --- |
 | `test_azure_key_vault_matches_rust_parity_fixture` | [parity_fixture_matches_python_backend_contract](../secrets-azure/tests/key_vault.rs) |
 
-## [tests/test_litellm/secret_managers/test_secret_managers_main.py](../../../tests/test_litellm/secret_managers/test_secret_managers_main.py)
+## [tests/unit/secret_managers/test_secret_managers_main.py](../../../tests/unit/secret_managers/test_secret_managers_main.py)
 
 | Python test | Rust coverage or boundary |
 | --- | --- |

litellm/integrations/levo/README.md

@@ -92,7 +92,7 @@ litellm/integrations/levo/
 
 ## Testing
 
-See the test files in `tests/test_litellm/integrations/levo/`:
+See the test files in `tests/unit/integrations/levo/`:
 - `test_levo.py`: Unit tests for configuration
 - `test_levo_integration.py`: Integration tests for callback registration
 

tests/test_litellm/integrations/levo/__init__.py

@@ -1 +0,0 @@
-# Levo integration tests

tests/unit/integrations/bitbucket/test_bitbucket_prompt_manager.py

@@ -307,37 +307,6 @@ def test_bitbucket_prompt_manager_render_template_not_found():
         manager.prompt_manager.render_template("nonexistent", {"some": "variable"})
 
 
-@patch("litellm.integrations.bitbucket.bitbucket_prompt_manager.BitBucketClient")
-def test_bitbucket_prompt_manager_integration(mock_client_class):
-    """Test BitBucketPromptManager integration with BitBucketClient."""
-    # Mock the BitBucket client
-    mock_client = MagicMock()
-    mock_client.get_file_content.return_value = """---
-model: gpt-4
-temperature: 0.7
----
-Hello {{name}}!"""
-    mock_client_class.return_value = mock_client
-
-    config = {
-        "workspace": "test-workspace",
-        "repository": "test-repo",
-        "access_token": "test-token",
-    }
-
-    manager = BitBucketPromptManager(config, prompt_id="test_prompt")
-
-    # Should have loaded the prompt
-    assert "test_prompt" in manager.prompt_manager.prompts
-    template = manager.prompt_manager.prompts["test_prompt"]
-    assert template.model == "gpt-4"
-    assert template.temperature == 0.7
-
-    # Test rendering
-    rendered = manager.prompt_manager.render_template("test_prompt", {"name": "World"})
-    assert rendered == "Hello World!"
-
-
 def test_bitbucket_prompt_manager_parse_prompt_to_messages():
     """Test parsing prompt content into messages."""
     config = {

tests/unit/integrations/conftest.py

@@ -1,6 +1,7 @@
 import functools
 import http.server
 import ipaddress
+import os
 import queue
 import ssl
 import threading
@@ -74,6 +75,14 @@ def write_self_signed_cert(directory: Path, stem: str) -> tuple[Path, Path]:
     return certificate_path, key_path
 
 
+@pytest.fixture(autouse=True)
+def restore_process_environment() -> Iterator[None]:
+    original: Final = dict(os.environ)
+    yield
+    os.environ.clear()
+    os.environ.update(original)
+
+
 @pytest.fixture
 def tls_sink(tmp_path: Path) -> Iterator[TlsSink]:
     certificate_path, key_path = write_self_signed_cert(tmp_path, "sink")

tests/unit/integrations/otel/test_otel_v2_components.py

@@ -42,7 +42,7 @@ from opentelemetry.trace.propagation.tracecontext import (  # noqa: E402
 )
 
 import litellm  # noqa: E402
-from conftest import TlsSink  # noqa: E402
+from tests.unit.integrations.conftest import TlsSink  # noqa: E402
 from litellm.integrations.otel.plumbing import context as ctx_mod  # noqa: E402
 from litellm.integrations.otel.plumbing import providers  # noqa: E402
 from litellm.integrations.otel.model.config import OpenTelemetryV2Config  # noqa: E402

tests/unit/integrations/otel/test_otel_v2_logger.py

@@ -2945,14 +2945,6 @@ def test_success_without_pre_call_emits_deferred_span():
     assert spans[0].end_time == 101_500_000_000
 
 
-def test_no_carrier_and_no_payload_is_noop():
-    logger, exporter = _logger()
-    asyncio.run(
-        logger.async_log_success_event({"litellm_params": {}}, None, None, None)
-    )
-    assert exporter.get_finished_spans() == ()
-
-
 def test_second_close_for_same_call_does_not_duplicate_span():
     """Success and failure can both fire on one logging object for the same call
     id. The first close pops the carrier and finishes the boundary span; the