From f07ea5921b361471198aced3c941d83fe7727ffb Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 16 Jul 2026 08:36:46 -0700 Subject: [PATCH 001/480] feat(proxy): enforce team isolation for provider-format batch ids and output files Managed batch ownership rows were already written for every create path, but retrieve/cancel/file-content checks only fired for unified (litellm_proxy-prefixed) ids, so model-encoded and raw provider batch ids bypassed isolation entirely. - enforce can_access_resource on retrieve/cancel for provider-format batch ids when an ownership row exists; ids with no row stay accessible so pass-through reads keep working - make batch sync operations update-only so a retrieve/cancel can never mint an ownership row attributed to the first caller - write ownership rows for a synced batch's provider-format output/error file ids, inherited from the owning batch row, and enforce them on file content/retrieve/delete --- .../proxy/hooks/managed_files.py | 142 ++++++ .../proxy/hooks/test_managed_files.py | 412 ++++++++++++++++++ 2 files changed, 554 insertions(+) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 3f42867d90e..27363f45505 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -30,10 +30,12 @@ from litellm.proxy._types import ( ) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, + decode_model_from_file_id, get_batch_id_from_unified_batch_id, get_content_type_from_file_object, get_model_id_from_unified_batch_id, get_models_from_unified_file_id, + get_original_file_id, normalize_mime_type_for_provider, ) from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue] @@ -165,7 +167,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_object_id: str, file_purpose: Literal["batch", "fine-tune", "response"], user_api_key_dict: UserAPIKeyAuth, + update_only: bool = False, ) -> None: + """Persist a managed object row. + + With ``update_only=True`` an existing row is refreshed but a missing + row is NOT created: sync operations (retrieve/cancel) must never mint + an ownership row attributed to whoever happened to call them first. + """ verbose_logger.info( f"Storing LiteLLM Managed {file_purpose} object with id={unified_object_id} in cache" ) @@ -175,6 +184,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): file_purpose=file_purpose, file_object=file_object, ) + + if update_only: + updated_count = ( + await self.prisma_client.db.litellm_managedobjecttable.update_many( + where={"unified_object_id": unified_object_id}, + data={ + "file_object": file_object.model_dump_json(), + "status": file_object.status, + "updated_by": user_api_key_dict.user_id, + }, + ) + ) + if updated_count == 0: + return + await self.internal_usage_cache.async_set_cache( + key=unified_object_id, + value=litellm_managed_object.model_dump(), + litellm_parent_otel_span=litellm_parent_otel_span, + ) + return + await self.internal_usage_cache.async_set_cache( key=unified_object_id, value=litellm_managed_object.model_dump(), @@ -284,6 +314,103 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): detail=f"Object not found: {unified_object_id}", ) + async def enforce_batch_object_access( + self, object_id: str, user_api_key_dict: UserAPIKeyAuth + ) -> None: + """Deny access to a provider-format batch id owned by another caller. + + Ids with no ownership row (batches created before ownership tracking, + or directly on the provider account) stay accessible so pass-through + reads keep working. + """ + if self.prisma_client is None: + return + managed_object = ( + await self.prisma_client.db.litellm_managedobjecttable.find_first( + where={"unified_object_id": object_id} + ) + ) + if managed_object is None: + return + if not can_access_resource( + user_api_key_dict=user_api_key_dict, + created_by=managed_object.created_by, + resource_team_id=managed_object.team_id, + ): + raise HTTPException( + status_code=403, + detail=f"User {user_api_key_dict.user_id} does not have access to the object {object_id}", + ) + + async def enforce_provider_file_access( + self, file_id: str, user_api_key_dict: UserAPIKeyAuth + ) -> None: + """Deny access to a provider-format file id owned by another caller. + + Ownership rows for provider-format ids are written when a managed + batch's output/error files are first synced; ids with no row stay + accessible so pass-through reads keep working. + """ + if self.prisma_client is None: + return + managed_file = ( + await self.prisma_client.db.litellm_managedfiletable.find_first( + where={"unified_file_id": file_id} + ) + ) + if managed_file is None: + return + if not can_access_resource( + user_api_key_dict=user_api_key_dict, + created_by=managed_file.created_by, + resource_team_id=managed_file.team_id, + ): + raise HTTPException( + status_code=403, + detail=f"User {user_api_key_dict.user_id} does not have access to the file {file_id}", + ) + + async def store_batch_output_file_ownership( + self, response: LiteLLMBatch, litellm_parent_otel_span: Optional[Span] + ) -> None: + """Record ownership rows for a batch's provider-format output/error + file ids, inherited from the owning batch row (never the caller), so + file reads can be isolation-checked.""" + provider_file_ids = tuple( + file_id + for file_id in ( + getattr(response, "output_file_id", None), + getattr(response, "error_file_id", None), + ) + if file_id and not _is_base64_encoded_unified_file_id(file_id) + ) + if not provider_file_ids: + return + if self.prisma_client is None: + return + batch_row = ( + await self.prisma_client.db.litellm_managedobjecttable.find_first( + where={"unified_object_id": response.id} + ) + ) + if batch_row is None or ( + batch_row.created_by is None and batch_row.team_id is None + ): + return + owner_identity = UserAPIKeyAuth( + user_id=batch_row.created_by, team_id=batch_row.team_id + ) + for file_id in provider_file_ids: + model_name = decode_model_from_file_id(file_id) + raw_file_id = get_original_file_id(file_id) + await self.store_unified_file_id( + file_id=file_id, + file_object=None, + litellm_parent_otel_span=litellm_parent_otel_span, + model_mappings={model_name: raw_file_id} if model_name else {}, + user_api_key_dict=owner_identity, + ) + async def list_user_batches( self, user_api_key_dict: UserAPIKeyAuth, @@ -397,6 +524,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): status_code=403, detail=f"User {user_api_key_dict.user_id} does not have access to the file {retrieve_file_id}", ) + if retrieve_file_id: + await self.enforce_provider_file_access( + retrieve_file_id, user_api_key_dict + ) return False async def check_file_ids_access( @@ -580,6 +711,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): data[accessor_key] = get_batch_id_from_unified_batch_id( potential_llm_object_id ) + elif retrieve_object_id and accessor_key == "batch_id": + await self.enforce_batch_object_access( + retrieve_object_id, user_api_key_dict + ) elif call_type == CallTypes.acreate_fine_tuning_job.value: input_file_id = cast(Optional[str], data.get("training_file")) if input_file_id: @@ -1183,6 +1318,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_mappings={model_id: provider_file_id}, user_api_key_dict=user_api_key_dict, ) + is_batch_create = "completion_window" in data or "input_file_id" in data await self.store_unified_object_id( unified_object_id=response.id, file_object=response, @@ -1190,7 +1326,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_object_id=original_response_id, file_purpose="batch", user_api_key_dict=user_api_key_dict, + update_only=not is_batch_create, ) + if not is_batch_create: + await self.store_batch_output_file_ownership( + response=response, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + ) # Only record batch creation metric on actual create (not retrieve/cancel). # unified_file_id in _hidden_params is only set by the create_batch endpoint. diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index 68dc3269f34..49d7a53d40a 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -11,6 +11,7 @@ from litellm.caching import DualCache from litellm.proxy._types import CallTypes from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, + encode_file_id_with_model, ) @@ -2367,3 +2368,414 @@ async def test_same_user_different_keys_can_access_batch(): assert "batch_id" in result2 # Both keys should get the same result assert result1["batch_id"] == result2["batch_id"] + + +MODEL_ENCODED_BATCH_ID = encode_file_id_with_model( + "batch_provider123", "gpt-4o-team-alias", id_type="batch" +) +MODEL_ENCODED_OUTPUT_FILE_ID = encode_file_id_with_model( + "file-output456", "gpt-4o-team-alias", id_type="file" +) +RAW_PROVIDER_BATCH_ID = "batch_provider123" +RAW_PROVIDER_FILE_ID = "file-output456" + + +def _owned_record(created_by, team_id): + record = MagicMock() + record.created_by = created_by + record.team_id = team_id + return record + + +def _batch_response(batch_id, output_file_id=None): + from litellm.types.utils import LiteLLMBatch + + return LiteLLMBatch( + id=batch_id, + completion_window="24h", + created_at=1700000000, + endpoint="/v1/chat/completions", + input_file_id="file-input789", + object="batch", + status="completed", + output_file_id=output_file_id, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["aretrieve_batch", "acancel_batch"]) +@pytest.mark.parametrize( + "batch_id", [MODEL_ENCODED_BATCH_ID, RAW_PROVIDER_BATCH_ID] +) +async def test_team_b_cannot_access_team_a_provider_format_batch( + call_type, batch_id +): + """ + Cross-team retrieve/cancel of a model-encoded or raw provider batch id + must 403 when an ownership row exists for another team. + + Regression test: before this check only unified (litellm_proxy-prefixed) + batch ids were enforced, so any key could read any model-encoded or raw + provider batch. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + with pytest.raises(HTTPException) as exc_info: + await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"batch_id": batch_id}, + call_type=call_type, + ) + + assert exc_info.value.status_code == 403 + prisma_client.db.litellm_managedobjecttable.find_first.assert_awaited_once_with( + where={"unified_object_id": batch_id} + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "caller_kwargs", + [ + {"user_id": "user_a", "team_id": "team_a"}, + {"user_id": "teammate_of_a", "team_id": "team_a"}, + {"user_id": "admin_user", "user_role": "proxy_admin"}, + ], +) +async def test_authorized_callers_can_access_provider_format_batch(caller_kwargs): + """ + The creator, a same-team member, and a proxy admin can all retrieve a + model-encoded batch owned by team_a. Data must pass through unmodified so + the endpoint's own routing still applies. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + parent_otel_span=MagicMock(), **caller_kwargs + ), + cache=MagicMock(), + data={"batch_id": MODEL_ENCODED_BATCH_ID}, + call_type="aretrieve_batch", + ) + + assert result["batch_id"] == MODEL_ENCODED_BATCH_ID + assert "model" not in result + + +@pytest.mark.asyncio +async def test_provider_format_batch_without_ownership_row_stays_accessible(): + """ + A provider-format batch id with no ownership row (created before ownership + tracking, or directly on the provider account) must stay retrievable. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.find_first.return_value = None + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"batch_id": RAW_PROVIDER_BATCH_ID}, + call_type="aretrieve_batch", + ) + + assert result["batch_id"] == RAW_PROVIDER_BATCH_ID + + +@pytest.mark.asyncio +async def test_fine_tuning_provider_format_id_not_enforced(): + """ + Provider-format fine-tuning job ids are deliberately out of scope for + ownership enforcement; only unified fine-tuning ids are checked. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"fine_tuning_job_id": "ftjob-abc123"}, + call_type="aretrieve_fine_tuning_job", + ) + + assert result["fine_tuning_job_id"] == "ftjob-abc123" + prisma_client.db.litellm_managedobjecttable.find_first.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_type", ["afile_content", "afile_retrieve", "afile_delete"] +) +@pytest.mark.parametrize( + "file_id", [MODEL_ENCODED_OUTPUT_FILE_ID, RAW_PROVIDER_FILE_ID] +) +async def test_team_b_cannot_access_team_a_provider_format_file( + call_type, file_id +): + """ + Cross-team content/retrieve/delete of a model-encoded or raw provider + file id must 403 when an ownership row exists for another team. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedfiletable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(), prisma_client=prisma_client + ) + + with pytest.raises(HTTPException) as exc_info: + await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"file_id": file_id}, + call_type=call_type, + ) + + assert exc_info.value.status_code == 403 + prisma_client.db.litellm_managedfiletable.find_first.assert_awaited_once_with( + where={"unified_file_id": file_id} + ) + + +@pytest.mark.asyncio +async def test_same_team_can_access_provider_format_file(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedfiletable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="teammate_of_a", team_id="team_a", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"file_id": MODEL_ENCODED_OUTPUT_FILE_ID}, + call_type="afile_content", + ) + + assert result["file_id"] == MODEL_ENCODED_OUTPUT_FILE_ID + + +@pytest.mark.asyncio +async def test_provider_format_file_without_ownership_row_stays_accessible(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedfiletable.find_first.return_value = None + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"file_id": RAW_PROVIDER_FILE_ID}, + call_type="afile_content", + ) + + assert result["file_id"] == RAW_PROVIDER_FILE_ID + + +@pytest.mark.asyncio +async def test_post_call_batch_create_stores_ownership_row(): + """ + Batch creation (request data carries completion_window/input_file_id) + must write an ownership row attributed to the creating key. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client + ) + + await proxy_managed_files.async_post_call_success_hook( + data={ + "input_file_id": "file-input789", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + user_api_key_dict=UserAPIKeyAuth( + user_id="user_a", team_id="team_a", parent_otel_span=MagicMock() + ), + response=_batch_response(MODEL_ENCODED_BATCH_ID), + ) + + upsert_call = prisma_client.db.litellm_managedobjecttable.upsert.await_args + assert upsert_call.kwargs["where"] == { + "unified_object_id": MODEL_ENCODED_BATCH_ID + } + create_data = upsert_call.kwargs["data"]["create"] + assert create_data["created_by"] == "user_a" + assert create_data["team_id"] == "team_a" + prisma_client.db.litellm_managedobjecttable.update_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_post_call_batch_sync_does_not_claim_ownership(): + """ + Retrieve/cancel of a batch with no ownership row must NOT create one: + otherwise the first foreign key to touch a legacy batch would become its + owner and lock out the real creator once enforcement is on. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.update_many.return_value = 0 + internal_usage_cache = MagicMock(async_set_cache=AsyncMock()) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + internal_usage_cache, prisma_client=prisma_client + ) + + await proxy_managed_files.async_post_call_success_hook( + data={"batch_id": MODEL_ENCODED_BATCH_ID}, + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + response=_batch_response(MODEL_ENCODED_BATCH_ID), + ) + + prisma_client.db.litellm_managedobjecttable.upsert.assert_not_awaited() + internal_usage_cache.async_set_cache.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_post_call_batch_sync_updates_existing_row(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.update_many.return_value = 1 + prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client + ) + + await proxy_managed_files.async_post_call_success_hook( + data={"batch_id": MODEL_ENCODED_BATCH_ID}, + user_api_key_dict=UserAPIKeyAuth( + user_id="user_a", team_id="team_a", parent_otel_span=MagicMock() + ), + response=_batch_response(MODEL_ENCODED_BATCH_ID), + ) + + update_call = prisma_client.db.litellm_managedobjecttable.update_many.await_args + assert update_call.kwargs["where"] == { + "unified_object_id": MODEL_ENCODED_BATCH_ID + } + assert update_call.kwargs["data"]["status"] == "completed" + prisma_client.db.litellm_managedobjecttable.upsert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_post_call_batch_sync_stores_output_file_ownership_from_batch_row(): + """ + When a synced batch reports a provider-format output file id, an + ownership row for that file must be written with the BATCH row's + created_by/team_id, not the caller's identity. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.update_many.return_value = 1 + prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client + ) + + await proxy_managed_files.async_post_call_success_hook( + data={"batch_id": MODEL_ENCODED_BATCH_ID}, + user_api_key_dict=UserAPIKeyAuth( + user_id="admin_user", + user_role="proxy_admin", + parent_otel_span=MagicMock(), + ), + response=_batch_response( + MODEL_ENCODED_BATCH_ID, output_file_id=MODEL_ENCODED_OUTPUT_FILE_ID + ), + ) + + file_upsert = prisma_client.db.litellm_managedfiletable.upsert.await_args + assert file_upsert.kwargs["where"] == { + "unified_file_id": MODEL_ENCODED_OUTPUT_FILE_ID + } + create_data = file_upsert.kwargs["data"]["create"] + assert create_data["created_by"] == "user_a" + assert create_data["team_id"] == "team_a" + assert create_data["flat_model_file_ids"] == ["file-output456"] + + +@pytest.mark.asyncio +async def test_post_call_batch_create_does_not_store_output_file_ownership(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client + ) + + await proxy_managed_files.async_post_call_success_hook( + data={ + "input_file_id": "file-input789", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + user_api_key_dict=UserAPIKeyAuth( + user_id="user_a", team_id="team_a", parent_otel_span=MagicMock() + ), + response=_batch_response( + MODEL_ENCODED_BATCH_ID, output_file_id=MODEL_ENCODED_OUTPUT_FILE_ID + ), + ) + + prisma_client.db.litellm_managedfiletable.upsert.assert_not_awaited() From 902a93d10d2159f66b9f8b5cacbde527d50e88b0 Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Tue, 4 Aug 2026 16:51:21 -0700 Subject: [PATCH 002/480] fix(router): honor ServiceUnavailableErrorRetries and InternalServerErrorRetries in retry policy --- litellm/router_utils/get_retry_from_policy.py | 8 ++ litellm/types/router.py | 1 + .../test_get_retry_from_policy.py | 102 ++++++++++++++++++ tests/test_litellm/test_router.py | 31 ++++++ .../components/ModelRetrySettingsTab.tsx | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 6 files changed, 145 insertions(+) create mode 100644 tests/test_litellm/router_utils/test_get_retry_from_policy.py diff --git a/litellm/router_utils/get_retry_from_policy.py b/litellm/router_utils/get_retry_from_policy.py index 1645e6776fc..fbcc3de83c0 100644 --- a/litellm/router_utils/get_retry_from_policy.py +++ b/litellm/router_utils/get_retry_from_policy.py @@ -8,7 +8,9 @@ from litellm.exceptions import ( AuthenticationError, BadRequestError, ContentPolicyViolationError, + InternalServerError, RateLimitError, + ServiceUnavailableError, Timeout, ) from litellm.types.router import RetryPolicy @@ -26,6 +28,8 @@ def get_num_retries_from_retry_policy( TimeoutErrorRetries: Optional[int] = None RateLimitErrorRetries: Optional[int] = None ContentPolicyViolationErrorRetries: Optional[int] = None + InternalServerErrorRetries: Optional[int] = None + ServiceUnavailableErrorRetries: Optional[int] = None """ # if we can find the exception then in the retry policy -> return the number of retries @@ -48,6 +52,10 @@ def get_num_retries_from_retry_policy( and retry_policy.ContentPolicyViolationErrorRetries is not None ): return retry_policy.ContentPolicyViolationErrorRetries + if isinstance(exception, ServiceUnavailableError) and retry_policy.ServiceUnavailableErrorRetries is not None: + return retry_policy.ServiceUnavailableErrorRetries + if isinstance(exception, InternalServerError) and retry_policy.InternalServerErrorRetries is not None: + return retry_policy.InternalServerErrorRetries if isinstance(exception, BadRequestError) and retry_policy.BadRequestErrorRetries is not None: return retry_policy.BadRequestErrorRetries diff --git a/litellm/types/router.py b/litellm/types/router.py index 21bed84a3a1..e0952d9dd02 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -95,6 +95,7 @@ class RetryPolicy(BaseModel): RateLimitErrorRetries: Optional[int] = None ContentPolicyViolationErrorRetries: Optional[int] = None InternalServerErrorRetries: Optional[int] = None + ServiceUnavailableErrorRetries: Optional[int] = None class UpdateRouterConfig(BaseModel): diff --git a/tests/test_litellm/router_utils/test_get_retry_from_policy.py b/tests/test_litellm/router_utils/test_get_retry_from_policy.py new file mode 100644 index 00000000000..a5e239b8595 --- /dev/null +++ b/tests/test_litellm/router_utils/test_get_retry_from_policy.py @@ -0,0 +1,102 @@ +import litellm +from litellm.router_utils.get_retry_from_policy import ( + get_num_retries_from_retry_policy, +) +from litellm.types.router import RetryPolicy + + +def _service_unavailable_error() -> litellm.ServiceUnavailableError: + return litellm.ServiceUnavailableError( + message="model is down", + llm_provider="openai", + model="gpt-5.6", + ) + + +def _internal_server_error() -> litellm.InternalServerError: + return litellm.InternalServerError( + message="upstream 500", + llm_provider="openai", + model="gpt-5.6", + ) + + +def test_service_unavailable_error_retries_honored(): + policy = RetryPolicy(ServiceUnavailableErrorRetries=0) + + assert ( + get_num_retries_from_retry_policy( + exception=_service_unavailable_error(), + retry_policy=policy, + ) + == 0 + ) + + +def test_service_unavailable_error_retries_nonzero(): + policy = RetryPolicy(ServiceUnavailableErrorRetries=4) + + assert ( + get_num_retries_from_retry_policy( + exception=_service_unavailable_error(), + retry_policy=policy, + ) + == 4 + ) + + +def test_internal_server_error_retries_honored(): + policy = RetryPolicy(InternalServerErrorRetries=0) + + assert ( + get_num_retries_from_retry_policy( + exception=_internal_server_error(), + retry_policy=policy, + ) + == 0 + ) + + +def test_service_unavailable_not_covered_by_internal_server_error_retries(): + policy = RetryPolicy(InternalServerErrorRetries=0) + + assert ( + get_num_retries_from_retry_policy( + exception=_service_unavailable_error(), + retry_policy=policy, + ) + is None + ) + + +def test_internal_server_error_not_covered_by_service_unavailable_retries(): + policy = RetryPolicy(ServiceUnavailableErrorRetries=0) + + assert ( + get_num_retries_from_retry_policy( + exception=_internal_server_error(), + retry_policy=policy, + ) + is None + ) + + +def test_service_unavailable_error_retries_from_dict_policy(): + assert ( + get_num_retries_from_retry_policy( + exception=_service_unavailable_error(), + retry_policy={"ServiceUnavailableErrorRetries": 0}, + ) + == 0 + ) + + +def test_service_unavailable_error_retries_from_model_group_policy(): + assert ( + get_num_retries_from_retry_policy( + exception=_service_unavailable_error(), + model_group="gpt-5.6", + model_group_retry_policy={"gpt-5.6": RetryPolicy(ServiceUnavailableErrorRetries=1)}, + ) + == 1 + ) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 46b5ce65c3f..c98be9c7d80 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -6574,3 +6574,34 @@ def test_model_info_is_active_for_environment_matrix(monkeypatch): monkeypatch.delenv("LITELLM_ENVIRONMENT") with pytest.raises(ValueError, match="LITELLM_ENVIRONMENT"): model_info_is_active_for_environment(model_info={"supported_environments": ["production"]}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy_retries,expected_calls", [(0, 1), (1, 2)]) +async def test_router_retry_policy_service_unavailable_retries(policy_retries, expected_calls): + from litellm.types.router import RetryPolicy + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-5.6", + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "fake-key"}, + } + ], + retry_policy=RetryPolicy(ServiceUnavailableErrorRetries=policy_retries), + disable_cooldowns=True, + ) + + error = litellm.ServiceUnavailableError( + message="model is down", + llm_provider="openai", + model="gpt-5.6", + ) + with patch.object(litellm, "acompletion", AsyncMock(side_effect=error)) as mock_acompletion: + with pytest.raises(litellm.ServiceUnavailableError): + await router.acompletion( + model="gpt-5.6", + messages=[{"role": "user", "content": "hi"}], + ) + + assert mock_acompletion.call_count == expected_calls diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx index a4e3c4b958c..a9e0b8eb051 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx @@ -30,6 +30,7 @@ const retryPolicyMap: Record = { "RateLimitError (429)": "RateLimitErrorRetries", "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries", "InternalServerError (500)": "InternalServerErrorRetries", + "ServiceUnavailableError (503)": "ServiceUnavailableErrorRetries", }; const ModelRetrySettingsTab = ({ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9133bfb5cf4..cf20f86a28d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30947,6 +30947,8 @@ export interface components { InternalServerErrorRetries?: number | null; /** Ratelimiterrorretries */ RateLimitErrorRetries?: number | null; + /** Serviceunavailableerrorretries */ + ServiceUnavailableErrorRetries?: number | null; /** Timeouterrorretries */ TimeoutErrorRetries?: number | null; }; From 26805f75fe44170f36b54d4e8e57d9ce7ad19b5b Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 5 Aug 2026 19:36:37 +0000 Subject: [PATCH 003/480] fix(bedrock_mantle): stop dropping the web_search tool on /v1/responses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../responses/transformation.py | 4 +- ...odel_prices_and_context_window_backup.json | 15 ++-- model_prices_and_context_window.json | 15 ++-- ...bedrock_mantle_responses_transformation.py | 90 +++++++++++++++++-- 4 files changed, 107 insertions(+), 17 deletions(-) diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index 92da5835b2d..3b01c7dbad0 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -44,7 +44,9 @@ _BASE_SUFFIXES_TO_STRIP: Final = ( ) # Per Bedrock Mantle Responses API validation errors. -_BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES = frozenset({"function", "mcp", "custom", "namespace", "tool_search"}) +_BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES: Final = frozenset( + {"function", "mcp", "custom", "namespace", "tool_search", "web_search"} +) _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"}) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 02b3cde217a..e85d49cfd12 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45247,7 +45247,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, @@ -45275,7 +45276,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, @@ -45303,7 +45305,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, @@ -45330,7 +45333,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.4": { "input_cost_per_token": 2.75e-06, @@ -45357,7 +45361,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/google.gemma-4-31b": { "input_cost_per_token": 1.4e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bc7330ec99c..7514f1596b0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45368,7 +45368,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, @@ -45396,7 +45397,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, @@ -45424,7 +45426,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, @@ -45451,7 +45454,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/openai.gpt-5.4": { "input_cost_per_token": 2.75e-06, @@ -45478,7 +45482,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "bedrock_mantle/google.gemma-4-31b": { "input_cost_per_token": 1.4e-07, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index bea979aec64..3810ef39062 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -339,7 +339,7 @@ class TestBedrockMantleResponsesTools: params = cfg.map_openai_params( response_api_optional_params={ "tools": [ - {"type": "web_search"}, + {"type": "file_search", "vector_store_ids": ["vs_123"]}, {"type": "function", "name": "exec_command"}, ] }, @@ -351,7 +351,7 @@ class TestBedrockMantleResponsesTools: def test_map_openai_params_removes_tools_when_all_unsupported(self): cfg = BedrockMantleResponsesAPIConfig() params = cfg.map_openai_params( - response_api_optional_params={"tools": [{"type": "web_search"}]}, + response_api_optional_params={"tools": [{"type": "file_search", "vector_store_ids": ["vs_123"]}]}, model="openai.gpt-5.5", drop_params=False, ) @@ -365,12 +365,86 @@ class TestBedrockMantleResponsesTools: "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning" ) as mock_warning: cfg.map_openai_params( - response_api_optional_params={"tools": [{"type": "web_search"}]}, + response_api_optional_params={"tools": [{"type": "file_search", "vector_store_ids": ["vs_123"]}]}, model="openai.gpt-5.5", drop_params=False, ) assert mock_warning.call_count == 1 - assert "web_search" in str(mock_warning.call_args) + assert "file_search" in str(mock_warning.call_args) + + +class TestBedrockMantleResponsesWebSearch: + """Web Search on Amazon Bedrock is a server-side built-in tool that Mantle runs + itself when the caller passes {"type": "web_search"} on the Responses path, so + the config must forward the tool and its options untouched instead of filtering + it out and returning an ungrounded answer.""" + + _WEB_SEARCH_TOOL = {"type": "web_search", "external_web_access": False} + + def test_web_search_survives_map_openai_params_with_its_options(self): + cfg = BedrockMantleResponsesAPIConfig() + params = cfg.map_openai_params( + response_api_optional_params={"tools": [self._WEB_SEARCH_TOOL]}, + model="openai.gpt-5.6-sol", + drop_params=False, + ) + assert params["tools"] == [self._WEB_SEARCH_TOOL] + + def test_web_search_reaches_outbound_body_alongside_function_tools(self): + cfg = BedrockMantleResponsesAPIConfig() + function_tool = {"type": "function", "name": "exec_command"} + params = cfg.map_openai_params( + response_api_optional_params={"tools": [self._WEB_SEARCH_TOOL, function_tool]}, + model="openai.gpt-5.6-sol", + drop_params=False, + ) + body = cfg.transform_responses_api_request( + model="openai.gpt-5.6-sol", + input="What did AWS announce today?", + response_api_optional_request_params=params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert body["tools"] == [self._WEB_SEARCH_TOOL, function_tool] + + def test_web_search_is_not_logged_as_dropped(self): + from unittest.mock import patch + + cfg = BedrockMantleResponsesAPIConfig() + with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning") as mock_warning: + cfg.map_openai_params( + response_api_optional_params={"tools": [self._WEB_SEARCH_TOOL]}, + model="openai.gpt-5.6-sol", + drop_params=False, + ) + assert mock_warning.call_count == 0 + + def test_hoisted_web_search_tool_survives(self): + cfg = BedrockMantleResponsesAPIConfig() + body = cfg.transform_responses_api_request( + model="openai.gpt-5.6-sol", + input=[ + {"type": "additional_tools", "role": "developer", "tools": [self._WEB_SEARCH_TOOL]}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ], + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert body["tools"] == [self._WEB_SEARCH_TOOL] + + @pytest.mark.parametrize( + "model", + [ + "bedrock_mantle/openai.gpt-5.6-sol", + "bedrock_mantle/openai.gpt-5.6-terra", + "bedrock_mantle/openai.gpt-5.6-luna", + "bedrock_mantle/openai.gpt-5.5", + "bedrock_mantle/openai.gpt-5.4", + ], + ) + def test_cost_map_advertises_web_search_support(self, model): + assert litellm.supports_web_search(model=model) is True def _codex_exec_tool(): @@ -558,7 +632,7 @@ class TestBedrockMantleCodexAdditionalTools: "type": "additional_tools", "role": "developer", "tools": [ - {"type": "web_search"}, + {"type": "file_search", "vector_store_ids": ["vs_123"]}, {"type": "function", "name": "wait"}, ], }, @@ -570,7 +644,11 @@ class TestBedrockMantleCodexAdditionalTools: def test_item_stripped_even_when_no_hoisted_tool_survives(self): body = self._transform( input=[ - {"type": "additional_tools", "role": "developer", "tools": [{"type": "web_search"}]}, + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "file_search", "vector_store_ids": ["vs_123"]}], + }, self._USER_MESSAGE, ] ) From e36e3d4a6f5b77329d53ac461ed5d3a081f0f36d Mon Sep 17 00:00:00 2001 From: eeshsaxena <139802361+eeshsaxena@users.noreply.github.com> Date: Sat, 8 Aug 2026 15:56:45 +0530 Subject: [PATCH 004/480] fix(proxy): raise (not return) proxy exception in get/delete credential endpoints --- litellm/proxy/credential_endpoints/endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 3b3e9692eda..106b43a722c 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -130,7 +130,7 @@ async def get_credentials( ] return {"success": True, "credentials": masked_credentials} except Exception as e: - return handle_exception_on_proxy(e) + raise handle_exception_on_proxy(e) @router.get( @@ -240,7 +240,7 @@ async def delete_credential( litellm.credential_list = [cred for cred in litellm.credential_list if cred.credential_name != credential_name] return {"success": True, "message": "Credential deleted successfully"} except Exception as e: - return handle_exception_on_proxy(e) + raise handle_exception_on_proxy(e) def update_db_credential( From 7b181ef1978cfc5d4169ff1397ebef65229bb595 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 21:53:25 +0000 Subject: [PATCH 005/480] fix(responses/mcp): make MCP follow-up calls stateless when store=false Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/responses/main.py | 9 +- .../mcp/litellm_proxy_mcp_handler.py | 31 +++- .../responses/mcp/mcp_streaming_iterator.py | 7 + .../mcp/test_litellm_proxy_mcp_handler.py | 149 ++++++++++++++++++ .../mcp/test_mcp_streaming_iterator.py | 78 +++++++++ 5 files changed, 270 insertions(+), 4 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index e0af363b1a5..0f6119ce94c 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -326,8 +326,13 @@ async def aresponses_api_with_mcp( ) if tool_results: + persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params) + follow_up_input: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( - response=response, tool_results=tool_results, original_input=input + response=response, + tool_results=tool_results, + original_input=input, + preserve_reasoning=persistence_disabled, ) # Prepare parameters for follow-up call (restores original stream setting) @@ -346,7 +351,7 @@ async def aresponses_api_with_mcp( follow_up_input=follow_up_input, model=model, all_tools=all_tools, - response_id=response.id, + response_id=None if persistence_disabled else response.id, **follow_up_call_params, ) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index c6e17502e5d..6271c5888db 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -951,11 +951,30 @@ class LiteLLM_Proxy_MCP_Handler: return follow_up_messages + @staticmethod + def _is_persistence_disabled(call_params: Mapping[str, object]) -> bool: + """Whether the caller opted out of server-side response persistence (store=false). + + Zero data retention callers send store=false, so the provider never persisted the + first response and previous_response_id cannot be used to link the follow-up call. + """ + return call_params.get("store") is False + + @staticmethod + def _extract_reasoning_items(response: ResponsesAPIResponse) -> tuple[Mapping[str, object], ...]: + """Reasoning output items, kept whole so reasoning.encrypted_content survives replay.""" + normalized: Final = tuple( + output_item if isinstance(output_item, dict) else output_item.model_dump(exclude_none=True) + for output_item in response.output + ) + return tuple(item for item in normalized if item.get("type") == "reasoning") + @staticmethod def _create_follow_up_input( response: ResponsesAPIResponse, tool_results: Sequence[Mapping[str, object]], original_input: str | ResponseInputParam | None = None, + preserve_reasoning: bool = False, ) -> list[object]: """Create follow-up input with tool results in proper format.""" follow_up_input: Final[list[object]] = [] @@ -1013,6 +1032,10 @@ class LiteLLM_Proxy_MCP_Handler: } ) + # Reasoning items must precede the function calls they produced + if preserve_reasoning: + follow_up_input.extend(LiteLLM_Proxy_MCP_Handler._extract_reasoning_items(response)) + # Add function calls (these can come directly after user message for LLM) for function_call in function_calls: follow_up_input.append(function_call) @@ -1034,10 +1057,14 @@ class LiteLLM_Proxy_MCP_Handler: follow_up_input: list[Any], model: str, all_tools: Sequence[ResponsesToolParam] | None, - response_id: str, + response_id: str | None, **call_params: Any, ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: - """Make follow-up response API call with tool results.""" + """Make follow-up response API call with tool results. + + response_id is None for stateless (store=false) requests, where the whole prior + turn is replayed in follow_up_input instead of linked by previous_response_id. + """ return await aresponses( input=follow_up_input, model=model, diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 186852f91c2..f1560e7ec84 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -774,10 +774,15 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): try: # Create follow-up input if self.collected_response is not None: + persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled( + self.original_request_params + ) + follow_up_input: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( response=self.collected_response, tool_results=self.tool_results, original_input=self.original_request_params.get("input"), + preserve_reasoning=persistence_disabled, ) # Make follow-up call with streaming @@ -788,6 +793,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): "stream": True, } ) + if persistence_disabled: + follow_up_params.pop("previous_response_id", None) else: return # Remove tool_choice to avoid forcing more tool calls diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index d60fff66c44..f08666f7b0d 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -9,10 +9,13 @@ from fastapi import HTTPException import importlib from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing +from litellm.responses import main as responses_main +from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_module from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from typing import Any, cast +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse from litellm.types.responses.main import OutputFunctionToolCall @@ -648,3 +651,149 @@ def test_extract_tool_call_details_still_prefers_openai_arguments(): assert name == "get_weather" assert call_id == "call_123" assert arguments == '{"city": "Paris"}' + + +def _response_with_reasoning_and_tool_call() -> Any: + """A first-turn response as a reasoning model returns it: reasoning item, then a function call.""" + return ResponsesAPIResponse( + id="resp_first", + created_at=1234567890, + model="gpt-5", + object="response", + status="completed", + output=[ + { + "type": "reasoning", + "id": "rs_1", + "summary": [], + "encrypted_content": "gAAAAA-opaque-blob", + }, + { + "type": "function_call", + "id": "fc_1", + "call_id": "call-1", + "name": "foo", + "arguments": "{}", + "status": "completed", + }, + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + + +def test_create_follow_up_input_preserves_reasoning_when_stateless(): + """ + Regression test (LIT-5427): a store=false follow-up has to replay the reasoning + item, including reasoning.encrypted_content, since the provider kept no state. + """ + follow_up = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( + response=_response_with_reasoning_and_tool_call(), + tool_results=[{"tool_call_id": "call-1", "name": "foo", "result": "done"}], + original_input="hi", + preserve_reasoning=True, + ) + + assert follow_up[1] == { + "type": "reasoning", + "id": "rs_1", + "summary": [], + "encrypted_content": "gAAAAA-opaque-blob", + } + # the reasoning item has to come before the function call it produced + assert follow_up[2] == { + "type": "function_call", + "call_id": "call-1", + "name": "foo", + "arguments": "{}", + } + assert follow_up[3] == { + "type": "function_call_output", + "call_id": "call-1", + "output": "done", + } + + +def test_create_follow_up_input_omits_reasoning_when_stateful(): + """With store=true the provider still holds the reasoning item, so don't resend it.""" + follow_up = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( + response=_response_with_reasoning_and_tool_call(), + tool_results=[{"tool_call_id": "call-1", "name": "foo", "result": "done"}], + original_input="hi", + ) + + assert not [item for item in follow_up if isinstance(item, dict) and item.get("type") == "reasoning"] + + +@pytest.mark.parametrize( + "call_params, expected", + [ + ({"store": False}, True), + ({"store": True}, False), + ({"store": None}, False), + ({}, False), + ], +) +def test_is_persistence_disabled(call_params: dict[str, Any], expected: bool): + assert LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params) is expected + + +@pytest.mark.parametrize( + "store, expected_previous_response_id", + [(False, None), (True, "resp_first")], +) +@pytest.mark.asyncio +async def test_mcp_follow_up_call_is_stateless_when_store_is_false( + monkeypatch: pytest.MonkeyPatch, store: bool, expected_previous_response_id: str | None +): + """ + Regression test (LIT-5427): linking the MCP follow-up call with + previous_response_id fails for zero data retention callers, because store=false + means the first response was never persisted. + """ + captured_calls: list[dict[str, Any]] = [] + first_response = _response_with_reasoning_and_tool_call() + + async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse: + captured_calls.append(kwargs) + return first_response if len(captured_calls) == 1 else ResponsesAPIResponse( + id="resp_follow_up", + created_at=1234567891, + model="gpt-5", + object="response", + status="completed", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + + async def fake_process(**kwargs: Any) -> tuple[list[Any], dict[str, str]]: + return ([], {"foo": "litellm_proxy"}) + + async def fake_execute(**kwargs: Any) -> list[dict[str, Any]]: + return [{"tool_call_id": "call-1", "name": "foo", "result": "done"}] + + monkeypatch.setattr(responses_main, "aresponses", fake_aresponses) + monkeypatch.setattr(mcp_handler_module, "aresponses", fake_aresponses) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, "_process_mcp_tools_without_openai_transform", staticmethod(fake_process) + ) + monkeypatch.setattr(LiteLLM_Proxy_MCP_Handler, "_execute_tool_calls", staticmethod(fake_execute)) + + await responses_main.aresponses_api_with_mcp( + input="hi", + model="gpt-5", + tools=[{"type": "mcp", "server_url": "litellm_proxy", "require_approval": "never"}], + store=store, + ) + + assert len(captured_calls) == 2 + follow_up_call = captured_calls[1] + assert follow_up_call["previous_response_id"] == expected_previous_response_id + + reasoning_items = [ + item for item in follow_up_call["input"] if isinstance(item, dict) and item.get("type") == "reasoning" + ] + assert bool(reasoning_items) is (store is False) diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py index 24edf12fffe..040ee26d796 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -257,3 +257,81 @@ async def test_initial_call_failure_is_stashed_for_eager_reraise(monkeypatch): assert iterator._initial_creation_error is not None assert "initial boom" in str(iterator._initial_creation_error) + + +def _reasoning_item(encrypted_content: str): + return {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": encrypted_content} + + +@pytest.mark.asyncio +async def test_streaming_follow_up_is_stateless_when_store_is_false(monkeypatch): + """ + Regression test (LIT-5427): with store=false the provider persisted nothing, so the + streaming follow-up must drop previous_response_id and replay the reasoning item + (carrying reasoning.encrypted_content) instead of pointing at a response id. + """ + _mock_mcp_environment(monkeypatch) + + aresponses_mock = AsyncMock(side_effect=[_text_only_stream("done")]) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + iterator = MCPEnhancedStreamingIterator( + base_iterator=_FakeAsyncStream( + [ + _output_item_added_chunk(), + _completed_chunk([_reasoning_item("gAAAAA-opaque-blob"), _function_call("call_1", "read_wiki_contents")]), + ] + ), + mcp_events=[], + tool_server_map={"read_wiki_contents": "deepwiki"}, + mcp_tools_with_litellm_proxy=[{"require_approval": "never"}], + user_api_key_auth=None, + original_request_params={ + "model": "gpt-5", + "input": "what is berriai/litellm?", + "tools": [{"type": "mcp"}], + "store": False, + "previous_response_id": "resp_prev", + }, + ) + + _ = [chunk async for chunk in iterator] + + assert aresponses_mock.call_count == 1 + follow_up_kwargs = aresponses_mock.call_args_list[0].kwargs + assert "previous_response_id" not in follow_up_kwargs + assert _reasoning_item("gAAAAA-opaque-blob") in follow_up_kwargs["input"] + + +@pytest.mark.asyncio +async def test_streaming_follow_up_keeps_previous_response_id_when_stored(monkeypatch): + """The stateful default is unchanged: previous_response_id still links the follow-up.""" + _mock_mcp_environment(monkeypatch) + + aresponses_mock = AsyncMock(side_effect=[_text_only_stream("done")]) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + iterator = MCPEnhancedStreamingIterator( + base_iterator=_FakeAsyncStream( + [ + _output_item_added_chunk(), + _completed_chunk([_reasoning_item("gAAAAA-opaque-blob"), _function_call("call_1", "read_wiki_contents")]), + ] + ), + mcp_events=[], + tool_server_map={"read_wiki_contents": "deepwiki"}, + mcp_tools_with_litellm_proxy=[{"require_approval": "never"}], + user_api_key_auth=None, + original_request_params={ + "model": "gpt-5", + "input": "what is berriai/litellm?", + "tools": [{"type": "mcp"}], + "previous_response_id": "resp_prev", + }, + ) + + _ = [chunk async for chunk in iterator] + + follow_up_kwargs = aresponses_mock.call_args_list[0].kwargs + assert follow_up_kwargs["previous_response_id"] == "resp_prev" + assert not [item for item in follow_up_kwargs["input"] if item.get("type") == "reasoning"] From fae91c2a1137c57f64106ceec02c1a6d8ad3938f Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 22:27:41 +0000 Subject: [PATCH 006/480] chore(responses/mcp): drop explanatory comments per repo policy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/responses/mcp/litellm_proxy_mcp_handler.py | 1 - .../test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py | 1 - 2 files changed, 2 deletions(-) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 6271c5888db..33ca99c1a7e 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -1032,7 +1032,6 @@ class LiteLLM_Proxy_MCP_Handler: } ) - # Reasoning items must precede the function calls they produced if preserve_reasoning: follow_up_input.extend(LiteLLM_Proxy_MCP_Handler._extract_reasoning_items(response)) diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index f08666f7b0d..dbb3ad9fd44 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -701,7 +701,6 @@ def test_create_follow_up_input_preserves_reasoning_when_stateless(): "summary": [], "encrypted_content": "gAAAAA-opaque-blob", } - # the reasoning item has to come before the function call it produced assert follow_up[2] == { "type": "function_call", "call_id": "call-1", From 779441b47ba564696c35e86a66527e683c1fad31 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:30:19 +0000 Subject: [PATCH 007/480] fix(bedrock_mantle): source per-request AWS credential params from litellm_params when signing chat completions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/custom_httpx/llm_http_handler.py | 26 ++++++- .../test_bedrock_mantle_transformation.py | 77 +++++++++++++++++++ .../custom_httpx/test_llm_http_handler.py | 19 +++++ 3 files changed, 120 insertions(+), 2 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 721b9545ac1..dfc234d08e0 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5,7 +5,7 @@ import ssl from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping from contextlib import asynccontextmanager from functools import lru_cache -from types import ModuleType +from types import MappingProxyType, ModuleType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, TypeVar, Union, cast, get_type_hints from urllib.parse import parse_qs, urlencode, urlparse, urlunparse @@ -20,6 +20,7 @@ from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.anthropic_messages.transformation import ( @@ -252,6 +253,24 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool: return False +def _aws_signing_overrides( + optional_params: Mapping[str, Any], litellm_params: Mapping[str, Any] +) -> Mapping[str, Any]: + """AWS credential params for SigV4 signers that read them off optional_params. + + Only `bedrock`/`sagemaker` keep `aws_*` in optional_params: every other provider + spreads optional_params into the request body, so the params are stripped there + and survive on litellm_params alone. + """ + return MappingProxyType( + { + key: litellm_params[key] + for key in AWS_CREDENTIAL_KWARGS_KEYS + if optional_params.get(key) is None and litellm_params.get(key) is not None + } + ) + + class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -495,7 +514,10 @@ class BaseLLMHTTPHandler: headers, signed_json_body = provider_config.sign_request( headers=headers, - optional_params=optional_params, + optional_params={ + **optional_params, + **_aws_signing_overrides(optional_params, litellm_params), + }, request_data=data, api_base=api_base, api_key=api_key, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 275fb460b9f..468cc9b9130 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -489,6 +489,83 @@ class TestBedrockMantleChatAuth: assert "/us-east-2/bedrock/aws4_request" in authorization assert requests[0]["url"].startswith("https://bedrock-mantle.us-east-2.api.aws") + def test_completion_per_request_role_reaches_signer_and_not_the_body( + self, monkeypatch + ): + # Per-request aws_role_name/aws_session_name are stripped from optional_params + # for non-bedrock providers, so they must be sourced from litellm_params at + # signing time, and must never be serialized into the provider request body. + from botocore.credentials import Credentials + + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + for var in ( + "BEDROCK_MANTLE_API_KEY", + "AWS_BEARER_TOKEN_BEDROCK", + "BEDROCK_MANTLE_API_BASE", + ): + monkeypatch.delenv(var, raising=False) + + credential_calls = [] + + def fake_get_credentials(self, **kwargs): + credential_calls.append(kwargs) + return Credentials( + access_key="ASIAEXAMPLE", + secret_key="YXNzdW1lZC1yb2xlLXNlY3JldC1hc3N1bWVk", + token="assumed-session-token", + ) + + monkeypatch.setattr(BaseAWSLLM, "get_credentials", fake_get_credentials) + + requests = [] + + def mock_post(self, url, data=None, headers=None, **kwargs): + raw_body = data.decode("utf-8") if isinstance(data, bytes) else data + requests.append({"headers": headers or {}, "body": json.loads(raw_body or "{}")}) + return httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1733529600, + "model": "google.gemma-4-31b", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + }, + request=httpx.Request("POST", url), + ) + + with patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post + ): + litellm.completion( + model="bedrock_mantle/google.gemma-4-31b", + messages=[{"role": "user", "content": "hello"}], + aws_role_name="arn:aws:iam::000000000000:role/attributed-role", + aws_session_name="user-123", + aws_region_name="us-east-1", + ) + + assert len(credential_calls) == 1 + assert ( + credential_calls[0]["aws_role_name"] + == "arn:aws:iam::000000000000:role/attributed-role" + ) + assert credential_calls[0]["aws_session_name"] == "user-123" + assert requests[0]["headers"]["Authorization"].startswith("AWS4-HMAC-SHA256") + assert not [key for key in requests[0]["body"] if key.startswith("aws_")] + class TestBedrockMantleProjectHeader: def test_validate_environment_sets_openai_project_header(self): diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index fddd8d09dfc..0906b39c514 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -2071,3 +2071,22 @@ async def test_anthropic_invalid_thinking_signature_retry_resigns_bedrock_reques retry_authorization = posts[1]["headers"]["Authorization"] assert retry_authorization.startswith("AWS4-HMAC-SHA256") assert retry_authorization != first_attempt_headers["Authorization"] + + +def test_aws_signing_overrides_only_fills_missing_credentials(): + from litellm.llms.custom_httpx.llm_http_handler import _aws_signing_overrides + + overrides = _aws_signing_overrides( + {"temperature": 0.2, "aws_region_name": "us-west-2"}, + { + "aws_role_name": "arn:aws:iam::000000000000:role/attributed", + "aws_session_name": "user-123", + "aws_region_name": "us-east-1", + "api_key": "not-an-aws-param", + }, + ) + + assert dict(overrides) == { + "aws_role_name": "arn:aws:iam::000000000000:role/attributed", + "aws_session_name": "user-123", + } From 2c7de60692d7a4fcd53964872d4355042d52b90b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:53:40 +0000 Subject: [PATCH 008/480] style: ruff format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/custom_httpx/llm_http_handler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index dfc234d08e0..cadf4c701e2 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -253,9 +253,7 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool: return False -def _aws_signing_overrides( - optional_params: Mapping[str, Any], litellm_params: Mapping[str, Any] -) -> Mapping[str, Any]: +def _aws_signing_overrides(optional_params: Mapping[str, Any], litellm_params: Mapping[str, Any]) -> Mapping[str, Any]: """AWS credential params for SigV4 signers that read them off optional_params. Only `bedrock`/`sagemaker` keep `aws_*` in optional_params: every other provider From 7bac4a41af549070d7084174f9e6e52fea282300 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:17:22 -0700 Subject: [PATCH 009/480] fix(prompts): key the in-memory prompt registry by environment LiteLLM_PromptTable is unique on (prompt_id, version, environment) and version numbering restarts at 1 per environment, but the in-memory registry keyed prompts as {prompt_id}.v{version} with no environment, so environments sharing a prompt id shadowed each other and only one environment's template ever served. Registry entries are now keyed {versioned_id}::{environment}, and serve time resolution goes through resolve_prompt_spec(base_id, version, environment): production > staging > development when no environment is requested, latest version within the chosen environment when no version is requested. Chat requests can pin an environment with a new optional prompt_environment body param, filtered from provider-bound params like prompt_id and prompt_version. The newest-updated_at dedupe in _init_prompts_in_db is dropped since registry keys can no longer collide, and the key-parsing serve helpers plus dead registry getters are removed --- litellm/proxy/prompts/prompt_endpoints.py | 247 +++--------------- litellm/proxy/prompts/prompt_registry.py | 174 +++++++++--- litellm/proxy/proxy_server.py | 11 +- litellm/proxy/utils.py | 28 +- litellm/types/utils.py | 1 + .../proxy/prompts/test_prompt_endpoints.py | 96 +------ .../prompts/test_prompt_endpoints_crud.py | 68 ++--- .../proxy/prompts/test_prompt_environment.py | 12 +- .../proxy/prompts/test_prompt_registry.py | 127 ++++++++- tests/test_litellm/proxy/test_proxy_server.py | 63 +++-- .../proxy_logging/test_guardrail_pipeline.py | 54 +++- 11 files changed, 413 insertions(+), 468 deletions(-) diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index 425ff7572d0..32fd1d53777 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -29,6 +29,12 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_utils import is_request_body_safe from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.path_utils import safe_filename +from litellm.proxy.prompts.prompt_registry import ( + DEFAULT_PROMPT_ENVIRONMENT, + get_base_prompt_id, + get_version_number, + prompt_environment_or_default, +) from litellm.repositories.table_repositories import PromptRepository from litellm.types.prompts.init_prompts import ( ListPromptsResponse, @@ -102,165 +108,20 @@ def _prompt_table(prisma_client: "PrismaClient") -> _PromptTableActions: return PromptRepository(prisma_client).table -def get_base_prompt_id(prompt_id: str) -> str: - """ - Extract the base prompt ID by stripping the version suffix if present. - - Args: - prompt_id: Prompt ID that may include version suffix (e.g., "jack_success.v1" or "jack_success_v1") - - Returns: - Base prompt ID without version suffix (e.g., "jack_success") - - Examples: - >>> get_base_prompt_id("jack_success.v1") - "jack_success" - >>> get_base_prompt_id("jack_success_v1") - "jack_success" - >>> get_base_prompt_id("jack_success") - "jack_success" - """ - # Try dot separator first (.v) - if ".v" in prompt_id: - return prompt_id.split(".v")[0] - # Try underscore separator (_v) - if "_v" in prompt_id: - return prompt_id.split("_v")[0] - return prompt_id - - -def get_version_number(prompt_id: str) -> int: - """ - Extract the version number from a versioned prompt ID. - - Args: - prompt_id: Prompt ID that may include version suffix (e.g., "jack_success.v2" or "jack_success_v2") - - Returns: - Version number (defaults to 1 if no version suffix or invalid format) - - Examples: - >>> get_version_number("jack_success.v2") - 2 - >>> get_version_number("jack_success_v2") - 2 - >>> get_version_number("jack_success") - 1 - """ - # Try dot separator first (.v) - if ".v" in prompt_id: - version_str = prompt_id.split(".v")[1] - try: - return int(version_str) - except ValueError: - pass - - # Try underscore separator (_v) - if "_v" in prompt_id: - version_str = prompt_id.split("_v")[1] - try: - return int(version_str) - except ValueError: - pass - - return 1 - - -def construct_versioned_prompt_id(prompt_id: str, version: int | None = None) -> str: - """ - Construct a versioned prompt ID from a base prompt_id and version number. - - Args: - prompt_id: Base prompt ID (e.g., "jack_success") - version: Version number (if None, returns the base prompt_id unchanged) - - Returns: - Versioned prompt ID (e.g., "jack_success.v4") - - Examples: - >>> construct_versioned_prompt_id("jack_success", 4) - "jack_success.v4" - >>> construct_versioned_prompt_id("jack_success", None) - "jack_success" - >>> construct_versioned_prompt_id("jack_success.v2", 4) - "jack_success.v4" - """ - if version is None: - return prompt_id - - # Strip any existing version suffix first - base_id: Final = get_base_prompt_id(prompt_id) - return f"{base_id}.v{version}" - - -def get_latest_version_prompt_id(prompt_id: str, all_prompt_ids: Mapping[str, object]) -> str: - """ - Find the latest version of a prompt from available prompt IDs. - - Args: - prompt_id: Base prompt ID or versioned prompt ID (e.g., "jack_success" or "jack_success.v2") - all_prompt_ids: Dictionary of all available prompt IDs (keys are prompt IDs) - - Returns: - The prompt ID with the highest version number, or the original prompt_id if no versions exist - - Examples: - >>> all_ids = {"jack.v1": {}, "jack.v2": {}, "jack.v3": {}} - >>> get_latest_version_prompt_id("jack", all_ids) - "jack.v3" - >>> get_latest_version_prompt_id("jack.v1", all_ids) - "jack.v3" - >>> all_ids = {"simple": {}} - >>> get_latest_version_prompt_id("simple", all_ids) - "simple" - """ - base_id: Final = get_base_prompt_id(prompt_id=prompt_id) - - # Find all versions of this prompt - matching_versions: Final = [] - for stored_prompt_id in all_prompt_ids: - if get_base_prompt_id(prompt_id=stored_prompt_id) == base_id: - version_num = get_version_number(prompt_id=stored_prompt_id) - matching_versions.append((version_num, stored_prompt_id)) - - # Use the highest version number - if matching_versions: - matching_versions.sort(reverse=True) - return matching_versions[0][1] - else: - # No versioned prompts found, use the base ID as-is - return prompt_id - - def get_latest_prompt_versions(prompts: list[PromptSpec]) -> list[PromptSpec]: """ - Filter a list of prompts to return only the latest version of each unique prompt. - - Args: - prompts: List of PromptSpec objects - - Returns: - List of PromptSpec objects with only the latest version of each prompt + Filter prompts down to the latest version per (base prompt id, environment). """ - latest_prompts: Final[dict[str, PromptSpec]] = {} - - for prompt in prompts: - base_id = get_base_prompt_id(prompt_id=prompt.prompt_id) - version = get_version_number(prompt_id=prompt.prompt_id) - - # Keep the prompt with the highest version number - if base_id not in latest_prompts: - latest_prompts[base_id] = prompt - else: - existing_version = get_version_number(prompt_id=latest_prompts[base_id].prompt_id) - if version > existing_version: - latest_prompts[base_id] = prompt - + sorted_prompts: Final = sorted(prompts, key=lambda prompt: get_version_number(prompt_id=prompt.prompt_id)) + latest_prompts: Final = { + (get_base_prompt_id(prompt_id=prompt.prompt_id), prompt_environment_or_default(prompt.environment)): prompt + for prompt in sorted_prompts + } return list(latest_prompts.values()) async def get_next_version_for_prompt( - prisma_client: "PrismaClient", prompt_id: str, environment: str = "development" + prisma_client: "PrismaClient", prompt_id: str, environment: str = DEFAULT_PROMPT_ENVIRONMENT ) -> int: """ Get the next version number for a prompt in a specific environment. @@ -403,11 +264,14 @@ async def list_prompts( if key_metadata is not None: prompts: Final = cast(list[str] | None, key_metadata.get("prompts", None)) if prompts is not None: - all_prompts = [ - IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[prompt_id] - for prompt_id in prompts - if prompt_id in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS + allowed_prompt_ids: Final = frozenset(prompts) + allowed_prompts: Final = [ + spec + for spec in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values() + if spec.prompt_id in allowed_prompt_ids + or get_base_prompt_id(prompt_id=spec.prompt_id) in allowed_prompt_ids ] + all_prompts = get_latest_prompt_versions(prompts=allowed_prompts) if environment: all_prompts = [p for p in all_prompts if p.environment == environment] prompt_list: Final = [] @@ -576,7 +440,7 @@ def _get_prompt_template(prompt_spec: PromptSpec, base_prompt_id: str) -> Prompt metadata=parsed.get("metadata"), ) else: - prompt_callback: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id(prompt_spec.prompt_id) + prompt_callback: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=prompt_spec) if prompt_callback is not None: integration_name: Final = prompt_callback.integration_name if integration_name == "dotprompt": @@ -690,15 +554,8 @@ async def get_prompt_info( if env_prompts: prompt_spec = create_versioned_prompt_spec(db_prompt=env_prompts[0]) - # Fallback: use in-memory registry (no environment filter) if prompt_spec is None and environment is None: - prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) - if prompt_spec is None: - latest_prompt_id: Final = get_latest_version_prompt_id( - prompt_id=prompt_id, - all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS, - ) - prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id) + prompt_spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec(prompt_id, version=requested_version) if prompt_spec is None: raise HTTPException( @@ -785,7 +642,7 @@ async def create_prompt( environment: Final = ( request.prompt_info.environment if request.prompt_info and request.prompt_info.environment - else "development" + else DEFAULT_PROMPT_ENVIRONMENT ) # Get next version number @@ -885,7 +742,7 @@ async def update_prompt( environment: Final = ( request.prompt_info.environment if request.prompt_info and request.prompt_info.environment - else "development" + else DEFAULT_PROMPT_ENVIRONMENT ) # Check if any version of this prompt exists (in any environment) @@ -897,9 +754,7 @@ async def update_prompt( detail=f"Prompt with ID {base_prompt_id} not found", ) - # Check if it's a config prompt - existing_in_memory: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) - if existing_in_memory and existing_in_memory.prompt_info.prompt_type == "config": + if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id): raise HTTPException( status_code=400, detail="Cannot update config prompts.", @@ -988,52 +843,24 @@ async def delete_prompt( raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) try: - # Try to get prompt directly first - existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) - - # If not found, try to find the latest version - if existing_prompt is None: - latest_prompt_id: Final = get_latest_version_prompt_id( - prompt_id=prompt_id, - all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS, - ) - existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id) - # Use the resolved prompt_id for deletion - prompt_id = latest_prompt_id + base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) + existing_prompt: Final = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec(prompt_id, environment=environment) if existing_prompt is None: raise HTTPException(status_code=404, detail=f"Prompt with ID {prompt_id} not found") - if existing_prompt.prompt_info.prompt_type == "config": + if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id): raise HTTPException( status_code=400, detail="Cannot delete config prompts.", ) - # Get the base prompt ID (without version suffix) for database deletion - base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) - - # Build delete filter; scope to environment if provided - delete_where: Final[dict[str, str]] = {"prompt_id": base_prompt_id} - if environment: - delete_where["environment"] = environment - - # Delete versions from the database (scoped to environment if provided) + delete_where: Final[dict[str, str]] = { + "prompt_id": base_prompt_id, + **({"environment": environment} if environment else {}), + } await _prompt_table(prisma_client).delete_many(where=delete_where) - - # Remove matching prompts from memory — scope to environment if provided - if environment: - prompts_to_delete: Final = [ - pid - for pid, prompt in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.items() - if get_base_prompt_id(prompt_id=pid) == base_prompt_id and prompt.environment == environment - ] - for pid in prompts_to_delete: - del IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[pid] - if pid in IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt: - del IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt[pid] - else: - IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id) + IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id=base_prompt_id, environment=environment) env_msg: Final = f" from {environment}" if environment else "" return {"message": f"Prompt {base_prompt_id} deleted successfully{env_msg}"} @@ -1105,7 +932,7 @@ async def patch_prompt( try: # Resolve the target row: find the latest version in the given environment base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) - env: Final = environment or "development" + env: Final = prompt_environment_or_default(environment) requested_version: Final = get_version_number(prompt_id=prompt_id) if prompt_id != base_prompt_id else None # Build query to find the exact row by composite unique key @@ -1129,11 +956,7 @@ async def patch_prompt( target_row: Final = db_rows[0] - # Check if prompt exists in memory - versioned_id: Final = f"{base_prompt_id}.v{target_row.version}" - existing_prompt: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(versioned_id) - - if existing_prompt and existing_prompt.prompt_info.prompt_type == "config": + if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id): raise HTTPException( status_code=400, detail="Cannot update config prompts.", diff --git a/litellm/proxy/prompts/prompt_registry.py b/litellm/proxy/prompts/prompt_registry.py index d4342773a85..b575184a229 100644 --- a/litellm/proxy/prompts/prompt_registry.py +++ b/litellm/proxy/prompts/prompt_registry.py @@ -1,6 +1,6 @@ import importlib import os -from collections.abc import Callable +from collections.abc import Callable, Sequence from pathlib import Path from typing import Final @@ -14,6 +14,77 @@ from litellm.types.prompts.init_prompts import ( prompt_initializer_registry = {} +DEFAULT_PROMPT_ENVIRONMENT: Final = "development" +PROMPT_ENVIRONMENT_SERVE_PRECEDENCE: Final = ("production", "staging", "development") + + +def get_base_prompt_id(prompt_id: str) -> str: + """ + Extract the base prompt ID by stripping the version suffix if present. + + Examples: + >>> get_base_prompt_id("jack_success.v1") + "jack_success" + >>> get_base_prompt_id("jack_success_v1") + "jack_success" + >>> get_base_prompt_id("jack_success") + "jack_success" + """ + if ".v" in prompt_id: + return prompt_id.split(".v")[0] + if "_v" in prompt_id: + return prompt_id.split("_v")[0] + return prompt_id + + +def get_version_number(prompt_id: str) -> int: + """ + Extract the version number from a versioned prompt ID (defaults to 1). + + Examples: + >>> get_version_number("jack_success.v2") + 2 + >>> get_version_number("jack_success_v2") + 2 + >>> get_version_number("jack_success") + 1 + """ + if ".v" in prompt_id: + version_str = prompt_id.split(".v")[1] + try: + return int(version_str) + except ValueError: + pass + + if "_v" in prompt_id: + version_str = prompt_id.split("_v")[1] + try: + return int(version_str) + except ValueError: + pass + + return 1 + + +def prompt_environment_or_default(environment: str | None) -> str: + return environment or DEFAULT_PROMPT_ENVIRONMENT + + +def registry_key_for_prompt(prompt: PromptSpec) -> str: + return f"{prompt.prompt_id}::{prompt_environment_or_default(prompt.environment)}" + + +def _spec_version(prompt: PromptSpec) -> int: + return prompt.version if prompt.version is not None else get_version_number(prompt_id=prompt.prompt_id) + + +def _default_serve_environment(prompts: Sequence[PromptSpec]) -> str: + present: Final = frozenset(prompt_environment_or_default(prompt.environment) for prompt in prompts) + ladder_pick: Final = next((env for env in PROMPT_ENVIRONMENT_SERVE_PRECEDENCE if env in present), None) + if ladder_pick is not None: + return ladder_pick + return min(present) if present else DEFAULT_PROMPT_ENVIRONMENT + def get_prompt_initializer_from_integrations(): """ @@ -113,17 +184,16 @@ class InMemoryPromptRegistry: """ import litellm - prompt_id: Final = prompt.prompt_id - if prompt_id in self.IN_MEMORY_PROMPTS: - verbose_proxy_logger.debug("prompt_id already exists in IN_MEMORY_PROMPTS") - return self.IN_MEMORY_PROMPTS[prompt_id] + registry_key: Final = registry_key_for_prompt(prompt) + if registry_key in self.IN_MEMORY_PROMPTS: + verbose_proxy_logger.debug("prompt already exists in IN_MEMORY_PROMPTS") + return self.IN_MEMORY_PROMPTS[registry_key] parsed_prompt, custom_prompt_callback = self._build_prompt_callback(prompt=prompt) litellm.logging_callback_manager.add_litellm_callback(custom_prompt_callback) - # store references to the prompt in memory - self.IN_MEMORY_PROMPTS[prompt_id] = parsed_prompt - self.prompt_id_to_custom_prompt[prompt_id] = custom_prompt_callback + self.IN_MEMORY_PROMPTS[registry_key] = parsed_prompt + self.prompt_id_to_custom_prompt[registry_key] = custom_prompt_callback return parsed_prompt @@ -166,57 +236,85 @@ class InMemoryPromptRegistry: import litellm parsed_prompt, new_callback = self._build_prompt_callback(prompt=prompt) - stale_callback: Final = self.prompt_id_to_custom_prompt.pop(prompt.prompt_id, None) - self.IN_MEMORY_PROMPTS.pop(prompt.prompt_id, None) + registry_key: Final = registry_key_for_prompt(parsed_prompt) + stale_callback: Final = self.prompt_id_to_custom_prompt.pop(registry_key, None) + self.IN_MEMORY_PROMPTS.pop(registry_key, None) if stale_callback is not None: litellm.logging_callback_manager.remove_callback_from_all_lists(stale_callback) litellm.logging_callback_manager.add_litellm_callback(new_callback) - self.IN_MEMORY_PROMPTS[prompt.prompt_id] = parsed_prompt - self.prompt_id_to_custom_prompt[prompt.prompt_id] = new_callback + self.IN_MEMORY_PROMPTS[registry_key] = parsed_prompt + self.prompt_id_to_custom_prompt[registry_key] = new_callback return parsed_prompt def sync_prompt_from_db(self, prompt: PromptSpec) -> PromptSpec | None: - existing: Final = self.IN_MEMORY_PROMPTS.get(prompt.prompt_id) + existing: Final = self.IN_MEMORY_PROMPTS.get(registry_key_for_prompt(prompt)) if existing is None: return self.initialize_prompt(prompt=prompt) if existing.litellm_params == prompt.litellm_params and existing.prompt_info == prompt.prompt_info: return existing return self.reload_prompt(prompt=prompt) - def get_prompt_by_id(self, prompt_id: str) -> PromptSpec | None: + def resolve_prompt_spec( + self, + prompt_id: str, + version: int | None = None, + environment: str | None = None, + ) -> PromptSpec | None: """ - Get a prompt by its ID from memory - """ - return self.IN_MEMORY_PROMPTS.get(prompt_id) + Resolve a prompt spec by base prompt id, optional version, and optional environment. - def get_prompt_callback_by_id(self, prompt_id: str) -> CustomPromptManagement | None: + With no environment, resolves within the default serve environment + (production > staging > development > alphabetical first present). + With no version, resolves to the highest version in the chosen environment. """ - Get a prompt callback by its ID from memory + base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) + base_matches: Final = tuple( + spec + for spec in self.IN_MEMORY_PROMPTS.values() + if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id + ) + if not base_matches: + return None + resolved_environment: Final = ( + environment if environment is not None else _default_serve_environment(base_matches) + ) + env_matches: Final = tuple( + spec for spec in base_matches if prompt_environment_or_default(spec.environment) == resolved_environment + ) + if not env_matches: + return None + if version is not None: + return next((spec for spec in env_matches if _spec_version(spec) == version), None) + return max(env_matches, key=_spec_version) + + def get_prompt_callback_for_prompt(self, prompt: PromptSpec) -> CustomPromptManagement | None: + return self.prompt_id_to_custom_prompt.get(registry_key_for_prompt(prompt)) + + def has_config_prompt(self, base_prompt_id: str) -> bool: + return any( + spec.prompt_info.prompt_type == "config" + for spec in self.IN_MEMORY_PROMPTS.values() + if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id + ) + + def delete_prompts_by_base_id(self, base_prompt_id: str, environment: str | None = None) -> list[str]: """ - return self.prompt_id_to_custom_prompt.get(prompt_id) + Delete matching prompts from memory, scoped to one environment when given. - def delete_prompts_by_base_id(self, base_prompt_id: str) -> list[str]: + Returns the registry keys that were deleted. """ - Delete all prompts matching the given base prompt ID from memory. - - Args: - base_prompt_id: The base prompt ID (without version suffix) - - Returns: - List of prompt IDs that were deleted - """ - from litellm.proxy.prompts.prompt_endpoints import get_base_prompt_id - - prompts_to_delete: Final = [ - pid for pid in self.IN_MEMORY_PROMPTS if get_base_prompt_id(prompt_id=pid) == base_prompt_id + keys_to_delete: Final = [ + key + for key, spec in self.IN_MEMORY_PROMPTS.items() + if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id + and (environment is None or prompt_environment_or_default(spec.environment) == environment) ] - for pid in prompts_to_delete: - del self.IN_MEMORY_PROMPTS[pid] - if pid in self.prompt_id_to_custom_prompt: - del self.prompt_id_to_custom_prompt[pid] + for key in keys_to_delete: + del self.IN_MEMORY_PROMPTS[key] + self.prompt_id_to_custom_prompt.pop(key, None) - return prompts_to_delete + return keys_to_delete IN_MEMORY_PROMPT_REGISTRY: Final = InMemoryPromptRegistry() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2cfe08fe332..bc4eaaab761 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7273,16 +7273,7 @@ class ProxyConfig: parsed_specs: Final[tuple[PromptSpec, ...]] = tuple( spec for row in prompts_in_db if (spec := parse_row(row)) is not None ) - newest_spec_per_id: Final[Mapping[str, PromptSpec]] = MappingProxyType( - { - spec.prompt_id: spec - for spec in sorted( - parsed_specs, - key=lambda s: s.updated_at.timestamp() if s.updated_at else float("-inf"), - ) - } - ) - for prompt_spec in newest_spec_per_id.values(): + for prompt_spec in parsed_specs: try: IN_MEMORY_PROMPT_REGISTRY.sync_prompt_from_db(prompt=prompt_spec) except Exception as prompt_sync_error: # noqa: BLE001 # one poisoned row must not block syncing the remaining prompts diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8cbf5b685fd..95f02279f19 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1397,27 +1397,26 @@ class ProxyLogging: ) -> None: """Process prompt template if applicable.""" - from litellm.proxy.prompts.prompt_endpoints import ( - construct_versioned_prompt_id, - get_latest_version_prompt_id, - ) from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY from litellm.utils import get_non_default_completion_params - if prompt_version is None: - lookup_prompt_id = get_latest_version_prompt_id( - prompt_id=prompt_id, - all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS, - ) - else: - lookup_prompt_id = construct_versioned_prompt_id(prompt_id=prompt_id, version=prompt_version) - - custom_logger: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id(lookup_prompt_id) - prompt_spec: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(lookup_prompt_id) + raw_prompt_environment: Final = data.get("prompt_environment", None) + prompt_environment: Final = raw_prompt_environment if isinstance(raw_prompt_environment, str) else None + prompt_spec: Final = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec( + prompt_id, + version=prompt_version, + environment=prompt_environment, + ) + custom_logger: Final = ( + IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=prompt_spec) + if prompt_spec is not None + else None + ) litellm_prompt_id: str | None = None if prompt_spec is not None: litellm_prompt_id = prompt_spec.litellm_params.prompt_id data.pop("prompt_id", None) + data.pop("prompt_environment", None) if custom_logger and prompt_spec is not None: ( @@ -1444,6 +1443,7 @@ class ProxyLogging: data.pop("prompt_variables", None) data.pop("prompt_label", None) data.pop("prompt_version", None) + data.pop("prompt_environment", None) def _process_guardrail_metadata(self, data: dict) -> None: """Process guardrails from metadata and add to applied_guardrails.""" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ef3586f2559..27437b74251 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3522,6 +3522,7 @@ all_litellm_params = ( "litellm_system_prompt", "provider_specific_header", "prompt_version", + "prompt_environment", "api_base", "force_timeout", "logger_fn", diff --git a/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py b/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py index 6ebb10eff76..35402db219f 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py @@ -104,97 +104,6 @@ class TestPromptVersioning: assert get_base_prompt_id(prompt_id="jack") == "jack" assert get_base_prompt_id(prompt_id="my_prompt.v10") == "my_prompt" - def test_get_latest_version_prompt_id(self): - """ - Test that get_latest_version_prompt_id returns the highest version - """ - from litellm.proxy.prompts.prompt_endpoints import get_latest_version_prompt_id - - # Mock prompt IDs dictionary - all_prompt_ids = { - "jack.v1": {}, - "jack.v2": {}, - "jack.v3": {}, - "jane.v1": {}, - "simple_prompt": {}, - } - - # Test with base prompt ID - should return latest version - assert ( - get_latest_version_prompt_id( - prompt_id="jack", all_prompt_ids=all_prompt_ids - ) - == "jack.v3" - ) - - # Test with versioned prompt ID - should still return latest version - assert ( - get_latest_version_prompt_id( - prompt_id="jack.v1", all_prompt_ids=all_prompt_ids - ) - == "jack.v3" - ) - - # Test with single version - assert ( - get_latest_version_prompt_id( - prompt_id="jane", all_prompt_ids=all_prompt_ids - ) - == "jane.v1" - ) - - # Test with non-versioned prompt - assert ( - get_latest_version_prompt_id( - prompt_id="simple_prompt", all_prompt_ids=all_prompt_ids - ) - == "simple_prompt" - ) - - # Test with non-existent prompt - assert ( - get_latest_version_prompt_id( - prompt_id="nonexistent", all_prompt_ids=all_prompt_ids - ) - == "nonexistent" - ) - - def test_construct_versioned_prompt_id(self): - """ - Test that construct_versioned_prompt_id correctly builds versioned IDs - """ - from litellm.proxy.prompts.prompt_endpoints import construct_versioned_prompt_id - - # Test with base prompt ID and version - assert ( - construct_versioned_prompt_id(prompt_id="jack_success", version=4) - == "jack_success.v4" - ) - - # Test with None version - should return base ID unchanged - assert ( - construct_versioned_prompt_id(prompt_id="jack_success", version=None) - == "jack_success" - ) - - # Test with existing versioned ID - should replace version - assert ( - construct_versioned_prompt_id(prompt_id="jack_success.v2", version=4) - == "jack_success.v4" - ) - - # Test with hyphenated prompt ID - assert ( - construct_versioned_prompt_id(prompt_id="my-prompt", version=1) - == "my-prompt.v1" - ) - - # Test with double-digit version - assert ( - construct_versioned_prompt_id(prompt_id="test_prompt", version=10) - == "test_prompt.v10" - ) - class TestPromptVersionsEndpoint: """ @@ -444,7 +353,7 @@ class TestAdminViewerReadAccess: "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry, ): - mock_registry.get_prompt_by_id.return_value = PromptSpec( + mock_registry.resolve_prompt_spec.return_value = PromptSpec( prompt_id="jack.v2", litellm_params=PromptLiteLLMParams( prompt_id="jack", @@ -453,8 +362,7 @@ class TestAdminViewerReadAccess: ), prompt_info=PromptInfo(prompt_type="db"), ) - mock_registry.IN_MEMORY_PROMPTS = {"jack.v1": {}, "jack.v2": {}} - mock_registry.get_prompt_callback_by_id.return_value = None + mock_registry.get_prompt_callback_for_prompt.return_value = None response = await get_prompt_info(prompt_id="jack", user_api_key_dict=viewer) diff --git a/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py b/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py index 688b739fb5a..6916a7163f8 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py @@ -52,8 +52,6 @@ async def test_delete_prompt_success(): with patch( "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry: - # User passes "test_prompt.v2" - # We simulate that get_prompt_by_id returns the prompt spec for v2 prompt_spec = PromptSpec( prompt_id="test_prompt.v2", litellm_params=PromptLiteLLMParams( @@ -61,7 +59,8 @@ async def test_delete_prompt_success(): ), prompt_info=PromptInfo(prompt_type="db"), ) - mock_registry.get_prompt_by_id.return_value = prompt_spec + mock_registry.resolve_prompt_spec.return_value = prompt_spec + mock_registry.has_config_prompt.return_value = False # Patch the prisma client in the endpoint module with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): @@ -79,7 +78,7 @@ async def test_delete_prompt_success(): # 2. Memory deletion should use base ID mock_registry.delete_prompts_by_base_id.assert_called_once_with( - expected_base_id + base_prompt_id=expected_base_id, environment=None ) assert response == { @@ -108,31 +107,14 @@ async def test_delete_prompt_by_base_id_success(): with patch( "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry: - # User passes "test_prompt" (base ID) - # 1. get_prompt_by_id("test_prompt") -> None (if it's not registered as base) - # 2. It calls get_latest_version_prompt_id -> returns "test_prompt.v3" - # 3. get_prompt_by_id("test_prompt.v3") -> returns Spec - - # Setup mocks behavior - def get_prompt_side_effect(prompt_id): - if prompt_id == "test_prompt": - return None - if prompt_id == "test_prompt.v3": - return PromptSpec( - prompt_id="test_prompt.v3", - litellm_params=PromptLiteLLMParams( - prompt_id="test_prompt", prompt_integration="dotprompt" - ), - prompt_info=PromptInfo(prompt_type="db"), - ) - return None - - mock_registry.get_prompt_by_id.side_effect = get_prompt_side_effect - mock_registry.IN_MEMORY_PROMPTS = { - "test_prompt.v1": {}, - "test_prompt.v2": {}, - "test_prompt.v3": {}, - } + mock_registry.resolve_prompt_spec.return_value = PromptSpec( + prompt_id="test_prompt.v3", + litellm_params=PromptLiteLLMParams( + prompt_id="test_prompt", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db"), + ) + mock_registry.has_config_prompt.return_value = False # Patch the prisma client in the endpoint module with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): @@ -150,7 +132,7 @@ async def test_delete_prompt_by_base_id_success(): # 2. Memory deletion should use base ID mock_registry.delete_prompts_by_base_id.assert_called_once_with( - expected_base_id + base_prompt_id=expected_base_id, environment=None ) assert response == { @@ -187,24 +169,8 @@ async def test_get_prompt_info_by_base_id(): prompt_info=PromptInfo(prompt_type="db"), ) - # When get_prompt_by_id is called with "test_prompt", return None (so it searches versions) - # When called with "test_prompt.v3", return the spec - def get_prompt_side_effect(prompt_id): - if prompt_id == "test_prompt": - return None - if prompt_id == "test_prompt.v3": - return prompt_spec_v3 - return None - - mock_registry.get_prompt_by_id.side_effect = get_prompt_side_effect - mock_registry.IN_MEMORY_PROMPTS = { - "test_prompt.v1": {}, - "test_prompt.v2": {}, - "test_prompt.v3": {}, - } - - # We also need to mock get_prompt_callback_by_id to avoid content extraction errors/logic - mock_registry.get_prompt_callback_by_id.return_value = None + mock_registry.resolve_prompt_spec.return_value = prompt_spec_v3 + mock_registry.get_prompt_callback_for_prompt.return_value = None response = await get_prompt_info( prompt_id="test_prompt", user_api_key_dict=mock_user_auth @@ -253,7 +219,7 @@ async def test_patch_prompt_row_deleted_mid_update_returns_404(): "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry, ): - mock_registry.get_prompt_by_id.return_value = existing_prompt + mock_registry.has_config_prompt.return_value = False with pytest.raises(HTTPException) as exc_info: await patch_prompt( @@ -294,7 +260,7 @@ async def test_patch_prompt_merges_unsent_fields_from_db_row_not_stale_memory(): "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry, ): - mock_registry.get_prompt_by_id.return_value = stale_in_memory + mock_registry.has_config_prompt.return_value = False mock_registry.reload_prompt.side_effect = lambda prompt: prompt response = await patch_prompt( @@ -456,7 +422,7 @@ async def test_patch_prompt_info_only_keeps_legacy_keyed_row_patchable(): "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry, ): - mock_registry.get_prompt_by_id.return_value = existing_prompt + mock_registry.has_config_prompt.return_value = False await patch_prompt( prompt_id="agent-prompt", diff --git a/tests/test_litellm/proxy/prompts/test_prompt_environment.py b/tests/test_litellm/proxy/prompts/test_prompt_environment.py index ecd89afefbe..3cb647dea13 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_environment.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_environment.py @@ -191,11 +191,7 @@ async def test_update_prompt_stores_environment_and_created_by(): with patch( "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry: - mock_registry.get_prompt_by_id.return_value = PromptSpec( - prompt_id="my_prompt.v1", - litellm_params=request.litellm_params, - prompt_info=PromptInfo(prompt_type="db"), - ) + mock_registry.has_config_prompt.return_value = False mock_registry.initialize_prompt.return_value = PromptSpec( prompt_id="my_prompt.v2", litellm_params=request.litellm_params, @@ -239,7 +235,8 @@ async def test_delete_prompt_scoped_to_environment(): prompt_info=PromptInfo(prompt_type="db"), environment="staging", ) - mock_registry.get_prompt_by_id.return_value = prompt_spec + mock_registry.resolve_prompt_spec.return_value = prompt_spec + mock_registry.has_config_prompt.return_value = False with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): await delete_prompt( @@ -251,3 +248,6 @@ async def test_delete_prompt_scoped_to_environment(): mock_prisma_client.db.litellm_prompttable.delete_many.assert_called_once_with( where={"prompt_id": "test_prompt", "environment": "staging"} ) + mock_registry.delete_prompts_by_base_id.assert_called_once_with( + base_prompt_id="test_prompt", environment="staging" + ) diff --git a/tests/test_litellm/proxy/prompts/test_prompt_registry.py b/tests/test_litellm/proxy/prompts/test_prompt_registry.py index 47f1ba13627..689cbd79873 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_registry.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_registry.py @@ -1,26 +1,35 @@ import pytest import litellm +from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry from litellm.types.prompts.init_prompts import PromptInfo, PromptLiteLLMParams, PromptSpec -def _db_prompt_spec(content: str) -> PromptSpec: +def _db_prompt_spec(content: str, environment: str = "development", version: int = 1) -> PromptSpec: return PromptSpec( - prompt_id="greeting.v1", + prompt_id=f"greeting.v{version}", litellm_params=PromptLiteLLMParams( prompt_id="greeting", prompt_integration="dotprompt", prompt_data={"content": content, "metadata": {}}, ), prompt_info=PromptInfo(prompt_type="db"), + version=version, + environment=environment, ) -def _served_content(registry: InMemoryPromptRegistry) -> str: - callback = registry.get_prompt_callback_by_id("greeting.v1") +def _resolved_callback(registry: InMemoryPromptRegistry, environment: str | None = None) -> CustomPromptManagement: + spec = registry.resolve_prompt_spec("greeting", environment=environment) + assert spec is not None + callback = registry.get_prompt_callback_for_prompt(prompt=spec) assert callback is not None - return callback.prompt_manager.get_prompt("greeting").content + return callback + + +def _served_content(registry: InMemoryPromptRegistry, environment: str | None = None) -> str: + return _resolved_callback(registry, environment=environment).prompt_manager.get_prompt("greeting").content @pytest.fixture @@ -32,32 +41,34 @@ def isolated_callbacks(monkeypatch: pytest.MonkeyPatch) -> list: def test_sync_prompt_from_db_reloads_row_edited_elsewhere(isolated_callbacks: list) -> None: registry = InMemoryPromptRegistry() registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY")) - stale_callback = registry.get_prompt_callback_by_id("greeting.v1") + stale_callback = _resolved_callback(registry) assert _served_content(registry) == "begin every reply with AHOY" registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY")) assert _served_content(registry) == "begin every reply with HOWDY" - assert registry.get_prompt_by_id("greeting.v1").litellm_params.prompt_data["content"] == "begin every reply with HOWDY" + reloaded_spec = registry.resolve_prompt_spec("greeting", environment="development") + assert reloaded_spec is not None + assert reloaded_spec.litellm_params.prompt_data["content"] == "begin every reply with HOWDY" assert stale_callback not in isolated_callbacks - assert isolated_callbacks == [registry.get_prompt_callback_by_id("greeting.v1")] + assert isolated_callbacks == [_resolved_callback(registry)] def test_sync_prompt_from_db_keeps_unchanged_row_in_place(isolated_callbacks: list) -> None: registry = InMemoryPromptRegistry() registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY")) - first_callback = registry.get_prompt_callback_by_id("greeting.v1") + first_callback = _resolved_callback(registry) registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY")) - assert registry.get_prompt_callback_by_id("greeting.v1") is first_callback + assert _resolved_callback(registry) is first_callback assert isolated_callbacks == [first_callback] def test_reload_prompt_replaces_callback_without_leaking_the_old_one(isolated_callbacks: list) -> None: registry = InMemoryPromptRegistry() registry.initialize_prompt(prompt=_db_prompt_spec("begin every reply with AHOY")) - stale_callback = registry.get_prompt_callback_by_id("greeting.v1") + stale_callback = _resolved_callback(registry) reloaded = registry.reload_prompt(prompt=_db_prompt_spec("begin every reply with HOWDY")) @@ -70,7 +81,7 @@ def test_reload_prompt_replaces_callback_without_leaking_the_old_one(isolated_ca def test_reload_prompt_keeps_the_old_template_when_the_replacement_fails(isolated_callbacks: list) -> None: registry = InMemoryPromptRegistry() registry.initialize_prompt(prompt=_db_prompt_spec("begin every reply with AHOY")) - old_callback = registry.get_prompt_callback_by_id("greeting.v1") + old_callback = _resolved_callback(registry) broken = PromptSpec( prompt_id="greeting.v1", @@ -80,11 +91,101 @@ def test_reload_prompt_keeps_the_old_template_when_the_replacement_fails(isolate prompt_data={"content": "begin every reply with HOWDY", "metadata": {}}, ), prompt_info=PromptInfo(prompt_type="db"), + version=1, + environment="development", ) with pytest.raises(ValueError, match="Unsupported prompt"): registry.reload_prompt(prompt=broken) - assert registry.get_prompt_callback_by_id("greeting.v1") is old_callback + assert _resolved_callback(registry) is old_callback assert _served_content(registry) == "begin every reply with AHOY" assert isolated_callbacks == [old_callback] + + +def test_environments_sharing_a_prompt_id_keep_separate_templates(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment="development")) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY", environment="production")) + + assert _served_content(registry, environment="development") == "begin every reply with AHOY" + assert _served_content(registry, environment="production") == "begin every reply with HOWDY" + assert _resolved_callback(registry, environment="development") is not _resolved_callback( + registry, environment="production" + ) + + +def test_default_resolution_prefers_production(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment="development")) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY", environment="production")) + + assert _served_content(registry) == "begin every reply with HOWDY" + + +@pytest.mark.parametrize("environment", ["staging", "qa"]) +def test_default_resolution_serves_the_only_environment_present(isolated_callbacks: list, environment: str) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment=environment)) + + assert _served_content(registry) == "begin every reply with AHOY" + + +def test_resolution_picks_exact_version_and_latest_within_an_environment(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment="development", version=1)) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with YO", environment="development", version=2)) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY", environment="production", version=1)) + + exact = registry.resolve_prompt_spec("greeting", version=1, environment="development") + assert exact is not None + assert exact.litellm_params.prompt_data["content"] == "begin every reply with AHOY" + + latest = registry.resolve_prompt_spec("greeting", environment="development") + assert latest is not None + assert latest.litellm_params.prompt_data["content"] == "begin every reply with YO" + + assert registry.resolve_prompt_spec("greeting", version=3, environment="development") is None + + +def test_resolution_returns_none_for_unknown_environment_or_prompt(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment="development")) + + assert registry.resolve_prompt_spec("greeting", environment="production") is None + assert registry.resolve_prompt_spec("no_such_prompt") is None + + +def test_delete_prompts_by_base_id_scoped_to_one_environment(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment="development")) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY", environment="production")) + + deleted = registry.delete_prompts_by_base_id(base_prompt_id="greeting", environment="development") + + assert deleted == ["greeting.v1::development"] + assert registry.resolve_prompt_spec("greeting", environment="development") is None + assert _served_content(registry, environment="production") == "begin every reply with HOWDY" + + deleted_rest = registry.delete_prompts_by_base_id(base_prompt_id="greeting") + + assert deleted_rest == ["greeting.v1::production"] + assert registry.resolve_prompt_spec("greeting") is None + + +def test_has_config_prompt_matches_any_version_of_the_base_id(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + config_spec = PromptSpec( + prompt_id="greeting", + litellm_params=PromptLiteLLMParams( + prompt_id="greeting", + prompt_integration="dotprompt", + prompt_data={"content": "begin every reply with AHOY", "metadata": {}}, + ), + prompt_info=PromptInfo(prompt_type="config"), + ) + registry.initialize_prompt(prompt=config_spec) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY", environment="production")) + + assert registry.has_config_prompt(base_prompt_id="greeting") is True + assert registry.has_config_prompt(base_prompt_id="other_prompt") is False diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 4266a13bf11..e2692a99732 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -11379,10 +11379,15 @@ async def test_init_prompts_in_db_reloads_rows_patched_on_another_worker(monkeyp } return row - def served_content() -> str: - callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("greeting_sync.v1") + def served_callback(): + spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec("greeting_sync") + assert spec is not None + callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=spec) assert callback is not None - return callback.prompt_manager.get_prompt("greeting_sync").content + return callback + + def served_content() -> str: + return served_callback().prompt_manager.get_prompt("greeting_sync").content prisma_client = MagicMock() try: @@ -11394,7 +11399,7 @@ async def test_init_prompts_in_db_reloads_rows_patched_on_another_worker(monkeyp await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client) assert served_content() == "Begin every reply with HOWDY" - assert litellm.callbacks == [IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("greeting_sync.v1")] + assert litellm.callbacks == [served_callback()] finally: IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id("greeting_sync") @@ -11433,22 +11438,25 @@ async def test_init_prompts_in_db_syncs_remaining_rows_when_one_row_fails(monkey ) await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client) - assert IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id("broken_sync.v1") is None - assert IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("healthy_sync.v1") is not None - assert litellm.callbacks == [IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("healthy_sync.v1")] + assert IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec("broken_sync") is None + healthy_spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec("healthy_sync") + assert healthy_spec is not None + healthy_callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=healthy_spec) + assert healthy_callback is not None + assert litellm.callbacks == [healthy_callback] finally: IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id("healthy_sync") IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id("broken_sync") @pytest.mark.asyncio -async def test_init_prompts_in_db_serves_the_newest_row_when_environments_collide_on_a_versioned_id(monkeypatch): +async def test_init_prompts_in_db_syncs_every_environment_sharing_a_versioned_id(monkeypatch): from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY from litellm.proxy.proxy_server import ProxyConfig monkeypatch.setattr(litellm, "callbacks", []) - def db_row(environment: str, content: str, updated_at: datetime) -> MagicMock: + def db_row(environment: str, content: str) -> MagicMock: row = MagicMock() row.model_dump.return_value = { "prompt_id": "greeting_env", @@ -11464,30 +11472,41 @@ async def test_init_prompts_in_db_serves_the_newest_row_when_environments_collid ), "prompt_info": json.dumps({"prompt_type": "db"}), "created_at": None, - "updated_at": updated_at, + "updated_at": None, } return row - freshly_patched = db_row( - "production", "Begin every reply with HOWDY", datetime(2026, 8, 26, 12, 0, tzinfo=timezone.utc) - ) - stale_sibling = db_row( - "development", "Begin every reply with AHOY", datetime(2026, 8, 26, 11, 0, tzinfo=timezone.utc) - ) + def served_content(environment: str | None) -> str: + spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec("greeting_env", environment=environment) + assert spec is not None + callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=spec) + assert callback is not None + return callback.prompt_manager.get_prompt("greeting_env").content prisma_client = MagicMock() try: - prisma_client.db.litellm_prompttable.find_many = AsyncMock(return_value=[freshly_patched, stale_sibling]) + prisma_client.db.litellm_prompttable.find_many = AsyncMock( + return_value=[ + db_row("development", "Begin every reply with AHOY"), + db_row("production", "Begin every reply with HOWDY"), + ] + ) await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client) - first_callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("greeting_env.v1") - assert first_callback is not None - assert first_callback.prompt_manager.get_prompt("greeting_env").content == "Begin every reply with HOWDY" + assert served_content("development") == "Begin every reply with AHOY" + assert served_content("production") == "Begin every reply with HOWDY" + assert served_content(None) == "Begin every reply with HOWDY" + prisma_client.db.litellm_prompttable.find_many = AsyncMock( + return_value=[ + db_row("development", "Begin every reply with YO"), + db_row("production", "Begin every reply with HOWDY"), + ] + ) await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client) - assert IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("greeting_env.v1") is first_callback - assert litellm.callbacks == [first_callback] + assert served_content("development") == "Begin every reply with YO" + assert served_content("production") == "Begin every reply with HOWDY" finally: IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id("greeting_env") diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 7df39b0ef82..69310124ed3 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -726,10 +726,7 @@ async def test_process_prompt_template_no_op_when_no_prompt_spec(proxy_logging, from litellm.proxy.prompts import prompt_registry monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_callback_by_id", lambda *a, **kw: None - ) - monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_by_id", lambda *a, **kw: None + prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: None ) data: Dict[str, Any] = {"messages": [{"role": "user"}], "model": "m", "temperature": 0.1} await proxy_logging._process_prompt_template( @@ -752,11 +749,11 @@ async def test_process_prompt_template_applies_when_spec_resolves(proxy_logging, monkeypatch.setattr( prompt_registry.IN_MEMORY_PROMPT_REGISTRY, - "get_prompt_callback_by_id", + "get_prompt_callback_for_prompt", lambda *a, **kw: custom_logger, ) monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_by_id", lambda *a, **kw: prompt_spec + prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: prompt_spec ) logging_obj = MagicMock() @@ -802,11 +799,11 @@ async def test_process_prompt_template_async_get_prompt_error_raises(proxy_loggi prompt_spec.litellm_params = MagicMock(prompt_id="x") monkeypatch.setattr( prompt_registry.IN_MEMORY_PROMPT_REGISTRY, - "get_prompt_callback_by_id", + "get_prompt_callback_for_prompt", lambda *a, **kw: custom_logger, ) monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_by_id", lambda *a, **kw: prompt_spec + prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: prompt_spec ) logging_obj = MagicMock() logging_obj.async_get_chat_completion_prompt = AsyncMock(side_effect=RuntimeError("bad prompt")) @@ -818,3 +815,44 @@ async def test_process_prompt_template_async_get_prompt_error_raises(proxy_loggi prompt_version=None, call_type="completion", ) + + +@pytest.mark.asyncio +async def test_process_prompt_template_resolves_the_requested_environment(proxy_logging, monkeypatch): + from litellm.proxy.prompts import prompt_registry + + prompt_spec = MagicMock() + prompt_spec.litellm_params = MagicMock(prompt_id="greeting") + resolve_calls: list[dict] = [] + + def fake_resolve(prompt_id, version=None, environment=None): + resolve_calls.append({"prompt_id": prompt_id, "version": version, "environment": environment}) + return prompt_spec + + monkeypatch.setattr(prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", fake_resolve) + monkeypatch.setattr( + prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_callback_for_prompt", lambda *a, **kw: MagicMock() + ) + logging_obj = MagicMock() + logging_obj.async_get_chat_completion_prompt = AsyncMock( + return_value=("m", [{"role": "user", "content": "rendered"}], {}) + ) + data: Dict[str, Any] = { + "messages": [{"role": "user", "content": "orig"}], + "model": "m", + "prompt_id": "greeting", + "prompt_version": 1, + "prompt_environment": "development", + } + await proxy_logging._process_prompt_template( + data=data, + litellm_logging_obj=logging_obj, + prompt_id="greeting", + prompt_version=1, + call_type="completion", + ) + + assert resolve_calls == [{"prompt_id": "greeting", "version": 1, "environment": "development"}] + assert "prompt_environment" not in data + assert "prompt_id" not in data + assert data["messages"] == [{"role": "user", "content": "rendered"}] From 6f94554713ea69f108b356ab6691ee2bc72896ac Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:36:16 -0700 Subject: [PATCH 010/480] fix(ui): key prompt table rows by environment and dedupe key prompt options --- .../prompts/_components/PromptTable.tsx | 4 +++- .../components/organisms/create_key_button.tsx | 2 +- .../components/templates/key_edit_view.test.tsx | 16 ++++++++++++++++ .../src/components/templates/key_edit_view.tsx | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx index 47d4f64f254..a64d7dbfe0c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx @@ -74,7 +74,9 @@ const PromptTable: React.FC = ({ prompt.prompt_id || String(index)} + getRowId={(prompt, index) => + prompt.prompt_id ? `${prompt.prompt_id}::${prompt.environment || "development"}` : String(index) + } sortingMode="client" sorting={sorting} onSortingChange={setSorting} diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index e1e6dcfa442..d0da4ddfb2b 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -349,7 +349,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp const fetchPrompts = async () => { try { const response = await getPromptsList(accessToken); - setPromptsList(response.prompts.map((prompt) => prompt.prompt_id)); + setPromptsList(Array.from(new Set(response.prompts.map((prompt) => prompt.prompt_id)))); } catch (error) { console.error("Failed to fetch prompts:", error); } diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index bf8f43b8b5f..08edce6eda1 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -425,6 +425,22 @@ describe("KeyEditView", () => { expect(screen.getByText("Policies")).toBeInTheDocument(); }); + it("lists a prompt existing in several environments once in the dropdown", async () => { + vi.mocked(getPromptsList).mockResolvedValueOnce({ + prompts: [ + { prompt_id: "envgreet", litellm_params: {}, prompt_info: { prompt_type: "db" }, environment: "development" }, + { prompt_id: "envgreet", litellm_params: {}, prompt_info: { prompt_type: "db" }, environment: "production" }, + ], + }); + + renderAs("Admin"); + + const prompts = await screen.findByLabelText(/Prompts/); + await userEvent.type(prompts, "envgreet"); + + expect(await screen.findAllByRole("option", { name: "envgreet" })).toHaveLength(1); + }); + it("should omit both fields and fire neither admin-only request for an internal user", async () => { renderAs("Internal User"); diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index df1af2ca8e9..d4a8a804f29 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -165,7 +165,7 @@ export function KeyEditView({ if (!accessToken) return; try { const response = await getPromptsList(accessToken); - setPromptsList(response.prompts.map((prompt) => prompt.prompt_id)); + setPromptsList(Array.from(new Set(response.prompts.map((prompt) => prompt.prompt_id)))); } catch (error) { console.error("Failed to fetch prompts:", error); } From dcba64ab72a259a14dcd1fd8b631973741338bda Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:16:16 -0700 Subject: [PATCH 011/480] fix(ui): scope prompt row delete and info view to the row's environment --- .../prompts/_components/PromptTable.test.tsx | 8 ++-- .../prompts/_components/PromptTable.tsx | 4 +- .../_components/PromptTableColumns.tsx | 20 ++++++-- .../prompts/_components/index.test.tsx | 48 ++++++++++++++----- .../(dashboard)/prompts/_components/index.tsx | 18 ++++--- .../prompts/_components/prompt_info.test.tsx | 29 +++++++++++ .../prompts/_components/prompt_info.tsx | 17 +++++-- .../src/components/networking.tsx | 7 ++- 8 files changed, 117 insertions(+), 34 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx index 52efd6407f8..edbb897fb2f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx @@ -65,11 +65,13 @@ describe("PromptTable", () => { expect(within(rows[1]).getByText("prompt-older")).toBeInTheDocument(); }); - it("should call onPromptClick when the prompt ID is clicked", async () => { + it("should call onPromptClick with the row's environment, defaulting to development", async () => { const user = userEvent.setup(); render(); await user.click(screen.getByRole("button", { name: "prompt-newer" })); - expect(mockOnPromptClick).toHaveBeenCalledWith("prompt-newer"); + expect(mockOnPromptClick).toHaveBeenCalledWith("prompt-newer", "production"); + await user.click(screen.getByRole("button", { name: "prompt-older" })); + expect(mockOnPromptClick).toHaveBeenCalledWith("prompt-older", "development"); }); it("should label the environment and default missing environments to development", () => { @@ -83,7 +85,7 @@ describe("PromptTable", () => { render(); await user.click(screen.getByTestId("prompt-actions-prompt-newer")); await user.click(await screen.findByTestId("prompt-action-delete")); - expect(mockOnDeleteClick).toHaveBeenCalledWith("prompt-newer", "prompt-newer"); + expect(mockOnDeleteClick).toHaveBeenCalledWith("prompt-newer", "prompt-newer", "production"); }); it("should copy the prompt ID through the actions menu", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx index a64d7dbfe0c..c766042ac44 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx @@ -13,8 +13,8 @@ import { ModelGroupInfo } from "./prompt_utils"; interface PromptTableProps { promptsList: PromptSpec[]; isLoading: boolean; - onPromptClick?: (id: string) => void; - onDeleteClick?: (id: string, name: string) => void; + onPromptClick?: (id: string, environment: string) => void; + onDeleteClick?: (id: string, name: string, environment: string) => void; accessToken: string | null; isAdmin: boolean; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx index ae584ef6df6..f927a6d1486 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx @@ -64,7 +64,7 @@ function PromptModelCell({ prompt, modelHubData }: { prompt: PromptSpec; modelHu interface PromptRowActionsProps { prompt: PromptSpec; isAdmin: boolean; - onDeleteClick?: (id: string, name: string) => void; + onDeleteClick?: (id: string, name: string, environment: string) => void; } function PromptRowActions({ prompt, isAdmin, onDeleteClick }: PromptRowActionsProps) { @@ -91,7 +91,13 @@ function PromptRowActions({ prompt, isAdmin, onDeleteClick }: PromptRowActionsPr onDeleteClick?.(prompt.prompt_id, prompt.prompt_id || "Unknown Prompt")} + onClick={() => + onDeleteClick?.( + prompt.prompt_id, + prompt.prompt_id || "Unknown Prompt", + prompt.environment || "development", + ) + } > Delete @@ -106,8 +112,8 @@ function PromptRowActions({ prompt, isAdmin, onDeleteClick }: PromptRowActionsPr interface PromptTableColumnsDeps { modelHubData: Map; isAdmin: boolean; - onPromptClick?: (id: string) => void; - onDeleteClick?: (id: string, name: string) => void; + onPromptClick?: (id: string, environment: string) => void; + onDeleteClick?: (id: string, name: string, environment: string) => void; } export const getPromptTableColumns = ({ @@ -128,7 +134,11 @@ export const getPromptTableColumns = ({ title={row.original.prompt_id} titleClassName="font-mono text-xs font-normal" className="max-w-60" - onClick={onPromptClick ? () => onPromptClick(row.original.prompt_id) : undefined} + onClick={ + onPromptClick + ? () => onPromptClick(row.original.prompt_id, row.original.environment || "development") + : undefined + } /> ), }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx index d6a4aaea2e1..334339e6f1d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx @@ -15,21 +15,31 @@ vi.mock("./PromptTable", () => ({ __esModule: true, default: ({ isLoading, + onPromptClick, onDeleteClick, }: { isLoading: boolean; - onDeleteClick: (id: string, name: string) => void; + onPromptClick: (id: string, environment: string) => void; + onDeleteClick: (id: string, name: string, environment: string) => void; }) => (
{isLoading ? "table-loading" : "table-loaded"} - +
), })); -vi.mock("./prompt_info", () => ({ __esModule: true, default: () =>
prompt-info-view
})); +vi.mock("./prompt_info", () => ({ + __esModule: true, + default: ({ initialEnvironment }: { initialEnvironment?: string }) => ( +
prompt-info-view:{initialEnvironment ?? "none"}
+ ), +})); vi.mock("./add_prompt_form", () => ({ __esModule: true, default: ({ visible }: { visible: boolean }) => (visible ?
add-prompt-form
: null), @@ -143,6 +153,22 @@ describe("PromptsPanel toolbar", () => { }); }); +describe("PromptsPanel row navigation", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetPromptsList.mockResolvedValue({ prompts: [] } as never); + }); + + it("should open the info view preselected to the clicked row's environment", async () => { + const user = userEvent.setup(); + renderPanel("Admin"); + + await user.click(await screen.findByRole("button", { name: "row-open" })); + + expect(screen.getByText("prompt-info-view:staging")).toBeInTheDocument(); + }); +}); + describe("PromptsPanel delete confirmation", () => { beforeEach(() => { vi.clearAllMocks(); @@ -156,13 +182,13 @@ describe("PromptsPanel delete confirmation", () => { await user.click(await screen.findByRole("button", { name: "row-delete" })); - expect(await screen.findByText(/delete prompt: my-prompt/i)).toBeInTheDocument(); + expect(await screen.findByText(/the staging copy of prompt: my-prompt/i)).toBeInTheDocument(); expect(screen.getByText(/cannot be undone/i)).toBeInTheDocument(); expect(mockDeletePromptCall).not.toHaveBeenCalled(); await user.click(screen.getByRole("button", { name: /^delete$/i })); - await waitFor(() => expect(mockDeletePromptCall).toHaveBeenCalledWith("sk-test", "prompt-1")); + await waitFor(() => expect(mockDeletePromptCall).toHaveBeenCalledWith("sk-test", "prompt-1", "staging")); }); it("should abandon the delete when the confirmation is dismissed", async () => { @@ -170,11 +196,11 @@ describe("PromptsPanel delete confirmation", () => { renderPanel("Admin"); await user.click(await screen.findByRole("button", { name: "row-delete" })); - await screen.findByText(/delete prompt: my-prompt/i); + await screen.findByText(/the staging copy of prompt: my-prompt/i); await user.click(screen.getByRole("button", { name: /cancel/i })); - await waitFor(() => expect(screen.queryByText(/delete prompt: my-prompt/i)).not.toBeInTheDocument()); + await waitFor(() => expect(screen.queryByText(/the staging copy of prompt: my-prompt/i)).not.toBeInTheDocument()); expect(mockDeletePromptCall).not.toHaveBeenCalled(); }); @@ -189,14 +215,14 @@ describe("PromptsPanel delete confirmation", () => { renderPanel("Admin"); await user.click(await screen.findByRole("button", { name: "row-delete" })); - await screen.findByText(/delete prompt: my-prompt/i); + await screen.findByText(/the staging copy of prompt: my-prompt/i); await user.click(screen.getByRole("button", { name: /^delete$/i })); - await waitFor(() => expect(mockDeletePromptCall).toHaveBeenCalledWith("sk-test", "prompt-1")); + await waitFor(() => expect(mockDeletePromptCall).toHaveBeenCalledWith("sk-test", "prompt-1", "staging")); await user.keyboard("{Escape}"); - expect(screen.getByText(/delete prompt: my-prompt/i)).toBeInTheDocument(); + expect(screen.getByText(/the staging copy of prompt: my-prompt/i)).toBeInTheDocument(); finishDelete(); - await waitFor(() => expect(screen.queryByText(/delete prompt: my-prompt/i)).not.toBeInTheDocument()); + await waitFor(() => expect(screen.queryByText(/the staging copy of prompt: my-prompt/i)).not.toBeInTheDocument()); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.tsx index 3594e933704..2d8d905c480 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.tsx @@ -41,11 +41,12 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { const [isLoading, setIsLoading] = useState(true); const [selectedEnvironment, setSelectedEnvironment] = useState(undefined); const [selectedPromptId, setSelectedPromptId] = useState(null); + const [selectedPromptEnvironment, setSelectedPromptEnvironment] = useState(undefined); const [isAddModalVisible, setIsAddModalVisible] = useState(false); const [showEditorView, setShowEditorView] = useState(false); const [editPromptData, setEditPromptData] = useState(null); const [isDeleting, setIsDeleting] = useState(false); - const [promptToDelete, setPromptToDelete] = useState<{ id: string; name: string } | null>(null); + const [promptToDelete, setPromptToDelete] = useState<{ id: string; name: string; environment: string } | null>(null); // Admin Viewer follows the read-parity rule: see prompts, no writes. const canModify = userRole ? isProxyAdminRole(userRole) : false; @@ -71,8 +72,9 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { fetchPrompts(); }, [accessToken, selectedEnvironment]); - const handlePromptClick = (promptId: string) => { + const handlePromptClick = (promptId: string, environment: string) => { setSelectedPromptId(promptId); + setSelectedPromptEnvironment(environment); }; const handleAddPrompt = () => { @@ -111,8 +113,8 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { setSelectedPromptId(null); }; - const handleDeleteClick = (promptId: string, promptName: string) => { - setPromptToDelete({ id: promptId, name: promptName }); + const handleDeleteClick = (promptId: string, promptName: string, environment: string) => { + setPromptToDelete({ id: promptId, name: promptName, environment }); }; const handleDeleteConfirm = async () => { @@ -120,8 +122,8 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { setIsDeleting(true); try { - await deletePromptCall(accessToken, promptToDelete.id); - toast.success(`Prompt "${promptToDelete.name}" deleted successfully`); + await deletePromptCall(accessToken, promptToDelete.id, promptToDelete.environment); + toast.success(`Prompt "${promptToDelete.name}" deleted successfully from ${promptToDelete.environment}`); fetchPrompts(); // Refresh the list } catch (error) { console.error("Error deleting prompt:", error); @@ -148,6 +150,7 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { ) : selectedPromptId ? ( setSelectedPromptId(null)} accessToken={accessToken} isAdmin={canModify} @@ -219,7 +222,8 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { Delete Prompt - Are you sure you want to delete prompt: {promptToDelete.name} ? This action cannot be undone. + Are you sure you want to delete the {promptToDelete.environment} copy of prompt: {promptToDelete.name}? + This action cannot be undone. diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx index bb29f12ac42..a5b195e5057 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx @@ -29,6 +29,35 @@ const promptWithoutTemplate = { environments: [], }; +describe("PromptInfoView environment scoping", () => { + beforeEach(() => { + vi.mocked(networking.getPromptInfo).mockReset().mockResolvedValue(promptWithoutTemplate); + vi.mocked(networking.getPromptVersions).mockReset().mockResolvedValue({ prompts: [] }); + }); + + it("fetches the initial environment it was opened with", async () => { + render( + , + ); + + await screen.findByRole("tab", { name: "Raw JSON" }); + expect(networking.getPromptInfo).toHaveBeenCalledWith("sk-test", "support-reply", "staging"); + }); + + it("fetches the serve default when opened without an environment", async () => { + render(); + + await screen.findByRole("tab", { name: "Raw JSON" }); + expect(networking.getPromptInfo).toHaveBeenCalledWith("sk-test", "support-reply", undefined); + }); +}); + describe("PromptInfoView tabs", () => { beforeEach(() => { vi.mocked(networking.getPromptInfo).mockReset().mockResolvedValue(promptWithoutTemplate); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx index c6a6f09fcaa..af4e3bf2121 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx @@ -20,6 +20,7 @@ import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from " export interface PromptInfoProps { promptId: string; + initialEnvironment?: string; onClose: () => void; accessToken: string | null; isAdmin: boolean; @@ -27,7 +28,15 @@ export interface PromptInfoProps { onEdit?: (promptData: any) => void; } -const PromptInfoView: React.FC = ({ promptId, onClose, accessToken, isAdmin, onDelete, onEdit }) => { +const PromptInfoView: React.FC = ({ + promptId, + initialEnvironment, + onClose, + accessToken, + isAdmin, + onDelete, + onEdit, +}) => { const [promptData, setPromptData] = useState(null); const [promptTemplate, setPromptTemplate] = useState(null); const [rawApiResponse, setRawApiResponse] = useState(null); @@ -43,7 +52,7 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo const [selectedVersion, setSelectedVersion] = useState(null); const [loadingVersions, setLoadingVersions] = useState(false); - // Initial fetch — no environment filter, gets default + all environments list + // Fetches the requested environment (or the serve-time default when omitted) plus the environments list const fetchPromptInfo = async (environment?: string) => { try { setLoading(true); @@ -89,7 +98,7 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo setSelectedEnv(null); setEnvironments([]); setVersionHistory([]); - fetchPromptInfo(); + fetchPromptInfo(initialEnvironment); }, [promptId, accessToken]); // When environment changes (user clicks tab), re-fetch — skip initial mount @@ -493,7 +502,7 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo Delete Prompt

- Are you sure you want to delete prompt: {basePromptId}? + Are you sure you want to delete prompt: {basePromptId} from every environment?

This action cannot be undone.

diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 5b6d70b4771..e297bcc5270 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -4625,9 +4625,12 @@ export const updatePromptCall = async (accessToken: string, promptId: string, pr } }; -export const deletePromptCall = async (accessToken: string, promptId: string) => { +export const deletePromptCall = async (accessToken: string, promptId: string, environment?: string) => { try { - const data = await apiClient.delete(`/prompts/${promptId}`, { accessToken }); + const data = await apiClient.delete(`/prompts/${promptId}`, { + accessToken, + query: { environment: environment || undefined }, + }); return data; } catch (error) { console.error("Failed to delete prompt:", error); From adfa42096d99e320e1413c37578158e6ec2d6465 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 23:07:18 -0700 Subject: [PATCH 012/480] fix(prompts): resolve config prompts in /prompts/{id}/info when environment is set --- litellm/proxy/prompts/prompt_endpoints.py | 6 +- .../proxy/prompts/test_prompt_endpoints.py | 92 +++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index 1b7932cdb18..b16611ac157 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -554,8 +554,10 @@ async def get_prompt_info( if env_prompts: prompt_spec = create_versioned_prompt_spec(db_prompt=env_prompts[0]) - if prompt_spec is None and environment is None: - prompt_spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec(prompt_id, version=requested_version) + if prompt_spec is None: + prompt_spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec( + prompt_id, version=requested_version, environment=environment + ) if prompt_spec is None: raise HTTPException( diff --git a/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py b/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py index 35402db219f..41c3003af34 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py @@ -368,3 +368,95 @@ class TestAdminViewerReadAccess: assert response.prompt_spec.prompt_id == "jack" assert response.prompt_spec.version == 2 + + +class TestConfigPromptInfoWithEnvironment: + """ + Regression: /prompts/{id}/info with an environment param must still resolve + config-file (in-memory) prompts on a DB-backed proxy instead of 400ing. + """ + + def _registry_with_config_prompt(self): + from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry + + registry = InMemoryPromptRegistry() + registry.IN_MEMORY_PROMPTS["envgreet::development"] = PromptSpec( + prompt_id="envgreet", + litellm_params=PromptLiteLLMParams( + prompt_id="envgreet", + prompt_integration="dotprompt", + dotprompt_content="AHOY {{user_message}}", + ), + prompt_info=PromptInfo(prompt_type="config"), + ) + return registry + + def _prisma_client_with_empty_prompt_table(self): + from unittest.mock import AsyncMock + + mock_prisma = MagicMock() + mock_prisma.db.litellm_prompttable.find_many = AsyncMock(return_value=[]) + return mock_prisma + + @pytest.mark.asyncio + async def test_get_prompt_info_with_environment_falls_back_to_registry(self): + from unittest.mock import patch + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.prompts.prompt_endpoints import get_prompt_info + + admin = UserAPIKeyAuth( + api_key="test_key", user_role=LitellmUserRoles.PROXY_ADMIN + ) + + with ( + patch( # test-quality-ok: endpoint reads prisma_client and the registry from module globals at call time; no injection seam + "litellm.proxy.proxy_server.prisma_client", + self._prisma_client_with_empty_prompt_table(), + ), + patch( # test-quality-ok: endpoint reads prisma_client and the registry from module globals at call time; no injection seam + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY", + self._registry_with_config_prompt(), + ), + ): + response = await get_prompt_info( + prompt_id="envgreet", + environment="development", + user_api_key_dict=admin, + ) + + assert response.prompt_spec.prompt_id == "envgreet" + assert response.prompt_spec.litellm_params.dotprompt_content == "AHOY {{user_message}}" + + @pytest.mark.asyncio + async def test_get_prompt_info_with_wrong_environment_still_400s(self): + from unittest.mock import patch + + from fastapi import HTTPException + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.prompts.prompt_endpoints import get_prompt_info + + admin = UserAPIKeyAuth( + api_key="test_key", user_role=LitellmUserRoles.PROXY_ADMIN + ) + + with ( + patch( # test-quality-ok: endpoint reads prisma_client and the registry from module globals at call time; no injection seam + "litellm.proxy.proxy_server.prisma_client", + self._prisma_client_with_empty_prompt_table(), + ), + patch( # test-quality-ok: endpoint reads prisma_client and the registry from module globals at call time; no injection seam + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY", + self._registry_with_config_prompt(), + ), + ): + with pytest.raises(HTTPException) as exc_info: + await get_prompt_info( + prompt_id="envgreet", + environment="production", + user_api_key_dict=admin, + ) + + assert exc_info.value.status_code == 400 + assert "environment production" in exc_info.value.detail From e5582b65c9b69adf636c8a9c739a9fa1b96e01a7 Mon Sep 17 00:00:00 2001 From: milan Date: Fri, 28 Aug 2026 20:33:53 +0000 Subject: [PATCH 013/480] fix(team_endpoints): let member_delete clear a team left on the user row Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_endpoints/team_endpoints.py | 38 +++--- .../test_team_endpoints.py | 118 ++++++++++++++++++ 2 files changed, 138 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index c6d7975b75e..88c58b8887a 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3303,9 +3303,6 @@ async def team_member_delete( data=data, ) - if not removed_team_members: - raise HTTPException(status_code=400, detail={"error": "User not found in team"}) - existing_team_row.members_with_roles = new_team_members _db_new_team_members: Final[list[dict]] = [m.model_dump() for m in new_team_members] @@ -3313,17 +3310,22 @@ async def team_member_delete( ## DELETE TEAM ID from USER ROW, IF EXISTS ## # get user row removed_user_ids: Final = frozenset(m.user_id for m in removed_team_members if m.user_id is not None) + addressed_user_ids: Final = removed_user_ids.union((data.user_id,) if data.user_id is not None else ()) key_val: Final[Mapping[str, object]] = ( - {"user_id": {"in": sorted(removed_user_ids)}} if removed_user_ids else {"user_email": data.user_email} + {"user_id": {"in": sorted(addressed_user_ids)}} if addressed_user_ids else {"user_email": data.user_email} ) member_tx: Final[_MemberDeleteTx] = tx existing_user_rows: Final = await member_tx.litellm_usertable.find_many(where=key_val) # Also clean up any existing team membership rows for this user and team - user_ids_to_delete: Final = removed_user_ids.union( - (data.user_id,) if data.user_id is not None else (), - (user.user_id for user in existing_user_rows if user.user_id), - ) + user_ids_to_delete: Final = addressed_user_ids.union(user.user_id for user in existing_user_rows if user.user_id) + + # A user row can outlive its roster entry, and until the team is off user.teams the user + # still sees it and still fails key creation against it, so removal has to clear it too + stale_user_rows: Final = tuple(user for user in existing_user_rows if data.team_id in user.teams) + + if not removed_team_members and not stale_user_rows: + raise HTTPException(status_code=400, detail={"error": "User not found in team"}) ## DELETE KEYS CREATED BY USER FOR THIS TEAM # Fetch keys before deletion so their audit records can be persisted alongside the delete. @@ -3335,17 +3337,17 @@ async def team_member_delete( } ) - await _team_tx_db(tx).update( - where={"team_id": data.team_id}, - data={"members_with_roles": json.dumps(_db_new_team_members)}, - ) + if removed_team_members: + await _team_tx_db(tx).update( + where={"team_id": data.team_id}, + data={"members_with_roles": json.dumps(_db_new_team_members)}, + ) - for existing_user in existing_user_rows: - if data.team_id in existing_user.teams: - await tx.litellm_usertable.update( - where={"user_id": existing_user.user_id}, - data={"teams": {"set": [team for team in existing_user.teams if team != data.team_id]}}, - ) + for existing_user in stale_user_rows: + await tx.litellm_usertable.update( + where={"user_id": existing_user.user_id}, + data={"teams": {"set": [team for team in existing_user.teams if team != data.team_id]}}, + ) for _uid in sorted(user_ids_to_delete): await tx.litellm_teammembership.delete_many(where={"team_id": data.team_id, "user_id": _uid}) diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index ffa6bc601e9..e5b58f4b38e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -4775,6 +4775,124 @@ async def test_team_member_delete_by_email_the_user_row_does_not_carry( ) +@pytest.mark.asyncio +async def test_team_member_delete_clears_team_left_on_the_user_row_without_a_roster_entry( + mock_db_client, mock_admin_auth +): + """ + A user row can keep a team (several times over, from older duplicate-prone adds) after the + roster entry is gone, which leaves the team listed on the user, offered in the key creation + dropdown, and rejected by key creation itself. Reporting "User not found in team" left that + residue unremovable, so the delete now cleans every copy of the team off the user row. + """ + from litellm.proxy._types import TeamMemberDeleteRequest + from litellm.proxy.management_endpoints.team_endpoints import team_member_delete + + test_team_id = "team-del-orphan-123" + test_user_id = "user-del-orphan-123" + + mock_team_row = MagicMock() + mock_team_row.model_dump.return_value = { + "team_id": test_team_id, + "members_with_roles": [], + "team_member_permissions": [], + "metadata": {}, + "models": [], + "spend": 0.0, + } + mock_db_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=mock_team_row + ) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_team_row) + + mock_user_row = MagicMock() + mock_user_row.user_id = test_user_id + mock_user_row.user_email = None + mock_user_row.teams = [test_team_id, "other-team", test_team_id] + mock_db_client.db.litellm_usertable.find_many = AsyncMock( + return_value=[mock_user_row] + ) + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_teammembership = MagicMock() + mock_db_client.db.litellm_teammembership.delete_many = AsyncMock( + return_value=MagicMock() + ) + + mock_db_client.db.litellm_verificationtoken = MagicMock() + mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_db_client.db.litellm_verificationtoken.delete_many = AsyncMock( + return_value=MagicMock() + ) + + _wire_member_delete_tx(mock_db_client) + + await team_member_delete( + data=TeamMemberDeleteRequest(team_id=test_team_id, user_id=test_user_id), + user_api_key_dict=mock_admin_auth, + ) + + mock_db_client.db.litellm_usertable.update.assert_awaited_once_with( + where={"user_id": test_user_id}, + data={"teams": {"set": ["other-team"]}}, + ) + mock_db_client.db.litellm_teammembership.delete_many.assert_awaited_once_with( + where={"team_id": test_team_id, "user_id": test_user_id} + ) + + +@pytest.mark.asyncio +async def test_team_member_delete_still_rejects_a_user_the_team_has_no_trace_of( + mock_db_client, mock_admin_auth +): + from litellm.proxy._types import TeamMemberDeleteRequest + from litellm.proxy.management_endpoints.team_endpoints import team_member_delete + + test_team_id = "team-del-absent-123" + test_user_id = "user-del-absent-123" + + mock_team_row = MagicMock() + mock_team_row.model_dump.return_value = { + "team_id": test_team_id, + "members_with_roles": [], + "team_member_permissions": [], + "metadata": {}, + "models": [], + "spend": 0.0, + } + mock_db_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=mock_team_row + ) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_team_row) + + mock_user_row = MagicMock() + mock_user_row.user_id = test_user_id + mock_user_row.user_email = None + mock_user_row.teams = ["other-team"] + mock_db_client.db.litellm_usertable.find_many = AsyncMock( + return_value=[mock_user_row] + ) + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_teammembership = MagicMock() + mock_db_client.db.litellm_teammembership.delete_many = AsyncMock( + return_value=MagicMock() + ) + + _wire_member_delete_tx(mock_db_client) + + with pytest.raises(HTTPException) as exc_info: + await team_member_delete( + data=TeamMemberDeleteRequest(team_id=test_team_id, user_id=test_user_id), + user_api_key_dict=mock_admin_auth, + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == {"error": "User not found in team"} + mock_db_client.db.litellm_usertable.update.assert_not_awaited() + mock_db_client.db.litellm_teammembership.delete_many.assert_not_awaited() + + class _InjectedMemberDeleteFailure(Exception): pass From df9b9f9ffb486561cd4cbbacffd2402b9b43ae16 Mon Sep 17 00:00:00 2001 From: milan Date: Fri, 28 Aug 2026 21:12:05 +0000 Subject: [PATCH 014/480] style(team_endpoints): apply ruff format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/management_endpoints/team_endpoints.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 88c58b8887a..fce109c0c8b 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3318,7 +3318,9 @@ async def team_member_delete( existing_user_rows: Final = await member_tx.litellm_usertable.find_many(where=key_val) # Also clean up any existing team membership rows for this user and team - user_ids_to_delete: Final = addressed_user_ids.union(user.user_id for user in existing_user_rows if user.user_id) + user_ids_to_delete: Final = addressed_user_ids.union( + user.user_id for user in existing_user_rows if user.user_id + ) # A user row can outlive its roster entry, and until the team is off user.teams the user # still sees it and still fails key creation against it, so removal has to clear it too From 3275459aec0935b03b60950ba5d625854e2d6c9b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:47:26 -0700 Subject: [PATCH 015/480] fix(mcp): cap tools preview and test-connection at the listing timeout and name the unreachable upstream --- .../mcp_server/rest_endpoints.py | 34 +++++--- .../mcp_server/test_rest_endpoints.py | 77 +++++++++++++++++-- 2 files changed, 96 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 3efb6429326..d73277f4417 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -4,10 +4,12 @@ from collections.abc import Awaitable, Callable, Mapping from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal +import anyio import httpx from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from litellm._logging import verbose_logger +from litellm.constants import MCP_TOOL_LISTING_TIMEOUT from litellm.exceptions import ( BlockedPiiEntityError, GuardrailRaisedException, @@ -68,7 +70,13 @@ _MCP_GUARDRAIL_REJECTIONS: Final = ( ) -def _connection_error_message(exc: BaseException) -> str: +def _connection_error_message(exc: BaseException, url: str | None, timeout_seconds: float) -> str: + if isinstance(exc, TimeoutError): + return ( + f"Failed to connect to MCP server: no response from {url or 'the server'} " + f"within {timeout_seconds:.0f}s. Check that the LiteLLM proxy can reach this URL " + "from its network (DNS, egress rules, firewalls) and that the server answers MCP requests." + ) if isinstance(exc, httpx.LocalProtocolError): return ( "Failed to connect to MCP server: a request header is malformed. " @@ -1136,6 +1144,7 @@ if MCP_AVAILABLE: mcp_auth_header: str | dict[str, str] | None = None, oauth2_headers: dict[str, str] | None = None, raw_headers: dict[str, str] | None = None, + timeout_seconds: float = MCP_TOOL_LISTING_TIMEOUT, ) -> Mapping[str, object]: """ Create a temporary MCP client from *request*, run *operation*, and return the result. @@ -1151,6 +1160,10 @@ if MCP_AVAILABLE: oauth2_headers: Headers extracted from the incoming request (may contain the litellm API key — must NOT be forwarded for M2M servers). raw_headers: Raw request headers forwarded for stdio env construction. + timeout_seconds: Cap on OAuth discovery, connect, handshake, and *operation* + combined. Defaults to ``MCP_TOOL_LISTING_TIMEOUT`` (30s, below common LB + timeouts) so an unreachable upstream yields this endpoint's JSON error + instead of an opaque load-balancer 504 with an empty body. Returns: The dict returned by *operation*, or an error dict on failure. @@ -1240,15 +1253,16 @@ if MCP_AVAILABLE: static_headers=request.static_headers, ) - client: Final = await global_mcp_server_manager._create_mcp_client( - server=server_model, - mcp_auth_header=mcp_auth_header, - extra_headers=merged_headers, - stdio_env=stdio_env, - cred_provider=preview_cred_provider, - ) + with anyio.fail_after(timeout_seconds): + client: Final = await global_mcp_server_manager._create_mcp_client( + server=server_model, + mcp_auth_header=mcp_auth_header, + extra_headers=merged_headers, + stdio_env=stdio_env, + cred_provider=preview_cred_provider, + ) - return await operation(client) + return await operation(client) except (KeyboardInterrupt, SystemExit, asyncio.CancelledError): raise @@ -1257,7 +1271,7 @@ if MCP_AVAILABLE: return { "status": "error", "error": True, - "message": _connection_error_message(e), + "message": _connection_error_message(e, request.url, timeout_seconds), } async def _preview_openapi_tools(spec_path: str) -> dict: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index ef5631218f3..51b946c11b7 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -1,4 +1,5 @@ import asyncio +import inspect import json import sys from datetime import datetime @@ -13,6 +14,7 @@ import pytest from fastapi import HTTPException from starlette.requests import Request +from litellm.constants import MCP_TOOL_LISTING_TIMEOUT from litellm.proxy._experimental.mcp_server import rest_endpoints from litellm.proxy._experimental.mcp_server.auth import ( user_api_key_auth_mcp as auth_mcp, @@ -109,6 +111,71 @@ class TestExecuteWithMcpClient: assert result["status"] == "error" assert "stack_trace" not in result + @pytest.mark.asyncio + async def test_timeout_caps_hanging_operation_and_names_url(self, monkeypatch): + async def fake_create_client(*args, **kwargs): + return object() + + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "_create_mcp_client", + fake_create_client, + ) + + async def hanging_operation(client): + await asyncio.Event().wait() + + payload = NewMCPServerRequest( + server_name="example", + url="https://mcp.example.com/mcp/", + auth_type=MCPAuth.none, + ) + + result = await asyncio.wait_for( + rest_endpoints._execute_with_mcp_client(payload, hanging_operation, timeout_seconds=0.05), + timeout=5, + ) + + assert result["error"] is True + assert "https://mcp.example.com/mcp/" in result["message"] + + @pytest.mark.asyncio + async def test_timeout_covers_client_creation(self, monkeypatch): + async def hanging_create_client(*args, **kwargs): + await asyncio.Event().wait() + + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "_create_mcp_client", + hanging_create_client, + ) + + async def unreached_operation(client): + return {"status": "ok"} + + payload = NewMCPServerRequest( + server_name="example", + url="https://mcp.example.com/mcp/", + auth_type=MCPAuth.none, + ) + + result = await asyncio.wait_for( + rest_endpoints._execute_with_mcp_client(payload, unreached_operation, timeout_seconds=0.05), + timeout=5, + ) + + assert result["error"] is True + assert "https://mcp.example.com/mcp/" in result["message"] + + def test_timeout_defaults_to_tool_listing_timeout(self): + default = inspect.signature(rest_endpoints._execute_with_mcp_client).parameters["timeout_seconds"].default + assert default == MCP_TOOL_LISTING_TIMEOUT + + def test_connection_error_message_timeout_names_url_and_budget(self): + message = rest_endpoints._connection_error_message(TimeoutError(), "https://api.example.com/mcp/", 30.0) + assert "https://api.example.com/mcp/" in message + assert "30s" in message + @pytest.mark.asyncio async def test_forwards_static_headers(self, monkeypatch): """Ensure static_headers are forwarded to the MCP client during test calls. @@ -2881,17 +2948,17 @@ class TestConnectionErrorMessage: secret = "Bearer sk-super-secret-token" exc = httpx.LocalProtocolError(f"Illegal header value b' {secret}'") - message = rest_endpoints._connection_error_message(exc) + message = rest_endpoints._connection_error_message(exc, "https://example.com", 30.0) assert "header" in message.lower() assert secret not in message def test_connect_error_points_at_reachability(self): - message = rest_endpoints._connection_error_message(httpx.ConnectError("All connection attempts failed")) + message = rest_endpoints._connection_error_message(httpx.ConnectError("All connection attempts failed"), "https://example.com", 30.0) assert "unreachable" in message.lower() def test_timeout_error_message(self): - message = rest_endpoints._connection_error_message(httpx.ConnectTimeout("timed out")) + message = rest_endpoints._connection_error_message(httpx.ConnectTimeout("timed out"), "https://example.com", 30.0) assert "unreachable" in message.lower() def test_http_status_error_includes_status_code(self): @@ -2901,11 +2968,11 @@ class TestConnectionErrorMessage: request=httpx.Request("POST", "http://x/"), response=response, ) - message = rest_endpoints._connection_error_message(exc) + message = rest_endpoints._connection_error_message(exc, "https://example.com", 30.0) assert "503" in message def test_unknown_error_falls_back_to_generic(self): - message = rest_endpoints._connection_error_message(RuntimeError("weird")) + message = rest_endpoints._connection_error_message(RuntimeError("weird"), "https://example.com", 30.0) assert "weird" not in message assert "proxy logs" in message.lower() From f4b5449c6a65cf167658fd5bb32695abda9633a2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:48:04 -0700 Subject: [PATCH 016/480] fix(openai_like): strip cache_control ttl before forwarding /v1/messages to non-Anthropic providers --- litellm/llms/anthropic/common_utils.py | 33 +++++ litellm/llms/openai_like/README.md | 5 +- .../openai_like/messages/transformation.py | 38 +++++- ..._like_anthropic_messages_transformation.py | 121 ++++++++++++++++++ 4 files changed, 195 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 681a8397f66..695e3a313ef 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1302,6 +1302,39 @@ def flatten_unencrypted_web_search_results_in_anthropic_messages( # mutable-ok: return [_flatten_web_search_results_in_message(m) for m in messages] # mutable-ok: JSON wire format +def _normalized_cache_control(cache_control: dict) -> dict: # mutable-ok: as sibling sanitizers + cache_type: Final = cache_control.get("type") + return {"type": cache_type if isinstance(cache_type, str) else "ephemeral"} # mutable-ok: JSON wire format + + +def _normalize_cache_control_value(value: object) -> object: + if isinstance(value, dict): + return normalize_cache_control_in_anthropic_payload(value) + if isinstance(value, list): + return [_normalize_cache_control_value(item) for item in value] # mutable-ok: JSON wire format + return value + + +def normalize_cache_control_in_anthropic_payload(payload: dict) -> dict: # mutable-ok: as sibling sanitizers + """ + Return a copy of an Anthropic /v1/messages payload with every + ``cache_control`` entry reduced to ``{"type": }``, + recursing through message content blocks, system blocks, and tools. + + Anthropic itself accepts prompt-caching extensions such as ``ttl``, but + strict non-Anthropic implementations of the Messages API validate the field + literally and reject the whole request (``cache_control.ttl: 1h is not + supported``, ``cache_control.type is required``), which 400s clients like + Claude Code that always send cache hints. Non-dict ``cache_control`` values + are dropped entirely. The caller's payload is never mutated. + """ + return { # mutable-ok: JSON wire format, as sibling sanitizers + key: _normalized_cache_control(value) if key == "cache_control" else _normalize_cache_control_value(value) + for key, value in payload.items() + if key != "cache_control" or isinstance(value, dict) + } + + def process_anthropic_headers(headers: httpx.Headers | dict) -> dict: openai_headers: Final = {} if "anthropic-ratelimit-requests-limit" in headers: diff --git a/litellm/llms/openai_like/README.md b/litellm/llms/openai_like/README.md index e9aaafe48a1..e1409b81c35 100644 --- a/litellm/llms/openai_like/README.md +++ b/litellm/llms/openai_like/README.md @@ -54,7 +54,10 @@ That's it! The provider will be automatically loaded and available. "constraints": { "temperature_max": 1.0, "temperature_min": 0.0, - "temperature_min_with_n_gt_1": 0.3 + "temperature_min_with_n_gt_1": 0.3, + // /v1/messages providers only: keep Anthropic cache_control extensions + // such as ttl instead of stripping them down to {"type": ...} + "cache_control_ttl": true }, // Optional: Special handling flags diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 11dc236064d..29973fe2101 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -1,11 +1,13 @@ from typing import Any, Final import litellm +from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.openai_like.json_loader import SimpleProviderConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" @@ -19,7 +21,9 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): ``"/v1/messages"``. The inbound Anthropic payload (system, cache_control, thinking, tools, ...) is forwarded essentially unchanged to ``{api_base}/v1/messages``, so Anthropic-only features that the - Anthropic->OpenAI translation would otherwise drop are preserved. Response + Anthropic->OpenAI translation would otherwise drop are preserved. The one + exception is ``cache_control``, whose Anthropic-only extensions (``ttl``) + are stripped unless ``supports_cache_control_ttl`` says otherwise. Response parsing and streaming are inherited from the native Anthropic config. """ @@ -53,6 +57,35 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): def should_filter_anthropic_beta_headers(self) -> bool: return False + def supports_cache_control_ttl(self) -> bool: + return False + + def transform_anthropic_messages_request( + self, + model: str, + messages: list[dict], # mutable-ok: matches dict-typed base signature + anthropic_messages_optional_request_params: dict, # mutable-ok: matches dict-typed base signature + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: matches dict-typed base signature + ) -> dict: # mutable-ok: matches dict-typed base signature + """ + Anthropic ignores prompt-caching hints it cannot honor, but strict + non-Anthropic implementations of the Messages API 400 the whole request + on Anthropic-only ``cache_control`` extensions (``cache_control.ttl: 1h + is not supported``), so unless the provider declares ttl support the + hints are reduced to their portable ``{"type": ...}`` core. + """ + request: Final = super().transform_anthropic_messages_request( + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + if self.supports_cache_control_ttl(): + return request + return normalize_cache_control_in_anthropic_payload(request) + def get_complete_url( self, api_base: str | None, @@ -91,6 +124,9 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig): def should_strip_billing_metadata(self) -> bool: return True + def supports_cache_control_ttl(self) -> bool: + return bool(self._provider.constraints.get("cache_control_ttl")) + def _resolve_api_key(self, api_key: str | None) -> str | None: return api_key or get_secret_str(self._provider.api_key_env) or litellm.api_key diff --git a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py index 33e677b000e..2cdd969b00f 100644 --- a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py @@ -317,3 +317,124 @@ def test_json_provider_messages_config_probes_capabilities_under_provider_slug() ) assert JSONProviderAnthropicMessagesConfig(provider).custom_llm_provider == "exampleprovider" assert OpenAILikeAnthropicMessagesConfig().custom_llm_provider == "anthropic" + + +def _cache_control_request_params() -> tuple[list, dict]: + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "write a regex for a US phone number", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + } + ] + optional_params = { + "max_tokens": 256, + "system": [ + { + "type": "text", + "text": "You are Claude Code.", + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + } + ], + "tools": [ + { + "name": "lookup", + "input_schema": {"type": "object"}, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + } + return messages, optional_params + + +def test_request_strips_cache_control_ttl_everywhere(config): + """Regression: Claude Code always sends ``cache_control: {type: ephemeral, + ttl: 1h}``, and strict non-Anthropic /v1/messages validators 400 the whole + request on the ttl extension (``cache_control.ttl: 1h is not supported``).""" + messages, optional_params = _cache_control_request_params() + + payload = config.transform_anthropic_messages_request( + model="some-model", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert payload["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + assert payload["system"][0]["cache_control"] == {"type": "ephemeral"} + assert payload["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert messages[0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + +def test_request_defaults_missing_cache_control_type_and_drops_non_dict(config): + payload = config.transform_anthropic_messages_request( + model="some-model", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "a", "cache_control": {"ttl": "1h"}}, + {"type": "text", "text": "b", "cache_control": None}, + ], + } + ], + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + blocks = payload["messages"][0]["content"] + assert blocks[0]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in blocks[1] + + +def test_native_anthropic_config_keeps_cache_control_ttl(): + """Anthropic itself accepts ttl, so the normalization must stay scoped to + the OpenAI-like passthrough and never reach the native Anthropic path.""" + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) + + messages, optional_params = _cache_control_request_params() + payload = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-20250514", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert payload["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert payload["system"][0]["cache_control"] == {"type": "ephemeral", "ttl": "5m"} + + +def test_json_provider_constraint_opts_into_cache_control_ttl(): + from litellm.llms.openai_like.json_loader import SimpleProviderConfig + from litellm.llms.openai_like.messages.transformation import ( + JSONProviderAnthropicMessagesConfig, + ) + + base_data = {"base_url": "https://api.example.com/v1", "api_key_env": "EXAMPLE_API_KEY"} + strict = JSONProviderAnthropicMessagesConfig(SimpleProviderConfig(slug="strictprov", data=base_data)) + lenient = JSONProviderAnthropicMessagesConfig( + SimpleProviderConfig(slug="lenientprov", data={**base_data, "constraints": {"cache_control_ttl": True}}) + ) + + def transform(provider_config): + messages, optional_params = _cache_control_request_params() + return provider_config.transform_anthropic_messages_request( + model="some-model", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert transform(strict)["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + assert transform(lenient)["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} From 3088db3f0ec77fd8256eff2b70dee882755f9a4c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:55:34 -0700 Subject: [PATCH 017/480] fix(prompts): accept a string prompt_version and carry the viewed environment into code snippets --- litellm/proxy/prompts/prompt_registry.py | 10 ++++++ litellm/proxy/utils.py | 5 +-- .../proxy/prompts/test_prompt_registry.py | 10 +++++- .../proxy_logging/test_guardrail_pipeline.py | 35 ++++++++++++++++++ .../PromptCodeSnippets.test.tsx | 24 +++++++++++++ .../prompt_editor_view/PromptCodeSnippets.tsx | 29 ++++++++------- .../PromptEditorHeader.test.tsx | 5 ++- .../prompt_editor_view/PromptEditorHeader.tsx | 1 + .../prompts/_components/prompt_info.test.tsx | 36 ++++++++++++++++++- .../prompts/_components/prompt_info.tsx | 1 + 10 files changed, 139 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/prompts/prompt_registry.py b/litellm/proxy/prompts/prompt_registry.py index ba53809690d..7803352e9e7 100644 --- a/litellm/proxy/prompts/prompt_registry.py +++ b/litellm/proxy/prompts/prompt_registry.py @@ -74,6 +74,16 @@ def registry_key_for_prompt(prompt: PromptSpec) -> str: return f"{prompt.prompt_id}::{prompt_environment_or_default(prompt.environment)}" +def parse_prompt_version(raw_version: object) -> int | None: + if isinstance(raw_version, bool): + return None + if isinstance(raw_version, int): + return raw_version + if isinstance(raw_version, str) and raw_version.isdigit(): + return int(raw_version) + return None + + def _spec_version(prompt: PromptSpec) -> int: return prompt.version if prompt.version is not None else get_version_number(prompt_id=prompt.prompt_id) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 09efa0cc72d..0263ce7598a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1749,7 +1749,6 @@ class ProxyLogging: litellm_logging_obj: Final = cast(Optional["LiteLLMLoggingObj"], data.get("litellm_logging_obj", None)) prompt_id: Final[str | None] = data.get("prompt_id", None) - prompt_version: Final[int | None] = data.get("prompt_version", None) ## PROMPT TEMPLATE CHECK ## @@ -1759,11 +1758,13 @@ class ProxyLogging: and prompt_id is not None and (call_type == "completion" or call_type == "acompletion" or call_type == "aresponses") ): + from litellm.proxy.prompts.prompt_registry import parse_prompt_version + await self._process_prompt_template( data=data, litellm_logging_obj=litellm_logging_obj, prompt_id=prompt_id, - prompt_version=prompt_version, + prompt_version=parse_prompt_version(data.get("prompt_version", None)), call_type=call_type, ) diff --git a/tests/test_litellm/proxy/prompts/test_prompt_registry.py b/tests/test_litellm/proxy/prompts/test_prompt_registry.py index 32930fbbdff..a0743b65dd7 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_registry.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_registry.py @@ -2,7 +2,7 @@ import pytest import litellm from litellm.integrations.custom_prompt_management import CustomPromptManagement -from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry +from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry, parse_prompt_version from litellm.types.prompts.init_prompts import PromptInfo, PromptLiteLLMParams, PromptSpec @@ -214,3 +214,11 @@ def test_remove_prompt_is_a_no_op_for_an_unknown_registry_key(isolated_callbacks assert registry.resolve_prompt_spec("greeting") is not None assert len(isolated_callbacks) == 1 + + +@pytest.mark.parametrize( + ("raw_version", "expected"), + [(2, 2), ("2", 2), (None, None), ("v2", None), (True, None), (2.0, None)], +) +def test_parse_prompt_version_accepts_integers_and_json_strings(raw_version: object, expected: int | None) -> None: + assert parse_prompt_version(raw_version) == expected diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 6589b14e53a..2ba58bd5644 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -858,6 +858,41 @@ async def test_process_prompt_template_resolves_the_requested_environment(proxy_ assert data["messages"] == [{"role": "user", "content": "rendered"}] +@pytest.mark.asyncio +async def test_pre_call_hook_matches_a_prompt_version_sent_as_a_json_string(proxy_logging, monkeypatch): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.prompts import prompt_registry + + prompt_spec = MagicMock() + prompt_spec.litellm_params = MagicMock(prompt_id="greeting") + resolve_calls: list[dict] = [] + + def fake_resolve(prompt_id, version=None, environment=None): + resolve_calls.append({"prompt_id": prompt_id, "version": version, "environment": environment}) + return prompt_spec + + monkeypatch.setattr(prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", fake_resolve) + monkeypatch.setattr( + prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_callback_for_prompt", lambda *a, **kw: MagicMock() + ) + logging_obj = MagicMock() + logging_obj.async_get_chat_completion_prompt = AsyncMock( + return_value=("m", [{"role": "user", "content": "rendered"}], {}) + ) + data: Dict[str, Any] = { + "messages": [{"role": "user", "content": "orig"}], + "model": "m", + "prompt_id": "greeting", + "prompt_version": "2", + "litellm_logging_obj": logging_obj, + } + + result = await proxy_logging.pre_call_hook(user_api_key_dict=UserAPIKeyAuth(), data=data, call_type="completion") + + assert resolve_calls == [{"prompt_id": "greeting", "version": 2, "environment": None}] + assert result["messages"] == [{"role": "user", "content": "rendered"}] + + @pytest.mark.asyncio async def test_process_prompt_template_aresponses_swaps_model_and_merges_input(proxy_logging, monkeypatch): from litellm.proxy.prompts import prompt_registry diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.test.tsx index a1b4ad52634..7fa44a4dfe5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.test.tsx @@ -44,4 +44,28 @@ describe("PromptCodeSnippets", () => { expect(screen.getByRole("combobox", { name: "Language" })).toHaveTextContent("Python (OpenAI SDK)"); }); + + it("includes the viewed environment in every generated request", async () => { + const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); + render( + , + ); + await user.click(screen.getByRole("button", { name: /get code/i })); + await screen.findByText("Generated Code"); + + await user.click(screen.getByRole("button", { name: /copy to clipboard/i })); + expect(await navigator.clipboard.readText()).toContain('"prompt_environment": "development"'); + + await user.click(screen.getByRole("tab", { name: "With Version" })); + await user.click(screen.getByRole("button", { name: /copy to clipboard/i })); + const versionSnippet = await navigator.clipboard.readText(); + expect(versionSnippet).toContain('"prompt_environment": "development"'); + expect(versionSnippet).toContain('"prompt_version": 2'); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.tsx index af7d6421265..a6adc160674 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.tsx @@ -22,6 +22,7 @@ interface PromptCodeSnippetsProps { promptVariables?: Record; accessToken: string | null; version?: string; + environment?: string; proxySettings?: { PROXY_BASE_URL?: string; LITELLM_UI_API_DOC_BASE_URL?: string | null; @@ -34,6 +35,7 @@ const PromptCodeSnippets: React.FC = ({ promptVariables = {}, accessToken, version = "1", + environment, proxySettings, }) => { const syntaxTheme = useSyntaxTheme(coy); @@ -64,6 +66,9 @@ const PromptCodeSnippets: React.FC = ({ // Generate code based on selected language and tab const generateCode = () => { const hasVariables = Object.keys(promptVariables).length > 0; + const curlEnvironment = environment ? `,\n "prompt_environment": "${environment}"` : ""; + const pythonEnvironment = environment ? `,\n "prompt_environment": "${environment}"` : ""; + const jsEnvironment = environment ? `,\n prompt_environment: "${environment}"` : ""; if (selectedLanguage === "curl") { if (selectedTab === "basic") { @@ -72,7 +77,7 @@ const PromptCodeSnippets: React.FC = ({ -H 'Authorization: Bearer ${effectiveApiKey}' \\ -d '{ "model": "${model}", - "prompt_id": "${promptId}"${ + "prompt_id": "${promptId}"${curlEnvironment}${ hasVariables ? `, "prompt_variables": ${JSON.stringify(promptVariables, null, 6).replace(/\n/g, "\n ")}` @@ -85,7 +90,7 @@ const PromptCodeSnippets: React.FC = ({ -H 'Authorization: Bearer ${effectiveApiKey}' \\ -d '{ "model": "${model}", - "prompt_id": "${promptId}"${ + "prompt_id": "${promptId}"${curlEnvironment}${ hasVariables ? `, "prompt_variables": ${JSON.stringify(promptVariables, null, 6).replace(/\n/g, "\n ")}` @@ -104,7 +109,7 @@ const PromptCodeSnippets: React.FC = ({ -H 'Authorization: Bearer ${effectiveApiKey}' \\ -d '{ "model": "${model}", - "prompt_id": "${promptId}", + "prompt_id": "${promptId}"${curlEnvironment}, "prompt_version": ${version}, "messages": [ { @@ -127,7 +132,7 @@ client = openai.OpenAI( response = client.chat.completions.create( model="${model}", extra_body={ - "prompt_id": "${promptId}"${ + "prompt_id": "${promptId}"${pythonEnvironment}${ hasVariables ? `, "prompt_variables": ${JSON.stringify(promptVariables, null, 8).replace(/\n/g, "\n ")}` @@ -145,7 +150,7 @@ response = client.chat.completions.create( {"role": "user", "content": "hi"} ], extra_body={ - "prompt_id": "${promptId}"${ + "prompt_id": "${promptId}"${pythonEnvironment}${ hasVariables ? `, "prompt_variables": ${JSON.stringify(promptVariables, null, 8).replace(/\n/g, "\n ")}` @@ -163,7 +168,7 @@ response = client.chat.completions.create( {"role": "user", "content": "Who are u"} ], extra_body={ - "prompt_id": "${promptId}", + "prompt_id": "${promptId}"${pythonEnvironment}, "prompt_version": ${version} } ) @@ -186,9 +191,9 @@ async function main() { model: "${model}", ${ hasVariables - ? `prompt_id: "${promptId}", + ? `prompt_id: "${promptId}"${jsEnvironment}, prompt_variables: ${JSON.stringify(promptVariables, null, 8).replace(/\n/g, "\n ")}` - : `prompt_id: "${promptId}"` + : `prompt_id: "${promptId}"${jsEnvironment}` } }); @@ -206,9 +211,9 @@ async function main() { ], ${ hasVariables - ? `prompt_id: "${promptId}", + ? `prompt_id: "${promptId}"${jsEnvironment}, prompt_variables: ${JSON.stringify(promptVariables, null, 8).replace(/\n/g, "\n ")}` - : `prompt_id: "${promptId}"` + : `prompt_id: "${promptId}"${jsEnvironment}` } }); @@ -224,7 +229,7 @@ async function main() { messages: [ { role: "user", content: "Who are u" } ], - prompt_id: "${promptId}", + prompt_id: "${promptId}"${jsEnvironment}, prompt_version: ${version} }); @@ -241,7 +246,7 @@ main();`; if (isModalVisible) { setGeneratedCode(generateCode()); } - }, [isModalVisible, selectedLanguage, selectedTab, promptId, model, promptVariables]); + }, [isModalVisible, selectedLanguage, selectedTab, promptId, model, promptVariables, version, environment]); return ( <> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.test.tsx index 5194cdd4e63..3afc00e37a5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.test.tsx @@ -2,7 +2,9 @@ import { fireEvent, render, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import PromptEditorHeader from "./PromptEditorHeader"; -vi.mock("./PromptCodeSnippets", () => ({ default: () => })); +vi.mock("./PromptCodeSnippets", () => ({ + default: ({ environment }: { environment?: string }) => , +})); describe("PromptEditorHeader", () => { it("preserves navigation, naming, and save actions", () => { @@ -48,5 +50,6 @@ describe("PromptEditorHeader", () => { ); expect(screen.getByRole("combobox", { name: "Environment" })).toHaveTextContent(label); + expect(screen.getByRole("button", { name: "Get Code" })).toHaveAttribute("data-environment", environment); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx index eea9755054f..04cac01365a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx @@ -89,6 +89,7 @@ const PromptEditorHeader: React.FC = ({ promptVariables={promptVariables} accessToken={accessToken} version={version?.replace("v", "") || "1"} + environment={environment} proxySettings={proxySettings} /> {editMode && onShowHistory && ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx index a5b195e5057..b14d5c3d91f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx @@ -12,7 +12,9 @@ vi.mock("@/components/networking", () => ({ })); vi.mock("./prompt_editor_view/PromptCodeSnippets", () => ({ - default: () =>
, + default: ({ environment }: { environment?: string }) => ( +
+ ), })); const promptWithoutTemplate = { @@ -58,6 +60,38 @@ describe("PromptInfoView environment scoping", () => { }); }); +describe("PromptInfoView code snippets", () => { + beforeEach(() => { + vi.mocked(networking.getPromptVersions).mockReset().mockResolvedValue({ prompts: [] }); + }); + + it.each([ + ["a prompt with several environments", "staging", ["development", "staging"]], + ["a config prompt with no environment list", "development", []], + ])("hands the viewed environment of %s to the code snippets", async (_label, environment, environments) => { + vi.mocked(networking.getPromptInfo) + .mockReset() + .mockResolvedValue({ + ...promptWithoutTemplate, + prompt_spec: { ...promptWithoutTemplate.prompt_spec, environment }, + environments, + }); + + render( + , + ); + + await screen.findByRole("tab", { name: "Raw JSON" }); + expect(screen.getByTestId("prompt-code-snippets")).toHaveAttribute("data-environment", environment); + }); +}); + describe("PromptInfoView tabs", () => { beforeEach(() => { vi.mocked(networking.getPromptInfo).mockReset().mockResolvedValue(promptWithoutTemplate); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx index af4e3bf2121..062e1d84a0a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx @@ -221,6 +221,7 @@ const PromptInfoView: React.FC = ({ promptVariables={extractTemplateVariables(promptTemplate?.content)} accessToken={accessToken} version={currentVersion} + environment={selectedEnv ?? promptData.environment} />
)} + {classifierType === "hybrid" && ( +
+ Boundary margin + handleHybridBoundaryMarginChange(event.target.value)} + onBlur={() => setDraft(null)} + className="w-full" + /> +

+ A score further than this from every tier boundary routes on the scorer's own tier, however expensive + that tier is. A score closer than this, and anything the scorer found no signal for at all, goes to the + classifier to break the tie +

+
+ )} +
How often to classify - classifierType === "llm" || classifierType === "heuristic_first"; + classifierType === "llm" || classifierType === "heuristic_first" || classifierType === "hybrid"; export type ClassifierFallback = "heuristic" | "default_model"; @@ -162,7 +162,8 @@ export const heuristicScoringRoleFor = ( classifierFallback: ClassifierFallback | undefined, ): HeuristicScoringRole => { if (classifierType === "heuristic_v2") return "never"; - if (classifierType === "heuristic" || classifierType === "heuristic_first") return "decides"; + if (classifierType === "heuristic" || classifierType === "heuristic_first" || classifierType === "hybrid") + return "decides"; return (classifierFallback ?? DEFAULT_CLASSIFIER_FALLBACK) === "heuristic" ? "fallback_only" : "never"; }; @@ -404,6 +405,8 @@ export interface ComplexityRouterConfigValue { classification_prompt?: string; /** Highest tier the scorer may decide alone under heuristic_first. Required by that type, rejected by the others. */ heuristic_first_max_tier?: string; + /** How near a tier boundary a score may land before hybrid defers to the classifier. Required by that type, rejected by the others. */ + hybrid_boundary_margin?: number; classification_mode?: ClassificationMode; session_affinity?: boolean; modality_routing?: boolean; @@ -516,6 +519,9 @@ export const effectiveTierLabel = (tier: keyof ComplexityTiers, tierLabels: Comp export const DEFAULT_HEURISTIC_FIRST_MAX_TIER = "SIMPLE"; +/** What the Hybrid radio starts at. Required by that type, so the form always has a value to send. */ +export const DEFAULT_HYBRID_BOUNDARY_MARGIN = 0.03; + /** * Tiers the heuristic_first threshold may name. The top tier is excluded because it would short * circuit every request and leave the classifier unreachable, which the backend rejects. diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 03b7432e4ec..81acb7b7e50 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -342,6 +342,7 @@ const AddAutoRouterTab: React.FC = ({ planModeMinTier: complexityRouterConfig.plan_mode_min_tier, classificationPrompt: complexityRouterConfig.classification_prompt, heuristicFirstMaxTier: complexityRouterConfig.heuristic_first_max_tier, + hybridBoundaryMargin: complexityRouterConfig.hybrid_boundary_margin, classificationMode: complexityRouterConfig.classification_mode, tierLabels: complexityRouterConfig.tier_labels, classifierType: complexityRouterConfig.classifier_type, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 2d8f55b2b64..c6370541a45 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -817,6 +817,39 @@ describe("heuristic_first", () => { }); }); +describe("hybrid", () => { + const hybridParams: BuildComplexityRouterConfigParams = { + ...baseParams, + classifierType: "hybrid", + hybridBoundaryMargin: 0.03, + classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 }, + classifierFallback: "default_model", + }; + + it("emits hybrid_boundary_margin, zero included since exactly-on-a-boundary is a real setting", () => { + expect(buildComplexityRouterConfig(hybridParams).hybrid_boundary_margin).toBe(0.03); + expect(buildComplexityRouterConfig({ ...hybridParams, hybridBoundaryMargin: 0 }).hybrid_boundary_margin).toBe(0); + }); + + it("keeps every classifier key the operator set, since hybrid still calls the classifier", () => { + const config = buildComplexityRouterConfig(hybridParams); + expect(config.classifier_type).toBe("hybrid"); + expect(config.classifier_llm_config).toEqual({ model: "gpt-4o-mini", timeout_ms: 3000 }); + expect(config.classifier_fallback).toBe("default_model"); + }); + + it("omits hybrid_boundary_margin on every other classifier type, which the backend rejects it on", () => { + for (const classifierType of ["heuristic", "llm", "heuristic_first"] as const) { + const config = buildComplexityRouterConfig({ + ...hybridParams, + classifierType, + ...(classifierType === "heuristic_first" && { heuristicFirstMaxTier: "SIMPLE" }), + }); + expect(config.hybrid_boundary_margin).toBeUndefined(); + } + }); +}); + describe("classification_mode", () => { it("emits user_turn", () => { const config = buildComplexityRouterConfig({ ...baseParams, classificationMode: "user_turn" }); @@ -924,10 +957,12 @@ describe("buildComplexityRouterConfig with an edited tier set", () => { dimensionWeights: { length: 1 }, reasoningOverrideMinScore: 0.5, heuristicFirstMaxTier: "SIMPLE", + hybridBoundaryMargin: 0.03, customTechnicalKeywords: ["kubernetes"], }; const emittingType = key === "heuristic_first_max_tier" ? "heuristic_first" : "llm"; - expect(buildComplexityRouterConfig({ ...baseParams, ...loaded, classifierType: emittingType })).toHaveProperty(key); + const typeForKey = key === "hybrid_boundary_margin" ? "hybrid" : emittingType; + expect(buildComplexityRouterConfig({ ...baseParams, ...loaded, classifierType: typeForKey })).toHaveProperty(key); expect(build(loaded)).not.toHaveProperty(key); }); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index af34dc92c0f..94c5badf6d1 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -107,6 +107,7 @@ export interface BuildComplexityRouterConfigParams { classifierFallback: ClassifierFallback | undefined; classificationPrompt: string | undefined; heuristicFirstMaxTier: string | undefined; + hybridBoundaryMargin?: number; classificationMode: ClassificationMode | undefined; sessionAffinity: boolean; modalityRouting?: boolean; @@ -163,6 +164,7 @@ export interface ComplexityRouterConfigPayload { classifier_fallback?: ClassifierFallback; classification_prompt?: string; heuristic_first_max_tier?: string; + hybrid_boundary_margin?: number; classification_mode: ClassificationMode; session_affinity: boolean; deployment_affinity: boolean; @@ -352,6 +354,7 @@ const classifierWireFields = ( classifierLlmConfig, classifierFallback, heuristicFirstMaxTier, + hybridBoundaryMargin, classifierContextWindowSize, classifierContextBudgetChars, classifierContextIncludeAssistantTurns, @@ -360,6 +363,7 @@ const classifierWireFields = ( | "classifierLlmConfig" | "classifierFallback" | "heuristicFirstMaxTier" + | "hybridBoundaryMargin" | "classifierContextWindowSize" | "classifierContextBudgetChars" | "classifierContextIncludeAssistantTurns" @@ -371,6 +375,8 @@ const classifierWireFields = ( classifierFallback !== undefined && { classifier_fallback: classifierFallback }), ...(effectiveType === "heuristic_first" && heuristicFirstMaxTier?.trim() && { heuristic_first_max_tier: heuristicFirstMaxTier }), + ...(effectiveType === "hybrid" && + hybridBoundaryMargin !== undefined && { hybrid_boundary_margin: hybridBoundaryMargin }), ...(usesLlmClassifier(effectiveType) && classifierContextWindowSize !== undefined && { classifier_context_window_size: classifierContextWindowSize, @@ -399,6 +405,7 @@ export const buildComplexityRouterConfig = ({ classifierFallback, classificationPrompt, heuristicFirstMaxTier, + hybridBoundaryMargin, classificationMode, sessionAffinity, modalityRouting, @@ -444,6 +451,7 @@ export const buildComplexityRouterConfig = ({ classifierLlmConfig, classifierFallback, heuristicFirstMaxTier, + hybridBoundaryMargin, classifierContextWindowSize, classifierContextBudgetChars, classifierContextIncludeAssistantTurns, diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts index d2706a82037..3c5b149f4da 100644 --- a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts @@ -122,10 +122,10 @@ export const CUSTOM_TIER_RESTRICTIONS = { reason: "Session pinning escalates along the built-in tier ladder, which your tier set replaces", }, heuristicClassifier: { - omit: ["heuristic_first_max_tier"], + omit: ["heuristic_first_max_tier", "hybrid_boundary_margin"], reason: "The heuristic scorer only produces the built-in tiers, so an edited set needs the LLM classifier. " + - "Heuristic first is out for the same reason: its local scorer decides the cheap traffic", + "Heuristic first and hybrid are out for the same reason: their local scorer decides the traffic it is sure of", }, heuristicScoring: { omit: [ diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 3e63cbd3b32..fae6744d3ac 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -520,16 +520,21 @@ describe("managed keys survive an untouched open-and-save", () => { }; // tier_definitions, fallback_tier and classification_prompt cannot sit beside heuristic_first, which - // this fixture uses, so no single stored config can hold every managed key. They get their own round - // trip below. - const CUSTOM_TIER_ONLY_KEYS = new Set(["tier_definitions", "fallback_tier", "classification_prompt"]); + // this fixture uses, and hybrid_boundary_margin belongs to the sibling hybrid type, so no single + // stored config can hold every managed key. Each gets its own round trip below. + const KEYS_ANOTHER_CLASSIFIER_TYPE_OWNS = new Set([ + "tier_definitions", + "fallback_tier", + "classification_prompt", + "hybrid_boundary_margin", + ]); it("carries every managed key a built-in router can hold through hydrate then save", () => { const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined); const saved = buildUpdatedComplexityRouterConfig(STORED_ALL_MANAGED, hydrated); const dropped = [...MANAGED_COMPLEXITY_ROUTER_KEYS] - .filter((key) => !CUSTOM_TIER_ONLY_KEYS.has(key)) + .filter((key) => !KEYS_ANOTHER_CLASSIFIER_TYPE_OWNS.has(key)) .filter((key) => saved[key] === undefined); expect(dropped).toEqual([]); }); @@ -583,6 +588,18 @@ describe("managed keys survive an untouched open-and-save", () => { ); }); + it("round-trips a hybrid router's margin, which save requires and the backend rejects without", () => { + const storedHybrid: Record = { + ...STORED_ALL_MANAGED, + classifier_type: "hybrid", + hybrid_boundary_margin: 0.05, + }; + delete storedHybrid.heuristic_first_max_tier; + const hydrated = hydrateComplexityRouterConfig(storedHybrid, undefined); + expect(hydrated.hybrid_boundary_margin).toBe(0.05); + expect(buildUpdatedComplexityRouterConfig(storedHybrid, hydrated).hybrid_boundary_margin).toBe(0.05); + }); + it("round-trips the heuristic_first threshold, which save requires and the backend rejects without", () => { const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined); expect(hydrated.heuristic_first_max_tier).toBe("SIMPLE"); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index e18582f77a0..7aa5f4beb1f 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -89,6 +89,7 @@ export interface StoredComplexityRouterConfig { plan_mode_min_tier?: unknown; classification_prompt?: unknown; heuristic_first_max_tier?: unknown; + hybrid_boundary_margin?: unknown; tier_labels?: unknown; classifier_type?: ClassifierType; classifier_llm_config?: ClassifierLLMConfig; @@ -167,6 +168,8 @@ export const hydrateComplexityRouterConfig = ( typeof parsedConfig.heuristic_first_max_tier === "string" && parsedConfig.heuristic_first_max_tier.trim() !== "" ? parsedConfig.heuristic_first_max_tier : undefined, + hybrid_boundary_margin: + typeof parsedConfig.hybrid_boundary_margin === "number" ? parsedConfig.hybrid_boundary_margin : undefined, classification_mode: parsedConfig.classification_mode === "user_turn" || parsedConfig.classification_mode === "every_request" ? parsedConfig.classification_mode @@ -214,6 +217,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "classifier_fallback", "classification_prompt", "heuristic_first_max_tier", + "hybrid_boundary_margin", "classification_mode", "session_affinity", "modality_routing", @@ -303,6 +307,7 @@ export const buildUpdatedComplexityRouterConfig = ( planModeMinTier: value.plan_mode_min_tier, classificationPrompt: value.classification_prompt, heuristicFirstMaxTier: value.heuristic_first_max_tier, + hybridBoundaryMargin: value.hybrid_boundary_margin, classificationMode: value.classification_mode, tierLabels: value.tier_labels, classifierType: value.classifier_type, diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index 66e80dcc655..ffef01010b7 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -86,6 +86,7 @@ const CONSTANT_CAUSE_LABELS: Record = { heuristic_scorer: "Heuristic scorer", heuristic_v2: "Heuristic v2", heuristic_first_short_circuit: "Heuristic scorer, classifier skipped", + hybrid_short_circuit: "Heuristic scorer, score clear of every boundary", classifier_plugin: "Custom classifier plugin", semantic_keyword_match: "Semantic keyword match", session_affinity_pin: "Pinned to session", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 79e44de4f1f..be81450b941 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -34560,7 +34560,7 @@ export interface components { * @enum {string} */ classifier_fallback: "heuristic" | "default_model"; - /** @description Configuration for the LLM classifier; required when classifier_type is 'llm' or 'heuristic_first' */ + /** @description Configuration for the LLM classifier; required when classifier_type is 'llm', 'heuristic_first' or 'hybrid' */ classifier_llm_config?: components["schemas"]["ClassifierLLMConfig"] | null; /** * Classifier Plugin @@ -34575,11 +34575,11 @@ export interface components { classifier_plugin_timeout_ms: number; /** * Classifier Type - * @description Classification strategy: local regex/keyword scoring, the bundled trained four-tier heuristic, an LLM call, a custom classifier plugin, or 'heuristic_first', which scores locally and only pays for the LLM classifier when the local scorer does not confidently land a cheap tier + * @description Classification strategy: local regex/keyword scoring, the bundled trained four-tier heuristic, an LLM call, a custom classifier plugin, 'heuristic_first', which scores locally and only pays for the LLM classifier when the local scorer does not confidently land a cheap tier, or 'hybrid', which trusts the local scorer everywhere except when its score lands near a tier boundary * @default heuristic * @enum {string} */ - classifier_type: "heuristic" | "heuristic_v2" | "llm" | "custom" | "heuristic_first"; + classifier_type: "heuristic" | "heuristic_v2" | "llm" | "custom" | "heuristic_first" | "hybrid"; /** * Code Keywords * @description Keywords indicating code-related content @@ -34651,6 +34651,11 @@ export interface components { * @description Additional case-sensitive literal sentinels that mark a request as client housekeeping, on top of the built-in conversation-title ones. For clients whose wording the built-ins don't cover, or after a client release changes its strings. */ housekeeping_patterns?: string[] | null; + /** + * Hybrid Boundary Margin + * @description How close to a tier boundary a heuristic score has to land before the LLM classifier breaks the tie; required when classifier_type is 'hybrid' and rejected otherwise. Everything further than this from every active boundary routes on the scorer's own tier with no classifier call, at any tier, which is what separates 'hybrid' from 'heuristic_first' and its cheap-tier ceiling. A prompt where no dimension fired still goes to the classifier, since the scorer has no opinion to be near a boundary with. 0 escalates only scores sitting exactly on a boundary. + */ + hybrid_boundary_margin?: number | null; /** * Keyword Tier Rules * @description Rules that force a specific tier when their keywords match the prompt @@ -35867,7 +35872,7 @@ export interface components { * Cause * @enum {string} */ - cause?: "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; + cause?: "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; /** Classifier Cost */ classifier_cost?: number; /** Classifier Model */ From 1df402b80d2d3f4fed8d504a5a73946162c133b6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:53:05 -0700 Subject: [PATCH 152/480] fix(vector-store): carry request metadata into the Router executor built from the router kwarg --- .../base_llm/vector_store/transformation.py | 16 ++++++- litellm/router.py | 11 +---- litellm/vector_stores/main.py | 13 ++++-- .../test_vector_store_endpoints.py | 2 +- .../test_milvus_vector_store.py | 46 +++++++++++++++++++ 5 files changed, 73 insertions(+), 15 deletions(-) diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 3547bf81448..9624a721870 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -62,6 +62,19 @@ class LiteLLMVectorStoreEmbeddingExecutor: ) +_REQUEST_METADATA: Final = TypeAdapter(dict[str, object]) + + +def vector_store_request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: + litellm_metadata: Final = kwargs.get("litellm_metadata") + if isinstance(litellm_metadata, dict): + return _REQUEST_METADATA.validate_python(litellm_metadata) + metadata: Final = kwargs.get("metadata") + if isinstance(metadata, dict): + return _REQUEST_METADATA.validate_python(metadata) + return MappingProxyType({}) + + @dataclass(frozen=True, slots=True) class RouterVectorStoreEmbeddingExecutor: router: Router @@ -312,11 +325,12 @@ class BaseQueryEmbeddingVectorStoreConfig(BaseVectorStoreConfig): def query_embedding_executor( embedding_executor: VectorStoreEmbeddingExecutor | None, router: Router | None, + request_metadata: Mapping[str, object] = MappingProxyType({}), ) -> VectorStoreEmbeddingExecutor: if embedding_executor is not None: return embedding_executor if router is not None: - return RouterVectorStoreEmbeddingExecutor(router=router, metadata=MappingProxyType({})) + return RouterVectorStoreEmbeddingExecutor(router=router, metadata=request_metadata) return LiteLLMVectorStoreEmbeddingExecutor() def embed_query( diff --git a/litellm/router.py b/litellm/router.py index 991f1bc2828..8b510e748f5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -87,6 +87,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( ) from litellm.llms.base_llm.vector_store.transformation import ( RouterVectorStoreEmbeddingExecutor, + vector_store_request_metadata, ) from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.router_strategy.budget_limiter import RouterBudgetLimiting @@ -6697,15 +6698,7 @@ class Router: @staticmethod def _vector_store_request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: - litellm_metadata: Final = kwargs.get("litellm_metadata") - if isinstance(litellm_metadata, dict): - return cast( # cast-ok: isinstance validates the runtime dict boundary - "dict[str, object]", litellm_metadata - ) - metadata: Final = kwargs.get("metadata") - if isinstance(metadata, dict): - return cast("dict[str, object]", metadata) # cast-ok: isinstance validates the runtime dict boundary - return MappingProxyType({}) + return vector_store_request_metadata(kwargs) async def _init_vector_store_api_endpoints( self, diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index d530a1c9d52..636bdd4b52e 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms.base_llm.vector_store.transformation import ( BaseQueryEmbeddingVectorStoreConfig, VectorStoreEmbeddingExecutor, + vector_store_request_metadata, ) from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.router import GenericLiteLLMParams @@ -42,10 +43,14 @@ base_llm_http_handler = BaseLLMHTTPHandler() ################################################# -def _direct_vector_store_embedding_executor(value: object, router: "Router | None") -> VectorStoreEmbeddingExecutor: +def _direct_vector_store_embedding_executor( + value: object, router: "Router | None", request_kwargs: Mapping[str, object] +) -> VectorStoreEmbeddingExecutor: if value is not None and not isinstance(value, VectorStoreEmbeddingExecutor): raise TypeError("Invalid direct vector store embedding executor") - return BaseQueryEmbeddingVectorStoreConfig.query_embedding_executor(value, router) + return BaseQueryEmbeddingVectorStoreConfig.query_embedding_executor( + value, router, vector_store_request_metadata(request_kwargs) + ) def mock_vector_store_search_response( @@ -300,7 +305,7 @@ async def asearch( Async: Search a vector store for relevant chunks based on a query and file attributes filter. """ embedding_executor: Final = _direct_vector_store_embedding_executor( - kwargs.pop("_direct_vector_store_embedding_executor", None), router + kwargs.pop("_direct_vector_store_embedding_executor", None), router, kwargs ) local_vars: Final = { # mutable-ok: exception logging requires a sanitized mutable snapshot key: value for key, value in locals().items() if key != "embedding_executor" @@ -386,7 +391,7 @@ def search( VectorStoreSearchResponse containing the search results. """ embedding_executor: Final = _direct_vector_store_embedding_executor( - kwargs.pop("_direct_vector_store_embedding_executor", None), router + kwargs.pop("_direct_vector_store_embedding_executor", None), router, kwargs ) local_vars: Final = { # mutable-ok: exception logging requires a sanitized mutable snapshot key: value for key, value in locals().items() if key != "embedding_executor" diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index a956aef51d6..1abbbe91e97 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -52,7 +52,7 @@ def _serialize_litellm_params(litellm_params): def test_direct_vector_store_embedding_executor_rejects_invalid_value(): with pytest.raises(TypeError, match="Invalid direct vector store embedding executor"): - _direct_vector_store_embedding_executor(object(), None) + _direct_vector_store_embedding_executor(object(), None, {}) def test_router_vector_store_search_injects_executor_and_request_metadata(): diff --git a/tests/vector_store_tests/test_milvus_vector_store.py b/tests/vector_store_tests/test_milvus_vector_store.py index 3b3bd444b20..2ba9168b49f 100644 --- a/tests/vector_store_tests/test_milvus_vector_store.py +++ b/tests/vector_store_tests/test_milvus_vector_store.py @@ -601,6 +601,52 @@ async def test_sdk_search_with_router_kwarg_resolves_bare_embedding_alias_async( _assert_alias_resolved(embedding_route, search_route, response) +def _team_alias_router(): + return Router( + model_list=[ + { + "model_name": "team-a-embedder", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "deployment-key", + }, + "model_info": {"team_id": "team-a", "team_public_model_name": "multilingual-e5-large"}, + } + ] + ) + + +@pytest.mark.asyncio +async def test_sdk_search_with_router_kwarg_resolves_team_alias_from_request_metadata( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_search_route(respx_mock) + + response = await litellm.vector_stores.asearch( + router=_team_alias_router(), metadata={"user_api_key_team_id": "team-a"}, **ALIAS_SEARCH_KWARGS + ) + + _assert_alias_resolved(embedding_route, search_route, response) + + +@pytest.mark.asyncio +async def test_sdk_search_with_router_kwarg_rejects_team_alias_without_team_metadata( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + _mock_search_route(respx_mock) + + with pytest.raises(litellm.APIConnectionError): + await litellm.vector_stores.asearch(router=_team_alias_router(), **ALIAS_SEARCH_KWARGS) + + assert embedding_route.call_count == 0 + + @pytest.mark.asyncio async def test_transform_uses_injected_executor_without_embedding_config(respx_mock: respx.MockRouter): executor = RecordingEmbeddingExecutor(ALIAS_EMBEDDING_RESPONSE) From b0fe71010b89425501587a3e7e66ede9a4e6e775 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:56:35 -0700 Subject: [PATCH 153/480] fix(ollama_chat): stamp finish_reason tool_calls when tool calls streamed before the done chunk (#39010) * fix(ollama_chat): stamp finish_reason tool_calls when tool calls streamed before the done chunk Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ollama_chat): trim finish_reason override comment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mubashir1osmani --- litellm/llms/ollama/chat/transformation.py | 7 +- .../test_streaming_iterator_stop_reason.py | 79 +++++++++++++++++++ .../ollama/test_ollama_chat_transformation.py | 40 ++++++++++ 3 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_stop_reason.py diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index de626b468f0..181894646e3 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -423,6 +423,7 @@ class OllamaChatConfig(BaseConfig): class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): started_reasoning_content: bool = False finished_reasoning_content: bool = False + seen_tool_calls: bool = False def _is_function_call_complete(self, function_args: str | dict) -> bool: if isinstance(function_args, dict): @@ -468,6 +469,7 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): # process tool calls - if complete function arg - add id to tool call tool_calls: Final = chunk["message"].get("tool_calls") if tool_calls is not None: + self.seen_tool_calls = True for tool_call in tool_calls: function_args = tool_call.get("function").get("arguments") if function_args is not None and len(function_args) > 0: @@ -508,9 +510,10 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): if chunk["done"] is True: finish_reason = chunk.get("done_reason") or "stop" - # Override finish_reason when tool_calls are present + # Override finish_reason when tool_calls appeared in any chunk # Fixes: https://github.com/BerriAI/litellm/issues/18922 - if tool_calls is not None: + # Fixes: https://github.com/BerriAI/litellm/issues/34692 + if self.seen_tool_calls: finish_reason = "tool_calls" choices = [ StreamingChoices( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_stop_reason.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_stop_reason.py new file mode 100644 index 00000000000..4b95b36fec3 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_stop_reason.py @@ -0,0 +1,79 @@ +""" +Regression tests for issue #34692. + +ollama_chat streams tool_calls in a mid-stream chunk while its final +(``done: true``) chunk carries only ``done_reason: "stop"``. The provider +iterator must remember the earlier tool_calls and stamp +``finish_reason="tool_calls"`` on the final chunk, so the Anthropic +``/v1/messages`` bridge emits ``stop_reason: "tool_use"``. Before the fix the +bridge emitted ``stop_reason: "end_turn"`` and Anthropic tool-runners +(Claude Code, ``messages.stream``) silently dropped the tool call. +""" + +import pytest + +from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( + AnthropicStreamWrapper, +) +from litellm.llms.ollama.chat.transformation import ( + OllamaChatCompletionResponseIterator, +) +from litellm.types.utils import ModelResponseStream + +_OLLAMA_TOOL_CHUNK = { + "model": "qwen3:8b", + "message": { + "role": "assistant", + "content": "", + "tool_calls": [{"function": {"name": "get_weather", "arguments": {"city": "San Francisco"}}}], + }, + "done": False, +} +_OLLAMA_DONE_CHUNK = { + "model": "qwen3:8b", + "message": {"role": "assistant", "content": ""}, + "done": True, + "done_reason": "stop", + "prompt_eval_count": 100, + "eval_count": 20, +} + + +def _ollama_streamed_chunks() -> list[ModelResponseStream]: + iterator = OllamaChatCompletionResponseIterator(streaming_response=iter([]), sync_stream=True) + return [iterator.chunk_parser(_OLLAMA_TOOL_CHUNK), iterator.chunk_parser(_OLLAMA_DONE_CHUNK)] + + +class _AsyncStream: + def __init__(self, items: list[ModelResponseStream]): + self._it = iter(items) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._it) + except StopIteration: + raise StopAsyncIteration + + +def _assert_tool_use_stop_reason(events: list[dict]) -> None: + block_types = [e["content_block"]["type"] for e in events if e.get("type") == "content_block_start"] + assert "tool_use" in block_types, f"no tool_use content block opened: {events}" + message_deltas = [e for e in events if e.get("type") == "message_delta"] + assert message_deltas, f"no message_delta emitted: {events}" + assert message_deltas[-1]["delta"]["stop_reason"] == "tool_use", ( + f"expected stop_reason 'tool_use', got: {message_deltas[-1]}" + ) + + +def test_ollama_mid_stream_tool_call_yields_tool_use_stop_reason_sync(): + wrapper = AnthropicStreamWrapper(completion_stream=iter(_ollama_streamed_chunks()), model="qwen3:8b") + _assert_tool_use_stop_reason(list(wrapper)) + + +@pytest.mark.asyncio +async def test_ollama_mid_stream_tool_call_yields_tool_use_stop_reason_async(): + wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(_ollama_streamed_chunks()), model="qwen3:8b") + _assert_tool_use_stop_reason([event async for event in wrapper]) diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index 8f3dbf7b0d9..25f9645faa0 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -615,6 +615,46 @@ class TestOllamaFinishReasonLength: result.choices[0].finish_reason == "stop" ), f"Expected 'stop' for natural finish, got '{result.choices[0].finish_reason}'" + def test_finish_reason_tool_calls_streamed_before_done_chunk(self): + """Streaming: tool_calls arriving mid-stream (not on the done chunk) must + still produce finish_reason='tool_calls' on the final chunk. + + Regression test for https://github.com/BerriAI/litellm/issues/34692: + Ollama emits tool_calls in an earlier chunk and the done chunk carries + none, which left finish_reason at 'stop' and made the Anthropic + /v1/messages bridge emit stop_reason 'end_turn' instead of 'tool_use'. + """ + iterator = OllamaChatCompletionResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + tool_chunk = { + "model": "qwen3:8b", + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + {"function": {"name": "get_weather", "arguments": {"city": "San Francisco"}}} + ], + }, + "done": False, + } + done_chunk = { + "model": "qwen3:8b", + "message": {"role": "assistant", "content": ""}, + "done": True, + "done_reason": "stop", + } + + tool_result = iterator.chunk_parser(tool_chunk) + assert tool_result.choices[0].delta.tool_calls is not None + + done_result = iterator.chunk_parser(done_chunk) + assert ( + done_result.choices[0].finish_reason == "tool_calls" + ), f"Expected 'tool_calls' when tool_calls were streamed earlier, got '{done_result.choices[0].finish_reason}'" + class TestOllamaReasoningContentStreaming: """Test that reasoning_content is properly extracted from all thinking chunks.""" From e641864e6f863ef03cd8724b1cb4d69097d447b0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:58:53 -0700 Subject: [PATCH 154/480] test(e2e/ui): give the seeded users passwords that pass the default password policy The default password policy from #39381 (12+ chars with upper, lower, number, and special) makes globalSetup's POST /user/update with password "test" fail with a 400, so every role login in the UI e2e suite has been failing since that merge. Seed the roles, the noteam@test.local login, and the second-admin spec with policy-compliant passwords and update the scrypt hash in seed.sql to match. --- tests/e2e/ui/constants.ts | 1 + tests/e2e/ui/fixtures/seed.sql | 20 +++++++++---------- tests/e2e/ui/fixtures/users.ts | 9 +++++---- .../internal-user/internalUserNoTeam.spec.ts | 3 ++- .../ui/tests/proxy-admin/secondAdmin.spec.ts | 2 +- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/tests/e2e/ui/constants.ts b/tests/e2e/ui/constants.ts index bb33c90ddf3..3a62252915c 100644 --- a/tests/e2e/ui/constants.ts +++ b/tests/e2e/ui/constants.ts @@ -30,6 +30,7 @@ export const E2E_PROXY_ADMIN_EMAIL = "admin@test.local"; export const E2E_INTERNAL_USER_ID = "e2e-internal-user"; export const E2E_INTERNAL_USER_EMAIL = "internal@test.local"; export const E2E_TEAM_ADMIN_USER_ID = "e2e-team-admin"; +export const E2E_SEEDED_USER_PASSWORD = "E2e-Test-Pass-2026!"; // Key aliases for seeded test keys (match seed.sql) export const E2E_UPDATE_LIMITS_KEY_ALIAS = "e2eUpdateLimitsKey"; diff --git a/tests/e2e/ui/fixtures/seed.sql b/tests/e2e/ui/fixtures/seed.sql index e77b4a16b3d..00ea668ed8f 100644 --- a/tests/e2e/ui/fixtures/seed.sql +++ b/tests/e2e/ui/fixtures/seed.sql @@ -24,18 +24,18 @@ INSERT INTO "LiteLLM_OrganizationTable" ( 'e2e-proxy-admin', 'e2e-proxy-admin' ); --- 4. Users (password hash is scrypt of "test") +-- 4. Users (password hash is scrypt of E2E_SEEDED_USER_PASSWORD from constants.ts) INSERT INTO "LiteLLM_UserTable" ("user_id", "user_email", "user_role", "teams", "password") VALUES - ('e2e-proxy-admin', 'admin@test.local', 'proxy_admin', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org","e2e-team-keygen"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-noteam', 'noteam@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-invitable-by-team-admin', 'invitable-team@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-removable-member', 'removable@test.local', 'internal_user', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'); + ('e2e-proxy-admin', 'admin@test.local', 'proxy_admin', '{"e2e-team-crud"}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', '{}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org","e2e-team-keygen"}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-internal-noteam', 'noteam@test.local', 'internal_user', '{}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-invitable-by-team-admin', 'invitable-team@test.local', 'internal_user', '{}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-removable-member', 'removable@test.local', 'internal_user', '{"e2e-team-crud"}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'); -- 5. Teams (members_with_roles is required JSON) INSERT INTO "LiteLLM_TeamTable" ( diff --git a/tests/e2e/ui/fixtures/users.ts b/tests/e2e/ui/fixtures/users.ts index 79ee237f334..0457361d9d4 100644 --- a/tests/e2e/ui/fixtures/users.ts +++ b/tests/e2e/ui/fixtures/users.ts @@ -1,6 +1,7 @@ import { ADMIN_STORAGE_PATH, ADMIN_VIEWER_STORAGE_PATH, + E2E_SEEDED_USER_PASSWORD, INTERNAL_USER_STORAGE_PATH, INTERNAL_VIEWER_STORAGE_PATH, TEAM_ADMIN_STORAGE_PATH, @@ -23,22 +24,22 @@ export const users: Record { // Log in via the form as the no-team seeded user. await page.goto("/ui/login"); await page.getByPlaceholder("Enter your username").fill("noteam@test.local"); - await page.getByPlaceholder("Enter your password").fill("test"); + await page.getByPlaceholder("Enter your password").fill(E2E_SEEDED_USER_PASSWORD); await page.getByRole("button", { name: "Login", exact: true }).click(); await expect(page.getByRole("complementary").getByText("Virtual Keys")).toBeVisible({ timeout: 30_000 }); expect(new URL(page.url()).pathname).not.toMatch(/\/connect$/); diff --git a/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts b/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts index 5a8bc84cc13..73263c844fa 100644 --- a/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts @@ -10,7 +10,7 @@ test.describe("Second proxy admin", () => { test("an invited admin can log in, mint a key, and call a model with it", async ({ page, browser, request }) => { const suffix = Date.now(); const email = `second-admin-${suffix}@test.local`; - const password = "e2e-second-admin-password"; + const password = "E2e-Second-Admin-Pass-1!"; const auth = { Authorization: `Bearer ${masterKey()}` }; const inviteAdminUser = async (): Promise => { From de643c028fce8d7634191550968eae7c4c991f04 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 3 Sep 2026 00:25:50 +0000 Subject: [PATCH 155/480] style: format handler after merge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/openai/responses/guardrail_translation/handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 0c19ecf909e..59ca601d8be 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -374,7 +374,9 @@ class OpenAIResponsesHandler(BaseTranslation): input_type="request", logging_obj=litellm_logging_obj, ) - self._apply_guardrailed_tools_to_data(data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")) + self._apply_guardrailed_tools_to_data( + data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools") + ) written_back: Final = self._written_back_request_fields(data, structured_messages, guardrailed_inputs) if written_back is not None: data["input"] = list(written_back.input) # mutable-ok: JSON body From 5ad330f6203af47ced983a6fc3852288d0a5cdc1 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:30:46 -0700 Subject: [PATCH 156/480] fix(http_handler): honor HTTP(S)_PROXY / NO_PROXY when force_ipv4 uses the httpx transport (#39443) * fix(http_handler): honor HTTP(S)_PROXY / NO_PROXY when force_ipv4 uses the httpx transport Passing an explicit transport to httpx.AsyncClient / httpx.Client disables its automatic environment proxy mounts, so force_ipv4 on the httpx path sent every LLM request direct and silently bypassed HTTPS_PROXY. Mount the same env-derived proxy transports next to the IPv4-pinned direct transport in AsyncHTTPHandler, HTTPHandler and the OpenAI async client factory. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(http_handler): carry the client's TLS verify and cert settings onto env proxy mounts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/custom_httpx/http_handler.py | 49 +++- litellm/llms/openai/common_utils.py | 12 +- .../llms/custom_httpx/test_http_handler.py | 233 ++++++++++++++++++ 3 files changed, 285 insertions(+), 9 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index b6e93f590ca..e1f0fc9e7d3 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -9,13 +9,15 @@ import threading import time from collections.abc import AsyncIterable, Callable, Iterable, Mapping from http.cookiejar import CookieJar, DefaultCookiePolicy -from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, Optional, TypeAlias, TypedDict +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, Optional, TypeAlias, TypedDict, TypeVar import certifi import httpx from aiohttp import ClientSession, DummyCookieJar, TCPConnector from httpx import USE_CLIENT_DEFAULT, AsyncHTTPTransport, HTTPTransport -from httpx._types import RequestFiles +from httpx._types import CertTypes, RequestFiles +from httpx._utils import get_environment_proxies import litellm from litellm._logging import verbose_logger @@ -66,6 +68,22 @@ _AddrInfo: TypeAlias = tuple[int | socket.AddressFamily, int | socket.SocketKind _RequestContent: TypeAlias = str | bytes | Iterable[bytes] | AsyncIterable[bytes] +_IPV4_LOCAL_ADDRESS: Final = "0.0.0.0" + +_HttpxTransportT = TypeVar("_HttpxTransportT", HTTPTransport, AsyncHTTPTransport) + + +def _environment_proxy_mounts( + build_proxy_transport: Callable[[str], _HttpxTransportT], +) -> Mapping[str, _HttpxTransportT | None]: + """httpx skips its own HTTP(S)_PROXY / NO_PROXY mounts whenever an explicit `transport=` is passed.""" + return MappingProxyType( + { + pattern: None if proxy_url is None else build_proxy_transport(proxy_url) + for pattern, proxy_url in get_environment_proxies().items() + } + ) + class _TCPConnectorKwargs(TypedDict, total=False): local_addr: tuple[str, int] | None @@ -607,6 +625,7 @@ class AsyncHTTPHandler: return httpx.AsyncClient( transport=transport, + mounts=AsyncHTTPHandler._create_httpx_proxy_mounts(transport, verify=ssl_config, cert=cert), event_hooks=event_hooks, timeout=timeout, verify=ssl_config, @@ -1191,10 +1210,22 @@ class AsyncHTTPHandler: - [Default] If force_ipv4 is False, it will return None """ if litellm.force_ipv4: - return AsyncHTTPTransport(local_address="0.0.0.0") + return AsyncHTTPTransport(local_address=_IPV4_LOCAL_ADDRESS) else: return None + @staticmethod + def _create_httpx_proxy_mounts( + transport: LiteLLMAiohttpTransport | AsyncHTTPTransport | None, + verify: VerifyTypes, + cert: CertTypes | None, + ) -> Mapping[str, AsyncHTTPTransport | None] | None: + if not isinstance(transport, AsyncHTTPTransport): + return None + return _environment_proxy_mounts( + lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert) + ) + class HTTPHandler: def __init__( @@ -1227,6 +1258,7 @@ class HTTPHandler: # Create a client with a connection pool return httpx.Client( transport=self._create_sync_transport(), + mounts=self._create_sync_proxy_mounts(verify=ssl_config, cert=cert), timeout=self.timeout if self.timeout is not None else _DEFAULT_TIMEOUT, verify=ssl_config, cert=cert, @@ -1507,10 +1539,19 @@ class HTTPHandler: Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them """ if litellm.force_ipv4: - return HTTPTransport(local_address="0.0.0.0") + return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS) else: return getattr(litellm, "sync_transport", None) + @staticmethod + def _create_sync_proxy_mounts( + verify: VerifyTypes, + cert: CertTypes | None, + ) -> Mapping[str, HTTPTransport | None] | None: + if not litellm.force_ipv4: + return None + return _environment_proxy_mounts(lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert)) + def get_async_httpx_client( llm_provider: LlmProviders | httpxSpecialProvider, diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 4d774f6f165..bcd4ea43243 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -305,14 +305,16 @@ class BaseOpenAILLM: # Get unified SSL configuration ssl_config: Final = get_ssl_configuration() + transport: Final = AsyncHTTPHandler._create_async_transport( + ssl_context=(ssl_config if isinstance(ssl_config, ssl.SSLContext) else None), + ssl_verify=ssl_config if isinstance(ssl_config, bool) else None, + shared_session=shared_session, + ) return httpx.AsyncClient( verify=ssl_config, - transport=AsyncHTTPHandler._create_async_transport( - ssl_context=(ssl_config if isinstance(ssl_config, ssl.SSLContext) else None), - ssl_verify=ssl_config if isinstance(ssl_config, bool) else None, - shared_session=shared_session, - ), + transport=transport, + mounts=AsyncHTTPHandler._create_httpx_proxy_mounts(transport, verify=ssl_config, cert=None), follow_redirects=True, ) diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index 16d57437043..9e64bfafa54 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -1314,3 +1314,236 @@ async def test_finalizer_on_live_loop_disposes_foreign_loop_session_without_sche assert AsyncHTTPHandler._finalizer_close_tasks == baseline_tasks assert session.closed + + +@pytest.fixture +def forward_proxy_server(): + """Plain HTTP forward proxy that records the absolute URIs it is asked to fetch.""" + from http.server import BaseHTTPRequestHandler, HTTPServer + from socketserver import ThreadingMixIn + + seen_uris: list[str] = [] + + class RecordingProxyHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self): + seen_uris.append(self.path) + self.send_response(200) + self.send_header("Content-Length", "9") + self.end_headers() + self.wfile.write(b"via-proxy") + + def log_message(self, format, *args): + pass + + class ThreadedServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + + server = ThreadedServer(("127.0.0.1", 0), RecordingProxyHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", seen_uris + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +# `.invalid` never resolves (RFC 6761), so the only way this request can succeed is through the proxy +_PROXY_ONLY_UPSTREAM_URL = "http://upstream.invalid/v1/models" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("disable_aiohttp_transport", [True, False]) +@pytest.mark.parametrize("force_ipv4", [True, False]) +async def test_async_handler_honours_proxy_env_for_every_transport( + forward_proxy_server, monkeypatch: pytest.MonkeyPatch, disable_aiohttp_transport: bool, force_ipv4: bool +): + proxy_url, seen_uris = forward_proxy_server + monkeypatch.setenv("HTTP_PROXY", proxy_url) + monkeypatch.delenv("NO_PROXY", raising=False) + monkeypatch.delenv("no_proxy", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", disable_aiohttp_transport) + monkeypatch.setattr(litellm, "force_ipv4", force_ipv4) + + handler = AsyncHTTPHandler() + try: + response = await handler.get(_PROXY_ONLY_UPSTREAM_URL) + finally: + await handler.close() + + assert response.text == "via-proxy" + assert seen_uris == [_PROXY_ONLY_UPSTREAM_URL] + + +@pytest.mark.parametrize("force_ipv4", [True, False]) +def test_sync_handler_honours_proxy_env(forward_proxy_server, monkeypatch: pytest.MonkeyPatch, force_ipv4: bool): + proxy_url, seen_uris = forward_proxy_server + monkeypatch.setenv("HTTP_PROXY", proxy_url) + monkeypatch.delenv("NO_PROXY", raising=False) + monkeypatch.delenv("no_proxy", raising=False) + monkeypatch.setattr(litellm, "force_ipv4", force_ipv4) + + handler = HTTPHandler() + try: + response = handler.get(_PROXY_ONLY_UPSTREAM_URL) + finally: + handler.close() + + assert response.text == "via-proxy" + assert seen_uris == [_PROXY_ONLY_UPSTREAM_URL] + + +@pytest.mark.asyncio +async def test_force_ipv4_httpx_transport_honours_no_proxy(keepalive_server, monkeypatch: pytest.MonkeyPatch): + """NO_PROXY hosts must still go direct when the proxy mounts are supplied by litellm instead of httpx.""" + monkeypatch.setenv("HTTP_PROXY", "http://proxy.invalid:3128") + monkeypatch.setenv("NO_PROXY", "127.0.0.1") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "force_ipv4", True) + + handler = AsyncHTTPHandler() + try: + response = await handler.get(keepalive_server) + finally: + await handler.close() + + assert response.text == "ok" + + +@pytest.fixture +def private_ca_tls_upstream(tmp_path: pathlib.Path): + """HTTPS server behind a CONNECT proxy, both on localhost; the server's cert is signed by a test-only CA.""" + import datetime + import select + import socket + from http.server import BaseHTTPRequestHandler, HTTPServer + from socketserver import ThreadingMixIn + + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import NameOID + + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "upstream.invalid")]) + now = datetime.datetime.now(datetime.timezone.utc) + cert = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(minutes=1)) + .not_valid_after(now + datetime.timedelta(hours=1)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("upstream.invalid")]), critical=False) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .sign(key, hashes.SHA256()) + ) + ca_pem = tmp_path / "ca.pem" + ca_pem.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_pem = tmp_path / "key.pem" + key_pem.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ) + ) + + class OkTlsHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self): + self.send_response(200) + self.send_header("Content-Length", "6") + self.end_headers() + self.wfile.write(b"ok-tls") + + def log_message(self, format, *args): + pass + + class ThreadedServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + + tls_server = ThreadedServer(("127.0.0.1", 0), OkTlsHandler) + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(str(ca_pem), str(key_pem)) + tls_server.socket = server_ctx.wrap_socket(tls_server.socket, server_side=True) + tls_port = tls_server.server_port + + class ConnectProxyHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_CONNECT(self): + upstream = socket.create_connection(("127.0.0.1", tls_port)) + self.send_response(200, "Connection established") + self.end_headers() + sockets = [self.connection, upstream] + while True: + readable, _, _ = select.select(sockets, [], [], 5) + if not readable: + break + for src in readable: + data = src.recv(65536) + if not data: + upstream.close() + return + (upstream if src is self.connection else self.connection).sendall(data) + + def log_message(self, format, *args): + pass + + proxy_server = ThreadedServer(("127.0.0.1", 0), ConnectProxyHandler) + threads = [ + threading.Thread(target=tls_server.serve_forever, daemon=True), + threading.Thread(target=proxy_server.serve_forever, daemon=True), + ] + for thread in threads: + thread.start() + try: + yield f"http://127.0.0.1:{proxy_server.server_port}", str(ca_pem) + finally: + for server in (proxy_server, tls_server): + server.shutdown() + server.server_close() + for thread in threads: + thread.join(timeout=5) + + +@pytest.mark.asyncio +async def test_force_ipv4_https_proxy_mount_uses_handler_ca_bundle( + private_ca_tls_upstream, monkeypatch: pytest.MonkeyPatch +): + proxy_url, ca_pem = private_ca_tls_upstream + monkeypatch.setenv("HTTPS_PROXY", proxy_url) + monkeypatch.delenv("NO_PROXY", raising=False) + monkeypatch.delenv("no_proxy", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "force_ipv4", True) + + handler = AsyncHTTPHandler(ssl_verify=ca_pem) + try: + response = await handler.get("https://upstream.invalid/v1/models") + finally: + await handler.close() + + assert response.text == "ok-tls" + + +def test_sync_force_ipv4_https_proxy_mount_uses_handler_ca_bundle( + private_ca_tls_upstream, monkeypatch: pytest.MonkeyPatch +): + proxy_url, ca_pem = private_ca_tls_upstream + monkeypatch.setenv("HTTPS_PROXY", proxy_url) + monkeypatch.delenv("NO_PROXY", raising=False) + monkeypatch.delenv("no_proxy", raising=False) + monkeypatch.setattr(litellm, "force_ipv4", True) + + handler = HTTPHandler(ssl_verify=ca_pem) + try: + response = handler.get("https://upstream.invalid/v1/models") + finally: + handler.close() + + assert response.text == "ok-tls" From 7978b9f7211784a0576c31386efc4ebb66cddae0 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 2 Sep 2026 17:32:00 -0700 Subject: [PATCH 157/480] fix(proxy): stop leaking internal exception details to clients (#39380) * fix(proxy): stop leaking internal exception details to clients Public error responses could disclose internal details in two places. A proxy-layer exception with no recognized provider status code (a bug in a custom callback, a hook, or litellm's own code) forwarded its raw str() text verbatim on a 5xx, including any embedded credential, filesystem path, or internal hostname, or a full stack trace; the same client-facing message now runs through a redaction layer built on top of the credential redaction that already runs on log output, so it also drops an embedded traceback and scrubs path-shaped and hostname-shaped substrings. It intentionally never runs on server-side logs, which must keep full detail for debugging. exception_type(), litellm's core exception mapper, is shared by direct SDK callers (litellm.completion()) and the proxy, and it deliberately embeds a traceback into an unmapped exception's message as a debugging aid for library users; a first pass at this fix stripped that traceback inside exception_type() itself and broke that convention (caught by tests asserting on the traceback frame). The traceback stays in exception_type()'s own output; only the proxy's client-facing response boundary (and the streaming response generator, which never needs to embed one at all) strips it. Full generic-message replacement for the unclassified-exception case was tried first and reverted too: several routes deliberately raise a bare exception as an informative, secret-free validation message (e.g. the OCR endpoint's rejection of provider-native file IDs), and replacing those wholesale broke that convention; targeted redaction leaves them untouched. Also stops the default uvicorn-based proxy from sending a Server response header. Resolves LIT-6747 * refactor(proxy): drop the unrelated error-message constant and trim redaction comments Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/_logging.py | 14 +++- .../litellm_core_utils/secret_redaction.py | 21 ++++++ litellm/proxy/common_request_processing.py | 9 +-- litellm/proxy/proxy_cli.py | 1 + .../test_exception_mapping_utils.py | 42 ++++++++++++ .../proxy/test_common_request_processing.py | 58 +++++++++++++++- tests/test_litellm/proxy/test_proxy_cli.py | 1 + .../test_redact_string_in_error_paths.py | 2 - tests/test_litellm/test_secret_redaction.py | 66 ++++++++++++++++++- 9 files changed, 202 insertions(+), 12 deletions(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index 14cda772234..c73b5175a31 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -17,7 +17,11 @@ from litellm.constants import ( from litellm.litellm_core_utils.env_utils import get_env_int from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.litellm_core_utils.secret_redaction import redact_string, redact_structured_value +from litellm.litellm_core_utils.secret_redaction import ( + redact_internal_details, + redact_string, + redact_structured_value, +) set_verbose = False @@ -89,6 +93,14 @@ def redact_secrets(value: str) -> str: return _redact_string(value) +def redact_internal_details_from_client_message(value: str) -> str: + """Public API: redact_secrets() plus filesystem paths, internal hostnames, and an + embedded traceback, for a string about to leave the process in an HTTP response.""" + if not _ENABLE_SECRET_REDACTION: + return value + return redact_internal_details(value) + + def _substituted_color_message(record: logging.LogRecord) -> str | None: """Render a record's ``color_message`` against its args, or None if absent. diff --git a/litellm/litellm_core_utils/secret_redaction.py b/litellm/litellm_core_utils/secret_redaction.py index e93ab155786..b62226a6a19 100644 --- a/litellm/litellm_core_utils/secret_redaction.py +++ b/litellm/litellm_core_utils/secret_redaction.py @@ -92,6 +92,27 @@ def redact_string(value: str) -> str: return _SECRET_RE.sub(_REDACTED, value) +_UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+" +_WINDOWS_DRIVE_PATH: Final = r"[A-Za-z]:\\[^\s'\"\)\]}>,]+" +_PRIVATE_OR_LOOPBACK_IPV4: Final = ( + r"\b(?:10(?:\.\d{1,3}){3}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|192\.168(?:\.\d{1,3}){2}|127(?:\.\d{1,3}){3})\b" +) +_INTERNAL_SUFFIX_HOSTNAME: Final = r"\b[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.(?:internal|local|corp|lan|intra|private)\b" +_INTERNAL_DETAIL_RE: Final = re.compile( + "|".join((_UNIX_SYSTEM_PATH, _WINDOWS_DRIVE_PATH, _PRIVATE_OR_LOOPBACK_IPV4, _INTERNAL_SUFFIX_HOSTNAME)), + re.IGNORECASE, +) +_TRACEBACK_MARKER: Final = "Traceback (most recent call last):" + + +def redact_internal_details(value: str) -> str: + """Drop an embedded traceback and scrub filesystem paths and internal hostnames, + on top of redact_string(). For client-facing messages only: server logs keep this detail.""" + marker_index: Final = value.find(_TRACEBACK_MARKER) + without_traceback: Final = value[:marker_index].rstrip() if marker_index != -1 else value + return _INTERNAL_DETAIL_RE.sub(_REDACTED, redact_string(without_traceback)) + + def redact_structured_value(key: str | None, value: str) -> str: """Scrub *value* as it appeared under *key* inside a structured record. diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 05ddef822f1..fc83c1ddeed 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3,7 +3,6 @@ import contextlib import json import logging import math -import traceback from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Mapping, Sequence from datetime import datetime from functools import lru_cache @@ -18,7 +17,7 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse from starlette.types import Receive, Scope, Send import litellm -from litellm._logging import _redact_string, verbose_proxy_logger +from litellm._logging import redact_internal_details_from_client_message, verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import ( DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE, @@ -3417,7 +3416,7 @@ class ProxyBaseLLMRequestProcessing: else: _code = status.HTTP_500_INTERNAL_SERVER_ERROR raise ProxyException( - message=getattr(e, "message", error_msg), + message=redact_internal_details_from_client_message(getattr(e, "message", error_msg)), type=getattr(e, "type", "None"), param=getattr(e, "param", "None"), openai_code=getattr(e, "code", None), @@ -3629,10 +3628,8 @@ class ProxyBaseLLMRequestProcessing: if isinstance(e, HTTPException): raise e - error_traceback: Final = _redact_string(traceback.format_exc()) - error_msg: Final = f"{e}\n\n{error_traceback}" proxy_exception: Final = ProxyException( - message=getattr(e, "message", error_msg), + message=redact_internal_details_from_client_message(getattr(e, "message", str(e))), type=getattr(e, "type", "None"), param=getattr(e, "param", "None"), code=getattr(e, "status_code", 500), diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 23932ba7c8c..ed247d52ce2 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -261,6 +261,7 @@ class ProxyInitializationHelpers: "app": "litellm.proxy.proxy_server:app", "host": host, "port": port, + "server_header": False, } if log_config is not None: print(f"Using log_config: {log_config}") diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index 15b7ae9d07a..1778eca25ef 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -1013,6 +1013,48 @@ def test_an_unmapped_exception_with_no_model_or_provider_is_a_connection_error(q assert "boom" in raised.value.message +def _raise_and_map( + model: str | None, original_exception: Exception, custom_llm_provider: str | None +) -> None: + """Calls exception_type() from inside the except block, as litellm/main.py does, + so traceback.format_exc() has a real stack.""" + try: + raise original_exception + except type(original_exception) as caught: + exception_type( + model=model, + original_exception=caught, + custom_llm_provider=custom_llm_provider, + ) + + +def test_an_unmapped_exception_message_keeps_traceback_for_sdk_callers(quiet_exception_mapping): + """Direct SDK callers debug unmapped provider exceptions with this traceback; + only the proxy's response boundary strips it.""" + with pytest.raises(litellm.APIConnectionError) as raised: + _raise_and_map( + model="MiniMax-M2.5", + original_exception=RuntimeError("socket hung up"), + custom_llm_provider="minimax", + ) + + assert "Traceback (most recent call last)" in raised.value.message + assert "test_exception_mapping_utils.py" in raised.value.message + + +def test_an_unmapped_exception_with_no_model_or_provider_message_keeps_traceback( + quiet_exception_mapping, +): + with pytest.raises(litellm.APIConnectionError) as raised: + _raise_and_map( + model=None, + original_exception=ValueError("boom"), + custom_llm_provider=None, + ) + + assert "Traceback (most recent call last)" in raised.value.message + + CONTEXT_WINDOW_MESSAGE = "This model's maximum context length is 4096 tokens." CONTENT_POLICY_MESSAGE = ( '{"error": {"type": "invalid_request_error", "code": "content_policy_violation"}}' diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index df14224af5c..6d6aad22ca3 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -3013,7 +3013,7 @@ class TestHandleLLMApiExceptionDictDetail: assert "NotFoundError" in proxy_exc.message async def test_exception_with_status_code_propagates(self): - """Exception with a statically-set status_code should propagate it.""" + """Exception with a statically-set status_code should propagate it and its message.""" from litellm.llms.vertex_ai.common_utils import VertexAIError exc = VertexAIError( @@ -3022,12 +3022,30 @@ class TestHandleLLMApiExceptionDictDetail: ) proxy_exc = await self._invoke(exc) assert proxy_exc.code == "429" + assert proxy_exc.message == "Rate limit exceeded" async def test_exception_without_status_code_defaults_to_500(self): - """Exception with no status_code attribute defaults to 500.""" + """Exception with no status_code attribute defaults to 500; a message with nothing + to redact still reaches the client, since routes raise plain exceptions as validation text.""" exc = ValueError("Something broke") proxy_exc = await self._invoke(exc) assert proxy_exc.code == "500" + assert proxy_exc.message == "Something broke" + + async def test_unclassified_exception_redacts_internal_details_from_client_message(self): + """Regression for LIT-6747: an unclassified exception's credential, path, and host + must not reach the client.""" + exc = RuntimeError( + "Failed to connect to postgresql://litellm_internal:S3cr3tPGPass@10.20.30.40:5432/litellm_prod " + "(config file /etc/litellm/secrets/db.yaml)" + ) + proxy_exc = await self._invoke(exc) + assert proxy_exc.code == "500" + assert "S3cr3tPGPass" not in proxy_exc.message + assert "litellm_internal" not in proxy_exc.message + assert "10.20.30.40" not in proxy_exc.message + assert "/etc/litellm/secrets/db.yaml" not in proxy_exc.message + assert "REDACTED" in proxy_exc.message async def test_already_normalized_proxy_exception_is_honored(self): """A ProxyException raised mid-request (e.g. a guardrail block) is already @@ -3244,6 +3262,42 @@ class TestStreamCloseOnDisconnect: assert upstream.aclosed + async def test_async_streaming_data_generator_redacts_internal_details_on_error( + self, + ): + """Regression for LIT-6747: a mid-stream exception must not hand its raw text or a + traceback to serialize_error.""" + + class FailingUpstream: + def __aiter__(self): + return self + + async def __anext__(self): + raise RuntimeError( + "Failed to connect to postgresql://litellm_internal:S3cr3tPGPass@10.20.30.40:5432/litellm_prod " + "(config file /etc/litellm/secrets/db.yaml)" + ) + + ProxyLogging._callback_capabilities_cache.clear() + captured: list = [] + gen = ProxyBaseLLMRequestProcessing.async_streaming_data_generator( + response=FailingUpstream(), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + request_data={"model": "mock-model"}, + proxy_logging_obj=ProxyLogging(user_api_key_cache=MagicMock()), + serialize_chunk=lambda c: "data: x\n\n", + serialize_error=lambda e: captured.append(e) or "data: error\n\n", + ) + + await gen.__anext__() + + assert len(captured) == 1 + message = captured[0].message + assert "S3cr3tPGPass" not in message + assert "10.20.30.40" not in message + assert "/etc/litellm/secrets/db.yaml" not in message + assert "Traceback (most recent call last)" not in message + @staticmethod def _request_that_disconnects() -> Request: async def receive(): diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 3e70dee23b7..7b3528f3a68 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -95,6 +95,7 @@ class TestProxyInitializationHelpers: assert args["app"] == "litellm.proxy.proxy_server:app" assert args["host"] == "localhost" assert args["port"] == 8000 + assert args["server_header"] is False # Test with log_config args = ProxyInitializationHelpers._get_default_unvicorn_init_args( diff --git a/tests/test_litellm/test_redact_string_in_error_paths.py b/tests/test_litellm/test_redact_string_in_error_paths.py index 6404db91acf..07d1ec5f523 100644 --- a/tests/test_litellm/test_redact_string_in_error_paths.py +++ b/tests/test_litellm/test_redact_string_in_error_paths.py @@ -172,8 +172,6 @@ class TestLLMHTTPHandlerRealtimeRedaction: class TestProxyStreamingDataGeneratorRedaction: - """Test _redact_string on traceback.format_exc() — the pattern at common_request_processing.py:1733.""" - def test_redact_traceback_format_exc(self): try: raise RuntimeError( diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index 303efcab9c7..9fa748edec1 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -2,6 +2,7 @@ import logging import logging.config import sys import time +import traceback from collections.abc import Callable from io import StringIO from typing import Final @@ -13,11 +14,12 @@ from litellm._logging import ( JsonFormatter, _redact_string, _secret_filter, + redact_internal_details_from_client_message, verbose_logger, verbose_proxy_logger, verbose_router_logger, ) -from litellm.litellm_core_utils.secret_redaction import redact_string +from litellm.litellm_core_utils.secret_redaction import redact_internal_details, redact_string SECRET = "sk-proj-abc123def456ghi789jklmnopqrst" @@ -657,3 +659,65 @@ def test_json_formatter_redacts_non_string_extra_values(extra): assert output.strip(), "no record captured" assert SECRET not in output, f"non-string extra leaked a secret: {output}" assert "REDACTED" in output + + +@pytest.mark.parametrize( + "text,leaked", + ( + ("config file /etc/litellm/secrets/db.yaml", "/etc/litellm/secrets/db.yaml"), + ("home dir /Users/admin/.litellm/master_key.txt", "/Users/admin/.litellm/master_key.txt"), + ("cache at /var/cache/litellm/tokens.db", "/var/cache/litellm/tokens.db"), + ("path C:\\Users\\admin\\secrets.env", "C:\\Users\\admin\\secrets.env"), + ("connecting to host 10.20.30.40", "10.20.30.40"), + ("connecting to host 192.168.1.5", "192.168.1.5"), + ("connecting to host 172.16.0.9", "172.16.0.9"), + ("connecting to host 127.0.0.1", "127.0.0.1"), + ("connecting to db-primary.internal", "db-primary.internal"), + ("connecting to redis.corp", "redis.corp"), + ), +) +def test_redact_internal_details_catches_paths_and_hostnames(text, leaked): + result = redact_internal_details(text) + assert leaked not in result, f"{leaked!r} was not redacted" + assert "REDACTED" in result + + +def test_redact_internal_details_leaves_public_hostnames_and_routes_alone(): + """litellm's own error messages rely on routes like /v1/models staying legible.""" + safe_strings = ( + "call https://api.openai.com/v1/chat/completions", + "/chat/completions: Invalid model name passed in model=gpt-9", + "Call `/v1/models` to view available models for your key", + "reducto:// file IDs are not accepted through the proxy OCR API", + ) + for text in safe_strings: + assert redact_internal_details(text) == text + + +def test_redact_internal_details_layers_on_top_of_credential_redaction(): + text = "postgresql://litellm_internal:S3cr3tPGPass@10.20.30.40:5432/litellm_prod" + result = redact_internal_details(text) + assert "S3cr3tPGPass" not in result + assert "10.20.30.40" not in result + + +def test_redact_internal_details_drops_embedded_traceback(): + """Regression for LIT-6747: the traceback exception_type() embeds for SDK callers + must never reach an HTTP client.""" + try: + raise RuntimeError("socket hung up") + except RuntimeError: + raw_tb = traceback.format_exc() + message = f"litellm.APIConnectionError: MinimaxException - socket hung up\n{raw_tb}" + + result = redact_internal_details(message) + + assert result == "litellm.APIConnectionError: MinimaxException - socket hung up" + assert "Traceback (most recent call last)" not in result + assert __file__.split("/")[-1] not in result + + +def test_redact_internal_details_from_client_message_respects_disable_flag(): + with patch("litellm._logging._ENABLE_SECRET_REDACTION", False): # test-quality-ok: the opt-out flag is the SUT + text = "config file /etc/litellm/secrets/db.yaml" + assert redact_internal_details_from_client_message(text) == text From 8e3566d2f7564438c8c33ab66ce831aee416059e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:38:43 -0700 Subject: [PATCH 158/480] fix(guardrails): forward mode and streaming params to crowdstrike_aidr handler (#39317) * fix(guardrails): forward mode and streaming params to crowdstrike_aidr handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): drop stream_chunk_builder patch from crowdstrike cadence test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): type test params and cover unsupported crowdstrike mode rejection Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crowdstrike_aidr/__init__.py | 13 +- .../crowdstrike_aidr/crowdstrike_aidr.py | 40 ++++- .../guardrail_hooks/crowdstrike_aidr.py | 13 +- .../guardrail_hooks/test_crowdstrike_aidr.py | 142 +++++++++++++++++- 4 files changed, 197 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/__init__.py index 5e75b7d4d94..c88e6e97a96 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/__init__.py @@ -1,8 +1,8 @@ from typing import TYPE_CHECKING, Final -from litellm.types.guardrails import GuardrailEventHooks, SupportedGuardrailIntegrations +from litellm.types.guardrails import SupportedGuardrailIntegrations -from .crowdstrike_aidr import CrowdStrikeAIDRHandler +from .crowdstrike_aidr import CrowdStrikeAIDRHandler, streaming_params_from_litellm_params if TYPE_CHECKING: from litellm.types.guardrails import Guardrail, LitellmParams @@ -15,17 +15,16 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" if not guardrail_name: raise ValueError("CrowdStrike AIDR guardrail name is required") + streaming_params: Final = streaming_params_from_litellm_params(litellm_params) _crowdstrike_aidr_callback: Final = CrowdStrikeAIDRHandler( guardrail_name=guardrail_name, api_base=litellm_params.api_base, api_key=litellm_params.api_key, - # Exclude during_call to prevent duplicate input events - event_hook=[ - GuardrailEventHooks.pre_call.value, - GuardrailEventHooks.post_call.value, - ], + event_hook=litellm_params.mode, default_on=litellm_params.default_on, fail_on_error=litellm_params.fail_on_error, + streaming_end_of_stream_only=streaming_params.streaming_end_of_stream_only, + streaming_sampling_rate=streaming_params.streaming_sampling_rate, ) litellm.logging_callback_manager.add_litellm_callback(_crowdstrike_aidr_callback) diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py index c8284fac440..f7f500b1adc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py @@ -24,8 +24,11 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, ) -from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.guardrails import GuardrailEventHooks, LitellmParams from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionToolParam +from litellm.types.proxy.guardrails.guardrail_hooks.crowdstrike_aidr import ( + CrowdStrikeAIDRGuardrailConfigModelOptionalParams, +) from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: @@ -153,6 +156,21 @@ def _merge_metadata_bags(request_data: Mapping[str, Any]) -> Mapping[str, Any] | return merged if present else None +def streaming_params_from_litellm_params( + litellm_params: LitellmParams, +) -> CrowdStrikeAIDRGuardrailConfigModelOptionalParams: + extras: Final[Mapping[str, object]] = litellm_params.model_extra or {} + nested: Final = litellm_params.optional_params + optional_params: Final[Mapping[str, object]] = {} if nested is None else nested.model_dump() + return CrowdStrikeAIDRGuardrailConfigModelOptionalParams.model_validate( + { + name: value + for name in CrowdStrikeAIDRGuardrailConfigModelOptionalParams.model_fields + if (value := optional_params.get(name, extras.get(name))) is not None + } + ) + + def _messages_since_last_assistant( messages: Sequence[AllMessageValues], ) -> _FilteredMessages: @@ -241,6 +259,8 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): api_key: str | None = None, api_base: str | None = None, fail_on_error: bool | None = True, + streaming_end_of_stream_only: bool | None = None, + streaming_sampling_rate: int | None = None, **kwargs, ) -> None: """ @@ -250,10 +270,19 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): guardrail_name (str): The name of the guardrail instance. api_key (str | None): The CrowdStrike AIDR API key. Reads from CS_AIDR_TOKEN env var if None. api_base (str | None): The CrowdStrike AIDR API base URL. Reads from CS_AIDR_BASE_URL env var if None. + streaming_end_of_stream_only (bool | None): Scan streamed output once at end of stream instead of + every streaming_sampling_rate chunks. Defaults to False. + streaming_sampling_rate (int | None): Scan the accumulated streamed output every Nth chunk. Defaults to 5. **kwargs: Additional arguments passed to the CustomGuardrail base class. """ self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) self.fail_on_error = True if fail_on_error is None else fail_on_error + self._set_streaming_params( + CrowdStrikeAIDRGuardrailConfigModelOptionalParams( + streaming_end_of_stream_only=streaming_end_of_stream_only, + streaming_sampling_rate=streaming_sampling_rate, + ) + ) self.api_key = api_key or os.environ.get("CS_AIDR_TOKEN") if not self.api_key: @@ -274,6 +303,15 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): "Initialized CrowdStrike AIDR Guardrail: name=%s, api_base=%s", guardrail_name, self.api_base ) + def _set_streaming_params(self, streaming_params: CrowdStrikeAIDRGuardrailConfigModelOptionalParams) -> None: + self.streaming_end_of_stream_only: bool = streaming_params.streaming_end_of_stream_only or False + self.streaming_sampling_rate: int = streaming_params.streaming_sampling_rate or 5 + + @override + def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: + super().update_in_memory_litellm_params(litellm_params) + self._set_streaming_params(streaming_params_from_litellm_params(litellm_params)) + async def _call_crowdstrike_aidr_guard( self, payload: dict[str, Any], hook_name: str ) -> _GuardChatCompletionsResult: diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/crowdstrike_aidr.py b/litellm/types/proxy/guardrails/guardrail_hooks/crowdstrike_aidr.py index f47c38af3e3..6beca030a3a 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/crowdstrike_aidr.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/crowdstrike_aidr.py @@ -4,7 +4,18 @@ from .base import GuardrailConfigModel class CrowdStrikeAIDRGuardrailConfigModelOptionalParams(BaseModel): - pass + streaming_end_of_stream_only: bool | None = Field( + default=None, + description="If False (default when unset), post_call scans the accumulated streamed response every " + "streaming_sampling_rate chunks and an in-flight block stops the stream. If True, the guard runs once " + "over the assembled response at end of stream, so flagged content may already have reached the client.", + ) + streaming_sampling_rate: int | None = Field( + default=None, + ge=1, + description="When streaming_end_of_stream_only is False, scan the accumulated streamed response every Nth " + "chunk. Defaults to 5 when unset.", + ) class CrowdStrikeAIDRGuardrailConfigModel(GuardrailConfigModel[CrowdStrikeAIDRGuardrailConfigModelOptionalParams]): diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py index ec7854b9a35..1b50ea53db2 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py @@ -3,7 +3,9 @@ from unittest.mock import patch import httpx import pytest from fastapi import HTTPException +from pydantic import ValidationError +import litellm from litellm.exceptions import Timeout from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket from litellm.proxy.guardrails.guardrail_hooks.crowdstrike_aidr import initialize_guardrail @@ -12,8 +14,8 @@ from litellm.proxy.guardrails.guardrail_hooks.crowdstrike_aidr.crowdstrike_aidr CrowdStrikeAIDRHandler, ) from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 -from litellm.types.guardrails import Guardrail, LitellmParams -from litellm.types.utils import GenericGuardrailAPIInputs, ModelResponse +from litellm.types.guardrails import Guardrail, GuardrailEventHooks, LitellmParams +from litellm.types.utils import Delta, GenericGuardrailAPIInputs, ModelResponse, ModelResponseStream @pytest.fixture @@ -1578,3 +1580,139 @@ async def test_unparseable_transformed_response_fails_closed_under_fail_open() - assert exc_info.value.status_code == 500 assert "failing closed" in exc_info.value.detail["error"] + + +def _initialize_from_config(**litellm_params_kwargs: object) -> CrowdStrikeAIDRHandler: + litellm_params = LitellmParams( + guardrail="crowdstrike_aidr", + api_key="pts_crowdstrike_tokenid", + api_base="https://api.crowdstrike.com/aidr/aiguard", + default_on=True, + **litellm_params_kwargs, + ) + guardrail = Guardrail(guardrail_name="crowdstrike-aidr-guard", litellm_params=litellm_params) + return initialize_guardrail(litellm_params=litellm_params, guardrail=guardrail) + + +@pytest.mark.parametrize( + ("mode", "runs_pre_call", "runs_post_call"), + [("post_call", False, True), ("pre_call", True, False), (["pre_call", "post_call"], True, True)], +) +def test_initialize_guardrail_honors_configured_mode( + mode: str | list[str], runs_pre_call: bool, runs_post_call: bool +) -> None: + handler = _initialize_from_config(mode=mode) + + assert handler.should_run_guardrail({}, GuardrailEventHooks.pre_call) is runs_pre_call + assert handler.should_run_guardrail({}, GuardrailEventHooks.post_call) is runs_post_call + + +def test_initialize_guardrail_rejects_unsupported_mode_instead_of_running_other_hooks() -> None: + with pytest.raises(ValueError, match="during_call is not in the supported event hooks"): + _initialize_from_config(mode="during_call") + + +def test_initialize_guardrail_defaults_streaming_params() -> None: + handler = _initialize_from_config(mode="post_call") + + assert handler.streaming_end_of_stream_only is False + assert handler.streaming_sampling_rate == 5 + + +@pytest.mark.parametrize( + "configured", + [ + {"streaming_end_of_stream_only": True, "streaming_sampling_rate": 50}, + {"optional_params": {"streaming_end_of_stream_only": True, "streaming_sampling_rate": 50}}, + ], +) +def test_initialize_guardrail_forwards_streaming_params(configured: dict[str, object]) -> None: + handler = _initialize_from_config(mode="post_call", **configured) + + assert handler.streaming_end_of_stream_only is True + assert handler.streaming_sampling_rate == 50 + + +def test_initialize_guardrail_rejects_non_positive_sampling_rate() -> None: + with pytest.raises(ValidationError): + _initialize_from_config(mode="post_call", streaming_sampling_rate=0) + + +def test_update_in_memory_litellm_params_reapplies_streaming_params() -> None: + handler = _initialize_from_config(mode="post_call") + + handler.update_in_memory_litellm_params( + LitellmParams( + guardrail="crowdstrike_aidr", + mode="post_call", + streaming_end_of_stream_only=True, + streaming_sampling_rate=7, + ) + ) + + assert handler.streaming_end_of_stream_only is True + assert handler.streaming_sampling_rate == 7 + + +def _stream_chunk(content: str, finish_reason: str | None) -> ModelResponseStream: + return ModelResponseStream( + model="gpt-4", + choices=[ + litellm.StreamingChoices( + index=0, delta=Delta(role="assistant", content=content), finish_reason=finish_reason + ) + ], + ) + + +async def _guard_calls_for_stream(handler: CrowdStrikeAIDRHandler, chunk_texts: list[str]) -> int: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import UnifiedLLMGuardrails + + async def stream(): + for i, content in enumerate(chunk_texts): + yield _stream_chunk(content, "stop" if i == len(chunk_texts) - 1 else None) + + calls = 0 + + def _allow(request: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + return httpx.Response( + status_code=200, json={"result": {"blocked": False, "transformed": False}}, request=request + ) + + request_data = { + "messages": [{"role": "user", "content": "hi"}], + "guardrail_to_apply": handler, + "metadata": {"guardrails": ["crowdstrike-aidr-guard"]}, + } + async with httpx.AsyncClient(transport=httpx.MockTransport(_allow)) as client: + await handler.async_handler.close() + handler.async_handler.client = client + async for _ in UnifiedLLMGuardrails().async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test", request_route="/chat/completions"), + response=stream(), + request_data=request_data, + ): + pass + return calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("configured", "expected_calls"), + [ + ({}, 3), + ({"streaming_sampling_rate": 2}, 6), + ({"streaming_end_of_stream_only": True}, 1), + ({"streaming_end_of_stream_only": True, "streaming_sampling_rate": 2}, 1), + ], +) +async def test_streaming_params_from_config_control_output_scan_cadence( + configured: dict[str, object], expected_calls: int +) -> None: + """10 chunks: default samples at 5 and 10 plus the final pass, rate 2 samples 5 times plus final, end-of-stream scans once.""" + handler = _initialize_from_config(mode="post_call", **configured) + + assert await _guard_calls_for_stream(handler, list("ABCDEFGHIJ")) == expected_calls From 8cbaba88635871b44096488d6264490eac700b9f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:39:36 -0700 Subject: [PATCH 159/480] fix(mcp): gate the connect-time OBO pre-flight on the key's allowed servers (#39447) The pre-flight token exchange resolved its target from the requested path, so a key not entitled to that server still drove an outbound exchange at the IdP and populated the credential cache before the later access check denied it. Resolve the target through the same allowed-server set the rest of the route uses and skip the exchange when the requested server is not in it. Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/_experimental/mcp_server/server.py | 7 ++ .../mcp_server/test_mcp_server.py | 73 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 989b08b929a..af3ff6714af 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -3855,6 +3855,13 @@ if MCP_AVAILABLE: and server.auth_type == MCPAuth.oauth2_token_exchange and oauth2_headers and len(mcp_servers or []) == 1 + and server.server_id + in frozenset( + allowed.server_id + for allowed in await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, mcp_servers=mcp_servers, client_ip=client_ip + ) + ) ): await global_mcp_server_manager.preflight_token_exchange( server=server, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 82f74cda835..3f6d8f8837c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -8198,6 +8198,79 @@ class TestPreemptive401ModeAware: await self._run(delegate, self.LITELLM_KEY_HEADERS, has_stored_token=False) +def _make_obo_server(alias: str) -> MCPServer: + return MCPServer( + server_id=f"id-{alias}", + name=alias, + alias=alias, + server_name=alias, + url=f"https://{alias}.test/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2_token_exchange, + token_exchange_endpoint="https://idp.test/token", + client_id="cid", + client_secret="csecret", + mcp_info={"server_name": alias}, + ) + + +class TestOboPreflightScopedToAllowedServers: + """The connect-time OBO exchange is an outbound IdP call whose result is cached, so it must + only run for a server the caller's key resolves to through the allowed set, not for any + server the requested path happens to name.""" + + SUBJECT_HEADERS = {"Authorization": "Bearer upstream-subject-token"} + + async def _run(self, requested: MCPServer, allowed: list[MCPServer], user_api_key_auth: UserAPIKeyAuth | None): + from litellm.proxy._experimental.mcp_server import server as server_module + + allowed_lookup = AsyncMock(return_value=allowed) + preflight = AsyncMock() + with ( + patch.object( # test-quality-ok: route handler reads the module-level manager, no injection seam + server_module.global_mcp_server_manager, "get_mcp_server_by_name", return_value=requested + ), + patch.object( # test-quality-ok: the exchanger is the observable; a real one would call an IdP + server_module.global_mcp_server_manager, "preflight_token_exchange", preflight + ), + patch.object( # test-quality-ok: allowed-set resolution needs the DB; the test controls its answer + server_module, "_get_allowed_mcp_servers", allowed_lookup + ), + ): + await server_module._raise_preemptive_401_for_unauthenticated_servers( + scope={"type": "http", "method": "POST", "path": f"/mcp/{requested.alias}", "headers": []}, + mcp_servers=[requested.alias], + oauth2_headers=self.SUBJECT_HEADERS, + mcp_server_auth_headers=None, + user_api_key_auth=user_api_key_auth, + client_ip="10.0.0.7", + ) + return allowed_lookup, preflight + + @pytest.mark.asyncio + async def test_unentitled_key_never_reaches_the_exchanger(self): + requested = _make_obo_server("obo_tools") + key = UserAPIKeyAuth(api_key="sk-plain-only") + + allowed_lookup, preflight = await self._run( + requested, allowed=[_make_obo_server("plain_tools")], user_api_key_auth=key + ) + + preflight.assert_not_awaited() + allowed_lookup.assert_awaited_once_with( + user_api_key_auth=key, mcp_servers=[requested.alias], client_ip="10.0.0.7" + ) + + @pytest.mark.asyncio + async def test_entitled_key_still_exchanges_at_connect(self): + requested = _make_obo_server("obo_tools") + key = UserAPIKeyAuth(api_key="sk-obo") + + _, preflight = await self._run(requested, allowed=[requested], user_api_key_auth=key) + + preflight.assert_awaited_once_with(server=requested, oauth2_headers=self.SUBJECT_HEADERS, user_api_key_auth=key) + + @pytest.mark.asyncio async def test_post_mcp_call_guardrails_return_the_rewritten_result(): """The result a post_mcp_call guardrail rewrote must be what the caller sends back.""" From c19d49d9192f26a7e15c9571d1c3a4e7ee45092a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:40:13 -0700 Subject: [PATCH 160/480] fix(responses): keep provider response headers in streaming logging callbacks (#38131) * fix(responses): keep provider response headers in streaming logging callbacks The responses streaming iterator captures the provider's HTTP response headers into its own _hidden_params, but never puts them on the completed response, and the model_validate(model_dump()) copy made for logging drops pydantic private attributes. Success callbacks and StandardLoggingPayload.hidden_params.additional_headers therefore saw an empty dict for streaming /v1/responses, so Azure's apim-request-id was unreadable from the callback payload. Restore the headers on the nested response of the logging copy, preferring any the provider transform already set (the fake_stream path) and falling back to the ones the iterator captured from the stream. Skipped when the copy fell back to the original event, so a serialization failure never leaves logging-only state on the caller's object. * fix: satisfy LIT002 mutable-collection gate in header restore --------- Co-authored-by: Yucheng Zhu --- litellm/responses/streaming_iterator.py | 40 +++++ .../responses/test_streaming_iterator.py | 161 +++++++++++++++++- 2 files changed, 200 insertions(+), 1 deletion(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 7871c85220c..f271655f5e3 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -17,6 +17,7 @@ from typing_extensions import TypeIs import litellm from litellm.constants import ( + EMPTY_MAPPING, LITELLM_MAX_STREAMING_DURATION_SECONDS, STREAM_SSE_DONE_STRING, ) @@ -273,6 +274,9 @@ class BaseResponsesAPIStreamingIterator: self._hidden_params["additional_headers"] = process_response_headers( self.response.headers or {} ) # GUARANTEE OPENAI HEADERS IN RESPONSE + self._raw_response_headers: Mapping[str, str] = MappingProxyType( + dict(self.response.headers or {}) # mutable-ok: immediately frozen by MappingProxyType + ) def _check_max_streaming_duration(self) -> None: """Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS.""" @@ -446,6 +450,7 @@ class BaseResponsesAPIStreamingIterator: except Exception: # Fallback to original if serialization fails pass + self._restore_provider_response_headers(logging_response) end_time: Final = datetime.now() if is_async: @@ -480,6 +485,41 @@ class BaseResponsesAPIStreamingIterator: ) self._run_post_success_hooks(end_time=end_time) + def _restore_provider_response_headers(self, logging_response: object) -> None: + """Re-apply the provider's response headers to the copy handed to logging callbacks. + + ``model_validate(model_dump())`` above drops pydantic private attributes, so the + ``_hidden_params`` the provider transform set on the nested response are lost. Returns early + when that copy fell back to the original event, so logging-only state never lands on the + object the caller is iterating. + """ + if logging_response is self.completed_response: + return + target: Final[object] = getattr(logging_response, "response", None) + existing_hidden: Final[object] = getattr(target, "_hidden_params", None) + if not isinstance(existing_hidden, Mapping): + return + existing: Final[Mapping[str, object]] = existing_hidden + source_hidden: Final[object] = getattr( + getattr(self.completed_response, "response", None), "_hidden_params", None + ) + source: Final[Mapping[str, object]] = source_hidden if isinstance(source_hidden, Mapping) else EMPTY_MAPPING + processed: Final[object] = source.get("additional_headers") or self._hidden_params.get("additional_headers") + raw: Final[object] = source.get("headers") or self._raw_response_headers + headers: Final[Mapping[str, object]] = processed if isinstance(processed, Mapping) else EMPTY_MAPPING + raw_headers: Final[Mapping[str, object]] = raw if isinstance(raw, Mapping) else EMPTY_MAPPING + # rebuild by value and let existing keys win: sharing the source dicts would alias what the proxy + # splats into the client's HTTP headers, and copying non-header keys would carry response_cost + setattr( # noqa: B010 # target is typed object here, so a plain attribute store does not type check + target, + "_hidden_params", + { # mutable-ok: the cost calculator writes optional_params into _hidden_params + "additional_headers": {**headers}, # mutable-ok: fresh copy, logging callbacks may mutate it + "headers": {**raw_headers}, # mutable-ok: fresh copy, logging callbacks may mutate it + **existing, + }, + ) + def _handle_logging_completed_response(self): """Base implementation - should be overridden by subclasses""" diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index 9edcaaef034..c226c0b4d09 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -6,7 +6,7 @@ completion_start_time = end_time.""" import json from datetime import datetime from typing import Optional -from unittest.mock import Mock +from unittest.mock import Mock, patch import httpx import pytest @@ -378,3 +378,162 @@ def test_stamp_responses_usage_cost_survives_calculator_failure(): _stamp_responses_usage_cost(response, logging_obj) assert getattr(response.usage, "cost", None) is None + + +def _capture_dispatch(logged: list): + """Record the object handed to the success handlers. + + ``Mock(spec=LiteLLMLoggingObj).dispatch_success_handlers`` is an AsyncMock whose side effect + only runs when the coroutine is awaited, so capture with a plain function instead. + """ + + async def _noop() -> None: + return None + + def _dispatch(result, **kwargs): + logged.append(result) + return _noop() + + return _dispatch + + +def _headers_config(*, transform_hidden_params: Optional[dict] = None) -> Mock: + """Config whose completed event carries a real ResponsesAPIResponse, so the logging copy + performs a genuine model_dump/model_validate round trip.""" + mock_config = Mock(spec=BaseResponsesAPIConfig) + + def _transform(model, parsed_chunk, logging_obj): + evt_type = parsed_chunk.get("type") + if evt_type != "response.completed": + stub = Mock() + stub.type = evt_type + return stub + response = ResponsesAPIResponse( + id="resp_headers", + created_at=1, + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + if transform_hidden_params is not None: + response._hidden_params.update(transform_hidden_params) + return ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=response, + ) + + mock_config.transform_streaming_response.side_effect = _transform + return mock_config + + +def _make_header_iterator( + *, + headers: dict, + config: Mock, + logging_obj: LiteLLMLoggingObj, +) -> ResponsesAPIStreamingIterator: + async def aiter_bytes(): + yield _sse_event({"type": "response.completed"}) + + mock_response = Mock() + mock_response.headers = headers + mock_response.aiter_bytes = aiter_bytes + + return ResponsesAPIStreamingIterator( + response=mock_response, + model="gpt-4o-mini", + responses_api_provider_config=config, + logging_obj=logging_obj, + litellm_metadata={}, + custom_llm_provider="azure", + ) + + +@pytest.mark.asyncio +async def test_streaming_logging_response_carries_provider_response_headers(): + """LIT-6055: the provider headers the iterator captured must reach the logged response, so + custom loggers can read Azure's apim-request-id from the callback payload.""" + logging_obj = _logging_obj_stub() + logged: list[object] = [] + logging_obj.dispatch_success_handlers = _capture_dispatch(logged) + + logging_obj._on_deferred_stream_complete = None + + iterator = _make_header_iterator( + headers={"apim-request-id": "azure-correlation-1", "x-ms-region": "East US 2"}, + config=_headers_config(), + logging_obj=logging_obj, + ) + async for _ in iterator: + pass + + assert len(logged) == 1 + hidden_params = logged[0].response._hidden_params + assert hidden_params["additional_headers"]["llm_provider-apim-request-id"] == "azure-correlation-1" + assert hidden_params["additional_headers"]["llm_provider-x-ms-region"] == "East US 2" + assert hidden_params["headers"]["apim-request-id"] == "azure-correlation-1" + # the proxy builds the client's response headers from the iterator's own dict, so the logged + # response must hold copies rather than alias it + assert hidden_params["additional_headers"] is not iterator._hidden_params["additional_headers"] + assert hidden_params["headers"] is not iterator._raw_response_headers + + +@pytest.mark.asyncio +async def test_streaming_logging_copy_preserves_transform_hidden_params(): + """LIT-6055: model_validate(model_dump()) drops pydantic private attributes, so headers a + provider transform already set on the response (fake_stream) must be re-applied.""" + logging_obj = _logging_obj_stub() + logged: list[object] = [] + logging_obj.dispatch_success_handlers = _capture_dispatch(logged) + + logging_obj._on_deferred_stream_complete = None + + iterator = _make_header_iterator( + headers={}, + config=_headers_config( + transform_hidden_params={ + "additional_headers": {"llm_provider-apim-request-id": "from-transform"}, + "headers": {"apim-request-id": "from-transform"}, + "response_cost": 0.5, + } + ), + logging_obj=logging_obj, + ) + async for _ in iterator: + pass + + assert len(logged) == 1 + hidden_params = logged[0].response._hidden_params + assert hidden_params["additional_headers"]["llm_provider-apim-request-id"] == "from-transform" + assert hidden_params["headers"]["apim-request-id"] == "from-transform" + assert iterator.completed_response is not logged[0] + # only the header keys travel: response_cost would short-circuit the cost calculator + assert "response_cost" not in hidden_params + + +@pytest.mark.asyncio +async def test_streaming_logging_copy_fallback_leaves_caller_event_untouched(): + """LIT-6055: when the logging copy falls back to the original event, the header restore must + not stamp logging-only state onto the object the caller is iterating.""" + logging_obj = _logging_obj_stub() + logged: list[object] = [] + logging_obj.dispatch_success_handlers = _capture_dispatch(logged) + logging_obj._on_deferred_stream_complete = None + + iterator = _make_header_iterator( + headers={"apim-request-id": "azure-correlation-1"}, + config=_headers_config(), + logging_obj=logging_obj, + ) + async for _ in iterator: + pass + + assert len(logged) == 1 + iterator._completed_response_logged = False + logged.clear() + with patch.object(type(iterator.completed_response), "model_dump", side_effect=ValueError("cannot serialize")): + iterator._log_completed_response(is_async=True) + + assert logged == [iterator.completed_response] + assert iterator.completed_response.response._hidden_params == {} From 44a6c659fb831bc3da929f020f2cd4f3e508e384 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 2 Sep 2026 17:45:58 -0700 Subject: [PATCH 161/480] fix(mcp): fence an outbound-token write against an overlapping invalidation (#35398) get_or_compute single-flights concurrent misses under a per-key lock, but invalidate() deletes outside it. A mint already awaiting the IdP when an invalidation ran wrote its result into the slot afterwards, so a bearer minted before the invalidation was served after it for its full TTL; the upstream-401 retry then re-presented the bearer the server had just rejected. The cache now keeps a per-key generation beside the single-flight lock. invalidate() bumps it and get_or_compute snapshots it inside the lock right before minting, skipping the cache write when it no longer matches. The token is still returned to the caller it was minted for, so only the caching is fenced. Generation and lock share one object that a running mint binds to a local, which pins the weak map's entry for exactly as long as a write can still land; a generation held weakly on its own would be collected across the await and the re-check would silently pass. --- .../outbound_credentials/token_endpoint.py | 67 ++++++++--- .../test_token_endpoint.py | 105 ++++++++++++++++++ 2 files changed, 158 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py index 84f714db449..6f1e5baa109 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py @@ -121,6 +121,23 @@ class TokenEndpointClient: return Ok(ExchangedToken(access_token=parsed.access_token, expires_in=parsed.expires_in)) +class _KeyGuard: + """The per-key single-flight lock plus the invalidation generation that lock protects. + + Both live on one object so their lifetimes cannot diverge. `get_or_compute` binds the guard to + a local for its whole critical section, which keeps the weak map's entry alive for as long as + that compute could still write; an `invalidate` overlapping the compute therefore reaches the + very same object and its bump is guaranteed to be observed. Conversely a guard nobody holds is + collectible precisely because no write is outstanding for it to fence. + """ + + __slots__ = ("__weakref__", "generation", "lock") + + def __init__(self) -> None: + self.lock = asyncio.Lock() + self.generation = 0 + + class ExchangedTokenCache: """Memoizes the final token string per key, single-flighting concurrent misses on one lock.""" @@ -129,7 +146,7 @@ class ExchangedTokenCache: max_size_in_memory=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE, default_ttl=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL, ) - self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = weakref.WeakValueDictionary() + self._guards: weakref.WeakValueDictionary[str, _KeyGuard] = weakref.WeakValueDictionary() async def get_or_compute( self, @@ -144,28 +161,50 @@ class ExchangedTokenCache: guaranteeing the token it gets back was minted for the *current* inputs: a stored entry whose fingerprint differs reads as a miss and is re-minted over. That keeps eviction addressable without the key having to encode the credential material it protects. + + An `invalidate` landing while `compute` is in flight wins over that compute's write. The + token is still returned to the caller it was minted for, but it is not stored, so the next + resolution re-mints rather than serving a bearer that predates the invalidation for the + rest of its TTL. """ cached = self._get(cache_key, fingerprint) if cached is not None: return Ok(cached) - async with self._lock(cache_key): + guard = self._guard(cache_key) + async with guard.lock: cached = self._get(cache_key, fingerprint) if cached is not None: return Ok(cached) + generation = guard.generation match await compute(): case Ok(token): - self._cache.set_cache( # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped - cache_key, - (fingerprint, token.access_token), - ttl=_cache_ttl_seconds(token.expires_in), - ) + if guard.generation == generation: + self._store(cache_key, fingerprint, token) return Ok(token.access_token) case Error(err): return Error(err) def invalidate(self, cache_key: str) -> None: - """Evict one cached token so the next `get_or_compute` re-mints (e.g. after an upstream 401).""" + """Evict one cached token so the next `get_or_compute` re-mints (e.g. after an upstream 401). + + Bumping the guard's generation is what makes the eviction stick against a compute already + awaiting the token endpoint: that compute snapshotted the old generation and so skips its + write. No guard means no compute is in flight, since an in-flight one pins its own. + + Stays synchronous: callers invalidate from plain `def`s. + """ self._cache.delete_cache(cache_key) # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped + guard = self._guards.get(cache_key) + if guard is None: + return + guard.generation += 1 + + def _store(self, cache_key: str, fingerprint: str, token: ExchangedToken) -> None: + self._cache.set_cache( # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped + cache_key, + (fingerprint, token.access_token), + ttl=_cache_ttl_seconds(token.expires_in), + ) def _get(self, cache_key: str, fingerprint: str) -> str | None: """The stored token, or None when absent or minted for different inputs. @@ -180,12 +219,12 @@ class ExchangedTokenCache: return None return token if stored_fingerprint == fingerprint else None - def _lock(self, cache_key: str) -> asyncio.Lock: - lock = self._locks.get(cache_key) - if lock is None: - lock = asyncio.Lock() - self._locks[cache_key] = lock - return lock + def _guard(self, cache_key: str) -> _KeyGuard: + guard = self._guards.get(cache_key) + if guard is None: + guard = _KeyGuard() + self._guards[cache_key] = guard + return guard def _cache_ttl_seconds(expires_in: int | None) -> int: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_endpoint.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_endpoint.py index 5f277db2f72..db3a1a386a3 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_endpoint.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_endpoint.py @@ -7,6 +7,7 @@ cache's hit/single-flight behavior. Each assertion fails under a real mutation o """ import asyncio +import gc import json from unittest.mock import AsyncMock, MagicMock, patch @@ -359,6 +360,110 @@ async def test_cache_invalidate_only_evicts_the_named_key(): assert calls == 2 +@pytest.mark.asyncio +async def test_cache_invalidate_mid_compute_is_not_overwritten_by_that_compute(): + """A bearer minted before an invalidation must never be served after it. + + The compute is suspended at the token endpoint when the invalidation lands, so its write is + the one that would resurrect the evicted bearer for the rest of its TTL. The caller it was + minted for still gets it; the *cache* is what the invalidation is about. + """ + cache = ExchangedTokenCache() + mint_started, release_mint = asyncio.Event(), asyncio.Event() + + async def slow_mint(): + mint_started.set() + await release_mint.wait() + return _ok_token("bearer-minted-before-invalidation") + + async def re_mint(): + return _ok_token("bearer-minted-after-invalidation") + + in_flight = asyncio.create_task(cache.get_or_compute("slot", slow_mint, fingerprint="fp")) + await mint_started.wait() + + assert not in_flight.done() + cache.invalidate("slot") + release_mint.set() + + raced = await in_flight + assert isinstance(raced, Ok) and raced.ok == "bearer-minted-before-invalidation" + + after = await cache.get_or_compute("slot", re_mint, fingerprint="fp") + assert isinstance(after, Ok) and after.ok == "bearer-minted-after-invalidation" + + +@pytest.mark.asyncio +async def test_cache_invalidate_mid_compute_survives_garbage_collection(): + """The record of an invalidation must outlive a collection cycle taken mid-compute. + + Per-key state is held weakly so idle keys do not accumulate. If the state a compute checks + before writing were collectible while that compute is suspended, the check would read as + "nothing was invalidated" and the stale write would land; the running compute has to pin it. + """ + cache = ExchangedTokenCache() + mint_started, release_mint = asyncio.Event(), asyncio.Event() + + async def slow_mint(): + mint_started.set() + await release_mint.wait() + return _ok_token("bearer-minted-before-invalidation") + + async def re_mint(): + return _ok_token("bearer-minted-after-invalidation") + + in_flight = asyncio.create_task(cache.get_or_compute("slot", slow_mint, fingerprint="fp")) + await mint_started.wait() + + assert not in_flight.done() + cache.invalidate("slot") + gc.collect() + release_mint.set() + await in_flight + + after = await cache.get_or_compute("slot", re_mint, fingerprint="fp") + assert isinstance(after, Ok) and after.ok == "bearer-minted-after-invalidation" + + +@pytest.mark.asyncio +async def test_cache_stores_a_compute_that_started_after_the_invalidation(): + """Only the mint that predates the invalidation loses its write. + + A caller queued behind the single-flight lock computes after the eviction, so its token is + fresh and must be cached; otherwise the fix would trade one stale bearer for re-minting on + every subsequent resolution. + """ + cache = ExchangedTokenCache() + mint_started, release_mint = asyncio.Event(), asyncio.Event() + + async def slow_mint(): + mint_started.set() + await release_mint.wait() + return _ok_token("bearer-minted-before-invalidation") + + async def re_mint(): + return _ok_token("bearer-minted-after-invalidation") + + async def must_not_run(): + pytest.fail("the mint that followed the invalidation should have been cached") + + in_flight = asyncio.create_task(cache.get_or_compute("slot", slow_mint, fingerprint="fp")) + await mint_started.wait() + queued = asyncio.create_task(cache.get_or_compute("slot", re_mint, fingerprint="fp")) + await asyncio.sleep(0) + + assert not queued.done() + cache.invalidate("slot") + release_mint.set() + + raced, fresh = await asyncio.gather(in_flight, queued) + assert isinstance(raced, Ok) and raced.ok == "bearer-minted-before-invalidation" + assert isinstance(fresh, Ok) and fresh.ok == "bearer-minted-after-invalidation" + + served = await cache.get_or_compute("slot", must_not_run, fingerprint="fp") + assert isinstance(served, Ok) and served.ok == "bearer-minted-after-invalidation" + + @pytest.mark.asyncio async def test_cache_does_not_store_a_failed_compute(): cache = ExchangedTokenCache() From 45f44fea6019e3e9afd75e7157992899f0846bd0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:53:39 -0700 Subject: [PATCH 162/480] fix(headroom): leave background responses requests uncompressed and unconverted --- .../guardrail_hooks/headroom/headroom.py | 6 ++- .../guardrail_hooks/test_headroom.py | 43 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index 5fbf98d0aae..9d993384461 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -728,6 +728,10 @@ class HeadroomGuardrail(CustomGuardrail): verbose_proxy_logger.debug("Headroom: %s header set; skipping compression", BYPASS_HEADER) return inputs + if request_data.get("background"): + verbose_proxy_logger.debug("Headroom: background request; skipping compression") + return inputs + structured_messages: Final = inputs.get("structured_messages") if not _is_object_list(structured_messages) or not structured_messages: return inputs @@ -831,7 +835,7 @@ class HeadroomGuardrail(CustomGuardrail): effective: Final = base_result if base_result is not None else kwargs if call_type not in _STREAM_CONVERTIBLE_CALL_TYPES: return base_result - if not effective.get("stream"): + if not effective.get("stream") or effective.get("background"): return base_result if not has_headroom_retrieve_tool(effective.get("tools")): return base_result diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index a940e4c3394..400eaf8ab3f 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -193,6 +193,33 @@ async def test_apply_guardrail_compresses_and_returns_structured_messages( assert "headroom" in _applied_guardrails(request_data) +@pytest.mark.asyncio +async def test_apply_guardrail_leaves_background_requests_uncompressed( + guardrail: HeadroomGuardrail, +): + inputs = GenericGuardrailAPIInputs( + texts=["A" * 5000], + structured_messages=ORIGINAL_MESSAGES, + ) + request_data = {"model": "gpt-4o", "background": True} + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + return_value=_make_compress_response(COMPRESSED_MESSAGES), + ) as post: + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + + assert result is inputs + post.assert_not_awaited() + assert _recorded_guardrail_entries(request_data) == [] + + def _recorded_guardrail_response(request_data: dict) -> dict: entries = request_data["metadata"]["standard_logging_guardrail_information"] assert len(entries) == 1 @@ -2074,6 +2101,22 @@ async def test_pre_call_deployment_hook_converts_stream_only_for_ccr_chat_comple assert kwargs["stream"] is True +@pytest.mark.asyncio +async def test_pre_call_deployment_hook_leaves_background_streams_alone(guardrail: HeadroomGuardrail): + kwargs = { + "model": "gpt-4o", + "stream": True, + "background": True, + "tools": [_responses_retrieve_tool_definition()], + } + + result = await guardrail.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.aresponses) + + assert result is kwargs + assert HEADROOM_CONVERTED_STREAM_KEY not in kwargs + assert kwargs["stream"] is True + + @pytest.mark.asyncio async def test_pre_call_deployment_hook_still_compresses_for_deployment_level_configs( guardrail: HeadroomGuardrail, From ff1f21aea92574b3df1b9a00b2cdd70fa5e17dda Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 2 Sep 2026 17:54:17 -0700 Subject: [PATCH 163/480] fix(ui): paginate request logs by session groups server-side (#39257) * fix(ui): paginate request logs by session groups server-side The logs table server-paginated raw spend logs and then collapsed multi-call sessions client-side, so a page could render 3 rows while the footer claimed 25 and sessions straddled pages. Adds an opt-in group_by_session param to /spend/logs/ui that pages and counts one representative row per session (DISTINCT ON, newest non-MCP call), keeps the bounded count contract, enriches whole-session llm/agent composition counts, and deletes the client-side collapse pipeline. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxT89fiygmzz2ALcjpu7Ve * feat(ui): add a 10 rows-per-page option and default request logs to it Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxT89fiygmzz2ALcjpu7Ve * fix(ui): key session aggregates per api key in the logs enrichment Grouped pagination splits a reused session id into one row per api key, but the enrichment still aggregated by session_id alone, so both rows showed combined spend and counts. The aggregate query now groups by (session_id, api_key), the count folds into it (the separate group_by query is deleted), and each row reads its own key's totals. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxT89fiygmzz2ALcjpu7Ve * fix(ui): treat an empty api_key as a real session group value The spend-log schema defaults api_key to an empty string; truthiness guards in the enrichment treated it as missing, so keyless multi-call sessions lost their count and spend. Only None means missing now. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QxT89fiygmzz2ALcjpu7Ve --------- Co-authored-by: Claude Fable 5 --- .../spend_management_endpoints.py | 118 ++++++++------ tests/e2e/ui/helpers/traffic.ts | 3 + .../e2e/ui/tests/logs/logsPagination.spec.ts | 150 ++++++++++++++++++ .../test_spend_management_endpoints.py | 145 +++++++++++++++-- .../test_spend_query_optimization.py | 106 +++++++++++++ .../src/components/networking.tsx | 1 + .../view_logs/RequestLogsPanel.test.tsx | 45 +++--- .../components/view_logs/RequestLogsPanel.tsx | 63 +------- .../components/view_logs/RequestLogsTable.tsx | 4 +- .../RequestLogsTableColumns.test.tsx | 14 +- .../view_logs/RequestLogsTableColumns.tsx | 6 +- .../src/components/view_logs/columns.tsx | 1 - .../src/components/view_logs/constants.ts | 3 + .../components/view_logs/log_filter_logic.tsx | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 15 files changed, 512 insertions(+), 152 deletions(-) create mode 100644 tests/e2e/ui/tests/logs/logsPagination.spec.ts diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index c48750cea72..bb7dfafb297 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -55,6 +55,10 @@ router: Final = APIRouter() SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000 +_SESSION_GROUP_KEY_SQL: Final = "COALESCE(NULLIF(session_id, ''), request_id), api_key" +_MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" +_AGENT_CALL_TYPE_SQL: Final = "'asend_message'" + _INTERNAL_HEALTH_CHECK_API_KEYS: Final = ( LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, hash_token(token=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME), @@ -144,21 +148,16 @@ class _DailyTagSpendRow(TypedDict): total_spend: float -class _SessionCountAggregate(TypedDict): - session_id: int - - -class _SessionCountRow(TypedDict): - session_id: str - _count: _SessionCountAggregate - - class _SessionSpendRow(TypedDict): session_id: str + api_key: ReadOnly[str] + session_total_count: ReadOnly[int] session_total_spend: float mcp_tool_call_count: int mcp_tool_call_spend: float session_cache_hit_count: ReadOnly[int] + session_llm_count: ReadOnly[int] + session_agent_count: ReadOnly[int] class _SpendSumAggregate(TypedDict, total=False): @@ -242,18 +241,6 @@ async def _count_spend_logs(prisma_client: PrismaClient, where: Mapping[str, obj return await _spend_logs_table(prisma_client).count(where=where) -async def _count_logs_per_session( - prisma_client: PrismaClient, session_ids: Sequence[str | None] -) -> Sequence[_SessionCountRow]: - """Count spend log rows per session for the given session ids.""" - rows: Final = await _spend_logs_table(prisma_client).group_by( - by=["session_id"], - where={"session_id": {"in": session_ids}}, - count={"session_id": True}, - ) - return cast(Sequence[_SessionCountRow], rows) # cast-ok: group_by(count=) shape is fixed by the by/count args - - async def _find_team_row(prisma_client: PrismaClient, team_id: str) -> _SupportsModelDump | None: """Read a single team row as a Prisma model instance.""" return await _team_table(prisma_client).find_unique(where={"team_id": team_id}) @@ -2290,6 +2277,10 @@ async def ui_view_spend_logs( default=False, description="Exclude LiteLLM internal health check requests from results", ), + group_by_session: bool = fastapi.Query( + default=False, + description="Paginate over sessions instead of raw logs: one representative row per session, total counts sessions", + ), ): """ View spend logs with pagination support. @@ -2644,12 +2635,16 @@ async def ui_view_spend_logs( else: _order_expr = order_column + joined_conditions: Final = " AND ".join(sql_conditions) + session_grouping: Final = group_by_session is True + count_group_clause: Final = f"GROUP BY {_SESSION_GROUP_KEY_SQL}" if session_grouping else "" count_query: Final = f""" SELECT COUNT(*) AS total_count FROM ( SELECT 1 FROM "LiteLLM_SpendLogs" - WHERE {" AND ".join(sql_conditions)} + WHERE {joined_conditions} + {count_group_clause} LIMIT ${p} ) AS bounded_matches """ @@ -2660,21 +2655,36 @@ async def ui_view_spend_logs( total_is_capped: Final = raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP total_records: Final = SPEND_LOGS_PAGINATION_COUNT_CAP if total_is_capped else raw_total - sql_query: Final = f""" - SELECT - request_id, call_type, api_key, spend, total_tokens, + select_columns: Final = """request_id, call_type, api_key, spend, total_tokens, prompt_tokens, completion_tokens, "startTime", "endTime", "completionStartTime", model, model_id, model_group, custom_llm_provider, api_base, "user", metadata, cache_hit, cache_key, request_tags, team_id, organization_id, end_user, requester_ip_address, session_id, status, mcp_namespaced_tool_name, agent_id, - COALESCE(request_duration_ms, (EXTRACT(EPOCH FROM ("endTime" - "startTime")) * 1000)::INTEGER) AS request_duration_ms + COALESCE(request_duration_ms, (EXTRACT(EPOCH FROM ("endTime" - "startTime")) * 1000)::INTEGER) AS request_duration_ms""" + sql_query: Final = ( + f""" + SELECT * FROM ( + SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) + {select_columns} + FROM "LiteLLM_SpendLogs" + WHERE {joined_conditions} + ORDER BY {_SESSION_GROUP_KEY_SQL}, call_type IN {_MCP_CALL_TYPES_SQL}, "startTime" DESC + ) AS session_representatives + ORDER BY {_order_expr} {_sql_dir}{_nulls_clause}, request_id + LIMIT ${p} OFFSET ${p + 1} + """ + if session_grouping + else f""" + SELECT + {select_columns} FROM "LiteLLM_SpendLogs" - WHERE {" AND ".join(sql_conditions)} + WHERE {joined_conditions} ORDER BY {_order_expr} {_sql_dir}{_nulls_clause} LIMIT ${p} OFFSET ${p + 1} """ + ) sql_params.extend([page_size, skip]) data: Final = await prisma_client.db.query_raw(sql_query, *sql_params) @@ -4075,11 +4085,12 @@ async def _build_ui_spend_logs_response( Build the paginated response for the UI spend-logs endpoint. When ``enrich_session_counts`` is ``True`` (the default for the v1/UI - endpoint), each row is enriched with ``session_total_count`` so the - frontend knows which sessions are expandable (multi-call sessions). - For every row that carries a ``session_id``, a single ``GROUP BY`` query - fetches the total number of logs in each referenced session. Rows without - a ``session_id`` default to ``1``. + endpoint), each row is enriched with ``session_total_count`` plus spend + and call-type aggregates so the frontend knows which sessions are + expandable (multi-call sessions). One ``GROUP BY (session_id, api_key)`` + query serves every referenced session, keyed per api key so two callers + reusing a session id never see each other's totals. Rows without a + ``session_id`` default to ``1``. When ``enrich_session_counts`` is ``False`` (v2 endpoint), rows are serialised without the extra query. @@ -4101,7 +4112,6 @@ async def _build_ui_spend_logs_response( A dict with ``data`` (enriched rows), ``total``, ``page``, ``page_size``, ``total_pages``, and ``total_is_capped``. """ - count_map: dict[str, int] = {} if enrich_session_counts: session_ids: Final[Sequence[str | None]] = list( { @@ -4110,15 +4120,8 @@ async def _build_ui_spend_logs_response( if (row.get("session_id") if isinstance(row, dict) else getattr(row, "session_id", None)) } ) - if session_ids: - # NOTE: This GROUP BY runs on every v1/UI page load. The IN clause - # is bounded by page_size (typically 25-50 distinct session IDs). - # If performance degrades at scale, consider short-lived caching or - # folding the count into the main query via a window function. - counts: Final = await _count_logs_per_session(prisma_client, session_ids) - count_map = {r["session_id"]: r["_count"]["session_id"] for r in counts if r.get("session_id")} - session_spend_map: dict[str, dict[str, int | float]] = {} + session_spend_map: dict[tuple[str, str], dict[str, int | float]] = {} if enrich_session_counts and session_ids: from prisma.errors import PrismaError @@ -4130,38 +4133,46 @@ async def _build_ui_spend_logs_response( { (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) for row in data - if (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) + if (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) is not None } ) rows: Final[Sequence[_SessionSpendRow]] = await _query_raw( prisma_client, - """ - SELECT session_id, + f""" + SELECT session_id, api_key, + COUNT(*)::int AS session_total_count, COALESCE(SUM(spend), 0)::double precision AS session_total_spend, COUNT(*) FILTER ( - WHERE call_type IN ('call_mcp_tool', 'list_mcp_tools') + WHERE call_type IN {_MCP_CALL_TYPES_SQL} )::int AS mcp_tool_call_count, COALESCE(SUM(spend) FILTER ( - WHERE call_type IN ('call_mcp_tool', 'list_mcp_tools') + WHERE call_type IN {_MCP_CALL_TYPES_SQL} ), 0)::double precision AS mcp_tool_call_spend, - COUNT(*) FILTER (WHERE LOWER(cache_hit) = 'true')::int AS session_cache_hit_count + COUNT(*) FILTER (WHERE LOWER(cache_hit) = 'true')::int AS session_cache_hit_count, + COUNT(*) FILTER ( + WHERE call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} + )::int AS session_llm_count, + COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count FROM "LiteLLM_SpendLogs" WHERE session_id = ANY($1::text[]) AND api_key = ANY($2::text[]) - GROUP BY session_id + GROUP BY session_id, api_key """, session_ids, authorized_api_keys, ) session_spend_map = { - row["session_id"]: { + (row["session_id"], row["api_key"]): { + "session_total_count": int(row.get("session_total_count") or 0), "session_total_spend": float(row.get("session_total_spend") or 0.0), "mcp_tool_call_count": int(row.get("mcp_tool_call_count") or 0), "mcp_tool_call_spend": float(row.get("mcp_tool_call_spend") or 0.0), "session_cache_hit_count": int(row.get("session_cache_hit_count") or 0), + "session_llm_count": int(row.get("session_llm_count") or 0), + "session_agent_count": int(row.get("session_agent_count") or 0), } for row in rows - if row.get("session_id") + if row.get("session_id") and row.get("api_key") is not None } except PrismaError: verbose_proxy_logger.debug( @@ -4174,14 +4185,17 @@ async def _build_ui_spend_logs_response( for row in data: row_dict = dict(row) if isinstance(row, dict) else row.model_dump() sid = row_dict.get("session_id") - row_dict["session_total_count"] = count_map.get(sid, 1) if sid else 1 - session_stats = session_spend_map.get(sid) if sid else None + row_api_key = row_dict.get("api_key") + session_stats = session_spend_map.get((sid, row_api_key)) if sid and row_api_key is not None else None + row_dict["session_total_count"] = int(session_stats["session_total_count"]) if session_stats else 1 if session_stats: row_dict["session_total_spend"] = session_stats["session_total_spend"] if session_stats["mcp_tool_call_count"]: row_dict["mcp_tool_call_count"] = session_stats["mcp_tool_call_count"] row_dict["mcp_tool_call_spend"] = session_stats["mcp_tool_call_spend"] row_dict["session_cache_hit_count"] = session_stats["session_cache_hit_count"] + row_dict["session_llm_count"] = session_stats["session_llm_count"] + row_dict["session_agent_count"] = session_stats["session_agent_count"] enriched.append(row_dict) response_data: list = enriched else: diff --git a/tests/e2e/ui/helpers/traffic.ts b/tests/e2e/ui/helpers/traffic.ts index 25eb671fd0e..7f8417cdffb 100644 --- a/tests/e2e/ui/helpers/traffic.ts +++ b/tests/e2e/ui/helpers/traffic.ts @@ -21,6 +21,8 @@ interface ChatOptions { apiKey?: string; /** Sent as `user`, which lands in the spend log's end_user column. */ endUser?: string; + /** Sent as `litellm_trace_id`, which lands in the spend log's session_id column. */ + traceId?: string; } /** POST /v1/chat/completions and return the completion id (the Logs Request ID). */ @@ -34,6 +36,7 @@ export async function sendChatCompletion(request: APIRequestContext, opts: ChatO model: opts.model, messages: [{ role: "user", content: opts.prompt }], ...(opts.endUser ? { user: opts.endUser } : {}), + ...(opts.traceId ? { litellm_trace_id: opts.traceId } : {}), }, }); expect(res.ok(), `chat completion for ${opts.model} failed (${res.status()}): ${await res.text()}`).toBe(true); diff --git a/tests/e2e/ui/tests/logs/logsPagination.spec.ts b/tests/e2e/ui/tests/logs/logsPagination.spec.ts new file mode 100644 index 00000000000..416e1c3171f --- /dev/null +++ b/tests/e2e/ui/tests/logs/logsPagination.spec.ts @@ -0,0 +1,150 @@ +import { test, expect, type APIRequestContext, type Locator, type Page as PlaywrightPage } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; +import { CHAT_MODEL_A, createVirtualKey, sendChatCompletion, waitForSpendLog } from "../../helpers/traffic"; + +/** + * Session-grouped pagination (#38060): a page of N rows must render exactly N session rows, a + * session must never straddle pages, and two callers reusing one session id stay separate rows. + * All traffic is generated per run behind a unique key alias or session id, so concurrent specs + * cannot decide the outcome. + */ + +const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + +/** Every tab stays mounted, so the DOM holds four tables at once; scope to the visible one. */ +const requestLogsRows = (page: PlaywrightPage): Locator => + page.locator("table").filter({ visible: true }).first().locator("tbody tr"); + +const visibleTestId = (page: PlaywrightPage, id: string): Locator => page.getByTestId(id).filter({ visible: true }); + +async function openLogs(page: PlaywrightPage): Promise { + await navigateToPage(page, Page.Logs); + await dismissFeedbackPopup(page); + await expect(visibleTestId(page, "datatable-search")).toBeVisible({ timeout: 20_000 }); +} + +async function openFilterDrawer(page: PlaywrightPage): Promise { + await visibleTestId(page, "datatable-filters-trigger").click(); + const drawer = page.getByRole("dialog", { name: "Filters" }); + await expect(drawer).toBeVisible({ timeout: 10_000 }); + return drawer; +} + +async function applyKeyAliasFilter(page: PlaywrightPage, drawer: Locator, alias: string): Promise { + await drawer.getByRole("combobox", { name: "Search a key alias" }).click(); + await page.keyboard.type(alias); + await page.getByRole("option", { name: alias, exact: true }).first().click(); + await drawer.getByRole("button", { name: "Apply Filters" }).click(); + await expect(drawer).not.toBeVisible({ timeout: 10_000 }); +} + +async function setRowsPerPage(page: PlaywrightPage, size: "25" | "50" | "100"): Promise { + await visibleTestId(page, "pagination-page-size").click(); + await page.getByRole("option", { name: size, exact: true }).click(); +} + +test.describe("Logs page session-grouped pagination", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("a 25-row page renders exactly 25 session rows and no session straddles pages", async ({ page, request }) => { + const suffix = uniqueSuffix(); + const alias = `e2e-logs-pgn-${suffix}`; + const mine = await createVirtualKey(request, { key_alias: alias }); + + const soloIds: string[] = []; + for (let i = 0; i < 26; i++) { + soloIds.push( + await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `logs-pgn-solo-${i}-${suffix}`, + apiKey: mine.key, + }), + ); + } + const sessionA = `sess-pgn-a-${suffix}`; + const sessionB = `sess-pgn-b-${suffix}`; + let lastSessionCallId = ""; + for (let i = 0; i < 7; i++) { + lastSessionCallId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `logs-pgn-a-${i}-${suffix}`, + apiKey: mine.key, + traceId: sessionA, + }); + } + for (let i = 0; i < 3; i++) { + lastSessionCallId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `logs-pgn-b-${i}-${suffix}`, + apiKey: mine.key, + traceId: sessionB, + }); + } + await waitForSpendLog(request, lastSessionCallId); + await waitForSpendLog(request, soloIds[soloIds.length - 1]); + + // 36 calls in 28 session groups: 26 solos plus sessions of 7 and 3. + await openLogs(page); + const drawer = await openFilterDrawer(page); + await applyKeyAliasFilter(page, drawer, alias); + await setRowsPerPage(page, "25"); + + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 28", { timeout: 30_000 }); + await expect(requestLogsRows(page)).toHaveCount(25); + // The sessions are the newest groups, so their single representative rows sit on page 1. + await expect(requestLogsRows(page).filter({ hasText: sessionA })).toHaveCount(1); + await expect(requestLogsRows(page).filter({ hasText: sessionA })).toContainText("7"); + await expect(requestLogsRows(page).filter({ hasText: sessionB })).toHaveCount(1); + + await visibleTestId(page, "pagination-next").click(); + + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 26-28 of 28", { timeout: 30_000 }); + await expect(requestLogsRows(page)).toHaveCount(3); + await expect(requestLogsRows(page).filter({ hasText: sessionA })).toHaveCount(0); + await expect(requestLogsRows(page).filter({ hasText: sessionB })).toHaveCount(0); + }); + + test("two keys reusing one session id stay separate rows", async ({ page, request }) => { + const suffix = uniqueSuffix(); + const mine = await createVirtualKey(request, { key_alias: `e2e-logs-pgn-mine-${suffix}` }); + const theirs = await createVirtualKey(request, { key_alias: `e2e-logs-pgn-theirs-${suffix}` }); + const sharedSession = `sess-pgn-shared-${suffix}`; + + let lastId = ""; + for (let i = 0; i < 2; i++) { + lastId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `logs-pgn-shared-mine-${i}-${suffix}`, + apiKey: mine.key, + traceId: sharedSession, + }); + } + lastId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `logs-pgn-shared-theirs-${suffix}`, + apiKey: theirs.key, + traceId: sharedSession, + }); + await waitForSpendLog(request, lastId); + + await openLogs(page); + const drawer = await openFilterDrawer(page); + await drawer.getByPlaceholder("Enter session ID…").fill(sharedSession); + await drawer.getByRole("button", { name: "Apply Filters" }).click(); + await expect(drawer).not.toBeVisible({ timeout: 10_000 }); + + // One row per caller: reusing a session id must not merge two keys' activity into one row. + await expect(requestLogsRows(page).filter({ hasText: sharedSession })).toHaveCount(2, { timeout: 30_000 }); + + // And each row carries ITS key's totals: two calls badge the first key's row, + // while the other key's single call renders as a plain LLM row. + const mineRow = requestLogsRows(page).filter({ hasText: sharedSession }).filter({ hasText: mine.token }); + const theirsRow = requestLogsRows(page).filter({ hasText: sharedSession }).filter({ hasText: theirs.token }); + await expect(mineRow).toHaveCount(1); + await expect(theirsRow).toHaveCount(1); + await expect(mineRow.getByText("2", { exact: true })).toBeVisible(); + await expect(theirsRow.getByText("LLM", { exact: true })).toBeVisible(); + }); +}); diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index a0dcbf802ef..30b086bab61 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3959,18 +3959,18 @@ async def test_build_ui_spend_logs_response_dict_rows_session_counts(): ] mock_prisma = MagicMock() - mock_prisma.db.litellm_spendlogs.group_by = AsyncMock( - return_value=[ - {"session_id": session_id, "_count": {"session_id": 2}}, - ] - ) + mock_prisma.db.litellm_spendlogs.group_by = AsyncMock() mock_prisma.db.query_raw = AsyncMock( return_value=[ { "session_id": session_id, + "api_key": api_key, + "session_total_count": 2, "session_total_spend": 15.0, "mcp_tool_call_count": 1, "mcp_tool_call_spend": 10.0, + "session_llm_count": 1, + "session_agent_count": 0, } ] ) @@ -3995,6 +3995,8 @@ async def test_build_ui_spend_logs_response_dict_rows_session_counts(): assert rows[0]["mcp_tool_call_spend"] == 10.0 assert rows[1]["mcp_tool_call_count"] == 1 assert rows[1]["mcp_tool_call_spend"] == 10.0 + assert rows[0]["session_llm_count"] == 1 + assert rows[0]["session_agent_count"] == 0 # Every row in the session carries the full session spend, not just its own assert rows[0]["session_total_spend"] == 15.0 @@ -4003,13 +4005,126 @@ async def test_build_ui_spend_logs_response_dict_rows_session_counts(): # Row without a session_id defaults to 1 assert rows[2]["session_total_count"] == 1 - # group_by should have been called with the session_id - mock_prisma.db.litellm_spendlogs.group_by.assert_called_once_with( - by=["session_id"], - where={"session_id": {"in": [session_id]}}, - count={"session_id": True}, + # The count is folded into the single aggregate query; no separate group_by call. + mock_prisma.db.litellm_spendlogs.group_by.assert_not_called() + + +@pytest.mark.asyncio +async def test_build_ui_spend_logs_response_key_split_session_gets_per_key_aggregates(): + """ + Two keys reusing one session id are separate rows under grouped pagination, + and each row must carry ITS key's totals, never the combined session's: + the aggregate query and its lookup are keyed by (session_id, api_key). + """ + from litellm.proxy.spend_tracking.spend_management_endpoints import ( + _build_ui_spend_logs_response, ) + session_id = "sess-shared" + dict_rows = [ + {"request_id": "req-a", "session_id": session_id, "call_type": "completion", "api_key": "key-a"}, + {"request_id": "req-b", "session_id": session_id, "call_type": "completion", "api_key": "key-b"}, + ] + + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + { + "session_id": session_id, + "api_key": "key-a", + "session_total_count": 2, + "session_total_spend": 0.2, + "mcp_tool_call_count": 0, + "mcp_tool_call_spend": 0.0, + "session_cache_hit_count": 1, + "session_llm_count": 2, + "session_agent_count": 0, + }, + { + "session_id": session_id, + "api_key": "key-b", + "session_total_count": 1, + "session_total_spend": 0.7, + "mcp_tool_call_count": 0, + "mcp_tool_call_spend": 0.0, + "session_cache_hit_count": 0, + "session_llm_count": 1, + "session_agent_count": 0, + }, + ] + ) + + result = await _build_ui_spend_logs_response( + prisma_client=mock_prisma, + data=dict_rows, + total_records=2, + page=1, + page_size=50, + total_pages=1, + enrich_session_counts=True, + ) + + rows = result["data"] + assert [(r["session_total_count"], r["session_total_spend"]) for r in rows] == [(2, 0.2), (1, 0.7)] + assert [r["session_cache_hit_count"] for r in rows] == [1, 0] + assert [r["session_llm_count"] for r in rows] == [2, 1] + + aggregate_sql = mock_prisma.db.query_raw.mock_calls[0][1][0] + assert "GROUP BY session_id, api_key" in aggregate_sql + + +@pytest.mark.asyncio +async def test_build_ui_spend_logs_response_empty_api_key_keeps_session_aggregates(): + """ + The spend-log schema defaults api_key to an empty string, which is a real + group value and not a missing one: a multi-call session logged under an + empty key must keep its count and spend instead of degrading to a plain + single-call row. + """ + from litellm.proxy.spend_tracking.spend_management_endpoints import ( + _build_ui_spend_logs_response, + ) + + session_id = "sess-keyless" + dict_rows = [ + {"request_id": "req-1", "session_id": session_id, "call_type": "completion", "api_key": ""}, + ] + + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + { + "session_id": session_id, + "api_key": "", + "session_total_count": 3, + "session_total_spend": 0.09, + "mcp_tool_call_count": 0, + "mcp_tool_call_spend": 0.0, + "session_cache_hit_count": 0, + "session_llm_count": 3, + "session_agent_count": 0, + } + ] + ) + + result = await _build_ui_spend_logs_response( + prisma_client=mock_prisma, + data=dict_rows, + total_records=1, + page=1, + page_size=50, + total_pages=1, + enrich_session_counts=True, + ) + + row = result["data"][0] + assert row["session_total_count"] == 3 + assert row["session_total_spend"] == 0.09 + + # The empty key must reach the aggregate's authorized-keys filter too. + _, call_args, _ = mock_prisma.db.query_raw.mock_calls[0] + assert call_args[2] == [""] + @pytest.mark.asyncio async def test_build_ui_spend_logs_response_sums_multi_round_session_spend(): @@ -4033,14 +4148,13 @@ async def test_build_ui_spend_logs_response_sums_multi_round_session_spend(): ] mock_prisma = MagicMock() - mock_prisma.db.litellm_spendlogs.group_by = AsyncMock( - return_value=[{"session_id": session_id, "_count": {"session_id": 3}}] - ) # The raw aggregate query returns the full session spend (0.01 + 0.02 + 0.03). mock_prisma.db.query_raw = AsyncMock( return_value=[ { "session_id": session_id, + "api_key": api_key, + "session_total_count": 3, "session_total_spend": 0.06, "mcp_tool_call_count": 0, "mcp_tool_call_spend": 0.0, @@ -4089,13 +4203,12 @@ async def test_build_ui_spend_logs_response_session_cache_hit_count(): ] mock_prisma = MagicMock() - mock_prisma.db.litellm_spendlogs.group_by = AsyncMock( - return_value=[{"session_id": session_id, "_count": {"session_id": 2}}] - ) mock_prisma.db.query_raw = AsyncMock( return_value=[ { "session_id": session_id, + "api_key": api_key, + "session_total_count": 2, "session_total_spend": 0.05, "mcp_tool_call_count": 0, "mcp_tool_call_spend": 0.0, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py index ef68d9ce178..27e633099f0 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py @@ -274,6 +274,9 @@ async def test_spend_logs_ui_uses_bounded_count_not_full_scan(monkeypatch): "the page query must not carry a window count that forces a full-window " f"scan. SQL was:\n{page_sql}" ) + assert "GROUP BY" not in count_sql and "DISTINCT ON" not in page_sql, ( + "without group_by_session the endpoint must keep raw per-call pagination" + ) assert response["total"] == 137 assert response["total_is_capped"] is False @@ -499,3 +502,106 @@ async def test_global_spend_report_team_group_forwards_team_id(monkeypatch): params = mock_prisma.db.query_raw.call_args[0][1:] assert "team_x" in params, "team_id must be forwarded into the DB query params" assert "sl.team_id = $3" in sql, f"team query must filter on team_id. SQL was:\n{sql}" + + +@pytest.mark.asyncio +async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): + """ + With group_by_session=true, /spend/logs/ui must page and count SESSIONS, + not raw calls: the page query returns one representative row per session + (DISTINCT ON the session group key, preferring non-MCP calls, newest + first) and the bounded count counts groups. Otherwise the UI collapses a + server page of N calls into fewer visible rows while the footer still + claims N (issue #38060). + """ + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.spend_tracking.spend_management_endpoints import ( + SPEND_LOGS_PAGINATION_COUNT_CAP, + ui_view_spend_logs, + ) + + page_rows = [ + {"request_id": "req-1", "metadata": "{}", "session_id": None}, + {"request_id": "req-2", "metadata": "{}", "session_id": None}, + ] + mock_prisma = _make_ui_spend_logs_mock(count_total=12, page_rows=page_rows) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin") + mock_request = MagicMock() + mock_request.url.path = "/spend/logs/ui" + + response = await ui_view_spend_logs( + request=mock_request, + api_key=None, + user_id=None, + request_id=None, + start_date="2026-02-16 00:00:00", + end_date="2026-02-16 23:59:59", + page=1, + page_size=50, + sort_by="startTime", + sort_order="desc", + user_api_key_dict=auth, + group_by_session=True, + ) + + group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key" + + count_call = mock_prisma.db.query_raw.call_args_list[0] + count_sql = count_call[0][0] + assert f"GROUP BY {group_key}" in count_sql, f"grouped total must count sessions. SQL was:\n{count_sql}" + assert "COUNT(*) OVER ()" not in count_sql + assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" + assert count_call[0][-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1 + + page_sql = mock_prisma.db.query_raw.call_args_list[1][0][0] + assert f"DISTINCT ON ({group_key})" in page_sql, f"page must return one row per session. SQL was:\n{page_sql}" + assert f"ORDER BY {group_key}, call_type IN ('call_mcp_tool', 'list_mcp_tools'), \"startTime\" DESC" in page_sql, ( + "the session representative must prefer the newest non-MCP call" + ) + assert "COUNT(*) OVER ()" not in page_sql + + assert response["total"] == 12 + assert response["total_is_capped"] is False + assert response["total_pages"] == 1 + + +@pytest.mark.asyncio +async def test_spend_logs_ui_request_id_lookup_with_grouping_returns_exact_row(monkeypatch): + """ + A request_id lookup with group_by_session=true must still resolve the + exact requested row: the filter runs before grouping, so the row is its + own group's representative and deep links keep working. + """ + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.spend_tracking.spend_management_endpoints import ui_view_spend_logs + + target_row = {"request_id": "req-deep-link", "metadata": "{}", "session_id": None} + mock_prisma = _make_ui_spend_logs_mock(count_total=1, page_rows=[target_row]) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin") + mock_request = MagicMock() + mock_request.url.path = "/spend/logs/ui" + + response = await ui_view_spend_logs( + request=mock_request, + api_key=None, + user_id=None, + request_id="req-deep-link", + start_date=None, + end_date=None, + page=1, + page_size=1, + sort_by="startTime", + sort_order="desc", + user_api_key_dict=auth, + group_by_session=True, + ) + + page_call = mock_prisma.db.query_raw.call_args_list[1] + assert "request_id = $" in page_call[0][0], "the request_id equality filter must survive grouping" + assert "req-deep-link" in page_call[0] + assert [row["request_id"] for row in response["data"]] == ["req-deep-link"] + assert response["total"] == 1 diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index db7baf5e171..63232afc46b 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2042,6 +2042,7 @@ interface UiSpendLogsParams { min_spend?: number; max_spend?: number; exclude_internal_health_checks?: boolean; + group_by_session?: boolean; } interface UiSpendLogsCallOptions { diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx index 22e3f635b50..c368a43dd1e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx @@ -138,33 +138,39 @@ describe("RequestLogsPanel", () => { respondWith([]); }); - describe("multi-call session collapsing", () => { - const sessionRows = [ - logEntry({ request_id: "req-mcp", call_type: "call_mcp_tool", session_id: "sess-1", session_total_count: 3 }), - logEntry({ request_id: "req-llm", call_type: "acompletion", session_id: "sess-1", session_total_count: 3 }), - logEntry({ request_id: "req-llm-2", call_type: "acompletion", session_id: "sess-1", session_total_count: 3 }), - ]; - - it("collapses a multi-call session to a single representative row", async () => { - respondWith(sessionRows); + describe("server-grouped session pagination (#38060)", () => { + it("requests session-grouped pages of 10 rows by default", async () => { renderPanel(); - await waitFor(() => expect(row("req-mcp") ?? row("req-llm") ?? row("req-llm-2")).not.toBeNull()); - - const rendered = ["req-mcp", "req-llm", "req-llm-2"].filter((id) => row(id) !== null); - expect(rendered).toHaveLength(1); + await waitFor(() => expect(uiSpendLogsCall).toHaveBeenCalled()); + expect(lastCall()?.params?.group_by_session).toBe(true); + expect(lastCall()?.page_size).toBe(10); }); - it("prefers an LLM call over an MCP call as the session's representative", async () => { - respondWith(sessionRows); + it("renders every row the server returns without client-side collapsing", async () => { + respondWith([ + logEntry({ request_id: "req-a", call_type: "acompletion", session_id: "sess-1", session_total_count: 3 }), + logEntry({ request_id: "req-b", call_type: "acompletion", session_id: "sess-1", session_total_count: 3 }), + logEntry({ request_id: "req-c", call_type: "acompletion", session_id: "sess-1", session_total_count: 3 }), + ]); renderPanel(); - await waitFor(() => expect(row("req-llm")).not.toBeNull()); - expect(row("req-mcp")).toBeNull(); + await waitFor(() => expect(row("req-a")).not.toBeNull()); + expect(row("req-b")).not.toBeNull(); + expect(row("req-c")).not.toBeNull(); }); - it("shows the session's call count and composition on the representative row", async () => { - respondWith(sessionRows); + it("shows the session's call count on the server-picked representative row", async () => { + respondWith([ + logEntry({ + request_id: "req-llm", + call_type: "acompletion", + session_id: "sess-1", + session_total_count: 3, + session_llm_count: 2, + mcp_tool_call_count: 1, + }), + ]); renderPanel(); await waitFor(() => expect(row("req-llm")).not.toBeNull()); @@ -296,6 +302,7 @@ describe("RequestLogsPanel", () => { if (!byIdCall) throw new Error("expected a by-id uiSpendLogsCall"); expect(byIdCall.page).toBe(1); expect(byIdCall.page_size).toBe(1); + expect(byIdCall.params?.group_by_session).toBeUndefined(); }); it("closing the drawer removes ?log_id= from the URL and closes the drawer", async () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx index 52ea78abf5e..2ebabf64ab0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx @@ -10,7 +10,7 @@ import type { KeyResponse } from "../key_team_helpers/key_list"; import { keyInfoV1Call, uiSpendLogsCall } from "../networking"; import KeyInfoView from "../templates/key_info_view"; import type { LogEntry } from "./columns"; -import { AGENT_CALL_TYPES, MCP_CALL_TYPES } from "./constants"; +import { LOGS_PAGE_SIZE_OPTIONS } from "./constants"; import { DEFAULT_LOGS_SORTING, formatLogsWindow, @@ -24,7 +24,7 @@ import { LogDetailsDrawer } from "./LogDetailsDrawer"; import { LiveTailBanner, LogsTableToolbar } from "./LogsTableToolbar"; import { RequestLogsTable } from "./RequestLogsTable"; -const PAGE_SIZE = 50; +const PAGE_SIZE = LOGS_PAGE_SIZE_OPTIONS[0]; const DEFAULT_INTERVAL = { value: 24, unit: "hours" }; interface RequestLogsPanelProps { @@ -35,12 +35,6 @@ interface RequestLogsPanelProps { isActive: boolean; } -interface SessionComposition { - llm: number; - agent: number; - mcp: number; -} - export default function RequestLogsPanel({ accessToken, token, userRole, userID, isActive }: RequestLogsPanelProps) { const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: PAGE_SIZE }); const [sorting, setSorting] = useState(DEFAULT_LOGS_SORTING); @@ -157,49 +151,7 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, const isDrawerOpen = displayLog !== null || displaySessionId !== null; - const rows = useMemo(() => { - const searchedLogs = filteredLogs.data; - - const sessionCompositionById = searchedLogs.reduce>((acc, log) => { - if (!log.session_id) return acc; - if (!acc[log.session_id]) { - acc[log.session_id] = { llm: 0, agent: 0, mcp: 0 }; - } - if (MCP_CALL_TYPES.includes(log.call_type)) { - acc[log.session_id].mcp += 1; - } else if (AGENT_CALL_TYPES.includes(log.call_type)) { - acc[log.session_id].agent += 1; - } else { - acc[log.session_id].llm += 1; - } - return acc; - }, {}); - - const sessionRepresentativeMap = new Map(); - for (const log of searchedLogs) { - if (!log.session_id || (log.session_total_count || 1) <= 1) continue; - const isMcp = MCP_CALL_TYPES.includes(log.call_type); - const existing = sessionRepresentativeMap.get(log.session_id); - if (!existing || (existing.isMcp && !isMcp)) { - sessionRepresentativeMap.set(log.session_id, { requestId: log.request_id, isMcp }); - } - } - - return searchedLogs - .map((log) => { - const sessionComposition = log.session_id ? sessionCompositionById[log.session_id] : undefined; - return { - ...log, - session_llm_count: sessionComposition?.llm ?? undefined, - session_mcp_count: sessionComposition?.mcp ?? undefined, - session_agent_count: sessionComposition?.agent ?? undefined, - }; - }) - .filter((log) => { - if (!log.session_id || (log.session_total_count || 1) <= 1) return true; - return sessionRepresentativeMap.get(log.session_id)?.requestId === log.request_id; - }); - }, [filteredLogs.data]); + const rows: LogEntry[] = filteredLogs.data; const searchTerm = useMemo(() => { const entry = columnFilters.find((filter) => filter.id === LOG_FILTER_IDS.REQUEST_ID); @@ -258,13 +210,12 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, ); const handleSessionClick = useCallback( - (sessionId: string) => { - if (!sessionId) return; - const log = rows.find((candidate) => candidate.session_id === sessionId) ?? null; + (log: LogEntry) => { + if (!log.session_id) return; setSelectedLog(log); - openSession(sessionId, log?.request_id ?? null); + openSession(log.session_id, log.request_id); }, - [rows, openSession], + [openSession], ); const handleSelectLog = useCallback( diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTable.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTable.tsx index 7146cf33847..4159b3b699b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTable.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTable.tsx @@ -8,6 +8,7 @@ import { DataTable, DataTableFilterDrawer, DataTableToolbar } from "@/components import type { Team } from "../key_team_helpers/key_list"; import type { LogEntry } from "./columns"; +import { LOGS_PAGE_SIZE_OPTIONS } from "./constants"; import { LOG_FILTER_LABELS, type LogsWindow } from "./log_filter_logic"; import { RequestLogsFilters } from "./RequestLogsFilters"; import { getRequestLogsTableColumns } from "./RequestLogsTableColumns"; @@ -28,7 +29,7 @@ interface RequestLogsTableProps { onRefresh: () => void; onRowClick: (log: LogEntry) => void; onKeyHashClick: (keyHash: string) => void; - onSessionClick: (sessionId: string) => void; + onSessionClick: (log: LogEntry) => void; teams: Team[]; logsWindow: LogsWindow; toolbarChildren?: ReactNode; @@ -91,6 +92,7 @@ export function RequestLogsTable({ paginationMode="server" pagination={pagination} onPaginationChange={onPaginationChange} + pageSizeOptions={LOGS_PAGE_SIZE_OPTIONS} rowCount={rowCount} filterMode="server" columnFilters={columnFilters} diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx index 3d0ea03c5d5..ce59c62f1c4 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx @@ -85,13 +85,19 @@ describe("row action cells", () => { expect(deps.onKeyHashClick).toHaveBeenCalledWith("sk-hash-9"); }); - it("reports the session id from the session cell", async () => { + it("reports the clicked row from the session cell, so two rows sharing a session id stay distinguishable", async () => { const user = userEvent.setup(); const deps = { onKeyHashClick: vi.fn(), onSessionClick: vi.fn() }; - renderRows([logEntry({ request_id: "req-sess", session_id: "sess-42" })], deps); + renderRows( + [ + logEntry({ request_id: "req-key-a", session_id: "sess-42", api_key: "key-a" }), + logEntry({ request_id: "req-key-b", session_id: "sess-42", api_key: "key-b" }), + ], + deps, + ); - await user.click(screen.getByText("sess-42")); - expect(deps.onSessionClick).toHaveBeenCalledWith("sess-42"); + await user.click(screen.getAllByText("sess-42")[1]); + expect(deps.onSessionClick).toHaveBeenCalledWith(expect.objectContaining({ request_id: "req-key-b" })); }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx index cf776515bd4..b9058d02a6b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx @@ -13,7 +13,7 @@ import { AgentBadge, AgentIcon, LlmBadge, McpBadge, SparkleIcon, WrenchIcon } fr export interface RequestLogsTableColumnsDeps { onKeyHashClick: (keyHash: string) => void; - onSessionClick: (sessionId: string) => void; + onSessionClick: (log: LogEntry) => void; } const readMetaString = (metadata: Record | undefined, key: string): string | undefined => { @@ -61,7 +61,7 @@ export const getRequestLogsTableColumns = ({ const isAgent = AGENT_CALL_TYPES.includes(log.call_type); const sessionLlmCount = log.session_llm_count ?? (isMcp || isAgent ? 0 : sessionCount); const sessionAgentCount = log.session_agent_count ?? (isAgent ? sessionCount : 0); - const sessionMcpCount = log.session_mcp_count ?? (isMcp ? sessionCount : 0); + const sessionMcpCount = log.mcp_tool_call_count ?? (isMcp ? sessionCount : 0); if (isMcp) return ; if (isAgent && sessionCount <= 1) return ; @@ -113,7 +113,7 @@ export const getRequestLogsTableColumns = ({ header: "Session ID", size: 120, enableSorting: false, - cell: ({ row }) => , + cell: ({ row }) => onSessionClick(row.original)} />, }, { id: "request_id", diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx index eef957922d7..2f3a3681352 100644 --- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx @@ -46,6 +46,5 @@ export type LogEntry = { mcp_tool_call_count?: number; mcp_tool_call_spend?: number; session_llm_count?: number; - session_mcp_count?: number; session_agent_count?: number; }; diff --git a/ui/litellm-dashboard/src/components/view_logs/constants.ts b/ui/litellm-dashboard/src/components/view_logs/constants.ts index 5b0b1d0fee3..1c17f398e35 100644 --- a/ui/litellm-dashboard/src/components/view_logs/constants.ts +++ b/ui/litellm-dashboard/src/components/view_logs/constants.ts @@ -12,6 +12,9 @@ export const ERROR_CODE_OPTIONS: { label: string; value: string }[] = [ { label: "529 - Overloaded", value: "529" }, ]; +/** Page sizes the logs tables offer; the first entry is the default. */ +export const LOGS_PAGE_SIZE_OPTIONS = [10, 25, 50, 100]; + /** Call types that represent MCP tool invocations (shared across columns, index, drawer). */ export const MCP_CALL_TYPES = ["call_mcp_tool", "list_mcp_tools"]; diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx index 3b8d96596de..acd5be06593 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx @@ -181,6 +181,7 @@ export function useLogFilterLogic({ sort_by: sortBy, sort_order: sortOrder, exclude_internal_health_checks: excludeInternalHealthChecks, + group_by_session: true, }, }); }, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index be81450b941..7583b6f6df1 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -56755,6 +56755,8 @@ export interface operations { sort_order?: string | null; /** @description Exclude LiteLLM internal health check requests from results */ exclude_internal_health_checks?: boolean; + /** @description Paginate over sessions instead of raw logs: one representative row per session, total counts sessions */ + group_by_session?: boolean; }; header?: never; path?: never; @@ -56867,6 +56869,8 @@ export interface operations { sort_order?: string | null; /** @description Exclude LiteLLM internal health check requests from results */ exclude_internal_health_checks?: boolean; + /** @description Paginate over sessions instead of raw logs: one representative row per session, total counts sessions */ + group_by_session?: boolean; }; header?: never; path?: never; From 8ed1da40de0124983636c4bf6216046cea331488 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:55:02 -0700 Subject: [PATCH 164/480] fix(rag): forward the managed vector store's params to the search call /v1/rag/query folded the registry store's litellm_params into retrieval_config, where the caller allowlist dropped api_key, api_base, and provider extras such as Milvus outputFields and milvus_text_field, so a managed Milvus store 500'd with MILVUS_API_KEY is not set while the direct search endpoint worked. The store's params now travel as a trusted vector_store_params argument straight to the search call, never through the completion kwargs, and the caller allowlist stays in place. --- litellm/proxy/rag_endpoints/endpoints.py | 1 + litellm/rag/main.py | 21 +++++- .../proxy/rag_endpoints/test_rag_endpoints.py | 69 +++++++++++++++++++ tests/test_litellm/rag/test_main.py | 60 ++++++++++++++++ 4 files changed, 149 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index e144ff965ae..c8c6c505375 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -761,6 +761,7 @@ async def rag_query( model=model, messages=messages, retrieval_config=merged_retrieval_config, + vector_store_params=store_data, rerank=rerank, stream=stream, router=llm_router, diff --git a/litellm/rag/main.py b/litellm/rag/main.py index 94bfc305a6a..8ddc4c231dd 100644 --- a/litellm/rag/main.py +++ b/litellm/rag/main.py @@ -11,7 +11,7 @@ __all__ = ["aingest", "aquery", "ingest", "query"] import asyncio import contextvars -from collections.abc import Coroutine, Iterator +from collections.abc import Coroutine, Iterator, Mapping from contextlib import contextmanager from functools import partial from types import MappingProxyType @@ -66,6 +66,10 @@ _FORWARDABLE_RETRIEVAL_CONFIG_KEYS: Final = frozenset( } ) +_SEARCH_ARGS_SET_BY_PIPELINE: Final = frozenset( + {"vector_store_id", "query", "max_num_results", "custom_llm_provider", "router"} +) + def get_ingestion_class(provider: str) -> type[BaseRAGIngestion]: """ @@ -225,6 +229,7 @@ async def _execute_query_pipeline( retrieval_config: dict[str, Any], rerank: dict[str, Any] | None = None, stream: bool = False, + vector_store_params: Mapping[str, object] | None = None, **kwargs, ) -> ModelResponse: """ @@ -245,7 +250,14 @@ async def _execute_query_pipeline( provider_search_params: Final = MappingProxyType( {k: v for k, v in retrieval_config.items() if k in _FORWARDABLE_RETRIEVAL_CONFIG_KEYS} ) - forwarded_search_params: Final = MappingProxyType({**provider_search_params, **kwargs}) + store_search_params: Final = MappingProxyType( + { + k: v + for k, v in (vector_store_params.items() if vector_store_params else ()) + if k not in _SEARCH_ARGS_SET_BY_PIPELINE + } + ) + forwarded_search_params: Final = MappingProxyType({**provider_search_params, **store_search_params, **kwargs}) with _suppressed_sub_call_billing(): search_response: Final = await litellm.vector_stores.asearch( vector_store_id=retrieval_config["vector_store_id"], @@ -339,6 +351,7 @@ async def aquery( retrieval_config: dict[str, Any], rerank: dict[str, Any] | None = None, stream: bool = False, + vector_store_params: Mapping[str, object] | None = None, **kwargs, ) -> ModelResponse: """ @@ -356,6 +369,7 @@ async def aquery( retrieval_config=retrieval_config, rerank=rerank, stream=stream, + vector_store_params=vector_store_params, **kwargs, ) @@ -386,6 +400,7 @@ def query( retrieval_config: dict[str, Any], rerank: dict[str, Any] | None = None, stream: bool = False, + vector_store_params: Mapping[str, object] | None = None, **kwargs, ) -> ModelResponse | Coroutine[None, None, ModelResponse]: """ @@ -402,6 +417,7 @@ def query( retrieval_config=retrieval_config, rerank=rerank, stream=stream, + vector_store_params=vector_store_params, **kwargs, ) else: @@ -412,6 +428,7 @@ def query( retrieval_config=retrieval_config, rerank=rerank, stream=stream, + vector_store_params=vector_store_params, **kwargs, ) ) diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 0085b6ebd36..e68a964e997 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -421,6 +421,75 @@ def test_rag_query_store_params_win_over_user_retrieval_config(client_internal_u assert forwarded_config["aws_region_name"] == "eu-west-1" +def test_rag_query_forwards_managed_store_credentials_to_search(client_internal_user): + """ + Regression for LIT-6773: the registry store's api_key / api_base and its + provider extras (Milvus outputFields, milvus_text_field) must reach the + vector store search the way the direct /v1/vector_stores/{id}/search + endpoint forwards them. Pre-fix the RAG path allowlisted them away and a + managed Milvus store 500'd with "MILVUS_API_KEY is not set". + """ + import litellm + from litellm import Router + from litellm.types.vector_stores import VectorStoreSearchResponse + + mock_vector_store = { + "vector_store_id": "customer_kb", + "custom_llm_provider": "milvus", + "litellm_params": { + "vector_store_id": "customer_kb", + "custom_llm_provider": "milvus", + "api_base": "http://127.0.0.1:19530", + "api_key": "root:Milvus", + "litellm_embedding_model": "multilingual-e5-large", + "milvus_text_field": "book_intro_text", + "outputFields": ["book_intro_text"], + }, + } + mock_registry = MagicMock() + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = mock_vector_store + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse(object="vector_store.search_results.page", search_query="q", data=[]) + ) + router = Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test", "mock_response": "hi"}, + } + ] + ) + + with patch( # test-quality-ok: asearch is the boundary the store-credential forwarding under test targets; the real aquery pipeline runs in between + "litellm.vector_stores.asearch", new=fake_search + ), patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry and a mock-response router so real store resolution and the completion step run + "litellm.proxy.proxy_server.llm_router", router + ), patch( # test-quality-ok: grants store access, which is not under test, so the endpoint reaches the search boundary + "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", + new=AsyncMock(return_value=True), + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "which database is built for similarity search?"}], + "retrieval_config": {"vector_store_id": "customer_kb", "custom_llm_provider": "milvus", "top_k": 2}, + }, + ) + + assert response.status_code == 200, response.json() + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["vector_store_id"] == "customer_kb" + assert search_kwargs["custom_llm_provider"] == "milvus" + assert search_kwargs["max_num_results"] == 2 + assert search_kwargs["api_base"] == "http://127.0.0.1:19530" + assert search_kwargs["api_key"] == "root:Milvus" + assert search_kwargs["litellm_embedding_model"] == "multilingual-e5-large" + assert search_kwargs["milvus_text_field"] == "book_intro_text" + assert search_kwargs["outputFields"] == ["book_intro_text"] + + @pytest.mark.parametrize( "blocked_key", ["embedding_model", "litellm_embedding_model", "litellm_embedding_config", "litellm_credential_name"], diff --git a/tests/test_litellm/rag/test_main.py b/tests/test_litellm/rag/test_main.py index 51d03544910..420b72b6a3d 100644 --- a/tests/test_litellm/rag/test_main.py +++ b/tests/test_litellm/rag/test_main.py @@ -388,6 +388,66 @@ async def test_aquery_does_not_forward_connection_override_keys_to_search(): assert not (blocked & set(search_kwargs.keys())) +@pytest.mark.asyncio +async def test_aquery_forwards_vector_store_params_to_search_but_not_completion(): + """ + Regression for LIT-6773: the server-trusted vector_store_params (a managed + store's litellm_params) must reach the search call wholesale, including the + connection keys the caller allowlist blocks, while the caller's own + retrieval_config overrides stay blocked and the completion never inherits + the store's connection params. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + fake_completion = AsyncMock( + return_value=ModelResponse( + id="chatcmpl-test", + choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + model="gpt-4o-mini", + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search), patch( # test-quality-ok: asearch and acompletion are the two boundaries the forwarding contract under test targets + "litellm.acompletion", new=fake_completion + ): + await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={ + "vector_store_id": "customer_kb", + "custom_llm_provider": "milvus", + "api_base": "https://attacker.example.com", + "api_key": "attacker-key", + }, + vector_store_params={ + "vector_store_id": "customer_kb", + "custom_llm_provider": "milvus", + "api_base": "http://127.0.0.1:19530", + "api_key": "root:Milvus", + "milvus_text_field": "book_intro_text", + "outputFields": ["book_intro_text"], + }, + ) + + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["vector_store_id"] == "customer_kb" + assert search_kwargs["custom_llm_provider"] == "milvus" + assert search_kwargs["api_base"] == "http://127.0.0.1:19530" + assert search_kwargs["api_key"] == "root:Milvus" + assert search_kwargs["milvus_text_field"] == "book_intro_text" + assert search_kwargs["outputFields"] == ["book_intro_text"] + fake_completion.assert_awaited_once() + store_only_keys = {"api_base", "api_key", "milvus_text_field", "outputFields"} + assert not (store_only_keys & set(fake_completion.await_args.kwargs)) + + def test_rag_call_types_are_registered(): """ query/aquery/ingest/aingest are @client-decorated entry points, so their From 993766be0e1a1ea0f173dc9445f85e4eab3665bc Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 2 Sep 2026 18:03:22 -0700 Subject: [PATCH 165/480] feat(proxy): serve the auto-router preset catalog at runtime (#39412) The dashboard's template picker imported autorouter_presets.json at build time, so every catalog change needed a dashboard rebuild and artifacts refresh. The catalog now lives in litellm/proxy/public_endpoints/ and GET /public/autorouter_presets serves it, fetching litellm.autorouter_presets_url (GitHub raw on main, 1h in-process cache, bundled fallback) so a merged catalog change propagates to running proxies like the model cost map does. The dashboard fetches it at runtime via useAutoRouterPresets and keeps no local copy. Resolves LIT-6764 --- litellm/__init__.py | 4 + .../public_endpoints}/autorouter_presets.json | 0 .../public_endpoints/public_endpoints.py | 85 ++++++- .../public_endpoints/public_endpoints.py | 42 ++- .../public_endpoints/test_public_endpoints.py | 240 ++++++++++++++++++ .../hooks/autoRouter/useAutoRouterPresets.ts | 16 ++ .../add_model/add_auto_router_tab.test.tsx | 59 ++++- .../add_model/add_auto_router_tab.tsx | 28 +- .../src/components/networking.tsx | 10 + .../src/lib/autorouter_presets.test.ts | 13 +- .../src/lib/autorouter_presets.ts | 13 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 90 +++++++ .../tests/mocks/autoRouterPresets.ts | 16 ++ 13 files changed, 592 insertions(+), 24 deletions(-) rename {ui/litellm-dashboard/src => litellm/proxy/public_endpoints}/autorouter_presets.json (100%) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets.ts create mode 100644 ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts diff --git a/litellm/__init__.py b/litellm/__init__.py index 44f2e7c1f02..41a3789ab0d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -424,6 +424,10 @@ anthropic_beta_headers_url: str = os.getenv( "LITELLM_ANTHROPIC_BETA_HEADERS_URL", "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json", ) +autorouter_presets_url: str = os.getenv( + "LITELLM_AUTOROUTER_PRESETS_URL", + "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/proxy/public_endpoints/autorouter_presets.json", +) suppress_debug_info: bool = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None diff --git a/ui/litellm-dashboard/src/autorouter_presets.json b/litellm/proxy/public_endpoints/autorouter_presets.json similarity index 100% rename from ui/litellm-dashboard/src/autorouter_presets.json rename to litellm/proxy/public_endpoints/autorouter_presets.json diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 4d58a974bb8..94a59828451 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -1,11 +1,13 @@ +import asyncio import json import os import re -from collections.abc import Awaitable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from importlib.resources import files from typing import TYPE_CHECKING, Final, Protocol from fastapi import APIRouter, HTTPException, Request +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict import litellm @@ -28,6 +30,7 @@ from litellm.types.proxy.management_endpoints.model_management_endpoints import ) from litellm.types.proxy.public_endpoints.public_endpoints import ( AgentCreateInfo, + AutoRouterPresetRecord, ComplexityScorerDefaults, ProviderCreateInfo, PublicModelHubInfo, @@ -464,6 +467,86 @@ async def get_litellm_blog_posts(): return BlogPostsResponse(posts=posts) +_AUTOROUTER_PRESETS_ADAPTER: Final = TypeAdapter(dict[str, AutoRouterPresetRecord]) + + +def _load_bundled_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord]: + raw: Final = json.loads( + files("litellm.proxy.public_endpoints").joinpath("autorouter_presets.json").read_text(encoding="utf-8") + ) + return _AUTOROUTER_PRESETS_ADAPTER.validate_python(raw) + + +async def _fetch_remote_autorouter_presets(url: str) -> Mapping[str, AutoRouterPresetRecord]: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.UI) + response: Final = await client.get(url, timeout=5.0) + response.raise_for_status() + presets: Final = _AUTOROUTER_PRESETS_ADAPTER.validate_python(response.json()) + if not presets: + raise ValueError("remote auto-router preset catalog is empty") + return presets + + +async def _resolve_autorouter_presets( + url: str, + fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]], +) -> Mapping[str, AutoRouterPresetRecord]: + if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true": + return _load_bundled_autorouter_presets() + try: + return await fetch(url) + except Exception as e: + verbose_logger.warning( + "LiteLLM: failed to fetch auto-router presets from %s: %s. Serving the bundled catalog for the life of this process.", + url, + str(e), + ) + return _load_bundled_autorouter_presets() + + +class _AutoRouterPresetsCache: + presets: Mapping[str, AutoRouterPresetRecord] | None = None + lock: asyncio.Lock | None = None + + +async def get_autorouter_presets( + url: str, + fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]] = _fetch_remote_autorouter_presets, +) -> Mapping[str, AutoRouterPresetRecord]: + cached: Final = _AutoRouterPresetsCache.presets + if cached is not None: + return cached + if _AutoRouterPresetsCache.lock is None: + _AutoRouterPresetsCache.lock = asyncio.Lock() + async with _AutoRouterPresetsCache.lock: + held: Final = _AutoRouterPresetsCache.presets + if held is not None: + return held + resolved: Final = await _resolve_autorouter_presets(url=url, fetch=fetch) + _AutoRouterPresetsCache.presets = resolved + return resolved + + +@router.get( + "/public/autorouter_presets", + tags=["public", "auto router"], # mutable-ok: FastAPI route tags take a list + response_model=dict[str, AutoRouterPresetRecord], +) +async def get_public_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord]: + """ + Return the auto-router preset catalog the dashboard's template picker renders. + + Resolved once per process, like the model cost map: fetched from ``litellm.autorouter_presets_url`` + (override with ``LITELLM_AUTOROUTER_PRESETS_URL``) on the first request, falling back to the + catalog bundled with the package on any failure. Set ``LITELLM_LOCAL_AUTOROUTER_PRESETS=True`` + to serve the bundled catalog only. A restart picks up a newly published catalog. + """ + return await get_autorouter_presets(url=litellm.autorouter_presets_url) + + @router.get( "/public/endpoints", tags=["public"], diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index c7f80a61e0f..fa73926305b 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -1,7 +1,7 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Any, Literal -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class PublicModelHubInfo(BaseModel): @@ -73,6 +73,44 @@ class SupportedEndpointsResponse(BaseModel): endpoints: list[SupportedEndpoint] +class AutoRouterPresetTiers(BaseModel): + """Exactly the four built-in tiers the dashboard's preset prefill can apply. + + extra="forbid" on purpose: a tier name this dashboard cannot apply would grey out or crash the + picker, so such a catalog is rejected wholesale and the bundled one serves instead. + """ + + model_config = ConfigDict(extra="forbid") + + SIMPLE: Sequence[str] + MEDIUM: Sequence[str] + COMPLEX: Sequence[str] + REASONING: Sequence[str] + + +class AutoRouterPresetConfig(BaseModel): + """The complexity_router_config a preset prefills. + + Only tiers is validated, because every dashboard consumer dereferences it; everything else + passes through verbatim with unknown fields kept (extra="allow"), so a catalog published after + this proxy shipped still serves its new fields intact. + """ + + model_config = ConfigDict(extra="allow") + + tiers: AutoRouterPresetTiers + + +class AutoRouterPresetRecord(BaseModel): + """One auto-router preset as served to the dashboard's template picker.""" + + model_config = ConfigDict(extra="allow") + + label: str + description: str + complexity_router_config: AutoRouterPresetConfig + + class ComplexityScorerDefaults(BaseModel): """The complexity router's shipped heuristic scorer defaults. diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 8006f64ba41..551e27a18f5 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -1077,3 +1077,243 @@ def test_public_mcp_hub_does_not_expose_upstream_url(): assert all("url" not in item for item in data) assert secret_url not in response.text app.dependency_overrides.clear() + + + +@pytest.fixture +def reset_autorouter_presets_cache(): + from litellm.proxy.public_endpoints.public_endpoints import _AutoRouterPresetsCache + + _AutoRouterPresetsCache.presets = None + _AutoRouterPresetsCache.lock = None + yield + _AutoRouterPresetsCache.presets = None + _AutoRouterPresetsCache.lock = None + + +def test_get_autorouter_presets_local_mode_serves_bundled_catalog( + monkeypatch, reset_autorouter_presets_cache +): + monkeypatch.setenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "True") + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + response = client.get("/public/autorouter_presets") + + assert response.status_code == 200 + payload = response.json() + assert "anthropic_family" in payload + for preset in payload.values(): + assert isinstance(preset["label"], str) + assert isinstance(preset["description"], str) + assert "tiers" in preset["complexity_router_config"] + + +@pytest.mark.asyncio +async def test_get_autorouter_presets_fetches_once_per_process( + monkeypatch, reset_autorouter_presets_cache +): + from litellm.proxy.public_endpoints.public_endpoints import ( + _AUTOROUTER_PRESETS_ADAPTER, + get_autorouter_presets, + ) + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + remote = _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "remote_only": { + "label": "Remote Only", + "description": "from the remote catalog", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + } + } + ) + calls = [] + + async def fake_fetch(url): + calls.append(url) + return remote + + first = await get_autorouter_presets(url="https://example.test/presets.json", fetch=fake_fetch) + second = await get_autorouter_presets(url="https://example.test/presets.json", fetch=fake_fetch) + + assert first == remote + assert second == remote + assert calls == ["https://example.test/presets.json"] + + +@pytest.mark.asyncio +async def test_get_autorouter_presets_single_flight_on_concurrent_cold_start( + monkeypatch, reset_autorouter_presets_cache +): + import asyncio + + from litellm.proxy.public_endpoints.public_endpoints import ( + _AUTOROUTER_PRESETS_ADAPTER, + get_autorouter_presets, + ) + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + remote = _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "remote_only": { + "label": "Remote Only", + "description": "from the remote catalog", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + } + } + ) + calls = [] + + async def slow_fetch(url): + calls.append(url) + await asyncio.sleep(0.05) + return remote + + results = await asyncio.gather( + get_autorouter_presets(url="https://example.test/presets.json", fetch=slow_fetch), + get_autorouter_presets(url="https://example.test/presets.json", fetch=slow_fetch), + get_autorouter_presets(url="https://example.test/presets.json", fetch=slow_fetch), + ) + + assert all(result == remote for result in results) + assert len(calls) == 1 + + +@pytest.mark.asyncio +async def test_get_autorouter_presets_caches_bundled_fallback_on_remote_failure( + monkeypatch, reset_autorouter_presets_cache +): + from litellm.proxy.public_endpoints.public_endpoints import get_autorouter_presets + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + calls = [] + + async def broken_fetch(url): + calls.append(url) + raise ValueError("remote catalog unavailable") + + first = await get_autorouter_presets(url="https://example.test/presets.json", fetch=broken_fetch) + second = await get_autorouter_presets(url="https://example.test/presets.json", fetch=broken_fetch) + + assert "anthropic_family" in first + assert second == first + assert len(calls) == 1 + + +@pytest.mark.asyncio +async def test_autorouter_presets_adapter_rejects_wrong_shapes(): + from pydantic import ValidationError + + from litellm.proxy.public_endpoints.public_endpoints import _AUTOROUTER_PRESETS_ADAPTER + + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python({"bad": {"label": "no description or config"}}) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python(["not", "a", "mapping"]) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + {"no_tiers": {"label": "L", "description": "D", "complexity_router_config": {}}} + ) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "missing_builtin_tier": { + "label": "L", + "description": "D", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"]}}, + } + } + ) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "unknown_tier_name": { + "label": "L", + "description": "D", + "complexity_router_config": { + "tiers": { + "SIMPLE": ["m1"], + "MEDIUM": ["m2"], + "COMPLEX": ["m3"], + "REASONING": ["m4"], + "ULTRA": ["m5"], + } + }, + } + } + ) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "bad_tiers": { + "label": "L", + "description": "D", + "complexity_router_config": {"tiers": "not-a-mapping"}, + } + } + ) + + +def test_get_autorouter_presets_passes_unknown_catalog_fields_through( + monkeypatch, reset_autorouter_presets_cache +): + from litellm.proxy.public_endpoints.public_endpoints import ( + _AUTOROUTER_PRESETS_ADAPTER, + _AutoRouterPresetsCache, + ) + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + _AutoRouterPresetsCache.presets = _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "future_preset": { + "label": "Future", + "description": "carries fields this proxy version does not know", + "complexity_router_config": { + "tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}, + "future_config_knob": 3, + }, + "icon": "sparkles", + } + } + ) + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + response = client.get("/public/autorouter_presets") + + assert response.status_code == 200 + served = response.json()["future_preset"] + assert served["icon"] == "sparkles" + assert served["complexity_router_config"]["future_config_knob"] == 3 + assert served["complexity_router_config"]["tiers"]["SIMPLE"] == ["m1"] + + +@pytest.mark.asyncio +async def test_fetch_remote_autorouter_presets_parses_and_rejects_empty(monkeypatch): + import litellm.llms.custom_httpx.http_handler as http_handler_module + from litellm.proxy.public_endpoints.public_endpoints import _fetch_remote_autorouter_presets + + catalog = { + "remote_only": { + "label": "Remote Only", + "description": "from the remote catalog", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + } + } + response = MagicMock() + response.raise_for_status = MagicMock() + response.json = MagicMock(return_value=catalog) + client = MagicMock() + client.get = AsyncMock(return_value=response) + monkeypatch.setattr(http_handler_module, "get_async_httpx_client", lambda llm_provider: client) + + presets = await _fetch_remote_autorouter_presets("https://example.test/presets.json") + assert presets["remote_only"].label == "Remote Only" + response.raise_for_status.assert_called_once() + + response.json = MagicMock(return_value={}) + with pytest.raises(ValueError, match="empty"): + await _fetch_remote_autorouter_presets("https://example.test/presets.json") diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets.ts new file mode 100644 index 00000000000..82f9cc2da99 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets.ts @@ -0,0 +1,16 @@ +import { AutoRouterPreset, hydratePresets } from "@/lib/autorouter_presets"; +import { getAutoRouterPresets } from "@/components/networking"; +import { useQuery } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; + +const presetKeys = createQueryKeys("autoRouterPresets"); + +export const useAutoRouterPresets = () => { + const options = { + queryKey: presetKeys.list({}), + queryFn: async () => hydratePresets(await getAutoRouterPresets()), + staleTime: 24 * 60 * 60 * 1000, + gcTime: 24 * 60 * 60 * 1000, + }; + return useQuery(options); +}; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index ba380662403..7769042a0c7 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -9,11 +9,19 @@ import { getSubmitBlockedReason } from "./add_auto_router_tab"; import { buildModelAvailability } from "@/lib/autorouter_presets"; import { testAutoRouterRouting } from "../networking"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; -import { getAllPresets, getPresetByKey, getRequiredModelsInPreset } from "@/lib/autorouter_presets"; +import { AutoRouterPreset, getRequiredModelsInPreset } from "@/lib/autorouter_presets"; +import { BUNDLED_PRESETS, LOADED_PRESETS_QUERY, useAutoRouterPresets } from "../../../tests/mocks/autoRouterPresets"; vi.mock( "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults", async () => await import("../../../tests/mocks/complexityScorerDefaults"), ); +vi.mock( + "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets", + async () => await import("../../../tests/mocks/autoRouterPresets"), +); + +const getAllPresets = (): AutoRouterPreset[] => BUNDLED_PRESETS; +const getPresetByKey = (key: string): AutoRouterPreset | undefined => BUNDLED_PRESETS.find((p) => p.key === key); const ANTHROPIC_PRESET = getPresetByKey("anthropic_family")!; const ANTHROPIC_TIERS = ANTHROPIC_PRESET.complexity_router_config.tiers; @@ -1142,3 +1150,52 @@ describe("getSubmitBlockedReason", () => { ); }); }); + +describe("preset catalog fetch states", () => { + afterEach(() => vi.mocked(useAutoRouterPresets).mockReturnValue(LOADED_PRESETS_QUERY)); + + it("keeps showing cached presets without the error banner when only a refetch fails", () => { + vi.mocked(useAutoRouterPresets).mockReturnValue({ + ...LOADED_PRESETS_QUERY, + isError: true, + } as never); + renderWithProviders(); + + expect(screen.queryByText(/Could not load templates/)).not.toBeInTheDocument(); + + openTemplateDropdown(); + expect(screen.queryAllByRole("option").length).toBeGreaterThan(1); + }); + + it("shows a loading hint while the catalog fetch is pending", () => { + vi.mocked(useAutoRouterPresets).mockReturnValue({ + ...LOADED_PRESETS_QUERY, + data: undefined, + isPending: true, + } as never); + renderWithProviders(); + + expect(screen.getByText("Loading templates...")).toBeInTheDocument(); + }); + + it("degrades to Custom Configuration with a retry hint that refetches the catalog", async () => { + const refetch = vi.fn(); + vi.mocked(useAutoRouterPresets).mockReturnValue({ + ...LOADED_PRESETS_QUERY, + data: undefined, + isError: true, + refetch, + } as never); + renderWithProviders(); + + expect(await screen.findByText(/Could not load templates/)).toBeInTheDocument(); + + openTemplateDropdown(); + const options = screen.queryAllByRole("option"); + expect(options).toHaveLength(1); + expect(options[0]).toHaveTextContent("Custom Configuration"); + + fireEvent.click(screen.getByRole("button", { name: "Retry" })); + expect(refetch).toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 81acb7b7e50..ab5ea4cbfc9 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -50,8 +50,6 @@ import AutoRouterConnectionTest from "./auto_router_connection_test"; import AutoRouterRoutingTest from "./AutoRouterRoutingTest"; import { toast } from "@/lib/toast"; import { - getAllPresets, - getPresetByKey, getMissingModelsInPreset, getReferencedModelsError, buildEmptyPrefill, @@ -62,6 +60,7 @@ import { PresetPrefill, AutoRouterPreset, } from "@/lib/autorouter_presets"; +import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; interface AddAutoRouterTabProps { @@ -102,9 +101,7 @@ const presetDisabledHint = (availability: PresetAvailability): string | null => // caller-specific missing-model reason gets the alarming red treatment. const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models"; -// getAllPresets() already returns a stable, module-level array (see autorouter_presets.ts), so -// this is resolved once at import time rather than re-called from inside the component every render. -const presets = getAllPresets(); +const NO_PRESETS: AutoRouterPreset[] = []; // A one-line summary of what's configured, shown when the detailed section is collapsed so a // caller can see the shape of the config without opening it. @@ -229,6 +226,14 @@ const AddAutoRouterTab: React.FC = ({ }); const modelsLoading = groupsLoading || deploymentsLoading; const modelInfo = React.useMemo(() => data ?? [], [data]); + const { + data: presetsData, + isPending: presetsPending, + isError: presetsError, + refetch: refetchPresets, + } = useAutoRouterPresets(); + const presets = presetsData ?? NO_PRESETS; + const presetsUnavailable = presetsError && presetsData === undefined; // react-query keeps the last successful list around when a later refetch fails, so isError alone // can't tell "never loaded" apart from "loaded, then a background refetch errored" - only the // former leaves us with nothing trustworthy to verify a preset's models against. @@ -277,7 +282,7 @@ const AddAutoRouterTab: React.FC = ({ presets .map((preset) => ({ preset, availability: presetAvailability(preset) })) .sort((a, b) => Number(b.availability.kind === "available") - Number(a.availability.kind === "available")), - [presetAvailability], + [presets, presetAvailability], ); const templateItems = React.useMemo( @@ -307,7 +312,7 @@ const AddAutoRouterTab: React.FC = ({ return; } - const preset = getPresetByKey(presetKey); + const preset = presets.find((p) => p.key === presetKey); // Refuse to apply a preset whose models are not verified available. The dropdown disables // these options, so this is a guard against a stale click resolving after the list changed. if (!preset) return; @@ -538,6 +543,15 @@ const AddAutoRouterTab: React.FC = ({
)} + {presetsPending &&
Loading templates...
} + {presetsUnavailable && ( +
+ Could not load templates, so only Custom Configuration is shown.{" "} + +
+ )}
{requiresTeamScope && ( diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 63232afc46b..eaffa2b4802 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -90,6 +90,7 @@ import type { } from "@/app/(dashboard)/caching/_components/coordination_redis_settings/types"; import { MCP_TOOLS_PREVIEW_FORBIDDEN_MESSAGE } from "./mcp_tools/constants"; import type { ComplexityRouterConfigPayload } from "./add_model/build_complexity_router_config"; +import type { AutoRouterPresetsResponse } from "@/lib/autorouter_presets"; import type { VectorStoreIndex } from "@/app/(dashboard)/vector-stores/_components/IndexesTab"; import type { RoutingDecision } from "./view_logs/LogDetailsDrawer/RoutingDecisionCard"; import { @@ -410,6 +411,15 @@ export const getComplexityScorerDefaults = async (): Promise => { + /** + * Fetch the auto-router preset catalog from the proxy's public endpoint. The template picker + * renders from this rather than from a copy in the dashboard, so a catalog update propagates + * without a dashboard release. + */ + return await apiClient.get(`/public/autorouter_presets`); +}; + export const getAgentCreateMetadata = async (): Promise => { /** * Fetch agent type metadata from the proxy's public endpoint. diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index b5802ffa2dc..e3132067f4c 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -1,7 +1,9 @@ import { describe, it, expect } from "vitest"; +import bundledPresets from "../../../../litellm/proxy/public_endpoints/autorouter_presets.json"; import { - getAllPresets, - getPresetByKey, + hydratePresets, + AutoRouterPreset, + AutoRouterPresetsResponse, getRequiredModelsInPreset, getMissingModelsInPreset, getRequiredModels, @@ -18,8 +20,13 @@ import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKe const groupsOnly = (models: Iterable) => buildModelAvailability(models, []); +// Hydrated from the real bundled catalog so a catalog edit flows into these expectations. +const PRESETS = hydratePresets(bundledPresets as AutoRouterPresetsResponse); +const getAllPresets = (): AutoRouterPreset[] => PRESETS; +const getPresetByKey = (key: string): AutoRouterPreset | undefined => PRESETS.find((p) => p.key === key); + describe("autorouter_presets", () => { - it("loads exactly the bundled presets", () => { + it("hydrates exactly the bundled presets", () => { const presets = getAllPresets(); expect(presets.map((p) => p.label).sort()).toEqual(["Anthropic Family", "Gemini Family", "Lite", "OpenAI Family"]); // Every preset carries all four fields the UI relies on; a JSON typo dropping one fails here. diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index 721bd6f2b2a..30f8ed99e74 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -20,7 +20,6 @@ import { } from "@/components/add_model/complexity_router_tiers"; import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords"; import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching"; -import presetsRaw from "@/autorouter_presets.json"; // `key` is the stable JSON object key (e.g. "anthropic_family"); `label` is display text and // never an identity. @@ -31,16 +30,10 @@ export interface AutoRouterPreset { complexity_router_config: ComplexityRouterConfigPayload; } -// The bundled JSON is a developer-authored, build-time asset, so it is trusted at the import -// boundary rather than re-validated at runtime (resolveJsonModule widens its string literals, -// hence this one cast). autorouter_presets.test.ts pins the parsed shape, so a JSON typo fails CI. -const RAW = presetsRaw as Record>; +export type AutoRouterPresetsResponse = Record>; -const PRESETS: AutoRouterPreset[] = Object.entries(RAW).map(([key, preset]) => ({ key, ...preset })); - -export const getAllPresets = (): AutoRouterPreset[] => PRESETS; - -export const getPresetByKey = (key: string): AutoRouterPreset | undefined => PRESETS.find((p) => p.key === key); +export const hydratePresets = (raw: AutoRouterPresetsResponse): AutoRouterPreset[] => + Object.entries(raw).map(([key, preset]) => ({ key, ...preset })); // Generalized over ComplexityRouterConfigPayload so the same accessors check either a preset's own // bundled config or a caller's actually-built config - the two need to agree, since a preset only diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7583b6f6df1..90c4f03bf08 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -12129,6 +12129,31 @@ export interface paths { patch?: never; trace?: never; }; + "/public/autorouter_presets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Public Autorouter Presets + * @description Return the auto-router preset catalog the dashboard's template picker renders. + * + * Resolved once per process, like the model cost map: fetched from ``litellm.autorouter_presets_url`` + * (override with ``LITELLM_AUTOROUTER_PRESETS_URL``) on the first request, falling back to the + * catalog bundled with the package on any failure. Set ``LITELLM_LOCAL_AUTOROUTER_PRESETS=True`` + * to serve the bundled catalog only. A restart picks up a newly published catalog. + */ + get: operations["get_public_autorouter_presets_public_autorouter_presets_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/public/complexity_router/scorer_defaults": { parameters: { query?: never; @@ -23391,6 +23416,49 @@ export interface components { /** Tier Definitions */ tier_definitions: components["schemas"]["TierDefinition"][]; }; + /** + * AutoRouterPresetConfig + * @description The complexity_router_config a preset prefills. + * + * Only tiers is validated, because every dashboard consumer dereferences it; everything else + * passes through verbatim with unknown fields kept (extra="allow"), so a catalog published after + * this proxy shipped still serves its new fields intact. + */ + AutoRouterPresetConfig: { + tiers: components["schemas"]["AutoRouterPresetTiers"]; + } & { + [key: string]: unknown; + }; + /** + * AutoRouterPresetRecord + * @description One auto-router preset as served to the dashboard's template picker. + */ + AutoRouterPresetRecord: { + complexity_router_config: components["schemas"]["AutoRouterPresetConfig"]; + /** Description */ + description: string; + /** Label */ + label: string; + } & { + [key: string]: unknown; + }; + /** + * AutoRouterPresetTiers + * @description Exactly the four built-in tiers the dashboard's preset prefill can apply. + * + * extra="forbid" on purpose: a tier name this dashboard cannot apply would grey out or crash the + * picker, so such a catalog is rejected wholesale and the bundled one serves instead. + */ + AutoRouterPresetTiers: { + /** Complex */ + COMPLEX: string[]; + /** Medium */ + MEDIUM: string[]; + /** Reasoning */ + REASONING: string[]; + /** Simple */ + SIMPLE: string[]; + }; /** * AutoRouterRoutingTestRequest * @description A single request to classify against a complexity-router config that need not be saved yet. @@ -54679,6 +54747,28 @@ export interface operations { }; }; }; + get_public_autorouter_presets_public_autorouter_presets_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + [key: string]: components["schemas"]["AutoRouterPresetRecord"]; + }; + }; + }; + }; + }; get_complexity_scorer_defaults_public_complexity_router_scorer_defaults_get: { parameters: { query?: never; diff --git a/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts b/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts new file mode 100644 index 00000000000..f73faaa70fe --- /dev/null +++ b/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts @@ -0,0 +1,16 @@ +import { vi } from "vitest"; +import bundledPresets from "../../../../litellm/proxy/public_endpoints/autorouter_presets.json"; +import { hydratePresets, type AutoRouterPresetsResponse } from "@/lib/autorouter_presets"; + +// Derived from the real bundled catalog so a preset edit there flows into test expectations +// instead of redding on a stale copy. Exported as vi.fn so a test can override the query state. +export const BUNDLED_PRESETS = hydratePresets(bundledPresets as AutoRouterPresetsResponse); + +export const LOADED_PRESETS_QUERY = { + data: BUNDLED_PRESETS, + isPending: false, + isError: false, + refetch: vi.fn(), +}; + +export const useAutoRouterPresets = vi.fn(() => LOADED_PRESETS_QUERY); From 91061675ae48644459bb3448d145dcec322e6ef7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:04:52 -0700 Subject: [PATCH 166/480] test(rag): wrap the store-forwarding patches so every new line fits in 120 chars --- .../proxy/rag_endpoints/test_rag_endpoints.py | 15 ++++++++------- tests/test_litellm/rag/test_main.py | 5 +++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index e68a964e997..a176e91eaa4 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -460,13 +460,14 @@ def test_rag_query_forwards_managed_store_credentials_to_search(client_internal_ ] ) - with patch( # test-quality-ok: asearch is the boundary the store-credential forwarding under test targets; the real aquery pipeline runs in between - "litellm.vector_stores.asearch", new=fake_search - ), patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry and a mock-response router so real store resolution and the completion step run - "litellm.proxy.proxy_server.llm_router", router - ), patch( # test-quality-ok: grants store access, which is not under test, so the endpoint reaches the search boundary - "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", - new=AsyncMock(return_value=True), + with ( + patch("litellm.vector_stores.asearch", new=fake_search), # test-quality-ok: the search boundary under test + patch.object(litellm, "vector_store_registry", mock_registry), # test-quality-ok: seeds the store under test + patch("litellm.proxy.proxy_server.llm_router", router), # test-quality-ok: mock-response router for completion + patch( # test-quality-ok: store access is not under test, so the request reaches the search boundary + "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", + new=AsyncMock(return_value=True), + ), ): response = client_internal_user.post( "/v1/rag/query", diff --git a/tests/test_litellm/rag/test_main.py b/tests/test_litellm/rag/test_main.py index 420b72b6a3d..f119088b0e9 100644 --- a/tests/test_litellm/rag/test_main.py +++ b/tests/test_litellm/rag/test_main.py @@ -413,8 +413,9 @@ async def test_aquery_forwards_vector_store_params_to_search_but_not_completion( model="gpt-4o-mini", ) ) - with patch("litellm.vector_stores.asearch", new=fake_search), patch( # test-quality-ok: asearch and acompletion are the two boundaries the forwarding contract under test targets - "litellm.acompletion", new=fake_completion + with ( + patch("litellm.vector_stores.asearch", new=fake_search), # test-quality-ok: the search boundary under test + patch("litellm.acompletion", new=fake_completion), # test-quality-ok: the completion boundary under test ): await litellm.aquery( model="gpt-4o-mini", From d33fe95d192a819eef63848468c3630903a135a2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:08:45 -0700 Subject: [PATCH 167/480] test(responses): expect the 404 OpenAI now returns for an unknown model --- .../test_e2e_openai_responses_api.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py index 1a7fb1f3e41..abae26e02cd 100644 --- a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py +++ b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py @@ -1,5 +1,5 @@ import httpx -from openai import OpenAI, BadRequestError, APIStatusError +from openai import OpenAI, BadRequestError, NotFoundError, APIStatusError import pytest @@ -105,10 +105,9 @@ def test_streaming_response(): assert len(collected_chunks) > 0 -def test_bad_request_error(): +def test_model_not_found_error(): client = get_test_client() - with pytest.raises(BadRequestError): - # Trigger error with invalid model name + with pytest.raises(NotFoundError): client.responses.create(model="non-existent-model", input="This should fail") From 87618b7da8e755cb007b3bf2470dd05b93154f6d Mon Sep 17 00:00:00 2001 From: yujonglee Date: Wed, 2 Sep 2026 18:12:03 -0700 Subject: [PATCH 168/480] doc (#39456) --- tests/rust-python-harness/AGENTS.md | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/rust-python-harness/AGENTS.md diff --git a/tests/rust-python-harness/AGENTS.md b/tests/rust-python-harness/AGENTS.md new file mode 100644 index 00000000000..e9b17027ddc --- /dev/null +++ b/tests/rust-python-harness/AGENTS.md @@ -0,0 +1,44 @@ +# Expected Structure + +```text +tests/rust-python-harness/ +├── __main__.py +│ +├── strategies/ +│ ├── e2e_parity/ +│ │ ├── runner.py +│ │ ├── sdk/ +│ │ │ ├── ocr/ +│ │ │ ├── messages/ +│ │ │ ├── chat_completions/ +│ │ │ └── responses/ +│ │ └── gateway/ +│ │ +│ ├── trace_parity/ +│ │ ├── runner.py +│ │ ├── sdk/ +│ │ └── gateway/ +│ │ +│ └── unit_tests/ +│ ├── runner.py +│ ├── mapping_validator.py +│ ├── python_runner.py +│ └── rust_runner.py +│ +└── shared/ + ├── parity/ + ├── tracing/ + └── reporting/ +``` + +- Run locally only; no CI integration +- `__main__.py` selects strategies and combines their reports; each strategy also runs independently +- `e2e_parity/` compares SDK objects, exceptions, callbacks, and streams, or gateway HTTP responses +- `trace_parity/` compares mapped operations, call counts, and required execution ordering +- E2E and trace runners share orchestration across `sdk/` and `gateway/`; surface-specific execution lives in those folders +- `unit_tests/runner.py` combines mapping validation, Python test runs, and native Rust test runs +- `mapping_validator.py` matches Python/Rust tests by agreed names or annotations and reports missing or ambiguous counterparts +- `python_runner.py` runs existing Python tests with Rust disabled and enabled in separate processes, verifies backend selection, and compares results +- `rust_runner.py` runs Cargo tests; native Rust unit tests stay beside their implementation +- `shared/` contains reusable parity, tracing, and reporting machinery +- Keep fixtures with their owning API and existing Python tests in their current locations From 4f7b20ec102bc4f01f32156bc9d0d8325af3a081 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:25:28 -0700 Subject: [PATCH 169/480] fix(guardrails): skip streaming guardrail rounds that re-scan cleared output (#39386) * fix(guardrails): skip streaming guardrail rounds that re-scan cleared output Streaming guardrails scanned the finished answer twice at end of stream whenever the chunk count landed on a multiple of the sampling rate, ran sampled rounds whose payload was identical to the previous one, and on /v1/messages could scan an empty text before the first content chunk. Every redundant round is a paid guardrail provider call. Each endpoint handler now exposes a scan key describing what a round would hand to apply_guardrail (the text so far, plus tool calls once the stream has ended), and the unified streaming hook skips a sampled or end-of-stream round whose key equals the last scanned one or carries nothing to scan yet. Rounds that carry tool calls are never skipped. * test(guardrails): expect one end-of-stream scan when the terminal chunk is sampled Update sampled cadence expectations and use tuple-backed scan state Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../a2a/chat/guardrail_translation/handler.py | 12 +- .../chat/guardrail_translation/handler.py | 25 +- .../guardrail_translation/base_translation.py | 19 ++ .../base_llm/guardrail_translation/utils.py | 12 + .../chat/guardrail_translation/handler.py | 37 ++- .../guardrail_translation/handler.py | 72 ++++- .../unified_guardrail/unified_guardrail.py | 40 ++- .../test_a2a_guardrail_handler.py | 35 +++ .../test_anthropic_guardrail_handler.py | 54 ++++ .../test_openai_guardrail_handler.py | 72 +++++ ...test_openai_responses_guardrail_handler.py | 90 ++++++ .../test_openai_moderation_streaming.py | 19 +- .../test_generic_guardrail_api.py | 18 +- .../test_unified_guardrail.py | 268 ++++++++++++++++++ 14 files changed, 733 insertions(+), 40 deletions(-) create mode 100644 tests/test_litellm/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py diff --git a/litellm/llms/a2a/chat/guardrail_translation/handler.py b/litellm/llms/a2a/chat/guardrail_translation/handler.py index f1c7451796d..5c30ff4747a 100644 --- a/litellm/llms/a2a/chat/guardrail_translation/handler.py +++ b/litellm/llms/a2a/chat/guardrail_translation/handler.py @@ -17,7 +17,10 @@ from typing import TYPE_CHECKING, Any, Final, Optional from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger -from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.llms.base_llm.guardrail_translation.base_translation import ( + BaseTranslation, + StreamingScanKey, +) from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: @@ -313,9 +316,14 @@ class A2AGuardrailHandler(BaseTranslation): return responses_so_far + def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: + _, valid_parsed = self._parse_streaming_responses(responses_so_far) + combined_text, _ = self._collect_text_from_parsed_chunks(valid_parsed) + return StreamingScanKey(texts=(combined_text,)) + def _parse_streaming_responses( self, - responses_so_far: list[object], + responses_so_far: Sequence[object], ) -> tuple[list[dict[str, object] | None], list[tuple[int, dict[str, object]]]]: """Parse JSON-RPC items, returning aligned parsed list and valid entries.""" parsed: Final[list[dict[str, object] | None]] = [None] * len(responses_so_far) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index c7d12e5cf3a..c23797f72af 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -26,7 +26,10 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im LiteLLMAnthropicMessagesAdapter, is_provider_native_tool_dict, ) -from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.llms.base_llm.guardrail_translation.base_translation import ( + BaseTranslation, + StreamingScanKey, +) from litellm.llms.base_llm.guardrail_translation.utils import ( anthropic_tool_name, anthropic_tool_names, @@ -36,6 +39,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( merge_guardrailed_scoped_messages, merge_returned_tools_into_request_tools, scoped_structured_message_indices, + stream_item_fingerprint, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, @@ -1176,6 +1180,25 @@ class AnthropicMessagesHandler(BaseTranslation): inputs["model"] = response_model return inputs + def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: + stream_ended: Final = self._check_streaming_has_ended(responses_so_far) + return StreamingScanKey( + texts=(self.get_streaming_string_so_far(responses_so_far),), + tool_calls=self._streamed_tool_use_fingerprints(responses_so_far) if stream_ended else (), + stream_ended=stream_ended, + ) + + @classmethod + def _streamed_tool_use_fingerprints(cls, responses_so_far: Sequence[object]) -> tuple[str, ...]: + return tuple( + stream_item_fingerprint(block) + for item in responses_so_far + for event in cls._iter_sse_events(item) + if event.get("type") == "content_block_start" + and isinstance(block := event.get("content_block"), Mapping) + and block.get("type") == "tool_use" + ) + def get_streaming_string_so_far(self, responses_so_far: Sequence[object]) -> str: """ Parse streaming responses and extract accumulated text content. diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 220fcedb0f8..b28daf73bc4 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -35,6 +35,22 @@ class StreamTransformSink: holdback_per_choice: dict[int, int] = field(default_factory=dict) +@dataclass(frozen=True, slots=True) +class StreamingScanKey: + """What a streaming guardrail round would hand to ``apply_guardrail``. Two keys + compare equal when the round would scan the same content again; ``stream_ended`` + stays out of the comparison and only says whether the handler is on its + end-of-stream path, where an empty payload is still scanned today.""" + + texts: tuple[str, ...] + tool_calls: tuple[str, ...] = () + stream_ended: bool = field(default=False, compare=False) + + @property + def has_nothing_to_scan(self) -> bool: + return not self.stream_ended and not any(self.texts) and not self.tool_calls + + class BaseTranslation(ABC): @staticmethod def transform_user_api_key_dict_to_metadata( @@ -151,6 +167,9 @@ class BaseTranslation(ABC): """ return responses_so_far + def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: + return None + def build_block_sse_chunks( self, exc: "ModifyResponseException", diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index 9b6f9c47105..8dee262001d 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -4,6 +4,8 @@ import json from collections.abc import Callable, Iterator, Sequence from typing import Any, Final, TypeVar +from pydantic import BaseModel + from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage @@ -130,6 +132,16 @@ def stream_item_field(item: object, field: str) -> object | None: return getattr(item, field, None) +def stream_item_fingerprint(item: object) -> str: + plain: Final = item.model_dump() if isinstance(item, BaseModel) else item + return json.dumps(plain, sort_keys=True, default=str) + + +def stream_item_items(item: object, field: str) -> tuple[object, ...]: + value: Final = stream_item_field(item, field) + return tuple(value) if isinstance(value, (list, tuple)) else () + + def blocked_chat_stream_usage(original_response: object) -> tuple[int, int]: """ ``(prompt_tokens, completion_tokens)`` for a synthetic guardrail-blocked diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 96a5ed663fc..d41c8557d72 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -26,6 +26,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, + StreamingScanKey, StreamTransformSink, ) from litellm.llms.base_llm.guardrail_translation.utils import ( @@ -39,6 +40,8 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( role_out_of_guardrail_scope, scoped_structured_message_indices, stream_item_field, + stream_item_fingerprint, + stream_item_items, ) from litellm.main import stream_chunk_builder from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -503,12 +506,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): """Block-only streaming path: run the guardrail so an in-flight BLOCK can terminate the stream. Text rewrites are not propagated to the client here (see ``_process_streaming_transform`` for the incremental_diff path).""" - # check if the stream has ended - has_stream_ended = False - for chunk in responses_so_far: - if chunk.choices and chunk.choices[0].finish_reason is not None: - has_stream_ended = True - break + has_stream_ended: Final = self._first_choice_has_finished(responses_so_far) if has_stream_ended: # convert to model response @@ -706,8 +704,33 @@ class OpenAIChatCompletionsHandler(BaseTranslation): indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback) } + def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: + chunks: Final = tuple(chunk for chunk in responses_so_far if isinstance(chunk, ModelResponseStream)) + stream_ended: Final = self._first_choice_has_finished(responses_so_far) + return StreamingScanKey( + texts=tuple(self._combine_streaming_texts(chunks).values()), + tool_calls=self._streamed_tool_call_fingerprints(responses_so_far) if stream_ended else (), + stream_ended=stream_ended, + ) + + @staticmethod + def _streamed_tool_call_fingerprints(responses_so_far: Sequence[object]) -> tuple[str, ...]: + return tuple( + stream_item_fingerprint(tool_call) + for chunk in responses_so_far + for choice in _stream_chunk_choices(chunk) + for tool_call in stream_item_items(stream_item_field(choice, "delta"), "tool_calls") + ) + + @staticmethod + def _first_choice_has_finished(responses_so_far: Sequence[object]) -> bool: + first_choices: Final = tuple( + choices[0] for choices in (_stream_chunk_choices(chunk) for chunk in responses_so_far) if choices + ) + return any(stream_item_field(choice, "finish_reason") is not None for choice in first_choices) + def _combine_streaming_texts( - self, responses_so_far: list["ModelResponseStream"] + self, responses_so_far: Sequence["ModelResponseStream"] ) -> dict[tuple[int, int | None], str]: """ Combine all streaming chunks into complete text per choice. diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 5a5970fb867..a0db7aadb9e 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -44,10 +44,15 @@ from litellm._logging import verbose_proxy_logger from litellm.completion_extras.litellm_responses_transformation.transformation import ( OpenAiResponsesToChatCompletionStreamIterator, ) -from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.llms.base_llm.guardrail_translation.base_translation import ( + BaseTranslation, + StreamingScanKey, +) from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_responses_stream_usage, stream_item_field, + stream_item_fingerprint, + stream_item_items, ) from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools from litellm.responses.litellm_completion_transformation.transformation import ( @@ -593,18 +598,55 @@ class OpenAIResponsesHandler(BaseTranslation): ) return responses_so_far - def _check_streaming_has_ended(self, responses_so_far: Sequence[ResponsesStreamChunk]) -> bool: + def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool: """ Check if the streaming has ended. """ if not responses_so_far: return False - terminal_types: Final = { - ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value, - ResponsesAPIStreamEvents.RESPONSE_FAILED.value, - ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value, - } - return responses_so_far[-1].get("type") in terminal_types + terminal_types: Final = frozenset( + ( + ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value, + ResponsesAPIStreamEvents.RESPONSE_FAILED.value, + ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value, + ) + ) + return stream_item_field(responses_so_far[-1], "type") in terminal_types + + def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: + if not responses_so_far or not hasattr(responses_so_far[-1], "get"): + return None + last_event: Final = responses_so_far[-1] + last_event_type: Final = stream_item_field(last_event, "type") + if last_event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE.value: + return None + if last_event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value: + return self._completed_response_scan_key(stream_item_field(last_event, "response")) + return StreamingScanKey( + texts=(self.get_streaming_string_so_far(responses_so_far),), + stream_ended=self._check_streaming_has_ended(responses_so_far), + ) + + @staticmethod + def _completed_response_scan_key(response: object) -> StreamingScanKey: + output_items: Final = stream_item_items(response, "output") + message_items: Final = tuple( + item for item in output_items if stream_item_field(item, "type") != "function_call" + ) + return StreamingScanKey( + texts=tuple( + text + for item in message_items + for part in stream_item_items(item, "content") + if isinstance(text := stream_item_field(part, "text"), str) and text + ), + tool_calls=tuple( + stream_item_fingerprint(item) + for item in output_items + if stream_item_field(item, "type") == "function_call" + ), + stream_ended=True, + ) def build_stream_error_items( self, @@ -629,7 +671,7 @@ class OpenAIResponsesHandler(BaseTranslation): ), ) - def get_streaming_string_so_far(self, responses_so_far: Sequence[ResponsesStreamChunk]) -> str: + def get_streaming_string_so_far(self, responses_so_far: Sequence[object]) -> str: """ Get the string so far from the responses so far. @@ -641,12 +683,16 @@ class OpenAIResponsesHandler(BaseTranslation): """ keyed_events: Final = tuple( ( - (event.get("item_id"), event.get("output_index"), event.get("content_index")), - event.get("text"), - event.get("delta"), + ( + stream_item_field(event, "item_id"), + stream_item_field(event, "output_index"), + stream_item_field(event, "content_index"), + ), + stream_item_field(event, "text"), + stream_item_field(event, "delta"), ) for event in responses_so_far - if isinstance(event.get("text"), str) or isinstance(event.get("delta"), str) + if isinstance(stream_item_field(event, "text"), str) or isinstance(stream_item_field(event, "delta"), str) ) def part_text(part_key: tuple[object, object, object]) -> str: diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 46b00829b74..c6b8df1b493 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -36,6 +36,7 @@ if TYPE_CHECKING: from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, + StreamingScanKey, ) # Call types that stream JSON-RPC events (A2A); guardrail HTTPException is emitted as in-stream error @@ -54,6 +55,9 @@ class _EndpointTranslation(Protocol): @property def process_output_streaming_response(self) -> "Callable[..., Awaitable[object]]": ... + @property + def get_streaming_scan_key(self) -> "Callable[[Sequence[object]], StreamingScanKey | None]": ... + @property def build_block_sse_chunks(self) -> "Callable[..., Sequence[bytes] | None]": ... @@ -70,6 +74,12 @@ def _chunk_choices(item: object) -> Sequence[object]: return choices +def _is_redundant_scan(scan_key: "StreamingScanKey | None", last_scan_key: "StreamingScanKey | None") -> bool: + if scan_key is None: + return False + return scan_key == last_scan_key or scan_key.has_nothing_to_scan + + class _StreamTerminated(Exception): """Internal signal that the incremental transform stream has already emitted its terminal chunks (block message or in-stream error) and must stop.""" @@ -1011,6 +1021,7 @@ class UnifiedLLMGuardrails(CustomLogger): # Drives how a block terminates the stream: continue the in-progress # message (True) vs emit a standalone block message (False, buffered). chunks_yielded = False + last_scan_key: StreamingScanKey | None = None # rebind-ok: replaced after every scan round async for item in response: chunk_counter += 1 @@ -1052,6 +1063,19 @@ class UnifiedLLMGuardrails(CustomLogger): # Process chunk based on sampling rate if chunk_counter % sampling_rate == 0: + endpoint_translation = endpoint_guardrail_translation_mappings[CallTypes(call_type)]() + scan_key = endpoint_translation.get_streaming_scan_key(responses_so_far) + if _is_redundant_scan(scan_key, last_scan_key): + verbose_proxy_logger.debug( + "Skipping streaming chunk %s for guardrail %s: nothing new to scan since the last round", + chunk_counter, + guardrail_to_apply.guardrail_name, + ) + chunks_yielded = True + responses_yielded.append(item) + yield item + continue + verbose_proxy_logger.debug( "Processing streaming chunk %s (sampling_rate=%s) with guardrail %s", chunk_counter, @@ -1067,8 +1091,6 @@ class UnifiedLLMGuardrails(CustomLogger): # string, permanently losing this chunk's content. original_item = copy.deepcopy(item) - endpoint_translation = endpoint_guardrail_translation_mappings[CallTypes(call_type)]() - try: await endpoint_translation.process_output_streaming_response( responses_so_far=responses_so_far, @@ -1110,6 +1132,8 @@ class UnifiedLLMGuardrails(CustomLogger): ): yield error_item return + if scan_key is not None: + last_scan_key = scan_key chunks_yielded = True responses_yielded.append(original_item) yield original_item @@ -1136,6 +1160,18 @@ class UnifiedLLMGuardrails(CustomLogger): # preserve the list, not clone every chunk (deepcopy would double # peak memory for large responses). buffered_items: Final = list(responses_so_far) if buffer_until_moderated else None + end_scan_key: Final = endpoint_translation.get_streaming_scan_key(responses_so_far) + if _is_redundant_scan(end_scan_key, last_scan_key): + verbose_proxy_logger.debug( + "Skipping end-of-stream scan for guardrail %s: the last sampled round already scanned it all", + guardrail_to_apply.guardrail_name, + ) + for buffered_item in buffered_items or (): + yield buffered_item + for pending_item in pending_end_of_stream_items: + responses_yielded.append(pending_item) + yield pending_item + return try: await endpoint_translation.process_output_streaming_response( diff --git a/tests/test_litellm/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py b/tests/test_litellm/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py new file mode 100644 index 00000000000..dd7e8fadcd8 --- /dev/null +++ b/tests/test_litellm/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py @@ -0,0 +1,35 @@ +"""Tests for litellm/llms/a2a/chat/guardrail_translation/handler.py.""" + +import json + +from litellm.llms.a2a.chat.guardrail_translation.handler import A2AGuardrailHandler +from litellm.llms.base_llm.guardrail_translation.base_translation import StreamingScanKey + + +def _text_event(text: str) -> str: + return json.dumps( + { + "jsonrpc": "2.0", + "id": "req-1", + "result": {"kind": "message", "role": "agent", "parts": [{"kind": "text", "text": text}]}, + } + ) + + +def _status_event() -> str: + return json.dumps({"jsonrpc": "2.0", "id": "req-1", "result": {"kind": "status-update", "status": {}}}) + + +class TestA2AGuardrailHandlerStreamingScanKey: + def test_key_joins_the_text_of_every_message_event(self): + key = A2AGuardrailHandler().get_streaming_scan_key([_text_event("hello "), _text_event("world")]) + assert key == StreamingScanKey(texts=("hello world",)) + + def test_events_without_text_leave_the_key_unchanged(self): + handler = A2AGuardrailHandler() + events = [_text_event("hello")] + assert handler.get_streaming_scan_key(events + [_status_event()]) == handler.get_streaming_scan_key(events) + + def test_unparseable_items_are_ignored(self): + key = A2AGuardrailHandler().get_streaming_scan_key([_text_event("hi"), "not json", b"bytes"]) + assert key.texts == ("hi",) diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 0fe7730e91e..3044a321aa6 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -13,6 +13,7 @@ import pytest from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.base_llm.guardrail_translation.base_translation import StreamingScanKey from litellm.llms.anthropic.chat.guardrail_translation.handler import ( AnthropicMessagesHandler, ) @@ -1991,3 +1992,56 @@ class TestStructuredWriteBackKeepsToolResults: } later_blocks = [b for m in messages[tool_use_index + 1 :] for b in self._blocks(m)] assert {"type": "text", "text": "Now fetch the page."} in later_blocks + + +class TestAnthropicMessagesHandlerStreamingScanKey: + """get_streaming_scan_key mirrors what process_output_streaming_response would scan""" + + @staticmethod + def _sse(event_type, data): + return f"event: {event_type}\ndata: {json.dumps(data)}\n\n".encode() + + def _text_delta(self, text): + return self._sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}, + ) + + def test_key_is_empty_before_any_text_arrives(self): + head = self._sse("message_start", {"type": "message_start", "message": {"stop_reason": None}}) + key = AnthropicMessagesHandler().get_streaming_scan_key([head]) + assert key == StreamingScanKey(texts=("",)) + + def test_key_accumulates_text_deltas(self): + key = AnthropicMessagesHandler().get_streaming_scan_key([self._text_delta("hello "), self._text_delta("world")]) + assert key.texts == ("hello world",) + assert key.stream_ended is False + + def _stop(self, stop_reason): + return self._sse( + "message_delta", + {"type": "message_delta", "delta": {"stop_reason": stop_reason, "stop_sequence": None}, "usage": {}}, + ) + + def test_stop_without_tool_use_scans_the_same_payload(self): + handler = AnthropicMessagesHandler() + open_key = handler.get_streaming_scan_key([self._text_delta("hi")]) + ended_key = handler.get_streaming_scan_key([self._text_delta("hi"), self._stop("end_turn")]) + assert ended_key.stream_ended is True + assert ended_key == open_key + + def test_tool_use_blocks_enter_the_key_once_the_stream_has_ended(self): + handler = AnthropicMessagesHandler() + tool_use = self._sse( + "content_block_start", + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {}}, + }, + ) + open_key = handler.get_streaming_scan_key([self._text_delta("hi"), tool_use]) + ended_key = handler.get_streaming_scan_key([self._text_delta("hi"), tool_use, self._stop("tool_use")]) + assert open_key == StreamingScanKey(texts=("hi",)) + assert len(ended_key.tool_calls) == 1 and "get_weather" in ended_key.tool_calls[0] + assert ended_key != open_key diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 7dd6065063a..cebab2512d0 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -12,6 +12,7 @@ import pytest from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.base_llm.guardrail_translation.base_translation import StreamingScanKey from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, ) @@ -1643,3 +1644,74 @@ class TestCheckStreamingHasEnded: ) ] assert handler._check_streaming_has_ended(chunks) is True + + +class TestStreamingScanKey: + """get_streaming_scan_key identifies what a sampled round would scan so the + unified hook can skip rounds that would re-scan already-cleared text""" + + @staticmethod + def _chunk(content, finish_reason=None, index=0): + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + return ModelResponseStream( + choices=[StreamingChoices(index=index, delta=Delta(content=content), finish_reason=finish_reason)] + ) + + def test_key_carries_accumulated_text_and_open_stream(self): + handler = OpenAIChatCompletionsHandler() + key = handler.get_streaming_scan_key([self._chunk("hel"), self._chunk("lo")]) + assert key == StreamingScanKey(texts=("hello",)) + + def test_chunks_without_text_leave_the_key_unchanged(self): + handler = OpenAIChatCompletionsHandler() + before = handler.get_streaming_scan_key([self._chunk("hel"), self._chunk("lo")]) + after = handler.get_streaming_scan_key([self._chunk("hel"), self._chunk("lo"), self._chunk(None)]) + assert after == before + + def test_finish_chunk_without_tool_calls_scans_the_same_payload(self): + handler = OpenAIChatCompletionsHandler() + open_key = handler.get_streaming_scan_key([self._chunk("hi")]) + ended_key = handler.get_streaming_scan_key([self._chunk("hi"), self._chunk(None, finish_reason="stop")]) + assert open_key.stream_ended is False + assert ended_key.stream_ended is True + assert ended_key == open_key + + def test_tool_calls_only_enter_the_key_once_the_stream_has_ended(self): + from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponseStream, + StreamingChoices, + ) + + handler = OpenAIChatCompletionsHandler() + tool_call = ChatCompletionDeltaToolCall( + id="call_1", index=0, type="function", function=Function(name="get_weather", arguments='{"city": "Paris"}') + ) + tool_chunk = ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content=None, tool_calls=[tool_call]), finish_reason=None)] + ) + open_key = handler.get_streaming_scan_key([self._chunk("hi"), tool_chunk]) + ended_key = handler.get_streaming_scan_key( + [self._chunk("hi"), tool_chunk, self._chunk(None, finish_reason="stop")] + ) + assert open_key == StreamingScanKey(texts=("hi",)) + assert ended_key.texts == ("hi",) + assert len(ended_key.tool_calls) == 1 and "get_weather" in ended_key.tool_calls[0] + assert ended_key != open_key + + def test_text_after_the_first_choice_finishes_still_changes_the_key(self): + handler = OpenAIChatCompletionsHandler() + first_done = [self._chunk("a", index=0), self._chunk("b", finish_reason="stop", index=0)] + key_at_first_finish = handler.get_streaming_scan_key(first_done) + key_after_more_text = handler.get_streaming_scan_key(first_done + [self._chunk("y", index=1)]) + assert key_at_first_finish.stream_ended is True + assert key_after_more_text.stream_ended is True + assert key_after_more_text != key_at_first_finish + + def test_non_stream_items_are_ignored(self): + handler = OpenAIChatCompletionsHandler() + key = handler.get_streaming_scan_key([self._chunk("hi"), b"data: [DONE]"]) + assert key.texts == ("hi",) diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index d071ef78c2d..295121167d6 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -1731,3 +1731,93 @@ class TestBuildBlockSseChunks: dones = [payload for payload in payloads if payload["type"] == "response.output_item.done"] assert len(dones) == 1 assert dones[0]["item"]["content"][0]["text"] == "Blocked by policy." + + +class TestOpenAIResponsesHandlerStreamingScanKey: + """get_streaming_scan_key mirrors what process_output_streaming_response would scan""" + + @staticmethod + def _delta(sequence_number, text): + return { + "type": "response.output_text.delta", + "sequence_number": sequence_number, + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": text, + } + + def test_no_events_yields_no_key(self): + assert OpenAIResponsesHandler().get_streaming_scan_key([]) is None + + def test_key_accumulates_deltas_while_the_stream_is_open(self): + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamingScanKey + + key = OpenAIResponsesHandler().get_streaming_scan_key([self._delta(0, "hel"), self._delta(1, "lo")]) + assert key == StreamingScanKey(texts=("hello",)) + + def test_typed_delta_events_accumulate_like_dicts(self): + from litellm.types.llms.openai import OutputTextDeltaEvent + + events = [ + OutputTextDeltaEvent( + type="response.output_text.delta", + item_id="msg_1", + output_index=0, + content_index=0, + delta=text, + sequence_number=i, + ) + for i, text in enumerate(("hel", "lo")) + ] + key = OpenAIResponsesHandler().get_streaming_scan_key(events) + assert key.texts == ("hello",) + assert key.stream_ended is False + + def test_events_without_text_leave_the_key_unchanged(self): + handler = OpenAIResponsesHandler() + events = [self._delta(0, "hi")] + quiet = events + [{"type": "response.in_progress", "sequence_number": 1}] + assert handler.get_streaming_scan_key(quiet) == handler.get_streaming_scan_key(events) + + @staticmethod + def _completed(sequence_number, output): + return {"type": "response.completed", "sequence_number": sequence_number, "response": {"output": output}} + + def test_completed_event_keys_on_the_final_output_text(self): + handler = OpenAIResponsesHandler() + message = {"type": "message", "content": [{"type": "output_text", "text": "hi"}]} + open_key = handler.get_streaming_scan_key([self._delta(0, "hi")]) + ended_key = handler.get_streaming_scan_key([self._delta(0, "hi"), self._completed(1, [message])]) + assert ended_key.stream_ended is True + assert ended_key == open_key + + def test_completed_event_with_a_function_call_changes_the_key(self): + handler = OpenAIResponsesHandler() + message = {"type": "message", "content": [{"type": "output_text", "text": "hi"}]} + function_call = {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": "{}"} + open_key = handler.get_streaming_scan_key([self._delta(0, "hi")]) + ended_key = handler.get_streaming_scan_key([self._delta(0, "hi"), self._completed(1, [message, function_call])]) + assert ended_key.texts == ("hi",) + assert len(ended_key.tool_calls) == 1 and "get_weather" in ended_key.tool_calls[0] + assert ended_key != open_key + + def test_completed_event_reads_every_output_text_part(self): + from litellm.types.responses.main import GenericResponseOutputItem, OutputText + + item = GenericResponseOutputItem( + type="message", + id="msg_1", + status="completed", + role="assistant", + content=[ + OutputText(type="output_text", text="one", annotations=[]), + OutputText(type="output_text", text="two", annotations=[]), + ], + ) + key = OpenAIResponsesHandler().get_streaming_scan_key([self._completed(0, [item])]) + assert key.texts == ("one", "two") + + def test_output_item_done_round_is_never_deduped(self): + done = {"type": "response.output_item.done", "sequence_number": 1, "item": {"type": "function_call"}} + assert OpenAIResponsesHandler().get_streaming_scan_key([self._delta(0, "hi"), done]) is None diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py index 476d443d8d8..cb6772977ec 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py @@ -310,8 +310,8 @@ def _make_stream_chunk(content: str, finish_reason=None): @pytest.mark.asyncio async def test_openai_moderation_streaming_default_uses_sampled_cadence(): """Default config samples every 5th streamed chunk and runs a final aggregate - pass after the stream ends. 10 chunks → sampled at chunks 5 and 10 → 2 in-stream - calls, plus 1 final = 3 total. + pass after the stream ends. 10 chunks are sampled at 5 and 10; the end-of-stream + round is skipped because chunk 10 already scanned the full text, for 2 total calls """ import litellm @@ -370,8 +370,9 @@ async def test_openai_moderation_streaming_default_uses_sampled_cadence(): ): pass - assert patched_make_request.await_count == 3, ( - f"Expected 3 moderation calls (2 sampled at chunks 5 / 10 + 1 final), " + assert patched_make_request.await_count == 2, ( + f"Expected 2 moderation calls (2 sampled at chunks 5 / 10; " + f"the end-of-stream round is skipped because chunk 10 already scanned the full text), " f"got {patched_make_request.await_count}" ) @@ -448,7 +449,8 @@ async def test_openai_moderation_streaming_end_of_stream_only_opt_in_calls_moder @pytest.mark.asyncio async def test_openai_moderation_streaming_sampled_when_end_of_stream_only_disabled(): """With streaming_end_of_stream_only=False and streaming_sampling_rate=2, - moderation runs every 2nd chunk during the stream, plus once more at end. + moderation runs every 2nd chunk during the stream. The terminal chunk scan covers + the final aggregate, for 3 total calls """ import litellm @@ -509,9 +511,8 @@ async def test_openai_moderation_streaming_sampled_when_end_of_stream_only_disab ): pass - # 6 chunks, sampling_rate=2 → in-stream calls at chunks 2, 4, 6 (3 calls), - # plus the final aggregate pass after the stream ends (1 call) = 4 total. - assert patched_make_request.await_count == 4, ( - f"Expected 4 moderation calls (3 sampled + 1 final aggregate), " + assert patched_make_request.await_count == 3, ( + f"Expected 3 moderation calls (3 sampled; the end-of-stream round is skipped " + f"because chunk 6 already scanned the full text), " f"got {patched_make_request.await_count}" ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index 523ec1a37b4..83cc9ae8bb9 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -1517,7 +1517,9 @@ class TestGenericGuardrailAPIStreamingViaUnified: @pytest.mark.asyncio async def test_streaming_default_uses_sampled_cadence(self): - """Default samples every 5th chunk + final pass: 10 chunks → calls at 5, 10, and final = 3.""" + """Default samples every 5th chunk. For 10 chunks, sampled scans at 5 and 10 + cover the full text, so the end-of-stream round is skipped and there are 2 calls + """ from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( UnifiedLLMGuardrails, ) @@ -1566,8 +1568,9 @@ class TestGenericGuardrailAPIStreamingViaUnified: ): pass - assert mock_post.await_count == 3, ( - f"Expected 3 guardrail calls (2 sampled at chunks 5 / 10 + 1 final), " + assert mock_post.await_count == 2, ( + f"Expected 2 guardrail calls (2 sampled at chunks 5 / 10; " + f"the end-of-stream round is skipped because chunk 10 already scanned the full text), " f"got {mock_post.await_count}" ) for call in mock_post.await_args_list: @@ -1631,7 +1634,9 @@ class TestGenericGuardrailAPIStreamingViaUnified: @pytest.mark.asyncio async def test_streaming_sampling_rate_override(self): - """sampling_rate=2 on 6 chunks → in-stream at 2,4,6 plus final = 4 calls.""" + """sampling_rate=2 on 6 chunks. Scans at 2, 4, and 6 cover the full text, so + the end-of-stream round is skipped and there are 3 calls + """ from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( UnifiedLLMGuardrails, ) @@ -1680,8 +1685,9 @@ class TestGenericGuardrailAPIStreamingViaUnified: ): pass - assert mock_post.await_count == 4, ( - f"Expected 4 guardrail calls (3 sampled + 1 final aggregate), " + assert mock_post.await_count == 3, ( + f"Expected 3 guardrail calls (3 sampled; the end-of-stream round is skipped " + f"because chunk 6 already scanned the full text), " f"got {mock_post.await_count}" ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 8cad1c634a9..a28a2a71613 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -1971,3 +1971,271 @@ class TestStreamingGuardrailInformationBucket: assert recorded[0]["guardrail_name"] == "audit-recorder" assert recorded[0]["guardrail_status"] == "success" assert request_data["metadata"]["user_api_key_user_id"] == "user-1" + + +class _ScanCountingGuardrail(CustomGuardrail): + """Pass-through guardrail that records every response-side scan payload.""" + + def __init__(self, *, sampling_rate=5, end_of_stream_only=False, buffer_until_moderated=False): + super().__init__(guardrail_name="scan-counter") + self.streaming_sampling_rate = sampling_rate + self.streaming_end_of_stream_only = end_of_stream_only + self.streaming_buffer_until_moderated = buffer_until_moderated + self.guardrail_config = {} + self.scans: tuple[dict[str, object], ...] = () + + def should_run_guardrail(self, data, event_type): # type: ignore[override] + return True + + async def apply_guardrail(self, inputs, request_data, input_type, **kwargs): + self.scans = ( + *self.scans, + { + "texts": list(inputs.get("texts") or []), + "tool_calls": list(inputs.get("tool_calls") or []), + "model": inputs.get("model"), + }, + ) + return inputs + + +def _responses_delta(sequence_number, text): + return { + "type": "response.output_text.delta", + "sequence_number": sequence_number, + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": text, + } + + +def _responses_tail(sequence_number, text): + return [ + { + "type": "response.output_text.done", + "sequence_number": sequence_number, + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "text": text, + }, + { + "type": "response.completed", + "sequence_number": sequence_number + 1, + "response": { + "model": "gpt-5.6", + "output": [{"type": "message", "content": [{"type": "output_text", "text": text}]}], + }, + }, + ] + + +class TestStreamingScanDedup: + """A sampled round whose scan payload matches the previous round (or carries + no text yet) is skipped, so a stream is never re-scanned for output the + guardrail already cleared. Regression for LIT-6692.""" + + @pytest.fixture(autouse=True) + def _use_real_mappings(self, monkeypatch): + monkeypatch.setattr( + unified_module, + "endpoint_guardrail_translation_mappings", + load_guardrail_translation_mappings(), + ) + + @pytest.mark.asyncio + async def test_chat_terminal_chunk_on_sampled_index_is_scanned_once(self): + guardrail = _ScanCountingGuardrail(sampling_rate=3) + chunks = [_stream_chunk("a"), _stream_chunk("b"), _stream_chunk("c", finish_reason="stop")] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert len(out) == 3 + assert [scan["texts"] for scan in guardrail.scans] == [["abc"]] + + @pytest.mark.asyncio + async def test_chat_round_with_unchanged_text_is_skipped(self): + guardrail = _ScanCountingGuardrail(sampling_rate=3) + chunks = [ + _stream_chunk("a"), + _stream_chunk("b"), + _stream_chunk("c"), + _stream_chunk(None), + _stream_chunk(None), + _stream_chunk(None), + _stream_chunk("d", finish_reason="stop"), + ] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert len(out) == 7 + assert [scan["texts"] for scan in guardrail.scans] == [["abc"], ["abcd"]] + + @pytest.mark.asyncio + async def test_chat_finish_chunk_right_after_a_sampled_round_is_not_rescanned(self): + guardrail = _ScanCountingGuardrail(sampling_rate=3) + chunks = [_stream_chunk("a"), _stream_chunk("b"), _stream_chunk("c"), _stream_chunk(None, finish_reason="stop")] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert len(out) == 4 + assert [scan["texts"] for scan in guardrail.scans] == [["abc"]] + + @pytest.mark.asyncio + async def test_chat_finish_chunk_carrying_tool_calls_is_still_scanned(self): + from litellm.types.utils import ChatCompletionDeltaToolCall, Function + + guardrail = _ScanCountingGuardrail(sampling_rate=3) + tool_call = ChatCompletionDeltaToolCall( + id="call_1", index=0, type="function", function=Function(name="get_weather", arguments='{"city": "Paris"}') + ) + finish = ModelResponseStream( + choices=[ + StreamingChoices(index=0, delta=Delta(content=None, tool_calls=[tool_call]), finish_reason="tool_calls") + ] + ) + chunks = [_stream_chunk("a"), _stream_chunk("b"), _stream_chunk("c"), finish] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert len(out) == 4 + assert [scan["texts"] for scan in guardrail.scans] == [["abc"], ["abc"]] + assert [call["function"]["name"] for call in guardrail.scans[1]["tool_calls"]] == ["get_weather"] + + @pytest.mark.asyncio + async def test_chat_second_choice_finishing_later_still_gets_the_end_scan(self): + guardrail = _ScanCountingGuardrail(sampling_rate=3) + chunks = [ + _stream_chunk("a", index=0), + _stream_chunk("x", index=1), + _stream_chunk("b", finish_reason="stop", index=0), + _stream_chunk("y", index=1), + _stream_chunk("z", finish_reason="stop", index=1), + ] + + await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert len(guardrail.scans) == 2 + assert any("yz" in text for text in guardrail.scans[-1]["texts"]) + + @pytest.mark.asyncio + async def test_responses_completed_event_on_sampled_index_is_scanned_once(self): + guardrail = _ScanCountingGuardrail(sampling_rate=5) + deltas = [_responses_delta(i, f"t{i}") for i in range(8)] + full_text = "".join(f"t{i}" for i in range(8)) + chunks = deltas + _responses_tail(8, full_text) + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses") + + assert len(out) == 10 + assert [scan["texts"] for scan in guardrail.scans] == [["t0t1t2t3t4"], [full_text]] + assert guardrail.scans[-1]["model"] == "gpt-5.6" + + @pytest.mark.asyncio + async def test_responses_completed_right_after_a_sampled_round_is_not_rescanned(self): + guardrail = _ScanCountingGuardrail(sampling_rate=5) + deltas = [_responses_delta(i, f"t{i}") for i in range(5)] + chunks = deltas + _responses_tail(5, "t0t1t2t3t4") + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses") + + assert len(out) == 7 + assert [scan["texts"] for scan in guardrail.scans] == [["t0t1t2t3t4"]] + + @pytest.mark.asyncio + async def test_responses_completed_carrying_a_function_call_is_still_scanned(self): + guardrail = _ScanCountingGuardrail(sampling_rate=5) + deltas = [_responses_delta(i, f"t{i}") for i in range(5)] + completed = { + "type": "response.completed", + "sequence_number": 5, + "response": { + "model": "gpt-5.6", + "output": [ + {"type": "message", "content": [{"type": "output_text", "text": "t0t1t2t3t4"}]}, + { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "status": "completed", + }, + ], + }, + } + chunks = deltas + [completed] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses") + + assert len(out) == 6 + assert [scan["texts"] for scan in guardrail.scans] == [["t0t1t2t3t4"], ["t0t1t2t3t4"]] + assert [call["function"]["name"] for call in guardrail.scans[1]["tool_calls"]] == ["get_weather"] + + @pytest.mark.asyncio + async def test_responses_round_with_unchanged_text_is_skipped(self): + guardrail = _ScanCountingGuardrail(sampling_rate=5) + deltas = [_responses_delta(i, f"t{i}") for i in range(5)] + quiet = [{"type": "response.in_progress", "sequence_number": i} for i in range(5, 10)] + chunks = deltas + quiet + _responses_tail(10, "t0t1t2t3t4") + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses") + + assert len(out) == 12 + assert guardrail.scans == ({"texts": ["t0t1t2t3t4"], "tool_calls": [], "model": None},) + + @pytest.mark.asyncio + async def test_responses_tool_call_done_event_is_still_scanned(self): + guardrail = _ScanCountingGuardrail(sampling_rate=2) + tool_call_done = { + "type": "response.output_item.done", + "sequence_number": 1, + "output_index": 1, + "item": { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "status": "completed", + }, + } + chunks = [_responses_delta(0, "hi"), tool_call_done] + _responses_tail(2, "hi") + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses") + + assert len(out) == 4 + assert len(guardrail.scans) == 2 + assert [call["function"]["name"] for call in guardrail.scans[0]["tool_calls"]] == ["get_weather"] + assert guardrail.scans[1]["texts"] == ["hi"] + + @pytest.mark.asyncio + async def test_anthropic_skips_empty_round_and_terminal_duplicate(self): + guardrail = _ScanCountingGuardrail(sampling_rate=2) + chunks = _anthropic_message_chunks(["hello ", "world"]) + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/messages") + + assert out == chunks + assert [scan["texts"] for scan in guardrail.scans] == [["hello world"]] + + @pytest.mark.asyncio + async def test_end_of_stream_only_still_scans_exactly_once(self): + guardrail = _ScanCountingGuardrail(sampling_rate=2, end_of_stream_only=True) + chunks = _anthropic_message_chunks(["hello ", "world"]) + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/messages") + + assert out == chunks + assert [scan["texts"] for scan in guardrail.scans] == [["hello world"]] + + @pytest.mark.asyncio + async def test_buffer_until_moderated_still_scans_exactly_once_and_releases_every_chunk(self): + guardrail = _ScanCountingGuardrail(sampling_rate=1, buffer_until_moderated=True) + chunks = [_stream_chunk("a"), _stream_chunk("b"), _stream_chunk("c", finish_reason="stop")] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert out == chunks + assert [scan["texts"] for scan in guardrail.scans] == [["abc"]] From 63482cfdbd4d6e623b984c9b65ab98d1f224d879 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 18:25:51 -0700 Subject: [PATCH 170/480] chore(deps): lower the pymongo floor for the mongodb extra to 4.9 4.17 was picked on the belief that dnspython only became a core pymongo dependency there, which is wrong: pymongo has declared dnspython>=1.16.0,<3.0.0 as a core requirement since well before that, so mongodb+srv:// URIs resolve at 4.9 too. The real floor is 4.9, the release AsyncMongoClient landed in, and 4.8 has no AsyncMongoClient at all. Verified against live Atlas on 4.9: sync and async search, list_search_indexes, same top hit and score as 4.17. Resolution is unchanged, pymongo 4.17.0 either way, so this only widens what an existing environment is allowed to bring. --- pyproject.toml | 9 +++------ uv.lock | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c2f0dc6ced7..e3e103e6d49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,12 +112,9 @@ utils = [ ] caching = ["diskcache>=5.6.3,<6.0"] mcp = ["mcp>=1.28.1,<2.0"] -# Driver for the MongoDB Atlas vector store. Atlas Vector Search has no HTTP query -# API, so that provider talks to the cluster over the wire protocol. Imported lazily -# and kept out of the base install, which never needs a MongoDB driver. The floor is -# 4.17 because that is where dnspython became a core dependency rather than the `srv` -# extra, and Atlas hands out mongodb+srv:// URIs that do not resolve without it. -mongodb = ["pymongo>=4.17,<5.0"] +# Driver for the MongoDB Atlas vector store; Atlas Vector Search has no HTTP query API. +# The floor is 4.9 because that is the release AsyncMongoClient landed in. +mongodb = ["pymongo>=4.9,<5.0"] # SAML SSO for the admin UI. python3-saml pulls in xmlsec/lxml, whose wheels # bundle the native libxmlsec1/libxml2 libraries, so no system packages are # required. Kept out of the base `proxy` extra so it stays optional. diff --git a/uv.lock b/uv.lock index 362bb490a2a..bb1927ce093 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-30T17:51:25.171404Z" +exclude-newer = "2026-08-31T00:55:41.895302Z" exclude-newer-span = "P3D" [manifest] @@ -4554,7 +4554,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.10.0,<3.0.0" }, { name = "pydantic-settings", specifier = ">=2.14.1,<3.0" }, { name = "pyjwt", marker = "extra == 'proxy'", specifier = ">=2.13.0,<3.0" }, - { name = "pymongo", marker = "extra == 'mongodb'", specifier = ">=4.17,<5.0" }, + { name = "pymongo", marker = "extra == 'mongodb'", specifier = ">=4.9,<5.0" }, { name = "pynacl", marker = "extra == 'proxy'", specifier = ">=1.6.2,<2.0" }, { name = "pypdf", marker = "extra == 'proxy-runtime'", specifier = ">=6.16.1,<7.0" }, { name = "pyroscope-io", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.8.16,<1.0" }, From 92edcb90dbb41175f526187826f70058e5a57f94 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:27:19 -0700 Subject: [PATCH 171/480] fix: keep litellm importable on Python 3.10 and guard 3.11-only typing imports in CI (#39448) * ci: guard against Python 3.10-incompatible typing imports Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: address Python 3.10 typing guard review Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ci): honor version-guard direction and scan litellm-proxy-extras in py310 typing check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-code-quality.yml | 33 ++++ .../websearch_interception/handler.py | 4 +- .../_experimental/mcp_server/tool_search.py | 4 +- litellm/proxy/agent_endpoints/endpoints.py | 4 +- .../proxy/common_utils/reset_budget_job.py | 4 +- .../batch_file_validation.py | 4 +- .../general_upload_validation.py | 4 +- .../types/llms/gemini_audio_transcription.py | 4 +- .../check_py310_typing_imports.py | 150 ++++++++++++++++++ .../test_check_py310_typing_imports.py | 86 ++++++++++ 10 files changed, 286 insertions(+), 11 deletions(-) create mode 100644 tests/code_coverage_tests/check_py310_typing_imports.py create mode 100644 tests/test_litellm/test_check_py310_typing_imports.py diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index c112bf2bb22..d02f5878396 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -128,6 +128,9 @@ jobs: - name: check_fastuuid_usage run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py + - name: check_py310_typing_imports + run: uv run --no-sync python ./tests/code_coverage_tests/check_py310_typing_imports.py + - name: check_e2e_no_raw_requests run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py @@ -145,3 +148,33 @@ jobs: - name: documentation_test_api_docs run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py + + python-310-import-smoke: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.10" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Install dependencies + run: uv sync --frozen --extra proxy --python 3.10 + + - run: uv run --no-sync python --version + + - name: Import litellm + run: uv run --no-sync python -c "import litellm" + + - name: Check litellm CLI + run: uv run --no-sync litellm --version diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 2d737bc34e7..587da997f94 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -10,9 +10,9 @@ import asyncio import math import uuid from collections.abc import AsyncIterator, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, Never, TypedDict, TypeVar, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast -from typing_extensions import ReadOnly +from typing_extensions import Never, ReadOnly import litellm from litellm._logging import verbose_logger diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py index 4f6305d88cf..af02c11ad86 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_search.py +++ b/litellm/proxy/_experimental/mcp_server/tool_search.py @@ -5,10 +5,10 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, TypedDict, assert_never +from typing import TYPE_CHECKING, Any, Final, TypedDict from pydantic import ValidationError -from typing_extensions import ReadOnly, Required +from typing_extensions import ReadOnly, Required, assert_never import litellm from litellm.proxy.agent_endpoints.agent_search import DEFAULT_AGENT_SEARCH_TOP_K diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index 3e4dc07a521..3b8151d1064 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -13,10 +13,10 @@ import os import uuid from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Annotated, Final, TypedDict, assert_never +from typing import Annotated, Final, TypedDict from fastapi import APIRouter, Depends, HTTPException, Query, Request -from typing_extensions import ReadOnly, Required +from typing_extensions import ReadOnly, Required, assert_never import litellm from litellm._logging import verbose_proxy_logger diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 1682cf12f4e..47f69732e95 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -7,7 +7,9 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from enum import Enum from types import MappingProxyType -from typing import Final, Literal, Protocol, TypeVar, assert_never +from typing import Final, Literal, Protocol, TypeVar + +from typing_extensions import assert_never import litellm from litellm._logging import verbose_proxy_logger diff --git a/litellm/proxy/openai_files_endpoints/batch_file_validation.py b/litellm/proxy/openai_files_endpoints/batch_file_validation.py index 0aee5e8cc54..a41bd36d510 100644 --- a/litellm/proxy/openai_files_endpoints/batch_file_validation.py +++ b/litellm/proxy/openai_files_endpoints/batch_file_validation.py @@ -2,7 +2,9 @@ import json from collections.abc import Iterator from dataclasses import dataclass from itertools import chain -from typing import BinaryIO, Final, NoReturn, assert_never +from typing import BinaryIO, Final, NoReturn + +from typing_extensions import assert_never from litellm.proxy._types import ProxyException diff --git a/litellm/proxy/openai_files_endpoints/general_upload_validation.py b/litellm/proxy/openai_files_endpoints/general_upload_validation.py index 9d450cb5b8d..8c59a520272 100644 --- a/litellm/proxy/openai_files_endpoints/general_upload_validation.py +++ b/litellm/proxy/openai_files_endpoints/general_upload_validation.py @@ -8,7 +8,9 @@ extensions, path-traversal filenames) regardless of purpose. from dataclasses import dataclass from pathlib import Path -from typing import BinaryIO, Final, NoReturn, assert_never +from typing import BinaryIO, Final, NoReturn + +from typing_extensions import assert_never from litellm.proxy._types import ProxyException from litellm.proxy.common_utils.path_utils import safe_filename diff --git a/litellm/types/llms/gemini_audio_transcription.py b/litellm/types/llms/gemini_audio_transcription.py index cb12e0f45b8..f7e74ba4bf8 100644 --- a/litellm/types/llms/gemini_audio_transcription.py +++ b/litellm/types/llms/gemini_audio_transcription.py @@ -1,7 +1,7 @@ -from typing import Literal, Required +from typing import Literal from pydantic import BaseModel, ConfigDict -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, Required, TypedDict class GeminiTranscriptionAudioInput(TypedDict): diff --git a/tests/code_coverage_tests/check_py310_typing_imports.py b/tests/code_coverage_tests/check_py310_typing_imports.py new file mode 100644 index 00000000000..0cd4d089890 --- /dev/null +++ b/tests/code_coverage_tests/check_py310_typing_imports.py @@ -0,0 +1,150 @@ +import ast +import os +import sys +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +PY311_PLUS_TYPING_NAMES: Final[frozenset[str]] = frozenset( + { + "NotRequired", + "Required", + "Self", + "LiteralString", + "Never", + "assert_never", + "assert_type", + "reveal_type", + "TypeVarTuple", + "Unpack", + "dataclass_transform", + "override", + "TypeAliasType", + "get_original_bases", + "ReadOnly", + "TypeIs", + "NoDefault", + "get_protocol_members", + "is_protocol", + "evaluate_forward_ref", + "TypeForm", + } +) + + +@dataclass(frozen=True, slots=True) +class TypingImportViolation: + file: str + line: int + name: str + + +def _walk_with_ancestors( + node: ast.AST, ancestors: tuple[tuple[ast.AST, str], ...] = () +) -> Iterator[tuple[ast.AST, tuple[tuple[ast.AST, str], ...]]]: + yield node, ancestors + for field_name, field_value in ast.iter_fields(node): + if isinstance(field_value, ast.AST): + yield from _walk_with_ancestors(field_value, (*ancestors, (node, field_name))) + elif isinstance(field_value, list): + for child in field_value: + if isinstance(child, ast.AST): + yield from _walk_with_ancestors(child, (*ancestors, (node, field_name))) + + +def _is_sys_version_info(node: ast.AST) -> bool: + return ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "sys" + and node.attr == "version_info" + ) + + +def _is_version_guarded(ancestors: tuple[tuple[ast.AST, str], ...]) -> bool: + nearest_if: Final[tuple[ast.If, str] | None] = next( + ( + (ancestor, field_name) + for ancestor, field_name in reversed(ancestors) + if isinstance(ancestor, ast.If) + ), + None, + ) + if nearest_if is None: + return False + enclosing_if, branch = nearest_if + test: Final[ast.expr] = enclosing_if.test + if not isinstance(test, ast.Compare) or len(test.ops) != 1 or not _is_sys_version_info(test.left): + return False + operator: Final[ast.cmpop] = test.ops[0] + return (isinstance(operator, (ast.Gt, ast.GtE)) and branch == "body") or ( + isinstance(operator, (ast.Lt, ast.LtE)) and branch == "orelse" + ) + + +def scan_file(file_path: str | os.PathLike[str]) -> tuple[TypingImportViolation, ...]: + path: Final[Path] = Path(file_path) + tree: Final[ast.Module] = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + return tuple( + violation + for node, ancestors in _walk_with_ancestors(tree) + if not _is_version_guarded(ancestors) + for violation in _violations_for_node(node, path) + ) + + +def _violations_for_node( + node: ast.AST, path: Path +) -> tuple[TypingImportViolation, ...]: + if isinstance(node, ast.ImportFrom) and node.module == "typing": + return tuple( + TypingImportViolation(file=str(path), line=node.lineno, name=alias.name) + for alias in node.names + if alias.name in PY311_PLUS_TYPING_NAMES + ) + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "typing" + and node.attr in PY311_PLUS_TYPING_NAMES + ): + return (TypingImportViolation(file=str(path), line=node.lineno, name=node.attr),) + return () + + +def scan_directory(base_dir: str | os.PathLike[str] = ".") -> tuple[TypingImportViolation, ...]: + base_path: Final[Path] = Path(base_dir) + return tuple( + violation + for directory in ( + base_path / "litellm", + base_path / "enterprise", + base_path / "litellm-proxy-extras" / "litellm_proxy_extras", + ) + if directory.exists() + for path in directory.rglob("*.py") + for violation in scan_file(path) + ) + + +def main() -> None: + violations: Final[tuple[TypingImportViolation, ...]] = scan_directory() + if violations: + message: Final[str] = "\n".join( + ( + "Python 3.10-incompatible typing imports found:", + *( + f"{violation.file}:{violation.line}: {violation.name} is unavailable in Python 3.10; " + "import it from typing_extensions instead because litellm supports Python 3.10" + for violation in violations + ), + ) + ) + sys.stdout.write(f"{message}\n") + raise RuntimeError("Import Python 3.10-incompatible typing names from typing_extensions instead") + sys.stdout.write("No Python 3.10-incompatible typing imports found.\n") + + +if __name__ == "__main__": + main() diff --git a/tests/test_litellm/test_check_py310_typing_imports.py b/tests/test_litellm/test_check_py310_typing_imports.py new file mode 100644 index 00000000000..de370326091 --- /dev/null +++ b/tests/test_litellm/test_check_py310_typing_imports.py @@ -0,0 +1,86 @@ +import sys +from pathlib import Path +from typing import Final + +_CODE_COVERAGE_DIR: Final[Path] = Path(__file__).resolve().parents[1] / "code_coverage_tests" +sys.path.insert(0, str(_CODE_COVERAGE_DIR)) # test-quality-ok: required to import checker from its source directory +import check_py310_typing_imports as checker # noqa: E402 # load checker from its source directory + + +def _scan(tmp_path: Path, source: str) -> tuple[object, ...]: + file_path = tmp_path / "fixture.py" + file_path.write_text(source, encoding="utf-8") + return checker.scan_file(file_path) + + +def test_typing_import_flags_python_311_name(tmp_path: Path) -> None: + violations = _scan(tmp_path, "from typing import NotRequired, TypedDict\n") + assert tuple(violation.name for violation in violations) == ("NotRequired",) + + +def test_typing_extensions_import_passes(tmp_path: Path) -> None: + assert _scan(tmp_path, "from typing_extensions import NotRequired\n") == () + + +def test_typing_attribute_flags_python_311_name(tmp_path: Path) -> None: + violations = _scan(tmp_path, "import typing\nx: typing.Self\n") + assert tuple(violation.name for violation in violations) == ("Self",) + + +def test_version_guarded_typing_import_passes(tmp_path: Path) -> None: + source = ( + "import sys\n" + "if sys.version_info >= (3, 11):\n" + " from typing import NotRequired\n" + "else:\n" + " from typing_extensions import NotRequired\n" + ) + assert _scan(tmp_path, source) == () + + +def test_python_310_branch_flags_typing_import(tmp_path: Path) -> None: + source = ( + "import sys\n" + "if sys.version_info >= (3, 11):\n" + " from typing_extensions import NotRequired\n" + "else:\n" + " from typing import NotRequired\n" + ) + violations = _scan(tmp_path, source) + assert tuple(violation.name for violation in violations) == ("NotRequired",) + + +def test_python_310_branch_is_exempt_for_less_than_guard(tmp_path: Path) -> None: + source = ( + "import sys\n" + "if sys.version_info < (3, 11):\n" + " from typing_extensions import NotRequired\n" + "else:\n" + " from typing import NotRequired\n" + ) + assert _scan(tmp_path, source) == () + + +def test_nearest_if_controls_version_guard(tmp_path: Path) -> None: + source = ( + "if sys.version_info >= (3, 11):\n" + " from typing import Self\n" + " x = 1\n" + "if True:\n" + " from typing import Self\n" + ) + violations = _scan(tmp_path, source) + assert tuple((violation.name, violation.line) for violation in violations) == (("Self", 5),) + + +def test_scan_directory_includes_proxy_extras(tmp_path: Path) -> None: + file_path = tmp_path / "litellm-proxy-extras" / "litellm_proxy_extras" / "m.py" + file_path.parent.mkdir(parents=True) + file_path.write_text("from typing import NotRequired\n", encoding="utf-8") + + violations = checker.scan_directory(tmp_path) + assert tuple((violation.name, violation.file) for violation in violations) == (("NotRequired", str(file_path)),) + + +def test_python_310_typing_name_passes(tmp_path: Path) -> None: + assert _scan(tmp_path, "from typing import Optional\n") == () From 8441dd6e8ca16a480e0766651f7c4ab99d22d5d5 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:28:06 -0700 Subject: [PATCH 172/480] fix(proxy): keep SpendLogs and callback session ids in sync when the request has none (#39450) * fix(proxy): keep SpendLogs and callback session ids in sync when the request has none Add general_settings.missing_session_id (generate | reject). In generate mode one id is stamped into litellm_session_id, litellm_trace_id and metadata.session_id before callbacks run, so LiteLLM_SpendLogs.session_id and the Langfuse session id match. In reject mode such requests get a 400. Unset keeps the legacy behavior. MCP routes are not affected Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(proxy): regenerate schema.d.ts and shorten mutable-ok comment for ruff format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): mark generated session ids so affinity consumers do not pin on them Fireworks x-session-affinity, the router session_affinity pre-call check and the complexity router session pin all read metadata.session_id as a caller-chosen stable key. A missing_session_id: generate id is fresh per request, so it now carries metadata.litellm_session_id_generated and those consumers skip it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/llms/fireworks_ai/common_utils.py | 11 +- litellm/proxy/_types.py | 4 + litellm/proxy/litellm_pre_call_utils.py | 55 ++++++ .../complexity_router/complexity_router.py | 8 +- .../deployment_affinity_check.py | 4 +- .../test_fireworks_ai_chat_transformation.py | 16 ++ .../proxy/test_litellm_pre_call_utils.py | 176 ++++++++++++++++++ .../router_strategy/test_complexity_router.py | 22 ++- .../test_session_id_affinity.py | 43 ++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 11 files changed, 335 insertions(+), 10 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index c7b74e176db..ef9329b9dfc 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1449,6 +1449,7 @@ RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl" CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" +SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated" LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = ( "Truncation is a DB storage safeguard. " diff --git a/litellm/llms/fireworks_ai/common_utils.py b/litellm/llms/fireworks_ai/common_utils.py index 8c306faa036..ac934ad0cb5 100644 --- a/litellm/llms/fireworks_ai/common_utils.py +++ b/litellm/llms/fireworks_ai/common_utils.py @@ -2,6 +2,7 @@ from typing import Final from httpx import Headers +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -16,16 +17,18 @@ def get_fireworks_session_id(litellm_params: dict) -> str | None: """ Session id to send as `x-session-affinity`, or None when the caller gave none. - Deliberately does not fall back to `litellm_trace_id`: that is generated per - request (`str(uuid.uuid4())` when absent), so using it pins every request to a - different Fireworks node and prompt caching never hits. + Deliberately does not fall back to `litellm_trace_id`, and ignores session ids the + proxy generated for a request that had none: both are per request, so using them + pins every request to a different Fireworks node and prompt caching never hits. """ params: Final = litellm_params + metadata: Final = params.get("metadata") + if isinstance(metadata, dict) and metadata.get(SESSION_ID_GENERATED_METADATA_KEY): + return None for key in ("litellm_session_id", "session_id"): value = params.get(key) if value: return str(value) - metadata: Final = params.get("metadata") if isinstance(metadata, dict): value = metadata.get("session_id") if value: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 2da7ceb2d50..849e54c65aa 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2594,6 +2594,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.", ) + missing_session_id: Literal["generate", "reject"] | None = Field( + None, + description="What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.", + ) enable_public_model_hub: bool = Field( default=False, description="Public model hub for users to see what models they have access to, supported openai params, etc.", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 20f83085286..1d440448c2f 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -16,6 +16,7 @@ from starlette.datastructures import Headers import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging +from litellm._uuid import uuid from litellm.constants import ( CONSUMED_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, @@ -23,6 +24,7 @@ from litellm.constants import ( OTEL_SERVICE_NAME_METADATA_KEYS, PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + SESSION_ID_GENERATED_METADATA_KEY, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -40,6 +42,7 @@ from litellm.proxy._types import ( AddTeamCallback, CommonProxyErrors, LitellmDataForBackendLLMCall, + LiteLLMRoutes, LitellmUserRoles, ProxyErrorTypes, ProxyException, @@ -47,6 +50,8 @@ from litellm.proxy._types import ( TeamCallbackMetadata, UserAPIKeyAuth, ) +from litellm.proxy.auth.auth_utils import get_request_route +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.callback_utils import ( decrypt_callback_vars, get_metadata_variable_name_from_kwargs, @@ -715,6 +720,50 @@ def _get_anthropic_session_id_from_metadata(metadata: object) -> str | None: return session_id +def _is_llm_inference_route(request: Request) -> bool: + route: Final = get_request_route(request) + return RouteChecks.is_llm_api_route(route=route) and not RouteChecks.check_route_access( + route=route, allowed_routes=LiteLLMRoutes.mcp_routes.value + ) + + +def apply_missing_session_id_policy( + data: dict[str, object], # mutable-ok: stamps session ids in place on the request body the pipeline threads through + _metadata_variable_name: str, + general_settings: Mapping[str, object] | None, + request: Request, +) -> None: + policy: Final = general_settings.get("missing_session_id") if general_settings else None + if policy is None or not _is_llm_inference_route(request): + return + metadata: Final = data.get(_metadata_variable_name) + if not isinstance(metadata, dict): + return + if data.get("litellm_session_id") or metadata.get("session_id"): + return + match policy: + case "generate": + session_id: Final = str(data.get("litellm_trace_id") or metadata.get("trace_id") or uuid.uuid4()) + data["litellm_session_id"] = session_id # rebind-ok: data is an out-param + data.setdefault("litellm_trace_id", session_id) + metadata["session_id"] = session_id + metadata[SESSION_ID_GENERATED_METADATA_KEY] = True + case "reject": + raise ProxyException( + message=( + "Request has no session id. Send an `x-litellm-session-id` header or `metadata.session_id`. " + "Required by `general_settings.missing_session_id: reject`." + ), + type=ProxyErrorTypes.bad_request_error, + param="session_id", + code=400, + ) + case _: + verbose_proxy_logger.warning( + "Ignoring unknown general_settings.missing_session_id=%r; expected 'generate' or 'reject'", policy + ) + + def is_claude_code_user_agent(user_agent: str) -> bool: """Claude Code identifies itself as ``claude-cli/ ...``; the IDE extensions and the Agent SDK run through the same CLI and share that prefix.""" @@ -1818,6 +1867,12 @@ async def add_litellm_data_to_request( data=data, _metadata_variable_name=_metadata_variable_name, ) + apply_missing_session_id_policy( + data=data, + _metadata_variable_name=_metadata_variable_name, + general_settings=general_settings, + request=request, + ) # Expose request headers under the metadata field for guardrails (fixes #17477) if _metadata_variable_name in data and isinstance(data[_metadata_variable_name], dict): diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index d205db90607..430efe339a2 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -26,7 +26,11 @@ from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast from pydantic import BaseModel, create_model from litellm._logging import verbose_router_logger -from litellm.constants import EMPTY_MAPPING, RETURN_RAW_MODEL_NAME_METADATA_KEY +from litellm.constants import ( + EMPTY_MAPPING, + RETURN_RAW_MODEL_NAME_METADATA_KEY, + SESSION_ID_GENERATED_METADATA_KEY, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata @@ -2712,7 +2716,7 @@ class ComplexityRouter(CustomLogger): """Resolve a client-supplied session_id.""" for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs): session_id = metadata.get("session_id") - if session_id is not None: + if session_id is not None and not metadata.get(SESSION_ID_GENERATED_METADATA_KEY): return str(session_id) return None diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index b1e9dbdefa8..39d3e25aacb 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -21,7 +21,7 @@ from typing_extensions import TypedDict from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache -from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY +from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger, Span from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import AllMessageValues @@ -265,7 +265,7 @@ class DeploymentAffinityCheck(CustomLogger): @staticmethod def _get_session_id_from_metadata_dict(metadata: dict) -> str | None: session_id: Final = metadata.get("session_id") - if session_id is None: + if session_id is None or metadata.get(SESSION_ID_GENERATED_METADATA_KEY): return None return str(session_id) diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index d7cc89868af..ec8725db5f7 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -8,6 +8,7 @@ import litellm from litellm import get_model_info, supports_reasoning, supports_vision from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -235,6 +236,21 @@ def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id(): ) +def test_get_fireworks_session_id_ignores_proxy_generated_session_id(): + """general_settings.missing_session_id: generate stamps a fresh id per request; sending it + as x-session-affinity would pin every request to a different node.""" + assert ( + get_fireworks_session_id( + { + "litellm_session_id": "generated-1", + "litellm_trace_id": "generated-1", + "metadata": {"session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True}, + } + ) + is None + ) + + def test_handle_message_content_with_tool_calls(): config = FireworksAIConfig() message = Message( diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index ee0e2014951..8366e5546a9 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -41,7 +41,9 @@ from litellm.litellm_core_utils.get_provider_specific_headers import ( from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( TRUSTED_CALLBACK_VARS_FIELD, ) +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.types.utils import CredentialItem @@ -7719,3 +7721,177 @@ def test_stamped_model_access_groups_survive_the_litellm_metadata_merge(): } assert get_litellm_metadata_from_kwargs(kwargs)[MODEL_ACCESS_GROUP_METADATA_KEY] == ["tier-a"] + + +def _request_for(path: str) -> MagicMock: + request = MagicMock(spec=Request) + request.scope = {"path": path} + request.url = MagicMock() + request.url.path = path + request.url.__str__.return_value = f"http://localhost{path}" + request.method = "POST" + request.query_params = {} + request.headers = {"Content-Type": "application/json"} + request.client = MagicMock() + request.client.host = "127.0.0.1" + return request + + +def _spend_log_session_id(data: dict[str, object]) -> str: + """Resolve session_id the way LiteLLM_SpendLogs does: standard_logging_payload.trace_id.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.proxy.spend_tracking.spend_tracking_utils import _get_session_id_for_spend_log + + metadata = data["metadata"] + assert isinstance(metadata, dict) + litellm_params = get_litellm_params( + litellm_session_id=str(data["litellm_session_id"]) if "litellm_session_id" in data else None, + litellm_trace_id=str(data["litellm_trace_id"]) if "litellm_trace_id" in data else None, + metadata=metadata, + ) + trace_id = StandardLoggingPayloadSetup.get_standard_logging_payload_trace_id( + logging_obj=SimpleNamespace(litellm_trace_id="per-call-random-trace-id"), + litellm_params=litellm_params, + ) + return _get_session_id_for_spend_log(kwargs={}, standard_logging_payload={"trace_id": trace_id}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_correlation_in_logs", [False, True]) +async def test_missing_session_id_generate_makes_spend_log_and_callback_session_ids_agree( + monkeypatch: pytest.MonkeyPatch, request_correlation_in_logs: bool +): + """Without a session header, SpendLogs.session_id and the metadata.session_id that Langfuse logs + must be the same generated id, so cross-referencing the two by session_id works. The id is marked + as generated so affinity consumers (Fireworks x-session-affinity, router session pins) skip it.""" + monkeypatch.setattr(litellm, "request_correlation_in_logs", request_correlation_in_logs) + data = {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]} + + updated = await add_litellm_data_to_request( + data=data, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "generate"}, + ) + + callback_session_id = updated["metadata"]["session_id"] + assert isinstance(callback_session_id, str) and len(callback_session_id) == 36 + assert _spend_log_session_id(updated) == callback_session_id + assert updated["metadata"][SESSION_ID_GENERATED_METADATA_KEY] is True + assert get_fireworks_session_id( + {"litellm_session_id": updated["litellm_session_id"], "metadata": updated["metadata"]} + ) is None + + +@pytest.mark.asyncio +async def test_missing_session_id_unset_keeps_legacy_divergence(): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + ) + + assert "session_id" not in updated["metadata"] + assert "litellm_session_id" not in updated + assert _spend_log_session_id(updated) == "per-call-random-trace-id" + + +@pytest.mark.asyncio +async def test_missing_session_id_generate_reuses_traceparent_trace_id(): + """A W3C traceparent already decides SpendLogs.session_id, so the callback session id must reuse it.""" + request = _request_for("/v1/chat/completions") + request.headers = {"traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"} + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "generate"}, + ) + + assert updated["metadata"]["session_id"] == "4bf92f3577b34da6a3ce929d0e0e4736" + assert _spend_log_session_id(updated) == "4bf92f3577b34da6a3ce929d0e0e4736" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["generate", "reject"]) +async def test_missing_session_id_policy_keeps_client_supplied_session_id(policy: str): + request = _request_for("/v1/chat/completions") + request.headers = {"x-litellm-session-id": "client-session-1"} + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": policy}, + ) + + assert updated["litellm_session_id"] == "client-session-1" + assert updated["metadata"]["session_id"] == "client-session-1" + assert _spend_log_session_id(updated) == "client-session-1" + assert SESSION_ID_GENERATED_METADATA_KEY not in updated["metadata"] + assert ( + get_fireworks_session_id({"litellm_session_id": "client-session-1", "metadata": updated["metadata"]}) + == "client-session-1" + ) + + +@pytest.mark.asyncio +async def test_missing_session_id_reject_accepts_body_metadata_session_id(): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": [], "metadata": {"session_id": "body-session-1"}}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "reject"}, + ) + + assert updated["metadata"]["session_id"] == "body-session-1" + + +@pytest.mark.asyncio +async def test_missing_session_id_reject_returns_400_without_session_id(): + with pytest.raises(ProxyException) as exc_info: + await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "reject"}, + ) + + assert exc_info.value.code == "400" + assert exc_info.value.param == "session_id" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/mcp/", "/mcp/tools", "/key/health"]) +async def test_missing_session_id_policy_skips_non_inference_routes(path: str): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o"}, + request=_request_for(path), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "reject"}, + ) + + assert "session_id" not in updated["metadata"] + + +@pytest.mark.asyncio +async def test_missing_session_id_unknown_value_is_ignored(): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "typo"}, + ) + + assert "session_id" not in updated["metadata"] diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 941d78085e4..d7d02544efb 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -16,7 +16,7 @@ import litellm from litellm import Router from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache -from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY +from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.router_strategy.complexity_router.complexity_router import ( _CLASSIFICATION_CURRENT_MESSAGE_ONLY, _CLASSIFICATION_WITH_CONVERSATION, @@ -4274,6 +4274,26 @@ class TestSessionAffinity: assert first.model == "o1-preview" assert second.model == "gpt-4o-mini" + @pytest.mark.asyncio + async def test_proxy_generated_session_id_never_pins(self, mock_router_instance, session_affinity_config): + """A session id the proxy generated for a request that had none is per request, so + it must not create a pin even with session_affinity enabled.""" + mock_router_instance.cache = DualCache() + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=session_affinity_config, + ) + request_kwargs = {"metadata": {"session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True}} + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs=request_kwargs, messages=self.REASONING_MESSAGE + ) + second = await router.async_pre_routing_hook( + model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE_MESSAGE + ) + assert first.model == "o1-preview" + assert second.model == "gpt-4o-mini" + @pytest.mark.asyncio async def test_can_be_enabled_to_pin_every_later_turn(self, mock_router_instance, session_affinity_config): """Regression: session_affinity=True is the opt-in, so a shared session_id reuses the diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py b/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py index a3772a276fa..cf48888600e 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py @@ -7,7 +7,7 @@ import json import litellm from litellm.caching.dual_cache import DualCache -from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY +from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, ) @@ -180,6 +180,47 @@ async def test_async_session_id_affinity_priority_over_user_key(): assert filtered[0]["model_info"]["id"] == "deployment-2" +@pytest.mark.asyncio +async def test_proxy_generated_session_id_does_not_pin_a_deployment(): + """A session id the proxy generated for a request that had none is per request, so a + pin stored under it must be ignored and none must be written.""" + cache = DualCache() + callback = DeploymentAffinityCheck( + cache=cache, + ttl_seconds=123, + enable_user_key_affinity=False, + enable_responses_api_affinity=False, + enable_session_id_affinity=True, + ) + healthy_deployments = [ + {"model_name": "model_group", "litellm_params": {"model": "model_1"}, "model_info": {"id": "deployment-1"}}, + {"model_name": "model_group", "litellm_params": {"model": "model_2"}, "model_info": {"id": "deployment-2"}}, + ] + await cache.async_set_cache( + DeploymentAffinityCheck.get_session_affinity_cache_key("model_group", "generated-1", user_key="user1"), + {"model_id": "deployment-2"}, + ) + request_kwargs = { + "metadata": {"user_api_key_hash": "user1", "session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True} + } + + filtered = await callback.async_filter_deployments( + model="model_group", healthy_deployments=healthy_deployments, messages=[], request_kwargs=request_kwargs + ) + await callback.async_pre_call_deployment_hook( + kwargs={ + "metadata": {**request_kwargs["metadata"], "deployment_model_name": "model_group"}, + "model_info": {"id": "deployment-1"}, + }, + call_type=None, + ) + + assert len(filtered) == 2 + assert await cache.async_get_cache( + DeploymentAffinityCheck.get_session_affinity_cache_key("model_group", "generated-1", user_key="user1") + ) == {"model_id": "deployment-2"} + + MOCK_RESPONSES_API_RESPONSE = { "id": "resp_mock-resp-456", "object": "response", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 90c4f03bf08..491c4fb6a44 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25772,6 +25772,11 @@ export interface components { * @description Number of trusted reverse proxies/load balancers in front of the gateway that append to X-Forwarded-For. When set (and mcp_trusted_proxy_ranges validates the direct peer), the client IP for MCP access control is read this many entries from the right of the chain instead of the spoofable leftmost value, defeating append-style X-Forwarded-For forgery. */ mcp_xff_num_trusted_hops?: number | null; + /** + * Missing Session Id + * @description What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id. + */ + missing_session_id?: ("generate" | "reject") | null; /** * Model List Healthy Only * @description When true, `/models`, `/v1/models/{id}` and `/model/info` hide models whose backing deployments are all unhealthy, for every caller, without needing `healthy_only=true` per request. Requires `background_health_checks: true`, and keeps deployment health state cached without turning on `enable_health_check_routing`, so routing is unaffected. With no health state nothing is hidden. Hiding is presentation-only, a hidden model can still be called. From 105f99cea214b615888e4c9046502a42395aa077 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:29:55 -0700 Subject: [PATCH 173/480] fix(proxy-extras): kill the whole Prisma process group when a command times out Every Prisma CLI call now goes through one runner that starts the command in its own session and SIGKILLs the process group on timeout, so the Node process and the Rust schema engine die together with the Python wrapper instead of being reparented to pid 1, where they kept applying migrations after the proxy had given up and held the Prisma advisory lock against every retry and every later boot. Tests that faked subprocess.run now fake the runner, and the fake Prisma CLI in the migration tests forks a grandchild that must not outlive a timed-out migrate deploy. --- .../litellm_proxy_extras/prisma_toolchain.py | 69 +++++++++++++++---- .../litellm_proxy_extras/replica_identity.py | 7 +- .../litellm_proxy_extras/utils.py | 59 +++++----------- .../tests/test_setup_database_fail_fast.py | 30 ++++---- .../test_litellm_proxy_extras_utils.py | 6 +- .../test_prisma_toolchain.py | 42 +++++++++++ .../proxy/db/test_replica_identity.py | 4 +- 7 files changed, 139 insertions(+), 78 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py b/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py index 2283814ab35..b51de9609d3 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py +++ b/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py @@ -18,10 +18,15 @@ recoverable one. constant: it grows with the number of pending migrations, so a fresh database that has to replay every migration this package ships overruns a per-command budget sized for the short bookkeeping commands, on a laptop as much as on a -slow CI runner. The Python ``prisma`` wrapper spawns Node and the schema engine -as separate children, so killing the wrapper on timeout leaves them running: -the retry then contends with that orphan for Prisma's advisory lock and cannot -finish any sooner. Migrate deploy therefore runs under its own budget. +slow CI runner. Migrate deploy therefore runs under its own budget. + +The Python ``prisma`` wrapper spawns Node, which spawns the Rust schema +engine, so killing only the wrapper on timeout leaves the engine running with +no parent: it keeps mutating the database after the proxy has given up, holds +Prisma's advisory lock so every retry and every later boot queues behind it, +and dies mid-migration once its pipes close, leaving a half-applied ledger row. +Every Prisma command therefore runs in a process group of its own, and a +timeout kills the whole group. All three budgets are overridable so an operator can widen them without a release: ``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install, @@ -35,10 +40,12 @@ the deploy override says otherwise. import math import os import shutil +import signal import subprocess +from collections.abc import Mapping, Sequence from dataclasses import dataclass from pathlib import Path -from typing import Optional +from typing import IO, Optional, Union from litellm_proxy_extras._logging import logger @@ -167,6 +174,49 @@ def heal_incomplete_nodeenv_cache() -> bool: return True +def _kill_process_group(process: "subprocess.Popen[str]") -> None: + if os.name == "nt": + process.kill() + return + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + return + + +def run_prisma( + argv: Sequence[str], + *, + timeout: float, + env: Mapping[str, str], + stdout: Union[IO[str], int, None] = subprocess.PIPE, + stderr: Optional[int] = subprocess.PIPE, +) -> "subprocess.CompletedProcess[str]": + """Run one Prisma CLI command in its own process group, bounded by ``timeout``. + + Raises ``subprocess.TimeoutExpired`` once the budget is spent, after killing + the command together with every process it spawned, and + ``subprocess.CalledProcessError`` on a non-zero exit. Output is captured as + text unless ``stdout``/``stderr`` say otherwise. + """ + with subprocess.Popen( + argv, + env=env, + stdout=stdout, + stderr=stderr, + text=True, + start_new_session=True, + ) as process: + try: + out, err = process.communicate(timeout=timeout) + except BaseException: + _kill_process_group(process) + raise + if process.returncode: + raise subprocess.CalledProcessError(process.returncode, process.args, out, err) + return subprocess.CompletedProcess(process.args, process.returncode, out, err) + + def ensure_prisma_toolchain( prisma_command: str, prisma_env: dict[str, str] ) -> ToolchainBootstrap: @@ -179,14 +229,7 @@ def ensure_prisma_toolchain( timeout = prisma_bootstrap_timeout() logger.info("Preparing the Prisma CLI toolchain (timeout %ss)", timeout) try: - subprocess.run( - [prisma_command, BOOTSTRAP_ARG], - timeout=timeout, - check=True, - capture_output=True, - text=True, - env=prisma_env, - ) + run_prisma([prisma_command, BOOTSTRAP_ARG], timeout=timeout, env=prisma_env) except subprocess.TimeoutExpired: logger.warning( "Preparing the Prisma CLI toolchain timed out after %ss. Raise %s " diff --git a/litellm-proxy-extras/litellm_proxy_extras/replica_identity.py b/litellm-proxy-extras/litellm_proxy_extras/replica_identity.py index 157d595404e..3a5865a54cd 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/replica_identity.py +++ b/litellm-proxy-extras/litellm_proxy_extras/replica_identity.py @@ -16,7 +16,7 @@ import tempfile from pathlib import Path from litellm_proxy_extras._logging import logger -from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout +from litellm_proxy_extras.prisma_toolchain import prisma_command_timeout, run_prisma REPLICA_IDENTITY_FULL_ENV_VAR = "LITELLM_SET_REPLICA_IDENTITY_FULL" @@ -66,7 +66,7 @@ def apply_replica_identity_full( with tempfile.TemporaryDirectory(prefix="litellm_replica_identity_") as tmp_dir: sql_path = Path(tmp_dir) / "replica_identity_full.sql" sql_path.write_text(REPLICA_IDENTITY_FULL_SQL) - subprocess.run( + run_prisma( [ prisma_command, "db", @@ -77,9 +77,6 @@ def apply_replica_identity_full( schema_path, ], timeout=prisma_command_timeout(), - check=True, - capture_output=True, - text=True, env=prisma_env, ) except subprocess.CalledProcessError as e: diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index d22484bc0e8..168c3febae2 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -9,6 +9,7 @@ import time from pathlib import Path from typing import Optional +from litellm_proxy_extras import prisma_toolchain from litellm_proxy_extras._logging import logger from litellm_proxy_extras.replica_identity import ( REPLICA_IDENTITY_FULL_ENV_VAR, @@ -198,7 +199,7 @@ class ProxyExtrasDBManager: # 1. Generate migration SQL file by comparing empty state to current db state logger.info("Generating baseline migration...") migration_file = init_dir / "migration.sql" - subprocess.run( + prisma_toolchain.run_prisma( [ _get_prisma_command(), "migrate", @@ -209,14 +210,13 @@ class ProxyExtrasDBManager: "--script", ], stdout=open(migration_file, "w"), - check=True, timeout=prisma_command_timeout(), env=prisma_env, ) # 3. Mark the migration as applied since it represents current state logger.info("Marking baseline migration as applied...") - subprocess.run( + prisma_toolchain.run_prisma( [ _get_prisma_command(), "migrate", @@ -224,7 +224,6 @@ class ProxyExtrasDBManager: "--applied", "0_init", ], - check=True, timeout=prisma_command_timeout(), env=prisma_env, ) @@ -253,7 +252,7 @@ class ProxyExtrasDBManager: """Mark a specific migration as rolled back""" # Set up environment for offline mode if configured prisma_env = _get_prisma_env() - subprocess.run( + prisma_toolchain.run_prisma( [ _get_prisma_command(), "migrate", @@ -262,8 +261,6 @@ class ProxyExtrasDBManager: migration_name, ], timeout=prisma_command_timeout(), - check=True, - capture_output=True, env=prisma_env, ) @@ -315,11 +312,9 @@ class ProxyExtrasDBManager: def _resolve_specific_migration(migration_name: str): """Mark a specific migration as applied""" prisma_env = _get_prisma_env() - subprocess.run( + prisma_toolchain.run_prisma( [_get_prisma_command(), "migrate", "resolve", "--applied", migration_name], timeout=prisma_command_timeout(), - check=True, - capture_output=True, env=prisma_env, ) @@ -403,7 +398,7 @@ class ProxyExtrasDBManager: try: logger.info("Generating migration diff between DB and schema.prisma...") with open(diff_sql_path, "w") as f: - subprocess.run( + prisma_toolchain.run_prisma( [ _get_prisma_command(), "migrate", @@ -414,7 +409,6 @@ class ProxyExtrasDBManager: schema_path, "--script", ], - check=True, timeout=prisma_command_timeout(), stdout=f, env=_get_prisma_env(), @@ -437,7 +431,7 @@ class ProxyExtrasDBManager: migration_files = sorted(Path(migrations_dir).glob("*/migration.sql")) for mig_file in migration_files: try: - subprocess.run( + prisma_toolchain.run_prisma( [ _get_prisma_command(), "db", @@ -448,9 +442,6 @@ class ProxyExtrasDBManager: schema_path, ], timeout=prisma_command_timeout(), - check=True, - capture_output=True, - text=True, env=_get_prisma_env(), ) logger.info(f"Applied migration: {mig_file.parent.name}") @@ -483,7 +474,7 @@ class ProxyExtrasDBManager: applied_ok = False try: logger.info("Running prisma db execute to apply the migration diff...") - result = subprocess.run( + result = prisma_toolchain.run_prisma( [ _get_prisma_command(), "db", @@ -494,9 +485,6 @@ class ProxyExtrasDBManager: schema_path, ], timeout=prisma_command_timeout(), - check=True, - capture_output=True, - text=True, env=_get_prisma_env(), ) logger.info(f"prisma db execute stdout: {result.stdout}") @@ -525,7 +513,7 @@ class ProxyExtrasDBManager: for migration_name in migration_names: try: logger.info(f"Resolving migration: {migration_name}") - subprocess.run( + prisma_toolchain.run_prisma( [ _get_prisma_command(), "migrate", @@ -534,9 +522,6 @@ class ProxyExtrasDBManager: migration_name, ], timeout=prisma_command_timeout(), - check=True, - capture_output=True, - text=True, env=_get_prisma_env(), ) logger.debug(f"Resolved migration: {migration_name}") @@ -726,11 +711,12 @@ class ProxyExtrasDBManager: original_dir = os.getcwd() os.chdir(migrations_dir) try: - subprocess.run( + prisma_toolchain.run_prisma( [_get_prisma_command(), "db", "push", "--accept-data-loss"], timeout=prisma_command_timeout(), - check=True, env=_get_prisma_env(), + stdout=None, + stderr=None, ) return True except ( @@ -752,12 +738,9 @@ class ProxyExtrasDBManager: try: for attempt in range(4): try: - result = subprocess.run( + result = prisma_toolchain.run_prisma( [_get_prisma_command(), "migrate", "deploy"], timeout=deploy_timeout, - check=True, - capture_output=True, - text=True, env=_get_prisma_env(), ) logger.info(f"prisma migrate deploy stdout: {result.stdout}") @@ -1007,12 +990,9 @@ class ProxyExtrasDBManager: logger.info("Running prisma migrate deploy") try: # Set migrations directory for Prisma - result = subprocess.run( + result = prisma_toolchain.run_prisma( [_get_prisma_command(), "migrate", "deploy"], timeout=prisma_migrate_deploy_timeout(), - check=True, - capture_output=True, - text=True, env=_get_prisma_env(), ) logger.info(f"prisma migrate deploy stdout: {result.stdout}") @@ -1084,7 +1064,7 @@ class ProxyExtrasDBManager: f"Found failed migration: {failed_migration}, marking as rolled back" ) # Mark the failed migration as rolled back - subprocess.run( + prisma_toolchain.run_prisma( [ _get_prisma_command(), "migrate", @@ -1093,9 +1073,6 @@ class ProxyExtrasDBManager: failed_migration, ], timeout=prisma_command_timeout(), - check=True, - capture_output=True, - text=True, env=_get_prisma_env(), ) logger.info( @@ -1220,10 +1197,12 @@ class ProxyExtrasDBManager: if ProxyExtrasDBManager.spend_logs_is_partitioned(): raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR) # Use prisma db push with increased timeout - subprocess.run( + prisma_toolchain.run_prisma( [_get_prisma_command(), "db", "push", "--accept-data-loss"], timeout=prisma_command_timeout(), - check=True, + stdout=None, + stderr=None, + env=_get_prisma_env(), ) return True except subprocess.TimeoutExpired: diff --git a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py index 406f07eb792..2fea48a57da 100644 --- a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py +++ b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py @@ -42,7 +42,7 @@ def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path): "Error: P3018\nMigration name: 20250326162113_baseline\n" "Database error code: 42501\npermission denied for schema public" ) - with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): + with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)): with pytest.raises(RuntimeError, match="permission"): ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) @@ -60,7 +60,7 @@ def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path): "Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n" 'Reason: syntax error at or near "BRKN" LINE 42' ) - with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): + with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)): with pytest.raises(RuntimeError, match="cannot be auto-recovered"): ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) @@ -124,7 +124,7 @@ def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path): def fake_resolve(*args, **kwargs): resolve_called["n"] += 1 - monkeypatch.setattr("subprocess.run", fake_run) + monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", fake_run) monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve) ok = ProxyExtrasDBManager.setup_database(use_migrate=True) # v2 flag NOT set @@ -139,7 +139,7 @@ def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_pat (tmp_path / "schema.prisma").write_text("// stub") stderr = "db push error" - with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): + with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)): with pytest.raises(RuntimeError, match="prisma db push failed"): ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True) @@ -209,7 +209,7 @@ def test_v2_resolve_specific_migration_failure_raises_runtime_error( "Error: P3009\nMigration `20260101000000_some_migration` failed\n" "relation already exists" ) - with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): + with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)): with pytest.raises( RuntimeError, match="Failed to mark migration .* as applied" ): @@ -228,7 +228,7 @@ def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path): stdout = "Applied migration.\n" stderr = "" - monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult()) + monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", lambda *a, **kw: FakeResult()) resolve_called = {"n": 0} monkeypatch.setattr( @@ -296,7 +296,7 @@ def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path): "_resolve_specific_migration", lambda name: pytest.fail("a deadlocked migration must never be marked applied"), ) - monkeypatch.setattr("subprocess.run", _succeed_after(1, _DEADLOCK_P3018_STDERR)) + monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, _DEADLOCK_P3018_STDERR)) ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) assert ok is True @@ -309,7 +309,7 @@ def test_v2_p3018_persistent_deadlock_exhausts_attempts(monkeypatch, tmp_path): monkeypatch.setattr(ProxyExtrasDBManager, "_roll_back_migration", lambda name: None) with patch( - "subprocess.run", + "litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, _DEADLOCK_P3018_STDERR), ): with pytest.raises(RuntimeError, match="after 4 attempts"): @@ -343,7 +343,7 @@ def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_ "_resolve_specific_migration", lambda name: pytest.fail("a deadlocked migration must never be marked applied"), ) - monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr)) + monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr)) ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) assert ok is True @@ -372,7 +372,7 @@ def test_v2_p3009_empty_ledger_logs_rolls_back_and_retries(monkeypatch, tmp_path "_resolve_specific_migration", lambda name: pytest.fail("a deadlocked migration must never be marked applied"), ) - monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr)) + monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr)) ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) assert ok is True @@ -395,7 +395,7 @@ def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path): "_roll_back_migration", lambda name: pytest.fail("an unreadable ledger must not trigger a retry"), ) - monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr)) + monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr)) with pytest.raises(RuntimeError, match="cannot be auto-recovered"): ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) @@ -417,7 +417,7 @@ def test_v2_p3009_non_deadlock_ledger_row_still_raises(monkeypatch, tmp_path): lambda name: 'ERROR: syntax error at or near "BRKN"', ) - with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): + with patch("litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr)): with pytest.raises(RuntimeError, match="cannot be auto-recovered"): ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) @@ -427,7 +427,7 @@ def test_v2_bare_deadlock_stderr_retries(monkeypatch, tmp_path): waiter as victim) is retried, not fatal.""" _stub_v2_env(monkeypatch, tmp_path) monkeypatch.setattr( - "subprocess.run", _succeed_after(1, "Database error: deadlock detected") + "litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, "Database error: deadlock detected") ) ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) @@ -446,7 +446,7 @@ def test_v2_advisory_lock_timeout_retries(monkeypatch, tmp_path): """v2: the advisory-lock waiter that times out while a peer's retry holds the lock retries instead of dying.""" _stub_v2_env(monkeypatch, tmp_path) - monkeypatch.setattr("subprocess.run", _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR)) + monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR)) ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) assert ok is True @@ -456,7 +456,7 @@ def test_v2_p1002_without_advisory_lock_context_still_raises(monkeypatch, tmp_pa """v2: a plain P1002 (database unreachable) stays fatal.""" _stub_v2_env(monkeypatch, tmp_path) stderr = "Error: P1002\n\nThe database server at `db`:`5432` was reached but timed out." - monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr)) + monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(1, stderr)) with pytest.raises(RuntimeError, match="cannot be auto-recovered"): ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index b3d457707b8..7917ec8c00f 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -567,7 +567,7 @@ class TestResolveAllMigrationsLedger: return _FakeCompleted() return _FakeCompleted() - monkeypatch.setattr(utils_module.subprocess, "run", fake_run) + monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", fake_run) ProxyExtrasDBManager._resolve_all_migrations(str(tmp_path), "schema.prisma") return calls @@ -604,9 +604,9 @@ class TestPartitionedSpendLogsPushGuard: import litellm_proxy_extras.utils as utils_module def fail_run(cmd, **kwargs): - raise AssertionError(f"subprocess.run should not be called, got: {cmd}") + raise AssertionError(f"run_prisma should not be called, got: {cmd}") - monkeypatch.setattr(utils_module.subprocess, "run", fail_run) + monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", fail_run) def test_v1_db_push_fails_fast_with_guidance(self, monkeypatch): monkeypatch.setattr( diff --git a/tests/proxy_migration_tests/test_prisma_toolchain.py b/tests/proxy_migration_tests/test_prisma_toolchain.py index 733870f3239..0ed33193a9b 100644 --- a/tests/proxy_migration_tests/test_prisma_toolchain.py +++ b/tests/proxy_migration_tests/test_prisma_toolchain.py @@ -18,6 +18,7 @@ import ast import json import logging import os +import signal import sys import time from collections.abc import Callable @@ -47,6 +48,7 @@ FAKE_PRISMA = """#!{python} import json import os import pathlib +import subprocess import sys import time @@ -66,6 +68,9 @@ with log_path.open("a") as log: time.sleep(float(os.environ.get("FAKE_PRISMA_SLEEP", "0"))) if args[:2] == ["migrate", "deploy"]: if earlier_same_command == 0: + if os.environ.get("FAKE_PRISMA_GRANDCHILD_PIDFILE"): + grandchild = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)"]) + pathlib.Path(os.environ["FAKE_PRISMA_GRANDCHILD_PIDFILE"]).write_text(str(grandchild.pid)) time.sleep(float(os.environ.get("FAKE_PRISMA_FIRST_DEPLOY_SLEEP", "0"))) elif os.environ.get("FAKE_PRISMA_LATER_DEPLOY_STDERR"): print(os.environ["FAKE_PRISMA_LATER_DEPLOY_STDERR"], file=sys.stderr) @@ -272,6 +277,43 @@ def test_migrate_deploy_stops_at_its_own_timeout( assert elapsed < 30 +def _process_is_gone(pid: int, within_seconds: float) -> bool: + deadline = time.monotonic() + within_seconds + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + time.sleep(0.05) + return False + + +def test_a_timed_out_migrate_deploy_takes_its_process_tree_with_it( + toolchain_env: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The real CLI forks Node and a schema engine; a timeout must not leave them running.""" + _, log_path = toolchain_env + pidfile = tmp_path / "grandchild.pid" + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, "1") + monkeypatch.setenv("FAKE_PRISMA_FIRST_DEPLOY_SLEEP", "60") + monkeypatch.setenv("FAKE_PRISMA_LATER_DEPLOY_STDERR", "Error: P3018 permission denied for schema public") + monkeypatch.setenv("FAKE_PRISMA_GRANDCHILD_PIDFILE", str(pidfile)) + + with pytest.raises(RuntimeError, match="insufficient permissions"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + grandchild_pid = int(pidfile.read_text()) + try: + assert len(_deploy_calls(log_path)) == 2 + assert _process_is_gone(grandchild_pid, within_seconds=5) + finally: + try: + os.kill(grandchild_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def test_db_push_timeout_hint_names_the_per_command_budget( toolchain_env: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/test_litellm/proxy/db/test_replica_identity.py b/tests/test_litellm/proxy/db/test_replica_identity.py index ecfc6433ab1..9738fc9bd98 100644 --- a/tests/test_litellm/proxy/db/test_replica_identity.py +++ b/tests/test_litellm/proxy/db/test_replica_identity.py @@ -29,7 +29,7 @@ def test_hands_the_alter_statement_to_the_prisma_cli(): return subprocess.CompletedProcess(cmd, 0) with patch( - "litellm_proxy_extras.replica_identity.subprocess.run", side_effect=capture + "litellm_proxy_extras.replica_identity.run_prisma", side_effect=capture ): applied = apply_replica_identity_full( schema_path="/somewhere/schema.prisma", @@ -60,7 +60,7 @@ def test_hands_the_alter_statement_to_the_prisma_cli(): ) def test_every_failure_is_reported_instead_of_raised(failure): with patch( - "litellm_proxy_extras.replica_identity.subprocess.run", side_effect=failure + "litellm_proxy_extras.replica_identity.run_prisma", side_effect=failure ): assert ( apply_replica_identity_full( From 78ff5ac9cd144770f9e53931d190ca2d480d9d98 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 2 Sep 2026 18:31:25 -0700 Subject: [PATCH 174/480] feat(router): arm safeguard-refusal fallback on generic chains when no content-policy list exists (#39274) --- litellm/router.py | 30 ++++- .../router_utils/fallback_event_handlers.py | 32 ++++++ ...test_router_anthropic_messages_fallback.py | 106 ++++++++++++++++++ 3 files changed, 166 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 0af514fe8a2..303b22c9484 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -150,8 +150,10 @@ from litellm.router_utils.fallback_event_handlers import ( _check_non_standard_fallback_format, clear_pre_routing_selection, fallback_lookup_groups, + fallbacks_disabled_for_request, get_fallback_model_group_for_lookup_groups, get_pre_routing_selection, + record_disable_fallbacks, record_pre_routing_selection, run_async_fallback, ) @@ -5193,7 +5195,7 @@ class Router: if not has_generated_content and error_event is None else None ) - if refusal_stop_details is not None and self._has_content_policy_fallback(model, initial_kwargs): + if refusal_stop_details is not None and self._refusal_fallback_available(model, initial_kwargs): refusal_error = safeguard_refusal_error(model=model, stop_details=refusal_stop_details) raise MidStreamFallbackError( message=refusal_error.message, @@ -7266,6 +7268,7 @@ class Router: _fallback_metadata["original_model_group"] = model_group include_fallback_errors: Final = kwargs.get("include_fallback_errors", False) is True disable_fallbacks: Final[bool | None] = kwargs.pop("disable_fallbacks", False) + record_disable_fallbacks(kwargs, disable_fallbacks is True) fallbacks: Final[list | None] = kwargs.get("fallbacks", self.fallbacks) context_window_fallbacks: list | None = kwargs.get("context_window_fallbacks", self.context_window_fallbacks) content_policy_fallbacks: list | None = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) @@ -8131,6 +8134,29 @@ class Router: ) return False + def _refusal_fallback_available(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: + """ + Whether a safeguard refusal can actually be recovered by the dispatcher. A configured + content-policy list is authoritative; with none configured at all, the dispatcher falls + through to the generic fallbacks lookup, so the gate mirrors that reachability and arms + on a resolving generic chain (tier first, then the requested group, then "*"). + """ + if fallbacks_disabled_for_request(kwargs): + return False + content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) + if content_policy_fallbacks is not None: + return self._has_content_policy_fallback(model_group, kwargs) + if self._has_default_fallbacks(): + return True + fallbacks: Final = kwargs.get("fallbacks", self.fallbacks) + if fallbacks is None: + return False + resolved, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=fallbacks, + lookup_groups=fallback_lookup_groups(kwargs, model_group), + ) + return resolved is not None + def _should_raise_content_policy_error(self, model: str, response: ModelResponse, kwargs: dict) -> bool: """ Determines if a content policy error should be raised. @@ -8162,7 +8188,7 @@ class Router: return False if get_safeguard_refusal_stop_details(response) is None: return False - return self._has_content_policy_fallback(model, kwargs) + return self._refusal_fallback_available(model, kwargs) def _get_healthy_deployments(self, model: str, parent_otel_span: Span | None): _all_deployments: list = [] diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index f7855cb38ff..601b32c4386 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -263,6 +263,38 @@ def get_pre_routing_selection(kwargs: Mapping[str, Any]) -> str | None: return next((selected for selected in selections if isinstance(selected, str) and selected), None) +DISABLE_FALLBACKS_METADATA_KEY: Final = "_disable_fallbacks" + + +def record_disable_fallbacks(request_kwargs: Mapping[str, Any] | None, disabled: bool) -> None: + """ + Write-or-clear the request's disable_fallbacks verdict into the router-internal metadata + bucket. The wrapper pops the raw kwarg before any downstream frame runs, so the refusal + gate (which decides whether to convert a refusal into a recoverable error) needs this + carrier to know recovery is impossible. + """ + from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs + + if request_kwargs is None: + return + bucket: Final = request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)) + if not isinstance(bucket, dict): + return + if disabled: + bucket[DISABLE_FALLBACKS_METADATA_KEY] = True + else: + bucket.pop(DISABLE_FALLBACKS_METADATA_KEY, None) + + +def fallbacks_disabled_for_request(kwargs: Mapping[str, Any]) -> bool: + """True when this request opted out of fallbacks, read from the raw kwarg (pre-pop + snapshots keep it) or the router-internal bucket the wrapper stamps after popping it.""" + if kwargs.get("disable_fallbacks") is True: + return True + buckets: Final = (kwargs.get(name) for name in _ROUTER_METADATA_BUCKETS) + return any(isinstance(bucket, dict) and bucket.get(DISABLE_FALLBACKS_METADATA_KEY) is True for bucket in buckets) + + def fallback_lookup_groups(kwargs: Mapping[str, Any], model_group: str | None) -> tuple[str, ...]: """ Ordered keys for resolving a fallback chain: the tier a pre-routing hook selected wins, diff --git a/tests/router_unit_tests/test_router_anthropic_messages_fallback.py b/tests/router_unit_tests/test_router_anthropic_messages_fallback.py index 0c4d1dfc21e..4812d199c06 100644 --- a/tests/router_unit_tests/test_router_anthropic_messages_fallback.py +++ b/tests/router_unit_tests/test_router_anthropic_messages_fallback.py @@ -338,6 +338,112 @@ def test_record_pre_routing_selection_writes_only_the_internal_bucket(): assert kwargs["metadata"] == {"user_id": "u1"} +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_generic_only_row_recovers_safeguard_refusal(stream): + """With no content-policy list configured, a generic fallback row covers safeguard refusals, + so the dashboard's generic fallbacks work without config-only content_policy rows.""" + fake = FakeAnthropicUpstream() + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"fable-tier": ["opus-target"]}]) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", max_tokens=16, stream=stream, messages=[{"role": "user", "content": "hi"}] + ) + body = await _collect(response) if stream else response + + if stream: + assert b'"refusal"' not in body + assert b"text_delta" in body + else: + assert body["stop_reason"] == "end_turn" + assert len(fake.calls) == 2 + assert "claude-opus-5" in fake.calls[1] + + +@pytest.mark.asyncio +async def test_configured_content_policy_list_stays_authoritative_over_generic_rows(): + fake = FakeAnthropicUpstream() + router = Router( + model_list=[FABLE_TIER, OPUS_TARGET], + fallbacks=[{"fable-tier": ["opus-target"]}], + content_policy_fallbacks=[{"unrelated-group": ["opus-target"]}], + ) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", max_tokens=16, messages=[{"role": "user", "content": "hi"}] + ) + + assert response["stop_reason"] == "refusal" + assert len(fake.calls) == 1 + + +def test_refusal_fallback_available_arms_on_generic_rows_only_without_content_policy(): + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"tier-group": ["opus-target"]}]) + stamped = {"litellm_metadata": {PRE_ROUTING_SELECTED_MODEL_KEY: "tier-group"}} + + assert router._refusal_fallback_available("router-group", stamped) is True + assert router._refusal_fallback_available("router-group", {}) is False + assert router._refusal_fallback_available("router-group", {"content_policy_fallbacks": [{"other": ["x"]}]}) is False + + +def test_chat_content_filter_gate_unchanged_by_generic_rows(): + """The generic-row arming is scoped to /v1/messages safeguard refusals; the chat surface's + content_filter gate keeps its long-standing content-policy-only semantics.""" + from litellm.types.utils import Choices, ModelResponse + + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"fable-tier": ["opus-target"]}]) + response = ModelResponse(choices=[Choices(finish_reason="content_filter")]) + + assert router._should_raise_content_policy_error(model="fable-tier", response=response, kwargs={}) is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_disable_fallbacks_returns_the_refusal_instead_of_raising(stream): + """A request that opted out of fallbacks must receive the provider's refusal response, + never a ContentPolicyViolationError the dispatcher refuses to recover.""" + fake = FakeAnthropicUpstream() + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"fable-tier": ["opus-target"]}]) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", + max_tokens=16, + stream=stream, + disable_fallbacks=True, + messages=[{"role": "user", "content": "hi"}], + ) + body = await _collect(response) if stream else response + + if stream: + assert b'"stop_reason": "refusal"' in body + else: + assert body["stop_reason"] == "refusal" + assert len(fake.calls) == 1 + + +@pytest.mark.asyncio +async def test_disable_fallbacks_beats_a_content_policy_row_too(): + fake = FakeAnthropicUpstream() + router = Router( + model_list=[FABLE_TIER, OPUS_TARGET], + content_policy_fallbacks=[{"fable-tier": ["opus-target"]}], + ) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", + max_tokens=16, + disable_fallbacks=True, + messages=[{"role": "user", "content": "hi"}], + ) + + assert response["stop_reason"] == "refusal" + assert len(fake.calls) == 1 + + def test_refusal_gate_keys_on_pre_routing_tier_stamp(): router = _router(content_policy_fallbacks=[{"tier-group": ["opus-target"]}]) From b8680120ce4f0a05c0529eb5b884a765365aba2d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:45:18 -0700 Subject: [PATCH 175/480] fix(helm): render ingress-nginx compatible path types via ingress.controller ingress-nginx's admission webhook (strict-validate-path-type, on by default from v1.12.0 until v1.12.6 / v1.13.2 allowed dots again) rejects the chart's /favicon.ico Exact and /eu.assemblyai Prefix rules, so helm install fails on any cluster it fronts. A new ingress.controller value (alb, the default, or nginx) renders dotted built-in paths as ImplementationSpecific under nginx, which serves them as plain prefix locations, and drops the ALB-only /*.txt wildcard rule there. The default render is unchanged --- helm/litellm/templates/_helpers.tpl | 13 ++ helm/litellm/templates/ingress.yaml | 38 +++--- .../tests/ingress_controller_tests.yaml | 129 ++++++++++++++++++ helm/litellm/values.yaml | 11 ++ 4 files changed, 175 insertions(+), 16 deletions(-) create mode 100644 helm/litellm/tests/ingress_controller_tests.yaml diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 72f7f74bcf6..be6b9093f53 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -428,3 +428,16 @@ envFrom: {{- end }} {{- end }} {{- end -}} + +{{/* +ingress-nginx's admission webhook rejects a dot in an Exact or Prefix path +(strict-validate-path-type) and serves ImplementationSpecific as a plain +prefix location, so a dotted path takes that type there. +*/}} +{{- define "litellm.ingress.pathType" -}} +{{- if and (eq .controller "nginx") (contains "." .path) -}} +ImplementationSpecific +{{- else -}} +{{- .pathType -}} +{{- end -}} +{{- end -}} diff --git a/helm/litellm/templates/ingress.yaml b/helm/litellm/templates/ingress.yaml index f77ef537b02..0a215de5a56 100644 --- a/helm/litellm/templates/ingress.yaml +++ b/helm/litellm/templates/ingress.yaml @@ -5,6 +5,10 @@ {{- $gatewayPort := .Values.gateway.service.port -}} {{- $backendPort := .Values.backend.service.port -}} {{- $uiPort := .Values.ui.service.port -}} +{{- $controller := .Values.ingress.controller | default "alb" -}} +{{- if not (has $controller (list "alb" "nginx")) }} +{{- fail (printf "ingress.controller: unknown controller %q, expected one of alb, nginx" $controller) }} +{{- end }} {{/* Backends addressable from ingress.extraPaths, keyed by the `service` field. */}} @@ -27,10 +31,11 @@ /litellm-asset-prefix, so without /*.txt they fall to the backend catch-all → 404 → client-side navigation never settles and the login flow spins in an infinite redirect loop (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt - from the export; the rule only routes the request to it. Needs an ingress - controller whose ImplementationSpecific path is a wildcard pattern - (AWS ALB: `*` = 0+ chars); this chart targets the AWS Load Balancer - Controller. + from the export; the rule only routes the request to it. It needs an + ingress controller whose ImplementationSpecific path is a wildcard pattern + (AWS ALB: `*` = 0+ chars), so it is rendered for ingress.controller=alb + only: ingress-nginx serves ImplementationSpecific as a literal prefix + location, where /*.txt can never match. */}} {{- $uiPaths := list (dict "path" "/" "pathType" "Exact") @@ -38,8 +43,10 @@ (dict "path" "/litellm-asset-prefix" "pathType" "Prefix") (dict "path" "/_next" "pathType" "Prefix") (dict "path" "/ui" "pathType" "Prefix") - (dict "path" "/*.txt" "pathType" "ImplementationSpecific") -}} +{{- if eq $controller "alb" }} +{{- $uiPaths = append $uiPaths (dict "path" "/*.txt" "pathType" "ImplementationSpecific") }} +{{- end }} {{/* Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py. Versioned paths are listed explicitly to avoid routing management routes @@ -83,12 +90,6 @@ adding to it. */}} {{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}} -{{- range $uiPaths }} -{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path .pathType) }} -{{- end }} -{{- range $gatewayPrefixes }} -{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }} -{{- end }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -115,8 +116,10 @@ spec: paths: # --- UI (Next.js static export) --- {{- range $uiPaths }} + {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" .path "pathType" .pathType) }} + {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path $pathType) }} - path: {{ .path }} - pathType: {{ .pathType }} + pathType: {{ $pathType }} backend: service: name: {{ $uiName }} @@ -134,8 +137,10 @@ spec: port: number: {{ $gatewayPort }} {{- range $gatewayPrefixes }} + {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }} + {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }} - path: {{ . }} - pathType: Prefix + pathType: {{ $pathType }} backend: service: name: {{ $gatewayName }} @@ -147,10 +152,11 @@ spec: Rendered after every built-in path so an entry can never take precedence over a default, and before the backend catch-all. Position only decides the match on controllers that honour manifest - order: the AWS Load Balancer Controller this chart targets sorts - Exact paths first and Prefix paths longest-first, but keeps + order: the AWS Load Balancer Controller (ingress.controller=alb) + sorts Exact paths first and Prefix paths longest-first, but keeps ImplementationSpecific paths in manifest order, which is what the - /*.txt rule above already depends on. + /*.txt rule above already depends on. ingress-nginx ignores order + and serves the longest matching location. */}} {{- range $idx, $extra := .Values.ingress.extraPaths }} {{- if not (kindIs "map" $extra) }} diff --git a/helm/litellm/tests/ingress_controller_tests.yaml b/helm/litellm/tests/ingress_controller_tests.yaml new file mode 100644 index 00000000000..a86271a02be --- /dev/null +++ b/helm/litellm/tests/ingress_controller_tests.yaml @@ -0,0 +1,129 @@ +suite: test ingress.controller +templates: + - ingress.yaml +values: + - ./values/required.yaml +tests: + - it: keeps the AWS Load Balancer Controller path types by default + set: + ingress.enabled: true + asserts: + - contains: + path: spec.rules[0].http.paths + content: + path: /favicon.ico + pathType: Exact + backend: + service: + name: RELEASE-NAME-litellm-ui + port: + number: 3000 + - contains: + path: spec.rules[0].http.paths + content: + path: /eu.assemblyai + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 + - contains: + path: spec.rules[0].http.paths + content: + path: /*.txt + pathType: ImplementationSpecific + backend: + service: + name: RELEASE-NAME-litellm-ui + port: + number: 3000 + + - it: renders no dotted Exact or Prefix path for ingress-nginx, whose admission webhook rejects them + set: + ingress.enabled: true + ingress.controller: nginx + asserts: + - notMatchRegexRaw: + pattern: 'path: /\S*\.\S*\n\s+pathType: (Exact|Prefix)\n' + - contains: + path: spec.rules[0].http.paths + content: + path: /favicon.ico + pathType: ImplementationSpecific + backend: + service: + name: RELEASE-NAME-litellm-ui + port: + number: 3000 + - contains: + path: spec.rules[0].http.paths + content: + path: /eu.assemblyai + pathType: ImplementationSpecific + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 + + - it: drops the /*.txt wildcard for ingress-nginx and keeps every other route as is + set: + ingress.enabled: true + ingress.controller: nginx + asserts: + - notContains: + path: spec.rules[0].http.paths + content: + path: /*.txt + any: true + - contains: + path: spec.rules[0].http.paths + content: + path: /ui + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm-ui + port: + number: 3000 + - contains: + path: spec.rules[0].http.paths + content: + path: /test + pathType: Exact + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 + - equal: + path: spec.rules[0].http.paths[-1] + value: + path: / + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm-backend + port: + number: 4001 + + - it: rejects an extraPaths entry that repeats a built-in path at the pathType ingress-nginx renders it with + set: + ingress.enabled: true + ingress.controller: nginx + ingress.extraPaths: + - path: /favicon.ico + service: ui + pathType: ImplementationSpecific + asserts: + - failedTemplate: + errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it" + + - it: rejects a controller it has no path types for + set: + ingress.enabled: true + ingress.controller: traefik + asserts: + - failedTemplate: + errorMessage: 'ingress.controller: unknown controller "traefik", expected one of alb, nginx' diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 378c3b7a618..592bb6d6131 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -10,6 +10,17 @@ imagePullSecrets: [] ingress: enabled: false className: "" + # Which ingress controller serves this Ingress. Controllers disagree on the + # pathTypes they accept, so this picks the pathType of a few built-in paths: + # alb AWS Load Balancer Controller (default): Exact and Prefix paths plus + # the /*.txt wildcard that routes the UI's RSC payloads. + # nginx ingress-nginx: its admission webhook rejects a dot in an Exact or + # Prefix path (strict-validate-path-type, on by default from v1.12.0 + # until v1.12.6 / v1.13.2 allowed dots again), so /favicon.ico and + # /eu.assemblyai render as ImplementationSpecific, which nginx serves + # as a plain prefix location. /*.txt is dropped: nginx has no + # wildcard pathType, so that rule could never match there. + controller: alb annotations: {} host: "" # optional; if set, becomes the rule's host tls: [] From 872e115295f8f66d0a85b4e3db4b9c4b231b9675 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:46:00 -0700 Subject: [PATCH 176/480] fix(containers): pass upstream error status through and forward list pagination params The container retrieve, list, delete, create and file routes validated the provider's error body against the success model, so a deleted or unknown container and a rejected API key surfaced as 500 pydantic errors instead of the upstream 404 or 401. The handlers now raise the provider error class with the upstream status and message before transforming the response. GET /v1/containers dropped after, limit and order before calling the provider, and GET /v1/containers/{id}/files dropped the same three, so paginated list calls ignored their pagination arguments. Both routes now forward their declared query params. --- .../llms/custom_httpx/container_handler.py | 14 +- litellm/llms/custom_httpx/llm_http_handler.py | 169 ++++++++++-------- .../proxy/container_endpoints/endpoints.py | 11 +- .../container_endpoints/handler_factory.py | 20 ++- .../custom_httpx/test_llm_http_handler.py | 88 +++++++++ .../proxy/container_endpoints/__init__.py | 0 .../container_endpoints/test_endpoints.py | 64 +++++++ .../test_handler_factory.py | 69 +++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 30 +++- 9 files changed, 387 insertions(+), 78 deletions(-) create mode 100644 tests/test_litellm/proxy/container_endpoints/__init__.py create mode 100644 tests/test_litellm/proxy/container_endpoints/test_endpoints.py create mode 100644 tests/test_litellm/proxy/container_endpoints/test_handler_factory.py diff --git a/litellm/llms/custom_httpx/container_handler.py b/litellm/llms/custom_httpx/container_handler.py index dd20a8c2ed4..de72735e4ec 100644 --- a/litellm/llms/custom_httpx/container_handler.py +++ b/litellm/llms/custom_httpx/container_handler.py @@ -139,7 +139,7 @@ def _build_query_params( return {name: value if isinstance(value, str) else str(value) for name, value in supplied if value is not None} -def _error_message_from_response(response: httpx.Response) -> str: +def error_message_from_response(response: httpx.Response) -> str: try: body: Final = response.json() except ValueError: @@ -153,6 +153,16 @@ def _error_message_from_response(response: httpx.Response) -> str: return response.text +def raise_for_error_status(response: httpx.Response, container_provider_config: "BaseContainerConfig") -> None: + if not httpx.codes.is_error(response.status_code): + return + raise container_provider_config.get_error_class( + error_message=error_message_from_response(response), + status_code=response.status_code, + headers=response.headers, + ) + + def _transform_response( response: httpx.Response, returns_binary: bool, @@ -163,7 +173,7 @@ def _transform_response( if httpx.codes.is_error(response.status_code): raise BaseLLMException( status_code=response.status_code, - message=_error_message_from_response(response), + message=error_message_from_response(response), headers=dict(response.headers), ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 0f6966b0ae2..9e4e1bd5f1b 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -77,6 +77,7 @@ from litellm.llms.base_llm.vector_store_files.transformation import ( BaseVectorStoreFilesConfig, ) from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.llms.custom_httpx.container_handler import raise_for_error_status from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -8754,17 +8755,19 @@ class BaseLLMHTTPHandler: json=data, timeout=timeout, ) - - return container_provider_config.transform_container_create_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_create_response( + raw_response=response, + logging_obj=logging_obj, + ) async def async_container_create_handler( self, @@ -8830,17 +8833,19 @@ class BaseLLMHTTPHandler: json=data, timeout=timeout, ) - - return container_provider_config.transform_container_create_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_create_response( + raw_response=response, + logging_obj=logging_obj, + ) def container_list_handler( self, @@ -8920,17 +8925,19 @@ class BaseLLMHTTPHandler: headers=headers, params=params or None, ) - - return container_provider_config.transform_container_list_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_list_response( + raw_response=response, + logging_obj=logging_obj, + ) async def async_container_list_handler( self, @@ -8997,17 +9004,19 @@ class BaseLLMHTTPHandler: headers=headers, params=params or None, ) - - return container_provider_config.transform_container_list_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_list_response( + raw_response=response, + logging_obj=logging_obj, + ) def container_retrieve_handler( self, @@ -9085,17 +9094,19 @@ class BaseLLMHTTPHandler: headers=headers, params=params or None, ) - - return container_provider_config.transform_container_retrieve_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_retrieve_response( + raw_response=response, + logging_obj=logging_obj, + ) async def async_container_retrieve_handler( self, @@ -9162,17 +9173,19 @@ class BaseLLMHTTPHandler: headers=headers, params=params or None, ) - - return container_provider_config.transform_container_retrieve_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_retrieve_response( + raw_response=response, + logging_obj=logging_obj, + ) def container_delete_handler( self, @@ -9250,17 +9263,19 @@ class BaseLLMHTTPHandler: headers=headers, params=params or None, ) - - return container_provider_config.transform_container_delete_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_delete_response( + raw_response=response, + logging_obj=logging_obj, + ) async def async_container_delete_handler( self, @@ -9327,17 +9342,19 @@ class BaseLLMHTTPHandler: headers=headers, params=params or None, ) - - return container_provider_config.transform_container_delete_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_delete_response( + raw_response=response, + logging_obj=logging_obj, + ) def container_file_list_handler( self, @@ -9419,17 +9436,19 @@ class BaseLLMHTTPHandler: headers=headers, params=params or None, ) - - return container_provider_config.transform_container_file_list_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_file_list_response( + raw_response=response, + logging_obj=logging_obj, + ) async def async_container_file_list_handler( self, @@ -9498,17 +9517,19 @@ class BaseLLMHTTPHandler: headers=headers, params=params or None, ) - - return container_provider_config.transform_container_file_list_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_file_list_response( + raw_response=response, + logging_obj=logging_obj, + ) def container_file_content_handler( self, @@ -9584,17 +9605,19 @@ class BaseLLMHTTPHandler: headers=headers, params=params or None, ) - - return container_provider_config.transform_container_file_content_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_file_content_response( + raw_response=response, + logging_obj=logging_obj, + ) async def async_container_file_content_handler( self, @@ -9660,17 +9683,19 @@ class BaseLLMHTTPHandler: headers=headers, params=params or None, ) - - return container_provider_config.transform_container_file_content_response( - raw_response=response, - logging_obj=logging_obj, - ) - except Exception as e: raise self._handle_error( e=e, provider_config=container_provider_config, ) + raise_for_error_status( + response=response, + container_provider_config=container_provider_config, + ) + return container_provider_config.transform_container_file_content_response( + raw_response=response, + logging_obj=logging_obj, + ) ###### VECTOR STORE HANDLER ###### @staticmethod diff --git a/litellm/proxy/container_endpoints/endpoints.py b/litellm/proxy/container_endpoints/endpoints.py index eaa3db336a9..9e0f6fa741f 100644 --- a/litellm/proxy/container_endpoints/endpoints.py +++ b/litellm/proxy/container_endpoints/endpoints.py @@ -173,6 +173,9 @@ async def list_containers( request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + after: str | None = None, + limit: int | None = None, + order: str | None = None, ): """ Container list endpoint for retrieving a list of containers. @@ -208,7 +211,13 @@ async def list_containers( # Read query parameters query_params: Final = dict(request.query_params) - data: Final[dict[str, Any]] = {"query_params": query_params, "model": query_params.get("model")} + data: Final[dict[str, Any]] = { + "query_params": query_params, + "model": query_params.get("model"), + "after": after, + "limit": limit, + "order": order, + } # Extract custom_llm_provider using priority chain custom_llm_provider: Final = ( diff --git a/litellm/proxy/container_endpoints/handler_factory.py b/litellm/proxy/container_endpoints/handler_factory.py index aaee1d3e264..cc742f0520b 100644 --- a/litellm/proxy/container_endpoints/handler_factory.py +++ b/litellm/proxy/container_endpoints/handler_factory.py @@ -6,7 +6,9 @@ FastAPI route handlers for ALL container file endpoints. """ import json +from collections.abc import Mapping, Sequence from pathlib import Path +from types import MappingProxyType from typing import Any, Final from fastapi import APIRouter, Depends, Request, Response @@ -56,6 +58,7 @@ def _create_handler_for_path_params( route_type: str, returns_binary: bool = False, is_multipart: bool = False, + query_param_names: Sequence[str] = (), ): """ Dynamically create a handler with the correct path parameter signature. @@ -114,6 +117,7 @@ def _create_handler_for_path_params( user_api_key_dict=user_api_key_dict, route_type=route_type, path_params={"container_id": container_id}, + query_param_names=query_param_names, ) return handler_container_id @@ -133,6 +137,7 @@ def _create_handler_for_path_params( user_api_key_dict=user_api_key_dict, route_type=route_type, path_params={"container_id": container_id, "file_id": file_id}, + query_param_names=query_param_names, ) return handler_container_file @@ -150,6 +155,7 @@ def _create_handler_for_path_params( user_api_key_dict=user_api_key_dict, route_type=route_type, path_params={}, + query_param_names=query_param_names, ) return handler_no_params @@ -351,12 +357,17 @@ async def _process_multipart_upload_request( ) +def _declared_query_params(query_params: Mapping[str, str], query_param_names: Sequence[str]) -> Mapping[str, str]: + return MappingProxyType({name: query_params[name] for name in query_param_names if name in query_params}) + + async def _process_request( request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, route_type: str, path_params: dict[str, str], + query_param_names: Sequence[str] = (), ): """Common request processing logic.""" from litellm.proxy.proxy_server import ( @@ -376,6 +387,7 @@ async def _process_request( query_params: Final = dict(request.query_params) data: Final[dict[str, Any]] = { "query_params": query_params, + **_declared_query_params(query_params, query_param_names), **path_params, } @@ -452,7 +464,13 @@ def register_container_file_endpoints(router: APIRouter) -> None: is_multipart = endpoint_config.get("is_multipart", False) # Create handler with correct signature for path params - handler = _create_handler_for_path_params(path_params, route_type, returns_binary, is_multipart) + handler = _create_handler_for_path_params( + path_params, + route_type, + returns_binary, + is_multipart, + query_param_names=endpoint_config.get("query_params", ()), + ) # Register routes route_method = getattr(router, method) diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 1d583c16ad7..023e2d8843f 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -3098,3 +3098,91 @@ async def test_a_provider_that_keeps_rejecting_is_not_retried_forever_on_the_asy ) assert len(recorder.bodies) == 2 + + +CONTAINER_NOT_FOUND_BODY = { + "error": { + "message": "Container with id 'cntr_gone' not found.", + "type": "invalid_request_error", + "param": None, + "code": None, + } +} + +INVALID_API_KEY_BODY = { + "error": { + "message": "Incorrect API key provided: sk-proj-***. You can find your API key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", + "param": None, + "code": "invalid_api_key", + }, + "status": 401, +} + +CONTAINER_LIST_BODY = { + "object": "list", + "data": [{"id": "cntr_a", "object": "container", "created_at": 1, "status": "running", "name": "a"}], + "first_id": "cntr_a", + "last_id": "cntr_a", + "has_more": True, +} + + +def _container_sync_client(response: httpx.Response) -> HTTPHandler: + client = HTTPHandler() + client.client = httpx.Client(transport=httpx.MockTransport(lambda _request: response)) + return client + + +def _container_async_client(response: httpx.Response) -> AsyncHTTPHandler: + client = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(lambda _request: response)) + return client + + +def test_container_retrieve_handler_raises_upstream_error_status_and_message(): + from litellm.llms.openai.containers.transformation import OpenAIContainerConfig + + with pytest.raises(BaseLLMException) as exc_info: + BaseLLMHTTPHandler().container_retrieve_handler( + container_id="cntr_gone", + container_provider_config=OpenAIContainerConfig(), + litellm_params=GenericLiteLLMParams(api_key="sk-test"), + logging_obj=Mock(), + client=_container_sync_client(httpx.Response(404, json=CONTAINER_NOT_FOUND_BODY)), + ) + + assert exc_info.value.status_code == 404 + assert exc_info.value.message == "Container with id 'cntr_gone' not found." + + +@pytest.mark.asyncio +async def test_async_container_list_handler_raises_upstream_error_status_and_message(): + from litellm.llms.openai.containers.transformation import OpenAIContainerConfig + + with pytest.raises(BaseLLMException) as exc_info: + await BaseLLMHTTPHandler().async_container_list_handler( + container_provider_config=OpenAIContainerConfig(), + litellm_params=GenericLiteLLMParams(api_key="sk-rejected"), + logging_obj=Mock(), + client=_container_async_client(httpx.Response(401, json=INVALID_API_KEY_BODY)), + ) + + assert exc_info.value.status_code == 401 + assert exc_info.value.message == INVALID_API_KEY_BODY["error"]["message"] + + +@pytest.mark.asyncio +async def test_async_container_list_handler_transforms_success_response(): + from litellm.llms.openai.containers.transformation import OpenAIContainerConfig + + response = await BaseLLMHTTPHandler().async_container_list_handler( + container_provider_config=OpenAIContainerConfig(), + litellm_params=GenericLiteLLMParams(api_key="sk-test"), + logging_obj=Mock(), + limit=1, + client=_container_async_client(httpx.Response(200, json=CONTAINER_LIST_BODY)), + ) + + assert [container.id for container in response.data] == ["cntr_a"] + assert response.has_more is True diff --git a/tests/test_litellm/proxy/container_endpoints/__init__.py b/tests/test_litellm/proxy/container_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/proxy/container_endpoints/test_endpoints.py b/tests/test_litellm/proxy/container_endpoints/test_endpoints.py new file mode 100644 index 00000000000..3604da65258 --- /dev/null +++ b/tests/test_litellm/proxy/container_endpoints/test_endpoints.py @@ -0,0 +1,64 @@ +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.container_endpoints import endpoints +from litellm.types.containers.main import ContainerListResponse + +PROXY_SERVER_STUB = SimpleNamespace( + general_settings={}, + prisma_client=None, + llm_router=None, + proxy_config=None, + proxy_logging_obj=None, + select_data_generator=None, + user_api_base=None, + user_max_tokens=None, + user_model=None, + user_request_timeout=None, + user_temperature=None, + version="test", +) + + +def _client() -> TestClient: + app = FastAPI() + app.include_router(endpoints.router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="user-1") + return TestClient(app) + + +def test_list_containers_forwards_typed_pagination_params(monkeypatch): + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", PROXY_SERVER_STUB) + upstream = ContainerListResponse(object="list", data=[], has_more=True) + captured = {} + + class FakeProcessor: + def __init__(self, data): + captured["data"] = data + + async def base_process_llm_request(self, **kwargs): + return upstream + + async def _handle_llm_api_exception(self, **kwargs): + raise kwargs["e"] + + monkeypatch.setattr(endpoints, "ProxyBaseLLMRequestProcessing", FakeProcessor) + monkeypatch.setattr(endpoints, "filter_container_list_response", AsyncMock(return_value=upstream)) + + response = _client().get( + "/v1/containers", + params={"limit": "1", "order": "desc", "after": "cntr_prev"}, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200 + assert response.json()["has_more"] is True + assert captured["data"]["limit"] == 1 + assert captured["data"]["order"] == "desc" + assert captured["data"]["after"] == "cntr_prev" diff --git a/tests/test_litellm/proxy/container_endpoints/test_handler_factory.py b/tests/test_litellm/proxy/container_endpoints/test_handler_factory.py new file mode 100644 index 00000000000..01a03f6ef03 --- /dev/null +++ b/tests/test_litellm/proxy/container_endpoints/test_handler_factory.py @@ -0,0 +1,69 @@ +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.container_endpoints import endpoints, handler_factory + +PROXY_SERVER_STUB = SimpleNamespace( + general_settings={}, + prisma_client=None, + llm_router=None, + proxy_config=None, + proxy_logging_obj=None, + select_data_generator=None, + user_api_base=None, + user_max_tokens=None, + user_model=None, + user_request_timeout=None, + user_temperature=None, + version="test", +) + + +def _client() -> TestClient: + app = FastAPI() + app.include_router(endpoints.router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="user-1") + return TestClient(app) + + +def test_list_container_files_forwards_declared_query_params(monkeypatch): + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", PROXY_SERVER_STUB) + monkeypatch.setattr( + handler_factory, + "assert_user_can_access_container", + AsyncMock(return_value=("cntr_123", "openai")), + ) + captured = {} + + class FakeProcessor: + def __init__(self, data): + captured["data"] = data + + async def base_process_llm_request(self, **kwargs): + captured["route_type"] = kwargs["route_type"] + return {"object": "list", "data": [], "has_more": True} + + async def _handle_llm_api_exception(self, **kwargs): + raise kwargs["e"] + + monkeypatch.setattr(handler_factory, "ProxyBaseLLMRequestProcessing", FakeProcessor) + + response = _client().get( + "/v1/containers/cntr_123/files", + params={"limit": "1", "order": "desc", "after": "cfile_prev", "unknown": "x"}, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200 + assert captured["route_type"] == "alist_container_files" + assert captured["data"]["container_id"] == "cntr_123" + assert captured["data"]["limit"] == "1" + assert captured["data"]["order"] == "desc" + assert captured["data"]["after"] == "cfile_prev" + assert "unknown" not in captured["data"] diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 90c4f03bf08..c6a09202669 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -44052,7 +44052,11 @@ export interface operations { }; list_containers_containers_get: { parameters: { - query?: never; + query?: { + after?: string | null; + limit?: number | null; + order?: string | null; + }; header?: never; path?: never; cookie?: never; @@ -44068,6 +44072,15 @@ export interface operations { "application/json": unknown; }; }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; }; }; create_container_containers_post: { @@ -61090,7 +61103,11 @@ export interface operations { }; list_containers_v1_containers_get: { parameters: { - query?: never; + query?: { + after?: string | null; + limit?: number | null; + order?: string | null; + }; header?: never; path?: never; cookie?: never; @@ -61106,6 +61123,15 @@ export interface operations { "application/json": unknown; }; }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; }; }; create_container_v1_containers_post: { From e03eb6961c990b716daa7d9b74814fd295ee8201 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:53:56 -0700 Subject: [PATCH 177/480] fix(deps): bump uvloop to 0.22.1 so the proxy boots on Python 3.14 --- pyproject.toml | 2 +- tests/test_litellm/proxy/test_proxy_cli.py | 8 +++ uv.lock | 66 +++++++++++++--------- 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4b727a782c2..24ea1d3836b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ proxy = [ "gunicorn>=23.0.0,<24.0", "uvicorn>=0.33.0,<1.0", "granian>=2.7.4,<3.0", - "uvloop>=0.21.0,<1.0; sys_platform != 'win32'", + "uvloop>=0.22.1,<1.0; sys_platform != 'win32'", "fastapi>=0.136.3,<1.0", "starlette>=1.0.1,<2.0", "backoff>=2.2.1,<3.0", diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 7b3528f3a68..5f4cbcc5775 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -15,6 +15,8 @@ import urllib.parse as urlparse import uvicorn import yaml +from uvicorn.config import LOOP_FACTORIES +from uvicorn.importer import import_from_string from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server @@ -462,6 +464,12 @@ class TestProxyInitializationHelpers: with patch("sys.platform", "linux"): assert ProxyInitializationHelpers._get_loop_type() == "uvloop" + def test_selected_loop_factory_imports_on_this_interpreter(self): + loop_type = ProxyInitializationHelpers._get_loop_type() + if loop_type is None: + pytest.skip("uvicorn picks the loop itself on this platform") + assert callable(import_from_string(LOOP_FACTORIES[loop_type])) + @patch.dict(os.environ, {}, clear=True) def test_database_url_construction_with_special_characters(self): # Setup environment variables with special characters that need escaping diff --git a/uv.lock b/uv.lock index 67aa1014dd3..28cdd7fa711 100644 --- a/uv.lock +++ b/uv.lock @@ -4575,7 +4575,7 @@ requires-dist = [ { name = "tiktoken", specifier = ">=0.8.0,<1.0" }, { name = "tokenizers", specifier = ">=0.21.0,<1.0" }, { name = "uvicorn", marker = "extra == 'proxy'", specifier = ">=0.33.0,<1.0" }, - { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.21.0,<1.0" }, + { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.22.1,<1.0" }, { name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" }, ] provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] @@ -9947,34 +9947,46 @@ wheels = [ [[package]] name = "uvloop" -version = "0.21.0" +version = "0.22.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/c0/854216d09d33c543f12a44b393c402e89a920b1a0a7dc634c42de91b9cf6/uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3", size = 2492741, upload-time = "2024-10-14T23:38:35.489Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/76/44a55515e8c9505aa1420aebacf4dd82552e5e15691654894e90d0bd051a/uvloop-0.21.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ec7e6b09a6fdded42403182ab6b832b71f4edaf7f37a9a0e371a01db5f0cb45f", size = 1442019, upload-time = "2024-10-14T23:37:20.068Z" }, - { url = "https://files.pythonhosted.org/packages/35/5a/62d5800358a78cc25c8a6c72ef8b10851bdb8cca22e14d9c74167b7f86da/uvloop-0.21.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:196274f2adb9689a289ad7d65700d37df0c0930fd8e4e743fa4834e850d7719d", size = 801898, upload-time = "2024-10-14T23:37:22.663Z" }, - { url = "https://files.pythonhosted.org/packages/f3/96/63695e0ebd7da6c741ccd4489b5947394435e198a1382349c17b1146bb97/uvloop-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f38b2e090258d051d68a5b14d1da7203a3c3677321cf32a95a6f4db4dd8b6f26", size = 3827735, upload-time = "2024-10-14T23:37:25.129Z" }, - { url = "https://files.pythonhosted.org/packages/61/e0/f0f8ec84979068ffae132c58c79af1de9cceeb664076beea86d941af1a30/uvloop-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87c43e0f13022b998eb9b973b5e97200c8b90823454d4bc06ab33829e09fb9bb", size = 3825126, upload-time = "2024-10-14T23:37:27.59Z" }, - { url = "https://files.pythonhosted.org/packages/bf/fe/5e94a977d058a54a19df95f12f7161ab6e323ad49f4dabc28822eb2df7ea/uvloop-0.21.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10d66943def5fcb6e7b37310eb6b5639fd2ccbc38df1177262b0640c3ca68c1f", size = 3705789, upload-time = "2024-10-14T23:37:29.385Z" }, - { url = "https://files.pythonhosted.org/packages/26/dd/c7179618e46092a77e036650c1f056041a028a35c4d76945089fcfc38af8/uvloop-0.21.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:67dd654b8ca23aed0a8e99010b4c34aca62f4b7fce88f39d452ed7622c94845c", size = 3800523, upload-time = "2024-10-14T23:37:32.048Z" }, - { url = "https://files.pythonhosted.org/packages/57/a7/4cf0334105c1160dd6819f3297f8700fda7fc30ab4f61fbf3e725acbc7cc/uvloop-0.21.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c0f3fa6200b3108919f8bdabb9a7f87f20e7097ea3c543754cabc7d717d95cf8", size = 1447410, upload-time = "2024-10-14T23:37:33.612Z" }, - { url = "https://files.pythonhosted.org/packages/8c/7c/1517b0bbc2dbe784b563d6ab54f2ef88c890fdad77232c98ed490aa07132/uvloop-0.21.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0878c2640cf341b269b7e128b1a5fed890adc4455513ca710d77d5e93aa6d6a0", size = 805476, upload-time = "2024-10-14T23:37:36.11Z" }, - { url = "https://files.pythonhosted.org/packages/ee/ea/0bfae1aceb82a503f358d8d2fa126ca9dbdb2ba9c7866974faec1cb5875c/uvloop-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9fb766bb57b7388745d8bcc53a359b116b8a04c83a2288069809d2b3466c37e", size = 3960855, upload-time = "2024-10-14T23:37:37.683Z" }, - { url = "https://files.pythonhosted.org/packages/8a/ca/0864176a649838b838f36d44bf31c451597ab363b60dc9e09c9630619d41/uvloop-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a375441696e2eda1c43c44ccb66e04d61ceeffcd76e4929e527b7fa401b90fb", size = 3973185, upload-time = "2024-10-14T23:37:40.226Z" }, - { url = "https://files.pythonhosted.org/packages/30/bf/08ad29979a936d63787ba47a540de2132169f140d54aa25bc8c3df3e67f4/uvloop-0.21.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:baa0e6291d91649c6ba4ed4b2f982f9fa165b5bbd50a9e203c416a2797bab3c6", size = 3820256, upload-time = "2024-10-14T23:37:42.839Z" }, - { url = "https://files.pythonhosted.org/packages/da/e2/5cf6ef37e3daf2f06e651aae5ea108ad30df3cb269102678b61ebf1fdf42/uvloop-0.21.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4509360fcc4c3bd2c70d87573ad472de40c13387f5fda8cb58350a1d7475e58d", size = 3937323, upload-time = "2024-10-14T23:37:45.337Z" }, - { url = "https://files.pythonhosted.org/packages/8c/4c/03f93178830dc7ce8b4cdee1d36770d2f5ebb6f3d37d354e061eefc73545/uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c", size = 1471284, upload-time = "2024-10-14T23:37:47.833Z" }, - { url = "https://files.pythonhosted.org/packages/43/3e/92c03f4d05e50f09251bd8b2b2b584a2a7f8fe600008bcc4523337abe676/uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2", size = 821349, upload-time = "2024-10-14T23:37:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/a6/ef/a02ec5da49909dbbfb1fd205a9a1ac4e88ea92dcae885e7c961847cd51e2/uvloop-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d", size = 4580089, upload-time = "2024-10-14T23:37:51.703Z" }, - { url = "https://files.pythonhosted.org/packages/06/a7/b4e6a19925c900be9f98bec0a75e6e8f79bb53bdeb891916609ab3958967/uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc", size = 4693770, upload-time = "2024-10-14T23:37:54.122Z" }, - { url = "https://files.pythonhosted.org/packages/ce/0c/f07435a18a4b94ce6bd0677d8319cd3de61f3a9eeb1e5f8ab4e8b5edfcb3/uvloop-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb", size = 4451321, upload-time = "2024-10-14T23:37:55.766Z" }, - { url = "https://files.pythonhosted.org/packages/8f/eb/f7032be105877bcf924709c97b1bf3b90255b4ec251f9340cef912559f28/uvloop-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f", size = 4659022, upload-time = "2024-10-14T23:37:58.195Z" }, - { url = "https://files.pythonhosted.org/packages/3f/8d/2cbef610ca21539f0f36e2b34da49302029e7c9f09acef0b1c3b5839412b/uvloop-0.21.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:bfd55dfcc2a512316e65f16e503e9e450cab148ef11df4e4e679b5e8253a5281", size = 1468123, upload-time = "2024-10-14T23:38:00.688Z" }, - { url = "https://files.pythonhosted.org/packages/93/0d/b0038d5a469f94ed8f2b2fce2434a18396d8fbfb5da85a0a9781ebbdec14/uvloop-0.21.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:787ae31ad8a2856fc4e7c095341cccc7209bd657d0e71ad0dc2ea83c4a6fa8af", size = 819325, upload-time = "2024-10-14T23:38:02.309Z" }, - { url = "https://files.pythonhosted.org/packages/50/94/0a687f39e78c4c1e02e3272c6b2ccdb4e0085fda3b8352fecd0410ccf915/uvloop-0.21.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ee4d4ef48036ff6e5cfffb09dd192c7a5027153948d85b8da7ff705065bacc6", size = 4582806, upload-time = "2024-10-14T23:38:04.711Z" }, - { url = "https://files.pythonhosted.org/packages/d2/19/f5b78616566ea68edd42aacaf645adbf71fbd83fc52281fba555dc27e3f1/uvloop-0.21.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816", size = 4701068, upload-time = "2024-10-14T23:38:06.385Z" }, - { url = "https://files.pythonhosted.org/packages/47/57/66f061ee118f413cd22a656de622925097170b9380b30091b78ea0c6ea75/uvloop-0.21.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd53ecc9a0f3d87ab847503c2e1552b690362e005ab54e8a48ba97da3924c0dc", size = 4454428, upload-time = "2024-10-14T23:38:08.416Z" }, - { url = "https://files.pythonhosted.org/packages/63/9a/0962b05b308494e3202d3f794a6e85abe471fe3cafdbcf95c2e8c713aabd/uvloop-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553", size = 4660018, upload-time = "2024-10-14T23:38:10.888Z" }, + { url = "https://files.pythonhosted.org/packages/eb/14/ecceb239b65adaaf7fde510aa8bd534075695d1e5f8dadfa32b5723d9cfb/uvloop-0.22.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef6f0d4cc8a9fa1f6a910230cd53545d9a14479311e87e3cb225495952eb672c", size = 1343335, upload-time = "2025-10-16T22:16:11.43Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ae/6f6f9af7f590b319c94532b9567409ba11f4fa71af1148cab1bf48a07048/uvloop-0.22.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7cd375a12b71d33d46af85a3343b35d98e8116134ba404bd657b3b1d15988792", size = 742903, upload-time = "2025-10-16T22:16:12.979Z" }, + { url = "https://files.pythonhosted.org/packages/09/bd/3667151ad0702282a1f4d5d29288fce8a13c8b6858bf0978c219cd52b231/uvloop-0.22.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac33ed96229b7790eb729702751c0e93ac5bc3bcf52ae9eccbff30da09194b86", size = 3648499, upload-time = "2025-10-16T22:16:14.451Z" }, + { url = "https://files.pythonhosted.org/packages/b3/f6/21657bb3beb5f8c57ce8be3b83f653dd7933c2fd00545ed1b092d464799a/uvloop-0.22.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:481c990a7abe2c6f4fc3d98781cc9426ebd7f03a9aaa7eb03d3bfc68ac2a46bd", size = 3700133, upload-time = "2025-10-16T22:16:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/09/e0/604f61d004ded805f24974c87ddd8374ef675644f476f01f1df90e4cdf72/uvloop-0.22.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a592b043a47ad17911add5fbd087c76716d7c9ccc1d64ec9249ceafd735f03c2", size = 3512681, upload-time = "2025-10-16T22:16:18.07Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ce/8491fd370b0230deb5eac69c7aae35b3be527e25a911c0acdffb922dc1cd/uvloop-0.22.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1489cf791aa7b6e8c8be1c5a080bae3a672791fcb4e9e12249b05862a2ca9cec", size = 3615261, upload-time = "2025-10-16T22:16:19.596Z" }, + { url = "https://files.pythonhosted.org/packages/c7/d5/69900f7883235562f1f50d8184bb7dd84a2fb61e9ec63f3782546fdbd057/uvloop-0.22.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c60ebcd36f7b240b30788554b6f0782454826a0ed765d8430652621b5de674b9", size = 1352420, upload-time = "2025-10-16T22:16:21.187Z" }, + { url = "https://files.pythonhosted.org/packages/a8/73/c4e271b3bce59724e291465cc936c37758886a4868787da0278b3b56b905/uvloop-0.22.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b7f102bf3cb1995cfeaee9321105e8f5da76fdb104cdad8986f85461a1b7b77", size = 748677, upload-time = "2025-10-16T22:16:22.558Z" }, + { url = "https://files.pythonhosted.org/packages/86/94/9fb7fad2f824d25f8ecac0d70b94d0d48107ad5ece03769a9c543444f78a/uvloop-0.22.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21", size = 3753819, upload-time = "2025-10-16T22:16:23.903Z" }, + { url = "https://files.pythonhosted.org/packages/74/4f/256aca690709e9b008b7108bc85fba619a2bc37c6d80743d18abad16ee09/uvloop-0.22.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56a2d1fae65fd82197cb8c53c367310b3eabe1bbb9fb5a04d28e3e3520e4f702", size = 3804529, upload-time = "2025-10-16T22:16:25.246Z" }, + { url = "https://files.pythonhosted.org/packages/7f/74/03c05ae4737e871923d21a76fe28b6aad57f5c03b6e6bfcfa5ad616013e4/uvloop-0.22.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:40631b049d5972c6755b06d0bfe8233b1bd9a8a6392d9d1c45c10b6f9e9b2733", size = 3621267, upload-time = "2025-10-16T22:16:26.819Z" }, + { url = "https://files.pythonhosted.org/packages/75/be/f8e590fe61d18b4a92070905497aec4c0e64ae1761498cad09023f3f4b3e/uvloop-0.22.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:535cc37b3a04f6cd2c1ef65fa1d370c9a35b6695df735fcff5427323f2cd5473", size = 3723105, upload-time = "2025-10-16T22:16:28.252Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" }, + { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, + { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, + { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/182a2a593195bfd39842ea68ebc084e20c850806117213f5a299dfc513d9/uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705", size = 1358611, upload-time = "2025-10-16T22:16:36.833Z" }, + { url = "https://files.pythonhosted.org/packages/d2/14/e301ee96a6dc95224b6f1162cd3312f6d1217be3907b79173b06785f2fe7/uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8", size = 751811, upload-time = "2025-10-16T22:16:38.275Z" }, + { url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562, upload-time = "2025-10-16T22:16:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890, upload-time = "2025-10-16T22:16:40.547Z" }, + { url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472, upload-time = "2025-10-16T22:16:41.694Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" }, + { url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067, upload-time = "2025-10-16T22:16:44.503Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423, upload-time = "2025-10-16T22:16:45.968Z" }, + { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" }, + { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101, upload-time = "2025-10-16T22:16:49.318Z" }, + { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158, upload-time = "2025-10-16T22:16:50.517Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360, upload-time = "2025-10-16T22:16:52.646Z" }, + { url = "https://files.pythonhosted.org/packages/79/7b/b01414f31546caf0919da80ad57cbfe24c56b151d12af68cee1b04922ca8/uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289", size = 1454790, upload-time = "2025-10-16T22:16:54.355Z" }, + { url = "https://files.pythonhosted.org/packages/d4/31/0bb232318dd838cad3fa8fb0c68c8b40e1145b32025581975e18b11fab40/uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3", size = 796783, upload-time = "2025-10-16T22:16:55.906Z" }, + { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548, upload-time = "2025-10-16T22:16:57.008Z" }, + { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065, upload-time = "2025-10-16T22:16:58.206Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384, upload-time = "2025-10-16T22:16:59.36Z" }, + { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, ] [[package]] From e0e249225b1fd2d6620eaad90a13a5a8e62f713d Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 2 Sep 2026 18:55:22 -0700 Subject: [PATCH 178/480] feat(azure): support credential chain for storage (#39229) * feat(azure): support credential chain for storage * test(azure): clarify credential seam suppressions * fix(azure): read chain tokens in a worker thread The credential chain walk (IMDS probe, CLI subprocess) is blocking I/O, so reading the provider inline in async set_valid_azure_ad_token stalls every request on the worker's event loop --- .../azure_storage/azure_storage.py | 49 +++- .../azure_storage/test_azure_storage.py | 241 +++++++++++++++--- .../files/test_azure_blob_storage_backend.py | 47 +++- 3 files changed, 282 insertions(+), 55 deletions(-) diff --git a/litellm/integrations/azure_storage/azure_storage.py b/litellm/integrations/azure_storage/azure_storage.py index cb7175691df..16ef6920114 100644 --- a/litellm/integrations/azure_storage/azure_storage.py +++ b/litellm/integrations/azure_storage/azure_storage.py @@ -1,7 +1,9 @@ import asyncio import os import time +from collections.abc import Callable from datetime import datetime, timedelta +from functools import cache from typing import Final from litellm._logging import verbose_logger @@ -19,21 +21,40 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) +from litellm.secret_managers.get_azure_ad_token_provider import ( + get_azure_ad_token_provider, +) +from litellm.types.secret_managers.get_azure_ad_token_provider import ( + AzureCredentialType, +) from litellm.types.utils import StandardLoggingPayload +AZURE_STORAGE_TOKEN_SCOPE: Final = "https://storage.azure.com/.default" + + +@cache +def _cached_credential_chain_token_provider() -> Callable[[], str]: + return get_azure_ad_token_provider( + azure_scope=AZURE_STORAGE_TOKEN_SCOPE, + azure_credential=AzureCredentialType.DefaultAzureCredential, + ) + class AzureBlobStorageLogger(CustomBatchLogger): def __init__( self, + build_credential_chain_token_provider: Callable[ + [], Callable[[], str] + ] = _cached_credential_chain_token_provider, **kwargs, ): try: verbose_logger.debug("AzureBlobStorageLogger: in init azure blob storage logger") # Env Variables used for Azure Storage Authentication - self.tenant_id = os.getenv("AZURE_STORAGE_TENANT_ID") - self.client_id = os.getenv("AZURE_STORAGE_CLIENT_ID") - self.client_secret = os.getenv("AZURE_STORAGE_CLIENT_SECRET") + self.tenant_id = os.getenv("AZURE_STORAGE_TENANT_ID") or None + self.client_id = os.getenv("AZURE_STORAGE_CLIENT_ID") or None + self.client_secret = os.getenv("AZURE_STORAGE_CLIENT_SECRET") or None self.azure_storage_account_key: str | None = os.getenv("AZURE_STORAGE_ACCOUNT_KEY") # Required Env Variables for Azure Storage @@ -55,6 +76,9 @@ class AzureBlobStorageLogger(CustomBatchLogger): # Internal variables used for Token based authentication self.azure_auth_token: str | None = None # the Azure AD token to use for Azure Storage API requests self.token_expiry: datetime | None = None # the expiry time of the currentAzure AD token + self._build_credential_chain_token_provider: Callable[[], Callable[[], str]] = ( + build_credential_chain_token_provider + ) asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() @@ -231,10 +255,15 @@ class AzureBlobStorageLogger(CustomBatchLogger): """ Wrapper to set self.azure_auth_token to a valid Azure AD token, refreshing if necessary - Refreshes the token when: - - Token is expired - - Token is not set + Without a service principal configured, the credential chain provider is read every + time; it caches internally and refreshes against the token's real expiry. The read runs + in a worker thread because the chain walk (IMDS probe, CLI subprocess) is blocking """ + if self.tenant_id is None and self.client_id is None and self.client_secret is None: + token_provider: Final = self._build_credential_chain_token_provider() + self.azure_auth_token = await asyncio.to_thread(token_provider) + return + # Check if token needs refresh if self._azure_ad_token_is_expired() or self.azure_auth_token is None: verbose_logger.debug("Azure AD token needs refresh") @@ -273,13 +302,9 @@ class AzureBlobStorageLogger(CustomBatchLogger): tenant_id=tenant_id, client_id=client_id, client_secret=client_secret, - scope="https://storage.azure.com/.default", + scope=AZURE_STORAGE_TOKEN_SCOPE, ) - token: Final = token_provider() - - verbose_logger.debug("azure auth token %s", token) - - return token + return token_provider() def _azure_ad_token_is_expired(self): """ diff --git a/tests/test_litellm/integrations/azure_storage/test_azure_storage.py b/tests/test_litellm/integrations/azure_storage/test_azure_storage.py index 16c518ff412..a96eae0f9c3 100644 --- a/tests/test_litellm/integrations/azure_storage/test_azure_storage.py +++ b/tests/test_litellm/integrations/azure_storage/test_azure_storage.py @@ -1,10 +1,15 @@ +import asyncio import sys +import threading from unittest.mock import AsyncMock, MagicMock, patch import pytest - -from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger +from litellm.integrations.azure_storage.azure_storage import ( + AzureBlobStorageLogger, + _cached_credential_chain_token_provider, +) +from litellm.types.secret_managers.get_azure_ad_token_provider import AzureCredentialType from litellm.types.utils import StandardLoggingPayload @@ -25,6 +30,26 @@ def mock_gov_env_vars(mock_env_vars, monkeypatch): monkeypatch.setenv("AZURE_STORAGE_ENDPOINT_SUFFIX", "core.usgovcloudapi.net") +@pytest.fixture +def workload_identity_env_vars(monkeypatch): + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_NAME", "test-account") + monkeypatch.setenv("AZURE_STORAGE_FILE_SYSTEM", "test-container") + for unset in ( + "AZURE_STORAGE_TENANT_ID", + "AZURE_STORAGE_CLIENT_ID", + "AZURE_STORAGE_CLIENT_SECRET", + "AZURE_STORAGE_ACCOUNT_KEY", + "AZURE_STORAGE_ENDPOINT_SUFFIX", + "AZURE_CLIENT_SECRET", + "AZURE_CREDENTIAL", + "AZURE_SCOPE", + ): + monkeypatch.delenv(unset, raising=False) + monkeypatch.setenv("AZURE_CLIENT_ID", "workload-identity-client-id") + monkeypatch.setenv("AZURE_TENANT_ID", "workload-identity-tenant-id") + monkeypatch.setenv("AZURE_FEDERATED_TOKEN_FILE", "/var/run/secrets/azure/tokens/azure-identity-token") + + @pytest.mark.asyncio async def test_async_upload_payload_to_azure_blob_storage(mock_env_vars): """ @@ -32,17 +57,12 @@ async def test_async_upload_payload_to_azure_blob_storage(mock_env_vars): a payload to Azure Blob Storage using the 3-step process (create, append, flush). """ with ( - patch( - "litellm.integrations.azure_storage.azure_storage.get_async_httpx_client" - ) as mock_get_client, - patch( - "litellm.llms.azure.common_utils.get_azure_ad_token_from_entra_id" - ) as mock_get_token, + patch("litellm.integrations.azure_storage.azure_storage.get_async_httpx_client") as mock_get_client, + patch("litellm.integrations.azure_storage.azure_storage.get_azure_ad_token_from_entra_id") as mock_get_token, ): # Create mock HTTP client mock_http_client = AsyncMock() - mock_response = AsyncMock() - mock_response.raise_for_status = AsyncMock() + mock_response = MagicMock() mock_http_client.put.return_value = mock_response mock_http_client.patch.return_value = mock_response mock_get_client.return_value = mock_http_client @@ -79,9 +99,7 @@ async def test_async_upload_payload_to_azure_blob_storage(mock_env_vars): put_call_args = mock_http_client.put.call_args assert put_call_args[0][0] == f"{expected_base_url}?resource=file" assert put_call_args[1]["headers"]["x-ms-version"] is not None - assert ( - put_call_args[1]["headers"]["Authorization"] == "Bearer mock-azure-ad-token" - ) + assert put_call_args[1]["headers"]["Authorization"] == "Bearer mock-azure-ad-token" # Step 2: Append data assert mock_http_client.patch.call_count == 2 # Called for append and flush @@ -89,9 +107,7 @@ async def test_async_upload_payload_to_azure_blob_storage(mock_env_vars): assert append_call[0][0] == f"{expected_base_url}?action=append&position=0" assert append_call[1]["headers"]["x-ms-version"] is not None assert append_call[1]["headers"]["Content-Type"] == "application/json" - assert ( - append_call[1]["headers"]["Authorization"] == "Bearer mock-azure-ad-token" - ) + assert append_call[1]["headers"]["Authorization"] == "Bearer mock-azure-ad-token" assert "test-log-id-123" in append_call[1]["data"] # Step 3: Flush data @@ -110,9 +126,7 @@ async def test_async_upload_payload_uses_configured_endpoint_suffix(mock_gov_env AZURE_STORAGE_ENDPOINT_SUFFIX must reach the Entra-ID REST upload path so a sovereign-cloud account is addressed instead of the commercial dfs host. """ - with patch( - "litellm.integrations.azure_storage.azure_storage.get_async_httpx_client" - ) as mock_get_client: + with patch("litellm.integrations.azure_storage.azure_storage.get_async_httpx_client") as mock_get_client: mock_http_client = AsyncMock() mock_response = MagicMock() mock_http_client.put.return_value = mock_response @@ -127,17 +141,10 @@ async def test_async_upload_payload_uses_configured_endpoint_suffix(mock_gov_env await logger.async_upload_payload_to_azure_blob_storage(test_payload) - expected_base_url = ( - "https://test-account.dfs.core.usgovcloudapi.net/test-container/gov-log-id.json" - ) + expected_base_url = "https://test-account.dfs.core.usgovcloudapi.net/test-container/gov-log-id.json" assert mock_http_client.put.call_args[0][0] == f"{expected_base_url}?resource=file" - assert ( - mock_http_client.patch.call_args_list[0][0][0] - == f"{expected_base_url}?action=append&position=0" - ) - assert mock_http_client.patch.call_args_list[1][0][0].startswith( - f"{expected_base_url}?action=flush" - ) + assert mock_http_client.patch.call_args_list[0][0][0] == f"{expected_base_url}?action=append&position=0" + assert mock_http_client.patch.call_args_list[1][0][0].startswith(f"{expected_base_url}?action=flush") @pytest.mark.asyncio @@ -148,9 +155,7 @@ async def test_service_client_uses_configured_endpoint_suffix(mock_gov_env_vars) """ fake_aio_module = MagicMock() - with patch.dict( - sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module} - ): + with patch.dict(sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module}): logger = AzureBlobStorageLogger() await logger.get_service_client() @@ -160,14 +165,180 @@ async def test_service_client_uses_configured_endpoint_suffix(mock_gov_env_vars) ) +@pytest.mark.asyncio +async def test_upload_authenticates_through_the_credential_chain_under_workload_identity( + workload_identity_env_vars, +): + build_provider = MagicMock(return_value=lambda: "workload-identity-token") + with patch( # test-quality-ok: REST client is created inside the method; assert emitted request headers + "litellm.integrations.azure_storage.azure_storage.get_async_httpx_client" + ) as mock_get_client: + mock_http_client = AsyncMock() + mock_http_client.put.return_value = MagicMock() + mock_http_client.patch.return_value = MagicMock() + mock_get_client.return_value = mock_http_client + + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=build_provider) + await logger.async_upload_payload_to_azure_blob_storage({"id": "wif-log-id"}) + + build_provider.assert_called_once_with() + assert logger.azure_auth_token == "workload-identity-token" + sent_headers = [mock_http_client.put.call_args[1]["headers"]] + [ + call[1]["headers"] for call in mock_http_client.patch.call_args_list + ] + assert len(sent_headers) == 3 + assert all(headers["Authorization"] == "Bearer workload-identity-token" for headers in sent_headers) + + +def test_default_chain_provider_is_storage_scoped_and_built_once_per_process(): + _cached_credential_chain_token_provider.cache_clear() + with ( + patch( # test-quality-ok: assert the default factory's fixed scope and credential type without constructing Azure SDK credentials + "litellm.integrations.azure_storage.azure_storage.get_azure_ad_token_provider", + return_value=lambda: "chain-token", + ) as mock_builder + ): + first = _cached_credential_chain_token_provider() + second = _cached_credential_chain_token_provider() + _cached_credential_chain_token_provider.cache_clear() + + assert first is second + assert first() == "chain-token" + mock_builder.assert_called_once_with( + azure_scope="https://storage.azure.com/.default", + azure_credential=AzureCredentialType.DefaultAzureCredential, + ) + + +@pytest.mark.asyncio +async def test_chain_tokens_are_read_from_the_provider_on_every_refresh( + workload_identity_env_vars, +): + provider = MagicMock(side_effect=["chain-token-1", "chain-token-2"]) + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=MagicMock(return_value=provider)) + await logger.set_valid_azure_ad_token() + first_token = logger.azure_auth_token + await logger.set_valid_azure_ad_token() + + assert first_token == "chain-token-1" + assert logger.azure_auth_token == "chain-token-2" + assert provider.call_count == 2 + + +@pytest.mark.asyncio +async def test_chain_token_read_yields_to_the_event_loop(workload_identity_env_vars): + """ + The chain walk is blocking I/O (IMDS probe, CLI subprocess), so reading the provider + inline would stall every request on the worker. Prove other coroutines run during the read. + """ + loop_was_free = threading.Event() + + def provider() -> str: + if not loop_was_free.wait(timeout=5): + raise TimeoutError("the event loop never ran the observer while the token was being read") + return "chain-token" + + async def observer(): + loop_was_free.set() + + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=MagicMock(return_value=provider)) + observer_task = asyncio.create_task(observer()) + await logger.set_valid_azure_ad_token() + await observer_task + + assert logger.azure_auth_token == "chain-token" + + +@pytest.mark.asyncio +async def test_empty_string_service_principal_vars_still_use_the_credential_chain( + workload_identity_env_vars, monkeypatch +): + for name in ("AZURE_STORAGE_TENANT_ID", "AZURE_STORAGE_CLIENT_ID", "AZURE_STORAGE_CLIENT_SECRET"): + monkeypatch.setenv(name, "") + + logger = AzureBlobStorageLogger( + build_credential_chain_token_provider=MagicMock(return_value=lambda: "workload-identity-token") + ) + await logger.set_valid_azure_ad_token() + + assert logger.azure_auth_token == "workload-identity-token" + + +@pytest.mark.asyncio +async def test_client_secret_auth_still_uses_the_storage_scoped_service_principal(mock_env_vars): + build_provider = MagicMock() + with ( + patch( # test-quality-ok: assert the storage scope passed to the shared token factory without making an external auth call + "litellm.integrations.azure_storage.azure_storage.get_azure_ad_token_from_entra_id", + return_value=lambda: "client-secret-token", + ) as mock_entra_id + ): + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=build_provider) + await logger.set_valid_azure_ad_token() + + assert logger.azure_auth_token == "client-secret-token" + build_provider.assert_not_called() + assert mock_entra_id.call_args.kwargs == { + "tenant_id": "test-tenant-id", + "client_id": "test-client-id", + "client_secret": "test-client-secret", + "scope": "https://storage.azure.com/.default", + } + + +@pytest.mark.parametrize( + "missing_var", + ["AZURE_STORAGE_TENANT_ID", "AZURE_STORAGE_CLIENT_ID", "AZURE_STORAGE_CLIENT_SECRET"], +) +@pytest.mark.asyncio +async def test_partially_configured_service_principal_still_names_the_missing_variable( + mock_env_vars, monkeypatch, missing_var +): + monkeypatch.delenv(missing_var) + + build_provider = MagicMock() + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=build_provider) + with pytest.raises(ValueError, match=f"Missing required environment variable: {missing_var}"): + await logger.set_valid_azure_ad_token() + + build_provider.assert_not_called() + + +@pytest.mark.asyncio +async def test_account_key_auth_never_requests_a_token(workload_identity_env_vars, monkeypatch): + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_KEY", "dGVzdC1rZXk=") + + file_client = MagicMock() + file_client.create_file = AsyncMock() + file_client.append_data = AsyncMock() + file_client.flush_data = AsyncMock() + directory_client = MagicMock() + directory_client.exists = AsyncMock(return_value=True) + directory_client.get_file_client = MagicMock(return_value=file_client) + file_system_client = MagicMock() + file_system_client.get_directory_client = MagicMock(return_value=directory_client) + service_client = MagicMock() + service_client.get_file_system_client = MagicMock(return_value=file_system_client) + fake_aio_module = MagicMock() + fake_aio_module.DataLakeServiceClient = MagicMock(return_value=service_client) + + build_provider = MagicMock() + with patch.dict(sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module}): + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=build_provider) + await logger.async_upload_payload_to_azure_blob_storage({"id": "account-key-log-id"}) + + build_provider.assert_not_called() + assert logger.azure_auth_token is None + file_client.flush_data.assert_awaited_once() + assert fake_aio_module.DataLakeServiceClient.call_args.kwargs["credential"] == "dGVzdC1rZXk=" + + @pytest.mark.asyncio async def test_service_client_defaults_to_commercial_endpoint(mock_env_vars): """Unset AZURE_STORAGE_ENDPOINT_SUFFIX keeps the pre-existing commercial host""" fake_aio_module = MagicMock() - with patch.dict( - sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module} - ): + with patch.dict(sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module}): logger = AzureBlobStorageLogger() await logger.get_service_client() diff --git a/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py b/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py index 7e1139ca79a..222ead92b54 100644 --- a/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py +++ b/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py @@ -27,6 +27,20 @@ def mock_gov_env_vars(mock_env_vars, monkeypatch): monkeypatch.setenv("AZURE_STORAGE_ENDPOINT_SUFFIX", GOV_SUFFIX) +@pytest.fixture +def credential_chain_env_vars(monkeypatch): + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_NAME", "test-account") + monkeypatch.setenv("AZURE_STORAGE_FILE_SYSTEM", "test-container") + for name in ( + "AZURE_STORAGE_TENANT_ID", + "AZURE_STORAGE_CLIENT_ID", + "AZURE_STORAGE_CLIENT_SECRET", + "AZURE_STORAGE_ACCOUNT_KEY", + "AZURE_STORAGE_ENDPOINT_SUFFIX", + ): + monkeypatch.delenv(name, raising=False) + + def _make_backend() -> AzureBlobStorageBackend: backend = AzureBlobStorageBackend() backend.azure_auth_token = "mock-azure-ad-token" @@ -42,6 +56,29 @@ def _mock_upload_client() -> AsyncMock: return client +@pytest.mark.asyncio +async def test_upload_file_with_credential_chain(credential_chain_env_vars): + client = _mock_upload_client() + build_provider = MagicMock(return_value=lambda: "workload-identity-token") + + with patch( # test-quality-ok: the backend creates its REST client internally; assert the emitted authorization header + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=client + ): + backend = AzureBlobStorageBackend(build_credential_chain_token_provider=build_provider) + storage_url = await backend.upload_file( + file_content=b"hello", + filename="report.json", + content_type="application/json", + path_prefix="logs", + file_naming_strategy="original_filename", + ) + + build_provider.assert_called_once_with() + assert storage_url == "https://test-account.blob.core.windows.net/test-container/logs/report.json" + assert client.put.call_args[1]["headers"]["Authorization"] == "Bearer workload-identity-token" + assert client.patch.call_count == 2 + + @pytest.mark.parametrize( "env_fixture, expected_suffix", [("mock_env_vars", "core.windows.net"), ("mock_gov_env_vars", GOV_SUFFIX)], @@ -125,10 +162,7 @@ async def test_download_file_accepts_url_persisted_before_the_suffix_was_set(moc ) assert content == b"file-bytes" - assert ( - client.get.call_args[0][0] - == f"https://test-account.blob.{GOV_SUFFIX}/test-container/logs/report.json" - ) + assert client.get.call_args[0][0] == f"https://test-account.blob.{GOV_SUFFIX}/test-container/logs/report.json" @pytest.mark.parametrize( @@ -178,10 +212,7 @@ async def test_download_file_drops_query_string_from_the_stored_url(mock_env_var "https://test-account.blob.core.windows.net/test-container/logs/report.json?sig=redacted&se=2026" ) - assert ( - client.get.call_args[0][0] - == "https://test-account.blob.core.windows.net/test-container/logs/report.json" - ) + assert client.get.call_args[0][0] == "https://test-account.blob.core.windows.net/test-container/logs/report.json" @pytest.mark.parametrize( From c15f4e066f9f5b66ea36909b760dc740d1b2b543 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:59:28 -0700 Subject: [PATCH 179/480] fix(rag): let the managed store's params win over caller kwargs on the search call --- litellm/rag/main.py | 5 +++-- tests/test_litellm/rag/test_main.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/litellm/rag/main.py b/litellm/rag/main.py index 8ddc4c231dd..1f63152632e 100644 --- a/litellm/rag/main.py +++ b/litellm/rag/main.py @@ -246,7 +246,8 @@ async def _execute_query_pipeline( # 2. Search vector store # Forward allowlisted provider retrieval_config extras (region, embedding - # model, bucket, credential refs) to the search call; kwargs win on conflict. + # model, bucket, credential refs) to the search call; the managed store's + # params win on conflict. provider_search_params: Final = MappingProxyType( {k: v for k, v in retrieval_config.items() if k in _FORWARDABLE_RETRIEVAL_CONFIG_KEYS} ) @@ -257,7 +258,7 @@ async def _execute_query_pipeline( if k not in _SEARCH_ARGS_SET_BY_PIPELINE } ) - forwarded_search_params: Final = MappingProxyType({**provider_search_params, **store_search_params, **kwargs}) + forwarded_search_params: Final = MappingProxyType({**provider_search_params, **kwargs, **store_search_params}) with _suppressed_sub_call_billing(): search_response: Final = await litellm.vector_stores.asearch( vector_store_id=retrieval_config["vector_store_id"], diff --git a/tests/test_litellm/rag/test_main.py b/tests/test_litellm/rag/test_main.py index f119088b0e9..264bcd6fb75 100644 --- a/tests/test_litellm/rag/test_main.py +++ b/tests/test_litellm/rag/test_main.py @@ -394,8 +394,9 @@ async def test_aquery_forwards_vector_store_params_to_search_but_not_completion( Regression for LIT-6773: the server-trusted vector_store_params (a managed store's litellm_params) must reach the search call wholesale, including the connection keys the caller allowlist blocks, while the caller's own - retrieval_config overrides stay blocked and the completion never inherits - the store's connection params. + retrieval_config overrides stay blocked, the caller's top-level api_key and + api_base stay on the completion only, and the completion never inherits the + store's connection params. """ from unittest.mock import AsyncMock @@ -420,6 +421,8 @@ async def test_aquery_forwards_vector_store_params_to_search_but_not_completion( await litellm.aquery( model="gpt-4o-mini", messages=[{"role": "user", "content": "hello"}], + api_key="sk-llm-key", + api_base="https://llm.example.com", retrieval_config={ "vector_store_id": "customer_kb", "custom_llm_provider": "milvus", @@ -445,8 +448,10 @@ async def test_aquery_forwards_vector_store_params_to_search_but_not_completion( assert search_kwargs["milvus_text_field"] == "book_intro_text" assert search_kwargs["outputFields"] == ["book_intro_text"] fake_completion.assert_awaited_once() - store_only_keys = {"api_base", "api_key", "milvus_text_field", "outputFields"} - assert not (store_only_keys & set(fake_completion.await_args.kwargs)) + completion_kwargs = fake_completion.await_args.kwargs + assert completion_kwargs["api_key"] == "sk-llm-key" + assert completion_kwargs["api_base"] == "https://llm.example.com" + assert not ({"milvus_text_field", "outputFields"} & set(completion_kwargs)) def test_rag_call_types_are_registered(): From f0f5f1ec78c5fca4272dca3ec66bf50495fd73ab Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:03:17 -0700 Subject: [PATCH 180/480] test(proxy-extras): wrap an over-long monkeypatch line --- litellm-proxy-extras/tests/test_setup_database_fail_fast.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py index 2fea48a57da..040d67d25e4 100644 --- a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py +++ b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py @@ -446,7 +446,10 @@ def test_v2_advisory_lock_timeout_retries(monkeypatch, tmp_path): """v2: the advisory-lock waiter that times out while a peer's retry holds the lock retries instead of dying.""" _stub_v2_env(monkeypatch, tmp_path) - monkeypatch.setattr("litellm_proxy_extras.prisma_toolchain.run_prisma", _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR)) + monkeypatch.setattr( + "litellm_proxy_extras.prisma_toolchain.run_prisma", + _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR), + ) ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) assert ok is True From 8065ede40bb380d8947939120f95a740f970614f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:11:08 +0000 Subject: [PATCH 181/480] test(guardrails): expect the deduped end-of-stream scan in crowdstrike cadence test (#39467) Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrails/guardrail_hooks/test_crowdstrike_aidr.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py index 1b50ea53db2..a07157396df 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py @@ -1703,8 +1703,8 @@ async def _guard_calls_for_stream(handler: CrowdStrikeAIDRHandler, chunk_texts: @pytest.mark.parametrize( ("configured", "expected_calls"), [ - ({}, 3), - ({"streaming_sampling_rate": 2}, 6), + ({}, 2), + ({"streaming_sampling_rate": 2}, 5), ({"streaming_end_of_stream_only": True}, 1), ({"streaming_end_of_stream_only": True, "streaming_sampling_rate": 2}, 1), ], @@ -1712,7 +1712,10 @@ async def _guard_calls_for_stream(handler: CrowdStrikeAIDRHandler, chunk_texts: async def test_streaming_params_from_config_control_output_scan_cadence( configured: dict[str, object], expected_calls: int ) -> None: - """10 chunks: default samples at 5 and 10 plus the final pass, rate 2 samples 5 times plus final, end-of-stream scans once.""" + """10 chunks: default samples at 5 and 10, rate 2 samples 5 times, end-of-stream scans once. + + The final pass is skipped because chunk 10 already scanned the complete output. + """ handler = _initialize_from_config(mode="post_call", **configured) assert await _guard_calls_for_stream(handler, list("ABCDEFGHIJ")) == expected_calls From 7a81ae98e68746aab8aadad58ee6706a2b04de00 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 2 Sep 2026 19:15:14 -0700 Subject: [PATCH 182/480] fix(model_armor): handle Anthropic Messages and Responses streams in post_call (#39181) * fix(model_armor): handle Anthropic Messages and Responses streams in post_call The post_call streaming hook buffered every chunk and fed it to stream_chunk_builder, which only understands chat-completion deltas. /v1/messages streams raw Anthropic SSE bytes and /v1/responses streams typed Responses events, so both raised litellm.APIError and surfaced to the client as a 500 on every streamed request. Assemble each surface with its own reader, frame guardrail failures as terminal items in that surface's wire format, and pass the stream through unscanned when it cannot be assembled instead of raising. * fix(model_armor): classify the stream surface and fail closed when it cannot be assembled Decide the wire format explicitly instead of inferring it from a boolean pair, so an opaque raw SSE stream (the Google :streamGenerateContent route) is never refused in Anthropic framing, and a stream that cannot be assembled is blocked rather than released unscanned unless fail_on_error is disabled. Also scan Responses tool-call arguments, read the body only off a terminal Responses event, and record the applied guardrail on the fail-closed path. * test(model_armor): pin the error-only stream predicate against content-carrying streams is_sse_error_stream decides whether a buffered stream is forwarded to the client untouched, so a stream that still carries content must not qualify: the frames-only join drops typed chunks, an empty stream is not a refusal, and a content event may carry an empty error field. * fix(model_armor): let a streamed de-identify match mask instead of blocking A de-identify template reports MATCH_FOUND for every redaction it makes. The streaming block check omitted allow_sanitization, so with mask_response_content enabled that match read as a refusal and the client got a 400 where the non-streaming sibling returned the redacted text. Pass the flag through, as the non-streaming hook already does, and stamp the logged status from the same decision so the spend row agrees with what the client received. Also drop Any from the chat-completion assembler's parameter; stream_chunk_builder takes a bare list, so list[object] carries the mutability requirement without erasing the element type. * fix(model_armor): fail closed when a streamed de-identify match cannot be applied Allowing sanitization past the streaming block check is a promise to apply the redaction Model Armor asked for. Two paths broke that promise and released the buffered original instead: a match that comes back with no sanitized text, and a surface with no assembled body to rewrite. The outcome is now resolved once, before it is recorded, so the status stamped on request metadata agrees with what the client receives rather than reporting the success the block check alone would have implied. * fix: scan the deltas when a Responses stream ends without a body response.failed and response.incomplete are terminal events like response.completed, but a turn that broke mid-generation reports an empty output while the deltas ahead of it already spelled the answer out to the client. Reading only the terminal body found nothing to scan there, and the empty-content shortcut then forwarded every buffered delta past the guardrail. Fall back to the text the delta events carry whenever a Responses stream assembles to nothing. * fix: read the Responses delta event types off the event enum The hand-listed set left out response.mcp_call_arguments.delta, so a turn that streamed only MCP tool arguments and then reported an empty body still took the no-content shortcut and forwarded those chunks unscanned. Deriving the set from ResponsesAPIStreamEvents keeps it complete as the enum grows, and the str guard in the reader already covers any event whose delta is not text. * fix(model_armor): scan responses deltas alongside the terminal body A /v1/responses stream spells out reasoning summaries and tool-call arguments in delta events that its terminal body never repeats, so scanning the body alone handed every summary delta to the client unscanned whenever the body carried text. * fix(model_armor): scan responses delta fields apart from each other A Responses turn spells out its reasoning summary, its visible answer and its tool-call arguments in separate delta events. Joining every delta into one string let a finding form across the boundary between two fields that each carry nothing to find, so a safe stream could be blocked. Group the deltas by the field they belong to, join a field's own deltas as they streamed, and keep the fields apart. * fix(model_armor): scan each responses field once, not twice Separating delta fields stopped the terminal body from matching the delta text, so a turn with two visible fields sent Model Armor both copies. Only the delta fields the body does not already carry are appended now. --------- Co-authored-by: yassin --- litellm/proxy/guardrails/anthropic_sse.py | 64 +- .../model_armor/model_armor.py | 479 +++++-- .../guardrail_hooks/test_model_armor.py | 1151 +++++++++++++++++ 3 files changed, 1589 insertions(+), 105 deletions(-) diff --git a/litellm/proxy/guardrails/anthropic_sse.py b/litellm/proxy/guardrails/anthropic_sse.py index 50c05daee11..28220f09f00 100644 --- a/litellm/proxy/guardrails/anthropic_sse.py +++ b/litellm/proxy/guardrails/anthropic_sse.py @@ -13,6 +13,19 @@ from typing import Final from litellm.types.utils import Choices, ModelResponse +_ANTHROPIC_EVENT_TYPES: Final = frozenset( + { + "message_start", + "message_delta", + "message_stop", + "content_block_start", + "content_block_delta", + "content_block_stop", + "ping", + "error", + } +) + def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool: return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks) @@ -30,23 +43,43 @@ def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None: return None -def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None: +def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]: from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, ) + return tuple( + event_data + for event in AnthropicPassthroughLoggingHandler._split_sse_chunk_into_events(sse_stream) # pyright: ignore[reportPrivateUsage] # same parser the assembler uses + if (event_data := AnthropicPassthroughLoggingHandler._extract_sse_data(event)) is not None # pyright: ignore[reportPrivateUsage] # same parser the assembler uses; a private import beats forking SSE parsing + ) + + +def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None: return next( ( message - for event in AnthropicPassthroughLoggingHandler._split_sse_chunk_into_events(sse_stream) # pyright: ignore[reportPrivateUsage] # same parser the assembler uses - if (event_data := AnthropicPassthroughLoggingHandler._extract_sse_data(event)) is not None # pyright: ignore[reportPrivateUsage] # same parser the assembler uses; a private import beats forking SSE parsing - and event_data.get("type") == "message_start" - and isinstance(message := event_data.get("message"), dict) + for event_data in _parsed_sse_events(sse_stream) + if event_data.get("type") == "message_start" and isinstance(message := event_data.get("message"), dict) ), None, ) +def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool: + """Whether raw SSE frames are Anthropic Messages events. + + ``is_raw_sse_stream`` only says the chunks are unparsed bytes, and ``/v1/messages`` is not the + only endpoint that streams those: the Google ``:streamGenerateContent`` route marks its own + stream raw too. Reading its frames as Anthropic ones would refuse the response in a wire format + its client cannot parse, so the surface is decided on the event types actually present. + """ + sse_stream: Final = _joined_sse_stream(all_chunks) + if sse_stream is None: + return False + return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream)) + + def assemble_anthropic_sse_stream( all_chunks: Sequence[object], *, restore_identity: bool = False ) -> ModelResponse | None: @@ -111,6 +144,27 @@ def anthropic_sse_error_frames(message: str) -> tuple[bytes, ...]: ) +def is_sse_error_stream(all_chunks: Sequence[object]) -> bool: + """Whether the buffered stream carries nothing but error frames. + + post_call guardrails run in a chain, so a hook can be handed the terminal error frames an + earlier guardrail emitted when it blocked. Those carry no message to assemble, and replacing + them would hide the refusal the client is owed. Covers both wire forms a guardrail emits: the + Anthropic ``error`` event and the chat-completions ``{"error": ...}`` payload. + """ + if not all(isinstance(chunk, (str, bytes)) for chunk in all_chunks): + # A stream mixing typed chunks with an error frame still carries content to scan, and the + # frames-only join below would drop exactly the part that has to be scanned + return False + sse_stream: Final = _joined_sse_stream(all_chunks) + if sse_stream is None: + return False + events: Final = _parsed_sse_events(sse_stream) + return len(events) > 0 and all( + event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events + ) + + def anthropic_sse_chunks_from_response(assembled: ModelResponse) -> tuple[bytes, ...]: from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index d187b5b12e9..7d88a037f4f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -1,4 +1,5 @@ from collections.abc import AsyncGenerator, Mapping, Sequence +from enum import Enum, auto from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -27,12 +28,25 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.guardrails.anthropic_sse import ( + anthropic_sse_chunks_from_response, + anthropic_sse_error_frames, + assemble_anthropic_sse_stream, + is_anthropic_sse_stream, + is_raw_sse_stream, + is_sse_error_stream, +) from litellm.proxy.guardrails.guardrail_hooks.model_armor.file_scanning import ( MODEL_ARMOR_MAX_FILE_SIZE_BYTES, plan_file_scans, ) from litellm.types.guardrails import GuardrailEventHooks, LitellmParams -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionToolCallChunk, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) from litellm.types.utils import ( CallTypes, CallTypesLiteral, @@ -41,10 +55,33 @@ from litellm.types.utils import ( ModelResponse, ModelResponseStream, StandardLoggingGuardrailInformation, + TextCompletionResponse, ) GUARDRAIL_NAME: Final = "model_armor" +# Only these carry the finished output; response.created carries an empty body +_RESPONSES_TERMINAL_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete", "response.failed"}) + +# Every event whose ``delta`` is model output already on its way to the client. Read off the event +# enum rather than listed, so an event added there cannot quietly fall out of the scan +_RESPONSES_DELTA_EVENT_TYPES: Final = frozenset( + event.value for event in ResponsesAPIStreamEvents if event.value.endswith(".delta") +) + +# What makes two delta events part of the same field of the turn, rather than two fields that merely +# streamed next to each other +_RESPONSES_DELTA_FIELD_ATTRS: Final = ("type", "item_id", "output_index", "content_index", "summary_index") + + +class _StreamSurface(Enum): + """Wire format of a buffered streaming response, which decides how it is read and how it is refused.""" + + CHAT_COMPLETIONS = auto() + ANTHROPIC_MESSAGES = auto() + RESPONSES = auto() + OPAQUE_SSE = auto() + class ModelArmorAPIError(Exception): """Model Armor API failure (non-2xx), distinct from a content-block decision so @@ -322,19 +359,9 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): else: return {"modelResponseData": {"byteItem": {"byteDataType": file_type, "byteData": base64_data}}} - def _should_block_content(self, armor_response: dict, allow_sanitization: bool = False) -> bool: + def _should_block_content(self, armor_response: Mapping[str, Any], allow_sanitization: bool = False) -> bool: """Check if Model Armor response indicates content should be blocked, including both inspectResult and deidentifyResult.""" - sanitization_result: Final = armor_response.get("sanitizationResult", {}) - filter_results: Final = sanitization_result.get("filterResults", {}) - - # filterResults can be a dict (named keys) or a list (array of filter result dicts) - filter_result_items = [] - if isinstance(filter_results, dict): - filter_result_items = list(filter_results.values()) - elif isinstance(filter_results, list): - filter_result_items = filter_results - - for filt in filter_result_items: + for filt in self._filter_result_items(armor_response): # Check RAI, PI/Jailbreak, Malicious URI, CSAM, Virus scan as before if filt.get("raiFilterResult", {}).get("matchState") == "MATCH_FOUND": return True @@ -358,22 +385,12 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): # Fallback dict code removed; all cases handled above return False - def _get_sanitized_content(self, armor_response: dict) -> str | None: + def _get_sanitized_content(self, armor_response: Mapping[str, Any]) -> str | None: """ Get the sanitized content from a Model Armor response, if available. Looks for sanitized text in deidentifyResult, and falls back to root-level fields if not found. """ - result: Final = armor_response.get("sanitizationResult", {}) - filter_results: Final = result.get("filterResults", {}) - - # filterResults can be a dict (single filter) or a list (multiple filters) - filters: Final = ( - list(filter_results.values()) - if isinstance(filter_results, dict) - else filter_results - if isinstance(filter_results, list) - else [] - ) + filters: Final = self._filter_result_items(armor_response) # Prefer sanitized text from deidentifyResult if present for filter_entry in filters: @@ -397,6 +414,61 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): # Fallback: if Model Armor put sanitized text at the root, use it return armor_response.get("sanitizedText") or armor_response.get("text") + @staticmethod + def _filter_result_items(armor_response: Mapping[str, Any]) -> Sequence[Any]: + """Every filter result in a scan response. + + filterResults is a dict of named filters on most templates and a list on some, so both + shapes are flattened to the same list of filter entries. + """ + filter_results: Final = armor_response.get("sanitizationResult", {}).get("filterResults", {}) + if isinstance(filter_results, dict): + return list(filter_results.values()) + if isinstance(filter_results, list): + return filter_results + return [] + + def _has_deidentify_match(self, armor_response: Mapping[str, Any]) -> bool: + """Whether an SDP de-identify filter matched, i.e. Model Armor owes this response a redaction.""" + for filter_entry in self._filter_result_items(armor_response): + sdp = filter_entry.get("sdpFilterResult") + if sdp and sdp.get("deidentifyResult", {}).get("matchState") == "MATCH_FOUND": + return True + return False + + def _resolve_streaming_outcome( + self, + armor_response: Mapping[str, Any], + assembled_response: object, + content: str, + ) -> tuple[bool, str | None]: + """Whether to block the buffered stream, and the rewrite to emit when it is not blocked. + + A de-identify match only reaches here unblocked because masking is on, so the redaction it + stands for has to be both resolvable and emittable. Where it is neither, the buffered + original still carries what Model Armor matched on, so this fails closed instead of + releasing it. + """ + if self._should_block_content(armor_response, allow_sanitization=self.mask_response_content): + return True, None + if not self.mask_response_content: + return False, None + + sanitized_content: Final = self._get_sanitized_content(armor_response) + if not sanitized_content: + # No rewrite to apply. Harmless unless a match is outstanding, in which case applying + # nothing would hand back the very content that matched + return self._has_deidentify_match(armor_response), None + if sanitized_content == content: + return False, None + if not isinstance(assembled_response, ModelResponse): + verbose_proxy_logger.warning( + "Model Armor: sanitized content cannot be re-emitted on this streaming endpoint, " + "blocking the response instead" + ) + return True, None + return False, sanitized_content + @staticmethod def _append_armor_response(existing: object, armor_response: Mapping[str, object]) -> object: """Accumulate scan responses so a later text scan does not drop an earlier file scan. @@ -831,6 +903,185 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): return response + @staticmethod + def _is_terminal_error_stream(all_chunks: Sequence[object]) -> bool: + """Whether the buffered stream is only the refusal an earlier guardrail in the chain emitted. + + post_call guardrails are composed, so this hook can be handed the terminal error items a + preceding one produced. They carry no message to scan, and replacing them would hide the + refusal the client is owed. + """ + if all(getattr(chunk, "type", None) == "error" for chunk in all_chunks): + return True + return is_sse_error_stream(all_chunks) + + @staticmethod + def _classify_stream(all_chunks: Sequence[object]) -> _StreamSurface: + """Wire format the buffered chunks belong to.""" + if is_raw_sse_stream(all_chunks): + return ( + _StreamSurface.ANTHROPIC_MESSAGES if is_anthropic_sse_stream(all_chunks) else _StreamSurface.OPAQUE_SSE + ) + if any( + isinstance(event_type := getattr(chunk, "type", None), str) and event_type.startswith("response.") + for chunk in all_chunks + ): + return _StreamSurface.RESPONSES + return _StreamSurface.CHAT_COMPLETIONS + + @staticmethod + def _final_responses_api_response(all_chunks: Sequence[object]) -> ResponsesAPIResponse | None: + """Response body carried by a terminal ``/v1/responses`` event. + + A stream cut short before it completes has to read as unassembled rather than as a clean + empty response: ``response.created`` also carries a body, but an empty one, and scanning + that would release every buffered delta unscanned. + """ + return next( + ( + body + for chunk in reversed(all_chunks) + if getattr(chunk, "type", None) in _RESPONSES_TERMINAL_EVENT_TYPES + and isinstance(body := getattr(chunk, "response", None), ResponsesAPIResponse) + ), + None, + ) + + @staticmethod + def _responses_api_response_text(response: ResponsesAPIResponse) -> str: + """Text to scan in a Responses API response, tool-call arguments included. + + Tool calls are folded in because ``get_content_from_model_response`` folds them into what + the chat surface scans, and a Responses turn can carry its whole payload in them. + """ + from litellm.llms.openai.responses.guardrail_translation.handler import ( + OpenAIResponsesHandler, + ) + + texts: Final[list[str]] = [] # mutable-ok: the shared extractor below appends into caller-owned lists + tool_calls: Final[list[ChatCompletionToolCallChunk]] = [] # mutable-ok: the same extractor's tool-call sink + handler: Final = OpenAIResponsesHandler() + for output_idx, output_item in enumerate(response.output or ()): + handler._extract_output_text_and_images( # pyright: ignore[reportPrivateUsage] # the shared Responses output extractor; forking it would duplicate per-item parsing + output_item=output_item, + output_idx=output_idx, + texts_to_check=texts, + images_to_check=[], # mutable-ok: the extractor's images sink, unused here + task_mappings=[], # mutable-ok: the extractor's task-mapping sink, unused here + tool_calls_to_check=tool_calls, + ) + return "".join((*texts, *(json.dumps(tool_call) for tool_call in tool_calls))) + + def _extract_streaming_content(self, assembled_response: object) -> str: + """Text to scan from an assembled stream, for every endpoint shape this hook serves.""" + if isinstance(assembled_response, ResponsesAPIResponse): + return self._responses_api_response_text(assembled_response) + return self._extract_content_from_response(assembled_response) + + @staticmethod + def _responses_delta_field(chunk: object) -> tuple[str, ...]: + """Which field of the turn a delta event belongs to.""" + return tuple(str(getattr(chunk, attr, None)) for attr in _RESPONSES_DELTA_FIELD_ATTRS) + + @staticmethod + def _responses_delta_field_texts(all_chunks: Sequence[object]) -> tuple[str, ...]: + """Text each field of a ``/v1/responses`` turn has already spelled out in its delta events. + + One field's deltas are joined as they streamed, since a finding can be split across them, + and separate fields stay apart, so a reasoning summary running into the visible answer + cannot spell out a finding that neither of them carries. + """ + deltas: Final = tuple( + (ModelArmorGuardrail._responses_delta_field(chunk), delta) + for chunk in all_chunks + if getattr(chunk, "type", None) in _RESPONSES_DELTA_EVENT_TYPES + and isinstance(delta := getattr(chunk, "delta", None), str) + ) + return tuple( + "".join(delta for field, delta in deltas if field == streamed_field) + for streamed_field in dict.fromkeys(field for field, _ in deltas) + ) + + def _streaming_content_to_scan( + self, + assembled_response: object, + all_chunks: Sequence[object], + surface: _StreamSurface, + ) -> str: + """Text to scan for a buffered stream, which is everything the client is about to receive. + + A ``/v1/responses`` stream also spells out reasoning summaries and tool-call arguments in + delta events that its terminal body never repeats, so every delta field the body does not + already carry is scanned after it. + """ + content: Final = self._extract_streaming_content(assembled_response) + if surface is not _StreamSurface.RESPONSES: + return content + unscanned: Final = tuple(text for text in self._responses_delta_field_texts(all_chunks) if text not in content) + return "\n".join(part for part in (content, *unscanned) if part) + + @staticmethod + def _apply_sanitized_content(assembled_response: ModelResponse, sanitized_content: str) -> None: + """Replace every non-empty choice message with the Model Armor sanitized text.""" + for choice in assembled_response.choices: + if isinstance(choice, Choices) and choice.message.content: + choice.message.content = sanitized_content + + @staticmethod + def _assemble_chat_completion_stream( + all_chunks: list[object], # mutable-ok: stream_chunk_builder only accepts a mutable list + ) -> ModelResponse | TextCompletionResponse | None: + """Assemble chat-completion chunks, returning ``None`` when they cannot be assembled.""" + from litellm.main import stream_chunk_builder + + try: + return stream_chunk_builder(chunks=all_chunks) + except Exception as exc: + verbose_proxy_logger.warning("Model Armor: chat-completion stream assembly failed (%s)", exc) + return None + + def _assemble_stream( + self, all_chunks: Sequence[object], surface: _StreamSurface + ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None: + """Assemble the buffered stream into the scannable response its surface produces.""" + if surface is _StreamSurface.ANTHROPIC_MESSAGES: + return assemble_anthropic_sse_stream(all_chunks, restore_identity=True) + if surface is _StreamSurface.RESPONSES: + return self._final_responses_api_response(all_chunks) + if surface is _StreamSurface.OPAQUE_SSE: + return None + return self._assemble_chat_completion_stream(list(all_chunks)) + + @staticmethod + def _error_payload(exc: HTTPException) -> Mapping[str, object]: + """Error object for a terminal stream item, carrying the status the frame would otherwise lose.""" + detail: Final = exc.detail if isinstance(exc.detail, Mapping) else {"message": str(exc.detail)} + error_value: Final = detail.get("error", detail) + return { + **(dict(error_value) if isinstance(error_value, Mapping) else {"message": str(error_value)}), + "code": str(exc.status_code), + } + + @staticmethod + def _build_responses_error_items(exc: HTTPException) -> Sequence[object] | None: + """Responses API error events for a failure discovered after the stream started.""" + from litellm.llms.openai.responses.guardrail_translation.handler import ( + OpenAIResponsesHandler, + ) + + return OpenAIResponsesHandler().build_stream_error_items(exc, responses_so_far=None) + + def _stream_error_items(self, exc: HTTPException, *, surface: _StreamSurface) -> Sequence[object]: + """Frame a guardrail failure as terminal stream items in this endpoint's wire format.""" + payload: Final = self._error_payload(exc) + if surface is _StreamSurface.ANTHROPIC_MESSAGES: + return anthropic_sse_error_frames(str(payload.get("message", ""))) + if surface is _StreamSurface.RESPONSES and (responses_items := self._build_responses_error_items(exc)): + return responses_items + # Also the fallback when a surface cannot frame its own error: create_response() reads the + # status back out of this form, so the refusal keeps its code instead of arriving as a 200 + return (f"data: {json.dumps({'error': payload})}\n\n",) + async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -840,97 +1091,125 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): """Process streaming response chunks.""" from litellm.llms.base_llm.base_model_iterator import MockResponseIterator - from litellm.main import stream_chunk_builder + from litellm.proxy.common_utils.callback_utils import ( + add_guardrail_to_applied_guardrails_header, + ) # Collect all chunks - all_chunks: Final[list[ModelResponseStream]] = [] + all_chunks: Final[list[Any]] = [] async for chunk in response: all_chunks.append(chunk) + if not all_chunks or self._is_terminal_error_stream(all_chunks): + for chunk in all_chunks: + yield chunk + return + + surface: Final = self._classify_stream(all_chunks) + # Build complete response - assembled_response: Final = stream_chunk_builder(chunks=all_chunks) + assembled_response: Final = self._assemble_stream(all_chunks, surface) - if isinstance(assembled_response, ModelResponse): - # Extract content - content: Final = self._extract_content_from_response(assembled_response) + if assembled_response is None: + if not self.optional_params.get("fail_on_error", True): + verbose_proxy_logger.warning( + "Model Armor: streamed response could not be assembled for scanning, " + "forwarding it unscanned because fail_on_error is disabled" + ) + for chunk in all_chunks: + yield chunk + return - if content: - try: - # Check with Model Armor - armor_response: Final = await self.make_model_armor_request( - content=content, - source="model_response", - request_data=request_data, - ) + # Forwarding an unscannable stream would silently disable the guardrail, so fail closed + add_guardrail_to_applied_guardrails_header(request_data=request_data, guardrail_name=self.guardrail_name) + for error_item in self._stream_error_items( + HTTPException( + status_code=500, + detail=f"{self.guardrail_name}: streamed response could not be assembled for scanning, blocking it", + ), + surface=surface, + ): + yield error_item + return - # Attach Model Armor response & status to this request's metadata to avoid race conditions - if isinstance(request_data, dict): - _, metadata = get_or_create_metadata_bucket(request_data) - metadata["_model_armor_response"] = self._build_logging_response(armor_response) - metadata["_model_armor_status"] = ( - "blocked" if self._should_block_content(armor_response) else "success" - ) + # Extract content + content: Final = self._streaming_content_to_scan( + assembled_response=assembled_response, all_chunks=all_chunks, surface=surface + ) - # Add guardrail to applied_guardrails BEFORE potential blocking - # This ensures guardrail is recorded even when it blocks the request - from litellm.proxy.common_utils.callback_utils import ( - add_guardrail_to_applied_guardrails_header, - ) + if not content: + verbose_proxy_logger.debug("Model Armor: No text content in streaming response, skipping guardrail") + for chunk in all_chunks: + yield chunk + return - add_guardrail_to_applied_guardrails_header( - request_data=request_data, guardrail_name=self.guardrail_name - ) + try: + # Check with Model Armor + armor_response: Final = await self.make_model_armor_request( + content=content, + source="model_response", + request_data=request_data, + ) - # Check if blocked - if self._should_block_content(armor_response): - raise HTTPException( - status_code=400, - detail=self._build_block_error_detail( - "Streaming response blocked by Model Armor", - armor_response, - ), - ) + # Decide the outcome before recording it. Mirrors the non-streaming sibling: with + # masking on, a de-identify match is a redaction to apply rather than a refusal, but + # that only holds while the redaction can actually be delivered + blocked, sanitized_content = self._resolve_streaming_outcome( + armor_response=armor_response, + assembled_response=assembled_response, + content=content, + ) - # Apply sanitization if enabled - if self.mask_response_content: - sanitized_content: Final = self._get_sanitized_content(armor_response) - if sanitized_content and sanitized_content != content: - # Update assembled response - for choice in assembled_response.choices: - if isinstance(choice, Choices): - if choice.message.content: - choice.message.content = sanitized_content + # Attach Model Armor response & status to this request's metadata to avoid race conditions + if isinstance(request_data, dict): + _, metadata = get_or_create_metadata_bucket(request_data) + metadata["_model_armor_response"] = self._build_logging_response(armor_response) + metadata["_model_armor_status"] = "blocked" if blocked else "success" - # Return sanitized stream - mock_response: Final = MockResponseIterator(model_response=assembled_response) - async for chunk in mock_response: - yield chunk - return + # Add guardrail to applied_guardrails BEFORE potential blocking + # This ensures guardrail is recorded even when it blocks the request + add_guardrail_to_applied_guardrails_header(request_data=request_data, guardrail_name=self.guardrail_name) - except ModelArmorAPIError as e: - if self.optional_params.get("fail_on_error", True): - error_obj = {"message": e.detail, "code": "500"} - yield f"data: {json.dumps({'error': error_obj})}\n\n" - return - except HTTPException as e: - # Yield error as SSE event so create_response() detects it and - # returns a proper JSON error response with the correct status code. - # (Raising from a generator hits create_response's generic except → 500.) - detail: Final = e.detail if isinstance(e.detail, dict) else {"message": str(e.detail)} - error_value: Final = detail.get("error", detail) - if isinstance(error_value, dict): - error_obj = dict(error_value) - else: - error_obj = {"message": str(error_value)} - error_obj["code"] = str(e.status_code) - yield f"data: {json.dumps({'error': error_obj})}\n\n" + if blocked: + raise HTTPException( + status_code=400, + detail=self._build_block_error_detail( + "Streaming response blocked by Model Armor", + armor_response, + ), + ) + + if sanitized_content is not None and isinstance(assembled_response, ModelResponse): + self._apply_sanitized_content(assembled_response, sanitized_content) + + # Return sanitized stream + if surface is _StreamSurface.ANTHROPIC_MESSAGES: + for sse_chunk in anthropic_sse_chunks_from_response(assembled_response): + yield sse_chunk return - except Exception as e: - verbose_proxy_logger.error("Model Armor streaming error: %s", str(e), exc_info=True) - if self.optional_params.get("fail_on_error", True): - raise - else: - verbose_proxy_logger.debug("Model Armor: No text content in streaming response, skipping guardrail") + mock_response: Final = MockResponseIterator(model_response=assembled_response) + async for chunk in mock_response: + yield chunk + return + + except ModelArmorAPIError as e: + if self.optional_params.get("fail_on_error", True): + for error_item in self._stream_error_items( + HTTPException(status_code=500, detail=e.detail), surface=surface + ): + yield error_item + return + except HTTPException as e: + # Yield the error as a terminal stream item so create_response() detects it and returns + # a proper JSON error response with the correct status code. Raising from a generator + # instead hits create_response's generic except and becomes a 500. + for error_item in self._stream_error_items(e, surface=surface): + yield error_item + return + except Exception as e: + verbose_proxy_logger.error("Model Armor streaming error: %s", str(e), exc_info=True) + if self.optional_params.get("fail_on_error", True): + raise # Return original chunks if no sanitization needed for chunk in all_chunks: diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index da66c36328e..47089b7b1b1 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -15,6 +15,7 @@ import litellm.types.utils from litellm._logging import verbose_proxy_logger from litellm.caching import DualCache from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError +from litellm.proxy.guardrails.anthropic_sse import anthropic_sse_error_frames from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.model_armor import ModelArmorGuardrail from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ( @@ -3778,3 +3779,1153 @@ async def test_moderation_hook_skips_chat_traffic_when_configured_for_during_mcp assert result == data mock_post.assert_not_called() + + +_ANTHROPIC_SSE_CHUNKS = ( + b'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_1","type":"message",' + b'"role":"assistant","model":"claude","content":[],"usage":{"input_tokens":5,"output_tokens":0}}}\n\n', + b'event: content_block_start\ndata: {"type":"content_block_start","index":0,' + b'"content_block":{"type":"text","text":""}}\n\n', + b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,' + b'"delta":{"type":"text_delta","text":"my card is 4111-1111-1111-1111"}}\n\n', + b'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', + b'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},' + b'"usage":{"output_tokens":9}}\n\n', + b'event: message_stop\ndata: {"type":"message_stop"}\n\n', +) + +_MODEL_ARMOR_CLEAN = {"sanitizationResult": {"filterMatchState": "NO_MATCH_FOUND"}} + +_MODEL_ARMOR_BLOCK = { + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "sdp": { + "sdpFilterResult": { + "inspectResult": { + "matchState": "MATCH_FOUND", + "findings": [ + {"infoType": "CREDIT_CARD_NUMBER", "likelihood": "VERY_LIKELY"} + ], + } + } + } + }, + } +} + +# The root-level sanitizedText fallback in _get_sanitized_content, i.e. a rewrite that trips no +# named filter +_MODEL_ARMOR_SANITIZED = { + "sanitizedText": "my card is [REDACTED]", + "sanitizationResult": {"filterMatchState": "NO_MATCH_FOUND"}, +} + +# The shape a real de-identify template returns: the SDP filter both matches and hands back the +# rewritten text, so whether it blocks or masks is decided by allow_sanitization alone +_MODEL_ARMOR_DEIDENTIFIED = { + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "sdp": { + "sdpFilterResult": { + "deidentifyResult": { + "matchState": "MATCH_FOUND", + "data": {"text": "my card is [REDACTED]"}, + } + } + } + }, + } +} + + +def _chat_completion_chunks(): + """The chat-completions surface: typed ModelResponseStream chunks.""" + return ( + litellm.types.utils.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + index=0, + delta=litellm.types.utils.Delta(content="my card is 4111-1111-1111-1111"), + ) + ] + ), + litellm.types.utils.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + index=0, + delta=litellm.types.utils.Delta(content=""), + finish_reason="stop", + ) + ] + ), + ) + + +def _surface_guardrail(**kwargs): + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + **kwargs, + ) + guardrail._ensure_access_token_async = AsyncMock( + return_value=("test-token", "test-project") + ) + return guardrail + + +def _armor_post_mock(payload): + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.json = AsyncMock(return_value=payload) + return AsyncMock(return_value=mock_response) + + +async def _anthropic_sse_stream(): + for chunk in _ANTHROPIC_SSE_CHUNKS: + yield chunk + + +def _responses_api_events(): + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + completed = ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-4o-mini", + object="response", + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "my card is 4111-1111-1111-1111"}], + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + return ( + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta="my card is 4111-1111-1111-1111", + ), + ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=completed, + ), + ) + + +async def _drain_surface_hook(guardrail, chunks, request_data=None): + async def _stream(): + for chunk in chunks: + yield chunk + + return [ + item + async for item in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_stream(), + request_data=request_data + if request_data is not None + else { + "model": "claude-haiku", + "messages": [{"role": "user", "content": "show me a card"}], + "metadata": {"guardrails": ["model-armor-test"]}, + }, + ) + ] + + +@pytest.mark.asyncio +async def test_streaming_hook_scans_raw_anthropic_sse_instead_of_crashing(): + """A /v1/messages stream arrives as raw SSE bytes and must be assembled, then scanned. + + Regression for the 500 `Error building chunks for logging/streaming usage calculation`: + stream_chunk_builder calls .get() on each chunk, which raises on bytes. + """ + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, _ANTHROPIC_SSE_CHUNKS) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert "my card is 4111-1111-1111-1111" in scanned + assert tuple(delivered) == _ANTHROPIC_SSE_CHUNKS + + +@pytest.mark.asyncio +async def test_streaming_hook_scans_responses_api_events_instead_of_crashing(): + """A /v1/responses stream arrives as typed Responses events, which stream_chunk_builder + cannot subscript. The final response.completed event carries the text to scan.""" + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + events = _responses_api_events() + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, events) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert scanned == "my card is 4111-1111-1111-1111" + assert tuple(delivered) == events + + +@pytest.mark.asyncio +async def test_streaming_block_emits_anthropic_error_frame(): + """A block on /v1/messages must terminate the stream in Anthropic's error format. + + The OpenAI-shaped `data: {"error": ...}` frame the chat surface uses is rejected by + Anthropic clients. + """ + guardrail = _surface_guardrail() + + with patch.object( + guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_BLOCK) + ): + delivered = await _drain_surface_hook(guardrail, _ANTHROPIC_SSE_CHUNKS) + + body = b"".join(delivered) + assert b"event: error" in body + assert b'"type": "error"' in body + assert b"guardrail_error" in body + assert b"Streaming response blocked by Model Armor" in body + assert b"4111-1111-1111-1111" not in body + + +@pytest.mark.asyncio +async def test_streaming_block_emits_responses_api_error_event(): + """A block on /v1/responses must terminate the stream with a Responses ErrorEvent.""" + from litellm.types.llms.openai import ErrorEvent + + guardrail = _surface_guardrail() + + with patch.object( + guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_BLOCK) + ): + delivered = await _drain_surface_hook(guardrail, _responses_api_events()) + + assert len(delivered) == 1 + error_event = delivered[0] + assert isinstance(error_event, ErrorEvent) + assert error_event.error.type == "guardrail_error" + assert error_event.error.code == "400" + assert error_event.error.message == "Streaming response blocked by Model Armor" + + +@pytest.mark.asyncio +async def test_streaming_masking_re_emits_anthropic_sse_with_sanitized_text(): + """mask_response_content on /v1/messages must ship the sanitized text, not the original.""" + guardrail = _surface_guardrail(mask_response_content=True) + + with patch.object( + guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_SANITIZED) + ): + delivered = await _drain_surface_hook(guardrail, _ANTHROPIC_SSE_CHUNKS) + + body = b"".join(delivered) + assert b"[REDACTED]" in body + assert b"4111-1111-1111-1111" not in body + + +@pytest.mark.asyncio +async def test_streaming_masking_blocks_responses_api_stream(): + """A Responses event stream cannot be rebuilt from sanitized text, so releasing it would + ship the content the guardrail just rewrote. It is blocked instead.""" + from litellm.types.llms.openai import ErrorEvent + + guardrail = _surface_guardrail(mask_response_content=True) + + with patch.object( + guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_SANITIZED) + ): + delivered = await _drain_surface_hook(guardrail, _responses_api_events()) + + assert len(delivered) == 1 + assert isinstance(delivered[0], ErrorEvent) + assert delivered[0].error.code == "400" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("surface", ["anthropic_sse", "responses"]) +async def test_streaming_api_failure_frames_error_per_surface(surface): + """A Model Armor outage with fail_on_error must terminate the stream in the endpoint's + own error format rather than leaking an OpenAI SSE frame onto it.""" + from litellm.types.llms.openai import ErrorEvent + + guardrail = _surface_guardrail(fail_on_error=True) + chunks = _ANTHROPIC_SSE_CHUNKS if surface == "anthropic_sse" else _responses_api_events() + + mock_response = AsyncMock() + mock_response.status_code = 500 + mock_response.text = "Internal Server Error" + + with patch.object( + guardrail.async_handler, "post", AsyncMock(return_value=mock_response) + ): + delivered = await _drain_surface_hook(guardrail, chunks) + + assert len(delivered) >= 1 + if surface == "anthropic_sse": + assert b"event: error" in b"".join(delivered) + else: + assert isinstance(delivered[0], ErrorEvent) + assert delivered[0].error.code == "500" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "chunks", + [ + pytest.param( + (b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,' + b'"delta":{"type":"text_delta","text":"hi"}}\n\n',), + id="anthropic-sse-without-message-start", + ), + pytest.param(None, id="responses-stream-without-completed-event"), + pytest.param("created", id="responses-stream-cut-off-after-response-created"), + ], +) +async def test_streaming_hook_fails_closed_when_a_surface_stream_cannot_be_assembled(chunks): + """Forwarding an unscannable /v1/messages or /v1/responses stream would silently disable the + guardrail, so the stream is refused in its own wire format instead of released unscanned.""" + from litellm.types.llms.openai import ( + ErrorEvent, + OutputTextDeltaEvent, + ResponsesAPIStreamEvents, + ) + + if chunks is None or chunks == "created": + delta = OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta="my card is 4111-1111-1111-1111", + ) + # response.created carries a ResponsesAPIResponse too, but an empty one: reading the body + # off it would scan "" and release every buffered delta unscanned + chunks = (delta,) if chunks is None else (_responses_created_event(), delta) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + assert tuple(delivered) != tuple(chunks) + if isinstance(chunks[0], bytes): + joined = b"".join(item.encode() if isinstance(item, str) else item for item in delivered).decode() + assert "event: error" in joined + assert "could not be assembled for scanning" in joined + return + assert len(delivered) == 1 + assert isinstance(delivered[0], ErrorEvent) + assert "could not be assembled for scanning" in delivered[0].error.message + + +@pytest.mark.asyncio +async def test_streaming_hook_forwards_a_preceding_guardrails_error_item(): + """A guardrail earlier in the post_call chain replaces the stream with its own terminal + error item. That item is not a chat delta, and feeding it to stream_chunk_builder is what + surfaced the ticket's 500, so it has to be forwarded untouched instead.""" + from litellm.types.llms.openai import ( + ErrorEvent, + ErrorEventError, + ResponsesAPIStreamEvents, + ) + + chunks = ( + ErrorEvent( + type=ResponsesAPIStreamEvents.ERROR, + sequence_number=1, + error=ErrorEventError( + type="guardrail_error", + code="400", + message="Streaming response blocked by Model Armor", + param=None, + ), + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + assert tuple(delivered) == chunks + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "chunks", + [ + pytest.param(None, id="anthropic-error-event"), + pytest.param( + ('data: {"error": {"message": "Streaming response blocked by the first guardrail", "code": "400"}}\n\n',), + id="chat-completions-error-payload", + ), + ], +) +async def test_streaming_hook_forwards_a_preceding_guardrails_error_frame(chunks): + """Chained post_call guardrails hand each other their output. An earlier guardrail's error + frame carries no message to assemble, and replacing it would hide the real refusal.""" + if chunks is None: + chunks = anthropic_sse_error_frames("Streaming response blocked by the first guardrail") + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + assert tuple(delivered) == chunks + + +@pytest.mark.asyncio +async def test_streaming_responses_error_falls_back_to_sse_when_the_handler_declines(): + """build_stream_error_items may return None, which must not swallow the block into a clean + 200: the refusal falls back to the chat-completions SSE form that still carries the status.""" + from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ( + _StreamSurface, + ) + + class _DecliningGuardrail(ModelArmorGuardrail): + @staticmethod + def _build_responses_error_items(exc): + return None + + guardrail = _DecliningGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + exc = HTTPException(status_code=400, detail={"message": "blocked"}) + + items = guardrail._stream_error_items(exc, surface=_StreamSurface.RESPONSES) + + assert len(items) == 1 + assert '"code": "400"' in items[0] + assert "blocked" in items[0] + + +def _responses_created_event(): + from litellm.types.llms.openai import ( + ResponseCreatedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + return ResponseCreatedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_CREATED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-4o-mini", + object="response", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + + +@pytest.mark.asyncio +async def test_streaming_hook_refuses_an_opaque_sse_stream_without_anthropic_framing(): + """/v1/messages is not the only endpoint that streams raw SSE: the Google generateContent + route marks its own stream raw too. Its frames carry no Anthropic event types, so refusing + them in Anthropic's format would hand a Google client a body it cannot parse.""" + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + chunks = (b'data: {"candidates":[{"content":{"parts":[{"text":"my card is 4111"}]}}]}\n\n',) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + assert tuple(delivered) != chunks + body = "".join(item.decode() if isinstance(item, bytes) else item for item in delivered) + assert "could not be assembled for scanning" in body + assert "event: error" not in body + assert '"code": "500"' in body + + +@pytest.mark.asyncio +async def test_streaming_unassemblable_stream_is_forwarded_when_fail_on_error_is_disabled(): + """fail_on_error: false is a deliberate choice to degrade open, and it governs every other + path in this hook. The fail-closed refusal has to honour it too.""" + guardrail = _surface_guardrail(fail_on_error=False) + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + chunks = ( + b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,' + b'"delta":{"type":"text_delta","text":"hi"}}\n\n', + ) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + assert tuple(delivered) == chunks + + +@pytest.mark.asyncio +async def test_streaming_fail_closed_records_the_applied_guardrail(): + """A refusal that no header or log attributes to the guardrail leaves on-call unable to tell + a guardrail block apart from a provider failure.""" + guardrail = _surface_guardrail() + request_data = { + "model": "claude-haiku", + "messages": [{"role": "user", "content": "show me a card"}], + "metadata": {"guardrails": ["model-armor-test"]}, + } + chunks = ( + b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,' + b'"delta":{"type":"text_delta","text":"hi"}}\n\n', + ) + + with patch.object(guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_CLEAN)): + await _drain_surface_hook(guardrail, chunks, request_data=request_data) + + assert request_data["metadata"]["applied_guardrails"] == ["model-armor-test"] + + +@pytest.mark.asyncio +async def test_streaming_responses_tool_call_output_is_scanned(): + """An agentic /v1/responses turn can carry its whole payload in tool-call arguments, which + is what the chat surface already folds into the scanned text.""" + from litellm.types.llms.openai import ( + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-4o-mini", + object="response", + output=[ + { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "send_email", + "arguments": '{"body": "my card is 4111-1111-1111-1111"}', + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, (completed,)) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert "4111-1111-1111-1111" in scanned + assert tuple(delivered) == (completed,) + + +@pytest.mark.asyncio +async def test_streaming_hook_refuses_a_content_stream_that_ends_with_an_error_frame(): + """The chain-aware passthrough must stay narrow. A stream carrying real content plus a + trailing error frame is not a bare refusal to forward: the assembler cannot read it, and + releasing it would ship the buffered content unscanned.""" + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + chunks = (*_ANTHROPIC_SSE_CHUNKS, *anthropic_sse_error_frames("upstream gave up")) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + body = b"".join(delivered) + assert b"4111-1111-1111-1111" not in body + assert b"could not be assembled for scanning" in body + + +@pytest.mark.parametrize( + "chunks, expected, case", + [ + (anthropic_sse_error_frames("blocked upstream"), True, "anthropic-error-frames-only"), + ((f"data: {json.dumps({'error': {'message': 'blocked'}})}\n\n",), True, "chat-error-payload-only"), + ((), False, "empty-stream"), + ( + (b'event: message_delta\ndata: {"type":"message_delta","error":null}\n\n',), + False, + "content-event-carrying-a-null-error-field", + ), + ( + ( + litellm.types.utils.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + index=0, + delta=litellm.types.utils.Delta(content="my card is 4111-1111-1111-1111"), + ) + ] + ), + *anthropic_sse_error_frames("upstream gave up"), + ), + False, + "typed-content-chunks-plus-a-trailing-error-frame", + ), + ], +) +def test_is_sse_error_stream_only_matches_a_stream_that_is_nothing_but_refusals(chunks, expected, case): + """The chain-aware passthrough turns on this predicate, so anything it calls error-only is + forwarded to the client untouched. A stream that still carries content must not qualify: the + frames-only join drops typed chunks, and a content event may carry an empty ``error`` field.""" + from litellm.proxy.guardrails.anthropic_sse import is_sse_error_stream + + assert is_sse_error_stream(chunks) is expected, case + + +@pytest.mark.asyncio +async def test_streaming_hook_does_not_forward_typed_chunks_that_end_with_an_error_frame(): + """A stream mixing buffered content with a trailing refusal is not the bare refusal the chain + passthrough exists for. Forwarding it would release the content no scanner ever saw.""" + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + chunks = ( + litellm.types.utils.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + index=0, + delta=litellm.types.utils.Delta(content="my card is 4111-1111-1111-1111"), + ) + ] + ), + *anthropic_sse_error_frames("upstream gave up"), + ) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + body = b"".join(item if isinstance(item, bytes) else str(item).encode() for item in delivered) + assert b"4111-1111-1111-1111" not in body + assert b"could not be assembled for scanning" in body + + +def _delivered_bytes(delivered): + return b"".join( + item + if isinstance(item, bytes) + else item.encode() + if isinstance(item, str) + else str(item.model_dump() if hasattr(item, "model_dump") else item).encode() + for item in delivered + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("chunks, case", [(None, "chat_completions"), (_ANTHROPIC_SSE_CHUNKS, "anthropic_sse")]) +async def test_streaming_deidentify_match_masks_when_masking_is_enabled(chunks, case): + """A de-identify template reports MATCH_FOUND for every redaction it makes, so reading that + match as a refusal makes mask_response_content unusable on a stream: the client gets an error + where its non-streaming sibling gets redacted text. The block check has to allow sanitization + exactly as the non-streaming hook does.""" + guardrail = _surface_guardrail(mask_response_content=True) + post = _armor_post_mock(_MODEL_ARMOR_DEIDENTIFIED) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook( + guardrail, _chat_completion_chunks() if chunks is None else chunks + ) + + body = _delivered_bytes(delivered) + assert b"[REDACTED]" in body, case + assert b"4111-1111-1111-1111" not in body, case + assert b"blocked by Model Armor" not in body, case + + +@pytest.mark.asyncio +@pytest.mark.parametrize("chunks, case", [(None, "chat_completions"), (_ANTHROPIC_SSE_CHUNKS, "anthropic_sse")]) +async def test_streaming_deidentify_match_still_blocks_when_masking_is_disabled(chunks, case): + """Without mask_response_content there is nowhere to put the rewritten text, so the same + de-identify match must still end the stream rather than release the original.""" + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_DEIDENTIFIED) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook( + guardrail, _chat_completion_chunks() if chunks is None else chunks + ) + + body = _delivered_bytes(delivered) + assert b"Streaming response blocked by Model Armor" in body, case + assert b"4111-1111-1111-1111" not in body, case + + +@pytest.mark.asyncio +async def test_streaming_deidentify_match_logs_masked_run_as_success_not_blocked(): + """The status stamped on request metadata feeds the spend log, so it has to agree with what + the client actually received: a masked stream is a success, not a block.""" + guardrail = _surface_guardrail(mask_response_content=True) + request_data = { + "model": "claude-haiku", + "messages": [{"role": "user", "content": "show me a card"}], + "metadata": {"guardrails": ["model-armor-test"]}, + } + + with patch.object(guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_DEIDENTIFIED)): + await _drain_surface_hook(guardrail, _chat_completion_chunks(), request_data=request_data) + + assert request_data["metadata"]["_model_armor_status"] == "success" + + +# A de-identify template that matched but handed back no rewrite, e.g. because the transformation +# itself failed. The match still says the buffered original carries what it matched on +_MODEL_ARMOR_DEIDENTIFIED_NO_TEXT = { + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "sdp": {"sdpFilterResult": {"deidentifyResult": {"matchState": "MATCH_FOUND"}}} + }, + } +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("chunks, case", [(None, "chat_completions"), (_ANTHROPIC_SSE_CHUNKS, "anthropic_sse")]) +async def test_streaming_deidentify_match_without_a_rewrite_fails_closed(chunks, case): + """Allowing sanitization past the block check is a promise to apply the redaction. When Model + Armor matches but returns no sanitized text there is nothing to apply, and yielding the + buffered chunks would hand back exactly what it matched on.""" + guardrail = _surface_guardrail(mask_response_content=True) + post = _armor_post_mock(_MODEL_ARMOR_DEIDENTIFIED_NO_TEXT) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook( + guardrail, _chat_completion_chunks() if chunks is None else chunks + ) + + body = _delivered_bytes(delivered) + assert b"4111-1111-1111-1111" not in body, case + assert b"Streaming response blocked by Model Armor" in body, case + + +@pytest.mark.asyncio +async def test_streaming_status_records_a_surface_that_cannot_carry_the_rewrite_as_blocked(): + """The Responses surface has no assembled body to rewrite, so a de-identify match ends as a + refusal. The status stamped on metadata feeds the spend log and has to say so rather than + reporting the success the block check alone would have implied.""" + guardrail = _surface_guardrail(mask_response_content=True) + request_data = { + "model": "gpt-4o-mini", + "input": "show me a card", + "metadata": {"guardrails": ["model-armor-test"]}, + } + + with patch.object(guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_DEIDENTIFIED)): + delivered = await _drain_surface_hook( + guardrail, _responses_api_events(), request_data=request_data + ) + + body = _delivered_bytes(delivered) + assert b"4111-1111-1111-1111" not in body + assert b"Streaming response blocked by Model Armor" in body + assert request_data["metadata"]["_model_armor_status"] == "blocked" + + +def _responses_api_events_truncated(terminal: str): + """A /v1/responses stream whose text went out as deltas and whose terminal event reports no body. + + ``response.failed`` and ``response.incomplete`` are terminal like ``response.completed``, but a + turn that broke mid-generation reports an empty ``output`` while the deltas ahead of it already + spelled the answer out to the client. + """ + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + empty_body = ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-4o-mini", + object="response", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + terminal_event = ( + ResponseFailedEvent(type=ResponsesAPIStreamEvents.RESPONSE_FAILED, response=empty_body) + if terminal == "failed" + else ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=empty_body) + ) + return ( + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta="my card is 4111-1111-1111-1111", + ), + terminal_event, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("terminal", ["failed", "incomplete"]) +async def test_streaming_responses_terminal_event_without_a_body_still_scans_the_deltas(terminal): + """A /v1/responses turn that broke mid-generation has still delivered its deltas. + + Reading only the terminal body would find nothing to scan and hand every buffered delta to the + client untouched, so the deltas themselves are what gets scanned. + """ + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_BLOCK) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, _responses_api_events_truncated(terminal)) + + post.assert_called_once() + assert "4111-1111-1111-1111" in post.call_args.kwargs["json"]["modelResponseData"]["text"] + rendered = "".join(str(item) for item in delivered) + assert "4111-1111-1111-1111" not in rendered + assert "Streaming response blocked by Model Armor" in rendered + + +@pytest.mark.asyncio +async def test_streaming_responses_mcp_argument_deltas_are_scanned_when_the_body_is_empty(): + """A turn that only streamed MCP tool arguments still handed the client a payload. + + The delta fallback is read off the event enum rather than listed by hand, so an argument event + that carries no `output_text` cannot fall out of the scan. + """ + from litellm.types.llms.openai import ( + MCPCallArgumentsDeltaEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + chunks = ( + MCPCallArgumentsDeltaEvent( + type=ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA, + output_index=0, + item_id="mcp_1", + delta='{"note": "my card is 4111-1111-1111-1111"}', + sequence_number=0, + ), + ResponseIncompleteEvent( + type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-4o-mini", + object="response", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_BLOCK) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_called_once() + assert "4111-1111-1111-1111" in post.call_args.kwargs["json"]["modelResponseData"]["text"] + rendered = "".join(str(item) for item in delivered) + assert "4111-1111-1111-1111" not in rendered + assert "Streaming response blocked by Model Armor" in rendered + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "with_output_text_delta", + [True, False], + ids=["summary-and-text-deltas", "summary-delta-only"], +) +async def test_streaming_responses_reasoning_summary_deltas_are_scanned_alongside_the_body(with_output_text_delta): + """A reasoning turn streams its summary in deltas the terminal body never repeats. + + Reading only the body scans the visible answer and hands the client every summary delta + unscanned, so the body and the deltas are scanned together. + """ + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ReasoningSummaryTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + answer = "the weather is fine" + summary_delta = ReasoningSummaryTextDeltaEvent( + type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, + item_id="rs_1", + output_index=0, + delta="the user said my card is 4111-1111-1111-1111", + ) + text_deltas = ( + ( + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=1, + content_index=0, + delta=answer, + ), + ) + if with_output_text_delta + else () + ) + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-5-mini", + object="response", + output=[ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": answer, "annotations": []}], + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_BLOCK) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, (summary_delta, *text_deltas, completed)) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert "4111-1111-1111-1111" in scanned + assert answer in scanned + assert scanned.count(answer) == 1 + rendered = "".join(str(item) for item in delivered) + assert "4111-1111-1111-1111" not in rendered + assert "Streaming response blocked by Model Armor" in rendered + + +@pytest.mark.asyncio +async def test_streaming_responses_deltas_of_separate_fields_do_not_form_a_finding_across_their_boundary(): + """Two fields of a turn are separate text, so what runs across their boundary is not model output. + + A reasoning summary ending in half a card number and an answer opening with the other half + each carry nothing to find, and joining them without a break would invent one. + """ + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ReasoningSummaryTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + answer = "1111-1111 is not a full card" + summary_delta = ReasoningSummaryTextDeltaEvent( + type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, + item_id="rs_1", + output_index=0, + delta="the prefix they gave me is 4111-1111-", + ) + text_delta = OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=1, + content_index=0, + delta=answer, + ) + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-5-mini", + object="response", + output=[ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": answer, "annotations": []}], + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, (summary_delta, text_delta, completed)) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert "4111-1111-" in scanned + assert answer in scanned + assert "4111-1111-1111-1111" not in scanned + rendered = "".join(str(item) for item in delivered) + assert "Streaming response blocked by Model Armor" not in rendered + + +@pytest.mark.asyncio +async def test_streaming_responses_one_fields_deltas_still_join_into_a_single_finding(): + """A card number split across two deltas of one field is still one card number to scan.""" + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + halves = ("my card is 4111-1111-", "1111-1111") + text_deltas = tuple( + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta=half, + ) + for half in halves + ) + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-5-mini", + object="response", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_BLOCK) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, (*text_deltas, completed)) + + post.assert_called_once() + assert "4111-1111-1111-1111" in post.call_args.kwargs["json"]["modelResponseData"]["text"] + rendered = "".join(str(item) for item in delivered) + assert "4111-1111-1111-1111" not in rendered + assert "Streaming response blocked by Model Armor" in rendered + + +@pytest.mark.asyncio +async def test_streaming_responses_fields_the_body_repeats_are_not_scanned_a_second_time(): + """A turn whose visible fields all reach the terminal body is scanned once, not twice. + + Two output_text fields stream as deltas and come back in the completed body, so scanning the + deltas on top of the body would send Model Armor two copies of everything the client sees. + """ + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + paragraphs = ("the first thing to know", "a second and separate point") + text_deltas = tuple( + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id=f"msg_{index}", + output_index=index, + content_index=0, + delta=paragraph, + ) + for index, paragraph in enumerate(paragraphs) + ) + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-5-mini", + object="response", + output=[ + { + "type": "message", + "id": f"msg_{index}", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": paragraph, "annotations": []}], + } + for index, paragraph in enumerate(paragraphs) + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, (*text_deltas, completed)) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert [scanned.count(paragraph) for paragraph in paragraphs] == [1, 1] + rendered = "".join(str(item) for item in delivered) + assert all(paragraph in rendered for paragraph in paragraphs) + + +def test_every_responses_delta_event_is_in_the_scanned_set(): + """Every ``.delta`` the Responses event enum defines is model output on its way to the client.""" + from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ( + _RESPONSES_DELTA_EVENT_TYPES, + ) + from litellm.types.llms.openai import ResponsesAPIStreamEvents + + missing = { + event.value + for event in ResponsesAPIStreamEvents + if event.value.endswith(".delta") and event.value not in _RESPONSES_DELTA_EVENT_TYPES + } + assert not missing + assert "response.mcp_call_arguments.delta" in _RESPONSES_DELTA_EVENT_TYPES From ad0f0af1922e226a34baf5e59bccef58761ada2c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:15:41 -0700 Subject: [PATCH 183/480] fix(helm): normalize ingress.extraPaths path types for ingress-nginx too A dotted extraPaths entry kept its requested Exact or Prefix type under ingress.controller=nginx, so the admission webhook rejected the render the option exists to avoid, and the duplicate check compared the raw type against the built-in paths' normalized one, letting a repeated /favicon.ico through. Extra paths now go through the same controller normalization before both the duplicate check and the render. --- helm/litellm/templates/ingress.yaml | 7 +- .../tests/ingress_controller_tests.yaml | 76 +++++++++++++++++++ helm/litellm/values.yaml | 6 +- 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/helm/litellm/templates/ingress.yaml b/helm/litellm/templates/ingress.yaml index 0a215de5a56..732564b280f 100644 --- a/helm/litellm/templates/ingress.yaml +++ b/helm/litellm/templates/ingress.yaml @@ -170,10 +170,11 @@ spec: {{- if not $target }} {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown service %q, expected one of backend, gateway, ui" $idx $extra.path $service) }} {{- end }} - {{- $pathType := $extra.pathType | default "Prefix" }} - {{- if not (has $pathType (list "Prefix" "Exact" "ImplementationSpecific")) }} - {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $pathType) }} + {{- $requestedPathType := $extra.pathType | default "Prefix" }} + {{- if not (has $requestedPathType (list "Prefix" "Exact" "ImplementationSpecific")) }} + {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $requestedPathType) }} {{- end }} + {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" $extra.path "pathType" $requestedPathType) }} {{- if eq $extra.path "/" }} {{- fail (printf "ingress.extraPaths[%d]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture" $idx) }} {{- end }} diff --git a/helm/litellm/tests/ingress_controller_tests.yaml b/helm/litellm/tests/ingress_controller_tests.yaml index a86271a02be..40790ba674a 100644 --- a/helm/litellm/tests/ingress_controller_tests.yaml +++ b/helm/litellm/tests/ingress_controller_tests.yaml @@ -127,3 +127,79 @@ tests: asserts: - failedTemplate: errorMessage: 'ingress.controller: unknown controller "traefik", expected one of alb, nginx' + + - it: rejects an extraPaths entry that repeats a built-in path once ingress-nginx normalizes its pathType + set: + ingress.enabled: true + ingress.controller: nginx + ingress.extraPaths: + - path: /favicon.ico + service: ui + asserts: + - failedTemplate: + errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it" + + - it: renders a dotted extraPaths entry as ImplementationSpecific for ingress-nginx + set: + ingress.enabled: true + ingress.controller: nginx + ingress.extraPaths: + - path: /eu.assemblyai.custom + service: gateway + - path: /robots.txt + service: ui + pathType: Exact + asserts: + - notMatchRegexRaw: + pattern: 'path: "?/\S*\.\S*"?\n\s+pathType: (Exact|Prefix)\n' + - contains: + path: spec.rules[0].http.paths + content: + path: /eu.assemblyai.custom + pathType: ImplementationSpecific + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 + - contains: + path: spec.rules[0].http.paths + content: + path: /robots.txt + pathType: ImplementationSpecific + backend: + service: + name: RELEASE-NAME-litellm-ui + port: + number: 3000 + + - it: keeps the requested pathType of a dotted extraPaths entry for the AWS Load Balancer Controller + set: + ingress.enabled: true + ingress.extraPaths: + - path: /eu.assemblyai.custom + service: gateway + - path: /robots.txt + service: ui + pathType: Exact + asserts: + - contains: + path: spec.rules[0].http.paths + content: + path: /eu.assemblyai.custom + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 + - contains: + path: spec.rules[0].http.paths + content: + path: /robots.txt + pathType: Exact + backend: + service: + name: RELEASE-NAME-litellm-ui + port: + number: 3000 diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 592bb6d6131..461330ba491 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -11,7 +11,8 @@ ingress: enabled: false className: "" # Which ingress controller serves this Ingress. Controllers disagree on the - # pathTypes they accept, so this picks the pathType of a few built-in paths: + # pathTypes they accept, so this picks the pathType of the dotted paths, the + # built-in ones and any dotted extraPaths entry alike: # alb AWS Load Balancer Controller (default): Exact and Prefix paths plus # the /*.txt wildcard that routes the UI's RSC payloads. # nginx ingress-nginx: its admission webhook rejects a dot in an Exact or @@ -37,7 +38,8 @@ ingress: # # path required; the HTTP path to route # service which component serves it: gateway (default), backend, or ui - # pathType Prefix (default), Exact, or ImplementationSpecific + # pathType Prefix (default), Exact, or ImplementationSpecific; a dotted + # path renders as ImplementationSpecific when controller is nginx # # The target component only answers paths its own route allowlist keeps, so # a path here still has to be one that component serves. From f81928f7aec870adc4491fd19025c16bef4a8cf6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:22:12 -0700 Subject: [PATCH 184/480] test(vector-store): accept embedding_executor in the Bedrock KB hook fake handler --- tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py index 98045725177..6b162ea90f1 100644 --- a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py +++ b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py @@ -370,6 +370,7 @@ async def test_bedrock_kb_request_body_has_transformed_filters( custom_llm_provider, litellm_params, logging_obj, + embedding_executor=None, extra_headers=None, extra_body=None, timeout=None, From a5639b8e2a9870fb0aeb7d2365a8a386ed4e0fe6 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:22:52 -0700 Subject: [PATCH 185/480] test: add OCR python-to-rust test parity ledger (WIP) (#39434) * test: add OCR python-to-rust test parity ledger * feat: add ledger loader for OCR test parity data * feat: add drift audit for OCR test parity ledger (WIP, untested) * fix: correct drift in OCR test-parity ledger Two entries referenced a typo'd Python test name, four duplicated entries already tracked under TestProxySecurityGuard, five real Python tests in test_rust_bridge.py were untracked, and three real Rust custom_logger tests were missing from rust_only_tests. Found by running validate_ledger.py's audit against the live repo. * test: add regression coverage for the OCR ledger and audit script Covers schema validation, AST/regex test enumeration, drift detection on both the Python and Rust sides, LedgerDriftError content, and a live-repo clean-audit guard against future drift. * test: simplify ledger test to one drift-guard assertion Replace the ledger-internals unit tests with a single test that runs the real audit against the live repo and asserts every OCR test is accounted for (mapped, unmapped-with-reason, or rust_only), printing the exact diff on failure. * chore: move OCR test-parity ledger to core/ocr validate_sub_methods/ mixes strategy-catalog metadata with the ledger. Ledger data belongs under a per-function core// path instead. * fix: point LEDGER_PATH at the new core/ocr location --- tests/rust-python-harness/cli.py | 48 +++- tests/rust-python-harness/shared/__init__.py | 0 .../shared/parity/__init__.py | 0 .../shared/parity/ledger.py | 136 ++++++++++++ .../strategies/__init__.py | 0 .../strategies/unit_tests/__init__.py | 0 .../ledgers/ocr/ocr_test_ledger.json | 208 ++++++++++++++++++ .../unit_tests/mapping_validator.py | 101 +++++++++ .../strategies/unit_tests/python_runner.py | 22 ++ .../strategies/unit_tests/rust_runner.py | 13 ++ tests/test_rust_python_harness.py | 55 +++++ 11 files changed, 581 insertions(+), 2 deletions(-) create mode 100644 tests/rust-python-harness/shared/__init__.py create mode 100644 tests/rust-python-harness/shared/parity/__init__.py create mode 100644 tests/rust-python-harness/shared/parity/ledger.py create mode 100644 tests/rust-python-harness/strategies/__init__.py create mode 100644 tests/rust-python-harness/strategies/unit_tests/__init__.py create mode 100644 tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json create mode 100644 tests/rust-python-harness/strategies/unit_tests/mapping_validator.py create mode 100644 tests/rust-python-harness/strategies/unit_tests/python_runner.py create mode 100644 tests/rust-python-harness/strategies/unit_tests/rust_runner.py diff --git a/tests/rust-python-harness/cli.py b/tests/rust-python-harness/cli.py index 41c46f9613a..f9e97d7ad43 100644 --- a/tests/rust-python-harness/cli.py +++ b/tests/rust-python-harness/cli.py @@ -9,9 +9,11 @@ from .catalog import load_catalog from .models import HarnessCase, Strategy from .runner import run_pytest from .ui import make_dashboard +from .strategies.unit_tests.mapping_validator import FunctionReport, build_function_report REPO_ROOT = Path(__file__).resolve().parents[2] COVERAGE_ROOT = REPO_ROOT / "target" / "rust-python-harness" +SDK_FUNCTION_CHOICES = ("ocr", "messages", "responses", "count_tokens") def _parser() -> argparse.ArgumentParser: @@ -40,9 +42,17 @@ def _parser() -> argparse.ArgumentParser: action="append", default=[], dest="sdk_functions", - choices=("ocr", "messages", "responses", "count_tokens"), + choices=SDK_FUNCTION_CHOICES, help="run only this SDK function", ) + parser.add_argument( + "--validate-ledger", + action="store_true", + help=( + "report Python<->Rust test-parity ledger gaps and drift instead of " + "running the dashboard; narrow with --function" + ), + ) parser.add_argument( "--plain", action="store_true", @@ -100,7 +110,7 @@ def _interactive_filters(strategies: Sequence[Strategy]) -> tuple[set[str], set[ ) sdk_functions = _pick_values( "SDK functions", - [(name, name) for name in ("ocr", "messages", "responses", "count_tokens")], + [(name, name) for name in SDK_FUNCTION_CHOICES], ) return strategy_ids, sdk_functions @@ -131,6 +141,38 @@ def _print_catalog(strategies: Sequence[Strategy]) -> None: print(f" {case.sdk_function:12} {case.coverage.value:14} {selectors}") +def _print_function_report(report: FunctionReport) -> None: + print(f"\n{report.sdk_function}") + if report.ledger is None or report.audit is None: + print(" no ledger yet") + return + ledger, audit = report.ledger, report.audit + print( + f" {ledger.mapped_count}/{ledger.total_count} python tests mapped to rust " + f"({ledger.percentage}%)" + ) + print(f" {len(ledger.rust_only_tests)} rust-only tests with no python counterpart") + if audit.is_clean: + print(" ledger is in sync with the live test files") + return + for label, items in ( + ("ledger references a python test that no longer exists", audit.missing_python_tests), + ("python test exists but is not tracked in the ledger", audit.stale_python_tests), + ("ledger references a rust test that no longer exists", audit.missing_rust_tests), + ("rust test exists but is not tracked in the ledger", audit.stale_rust_tests), + ): + for item in items: + print(f" {label}: {item}") + + +def _validate_ledger(sdk_functions: set[str]) -> int: + functions = sdk_functions or set(SDK_FUNCTION_CHOICES) + reports = tuple(build_function_report(function) for function in sorted(functions)) + for report in reports: + _print_function_report(report) + return 0 if all(report.is_clean for report in reports) else 1 + + def main(argv: Sequence[str] | None = None) -> int: args = _parser().parse_args(argv) if args.coverage and importlib.util.find_spec("pytest_cov") is None: @@ -138,6 +180,8 @@ def main(argv: Sequence[str] | None = None) -> int: "--coverage requires the project's pytest-cov dependency; run with " "`poetry run python -m tests.rust-python-harness --coverage`" ) + if args.validate_ledger: + return _validate_ledger(set(args.sdk_functions)) strategies = load_catalog() if args.list: _print_catalog(strategies) diff --git a/tests/rust-python-harness/shared/__init__.py b/tests/rust-python-harness/shared/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/shared/parity/__init__.py b/tests/rust-python-harness/shared/parity/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/shared/parity/ledger.py b/tests/rust-python-harness/shared/parity/ledger.py new file mode 100644 index 00000000000..40dfed583ae --- /dev/null +++ b/tests/rust-python-harness/shared/parity/ledger.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +@dataclass(frozen=True, slots=True) +class LedgerEntry: + python_file: str + python_test: str + status: str + rust_file: str + rust_test: str + justification: str + reason: str + + +@dataclass(frozen=True, slots=True) +class RustOnlyEntry: + rust_file: str + rust_test: str + reason: str + + +@dataclass(frozen=True, slots=True) +class TestLedger: + sdk_function: str + python_scope: tuple[str, ...] + rust_scope: tuple[str, ...] + entries: tuple[LedgerEntry, ...] + rust_only_tests: tuple[RustOnlyEntry, ...] + + @property + def mapped_count(self) -> int: + return sum(1 for entry in self.entries if entry.status == "mapped") + + @property + def total_count(self) -> int: + return len(self.entries) + + @property + def percentage(self) -> float: + if self.total_count == 0: + return 0.0 + return round(100.0 * self.mapped_count / self.total_count, 1) + + +def _require_string(value: Any, field: str, source: Path) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{source}: {field} must be a non-empty string") + return value + + +def _require_string_list(value: Any, field: str, source: Path) -> tuple[str, ...]: + if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value): + raise ValueError(f"{source}: {field} must be a list of non-empty strings") + return tuple(value) + + +def _load_entry(data: Any, index: int, source: Path) -> LedgerEntry: + if not isinstance(data, dict): + raise ValueError(f"{source}: entries[{index}] must be an object") + python_file = _require_string(data.get("python_file"), f"entries[{index}].python_file", source) + python_test = _require_string(data.get("python_test"), f"entries[{index}].python_test", source) + status = data.get("status") + if status not in ("mapped", "unmapped"): + raise ValueError(f"{source}: entries[{index}].status must be 'mapped' or 'unmapped'") + + if status == "mapped": + rust_file = _require_string(data.get("rust_file"), f"entries[{index}].rust_file", source) + rust_test = _require_string(data.get("rust_test"), f"entries[{index}].rust_test", source) + justification = _require_string( + data.get("justification"), f"entries[{index}].justification", source + ) + return LedgerEntry( + python_file=python_file, + python_test=python_test, + status=status, + rust_file=rust_file, + rust_test=rust_test, + justification=justification, + reason="", + ) + + reason = _require_string(data.get("reason"), f"entries[{index}].reason", source) + return LedgerEntry( + python_file=python_file, + python_test=python_test, + status=status, + rust_file="", + rust_test="", + justification="", + reason=reason, + ) + + +def _load_rust_only_entry(data: Any, index: int, source: Path) -> RustOnlyEntry: + if not isinstance(data, dict): + raise ValueError(f"{source}: rust_only_tests[{index}] must be an object") + return RustOnlyEntry( + rust_file=_require_string(data.get("rust_file"), f"rust_only_tests[{index}].rust_file", source), + rust_test=_require_string(data.get("rust_test"), f"rust_only_tests[{index}].rust_test", source), + reason=_require_string(data.get("reason"), f"rust_only_tests[{index}].reason", source), + ) + + +def load_ledger(path: Path) -> TestLedger: + with path.open(encoding="utf-8") as stream: + data = json.load(stream) + + sdk_function = _require_string(data.get("sdk_function"), "sdk_function", path) + python_scope = _require_string_list(data.get("python_scope"), "python_scope", path) + rust_scope = _require_string_list(data.get("rust_scope"), "rust_scope", path) + + entries_data = data.get("entries") + if not isinstance(entries_data, list): + raise ValueError(f"{path}: entries must be a list") + entries = tuple( + _load_entry(entry, index, path) for index, entry in enumerate(entries_data) + ) + + rust_only_data = data.get("rust_only_tests") + if not isinstance(rust_only_data, list): + raise ValueError(f"{path}: rust_only_tests must be a list") + rust_only_tests = tuple( + _load_rust_only_entry(entry, index, path) for index, entry in enumerate(rust_only_data) + ) + + return TestLedger( + sdk_function=sdk_function, + python_scope=python_scope, + rust_scope=rust_scope, + entries=entries, + rust_only_tests=rust_only_tests, + ) diff --git a/tests/rust-python-harness/strategies/__init__.py b/tests/rust-python-harness/strategies/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/unit_tests/__init__.py b/tests/rust-python-harness/strategies/unit_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json b/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json new file mode 100644 index 00000000000..799a1320463 --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json @@ -0,0 +1,208 @@ +{ + "sdk_function": "ocr", + "python_scope": [ + "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", + "tests/test_litellm/ocr/test_rust_bridge.py", + "tests/test_litellm/ocr/test_ocr_file_input.py", + "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", + "tests/test_litellm/ocr/test_ocr_native_format.py", + "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py" + ], + "rust_scope": [ + "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", + "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", + "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", + "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs" + ], + "entries": [ + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_should_encode_azure_document_intelligence_model_id", "status": "unmapped", "reason": "model-id URL percent-encoding has no Rust test; Rust only tests pages/features query building"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_should_reject_dot_segment_azure_document_intelligence_model_id", "status": "unmapped", "reason": "model-id dot-segment validation has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_preserves_azure_native_fields", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_response_normalizes_pages", "justification": "both assert page markdown, dimension (inch-to-pixel) normalization, and usage_info.pages_processed from the same Azure succeeded response shape"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_async_transform_ocr_response_preserves_azure_native_fields", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_response_normalizes_pages", "justification": "async twin of the sync case above, same underlying transform is exercised on the Rust side"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_tolerates_missing_native_fields", "status": "unmapped", "reason": "tables/keyValuePairs absence tolerance is not asserted by the Rust response test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_non_succeeded_status_raises", "status": "unmapped", "reason": "no Rust test asserts on a non-succeeded Azure DI status"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_supported_ocr_params_includes_features", "status": "unmapped", "reason": "supported-params list content has no Rust equivalent for Azure"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_native_format_carries_raw_operation", "status": "unmapped", "reason": "native req_format raw-operation passthrough is not tested in Rust"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_async_transform_ocr_response_native_format_carries_raw_operation", "status": "unmapped", "reason": "native req_format raw-operation passthrough is not tested in Rust"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_default_format_omits_raw_operation", "status": "unmapped", "reason": "req_format gating of raw-operation output has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_passes_through_req_format", "status": "unmapped", "reason": "req_format passthrough in map_ocr_params has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_rejects_unknown_req_format_as_bad_request", "status": "unmapped", "reason": "req_format validation error path has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_omits_req_format_query_param", "status": "unmapped", "reason": "no Rust test asserts req_format is excluded from the built URL"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_features", "status": "unmapped", "reason": "features-string normalization in map_ocr_params has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_empty_features_list_omitted", "status": "unmapped", "reason": "empty-features omission has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_invalid_features_raises", "status": "unmapped", "reason": "features validation error path has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_appends_features_query", "status": "unmapped", "reason": "features query-param construction has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_combines_pages_and_features", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_url_normalizes_zero_based_pages", "justification": "both assert 0-based, duplicate page indices are deduped, sorted, and rewritten 1-based into the request URL"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_validate_environment_uses_subscription_key", "status": "unmapped", "reason": "Python-side header derivation from litellm_params; Rust's poll test only checks the header is present, not how it was resolved"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_validate_environment_falls_back_to_entra_token", "status": "unmapped", "reason": "Entra bearer-token fallback logic has no Rust test"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_doc_intelligence_route", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_documentintelligence_and_is_case_insensitive", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_does_not_match_mistral_route", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_does_not_hijack_doc_intelligence", "status": "unmapped", "reason": "api_base resolution from the secret manager runs before the Rust bridge is called, no Rust test exists for it"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_explicit_api_base_is_honoured_for_doc_intelligence", "status": "unmapped", "reason": "api_base precedence resolution is Python-only"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_still_applies_to_mistral_ocr", "status": "unmapped", "reason": "api_base precedence resolution is Python-only"}, + + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_use_litellm_rust_toggles_flag", "status": "unmapped", "reason": "bridge-plumbing: Python-side feature-flag toggle, no Rust equivalent"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_env_var_enables_rust_ocr", "status": "unmapped", "reason": "bridge-plumbing: Python-side env-var flag gating"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_returns_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: dependency-injection test hook, not provider behavior"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_loader_returns_none_when_extension_absent", "status": "unmapped", "reason": "bridge-plumbing: native-extension import/loader fallback"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_loader_caches_absent_extension", "status": "unmapped", "reason": "bridge-plumbing: loader caching behavior"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_available_reflects_loader", "status": "unmapped", "reason": "bridge-plumbing: loader availability check"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_aocr_returns_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: dependency-injection test hook"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_toggle_without_ocr_arg_preserves_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: injected-impl state retention regression"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_explicit_ocr_none_clears_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: injected-impl clearing behavior"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_none_when_extension_absent", "status": "unmapped", "reason": "bridge-plumbing: degrade path when the native extension is missing"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_uses_compiled_extension", "status": "unmapped", "reason": "bridge-plumbing: native module resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_timeout_to_seconds_handles_float_timeout_and_none", "status": "unmapped", "reason": "bridge-plumbing: Python-side timeout normalization helper"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_bridge_wrapper_forwards_prepared_args_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: wrapper argument forwarding, asserted against a fake bridge not the real Rust code"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: async wrapper argument forwarding"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_prepares_request_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: request preparation and response wrapping in Python"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_resolves_key_via_secret_manager_when_missing", "status": "unmapped", "reason": "secret-manager: API key resolution happens in Python before the bridge is invoked"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_prefers_explicit_key_over_resolver", "status": "unmapped", "reason": "secret-manager: key precedence resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_uses_provider_api_key_env_var", "status": "unmapped", "reason": "secret-manager: provider-specific env var name resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_forwards_vertex_routing_metadata", "status": "unmapped", "reason": "secret-manager: vertex routing metadata merge happens in Python"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_manager", "status": "unmapped", "reason": "secret-manager: vertex project/location resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager", "status": "unmapped", "reason": "secret-manager: azure_ai api_base resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint", "status": "unmapped", "reason": "secret-manager: doc-intelligence endpoint resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_runs_pre_call_logging", "status": "unmapped", "reason": "bridge-plumbing: Python logging-object pre_call invocation"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_routes_to_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: routing to a fake bridge, not the real Rust transform"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_routes_azure_ai_to_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: provider-prefix stripping before routing"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_rust_path_converts_file_document_before_bridge", "status": "unmapped", "reason": "file-normalization: raw-bytes-to-data-URI conversion happens in Python before the bridge call"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_exception_type_uses_resolved_provider_context", "status": "unmapped", "reason": "bridge-plumbing: Python exception-type mapping on bridge failure"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_aocr_routes_to_async_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: async routing to a fake bridge"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_aocr_exception_type_uses_resolved_provider_context", "status": "unmapped", "reason": "bridge-plumbing: async exception-type mapping on bridge failure"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_does_not_route_to_rust_when_disabled", "status": "unmapped", "reason": "bridge-plumbing: Python control flow for the toggle-disabled branch, no Rust-owned behavior runs"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_falls_back_to_python_when_bridge_unavailable", "status": "unmapped", "reason": "bridge-plumbing: Python-only fallback when the compiled Rust extension is absent, Rust cannot test its own absence"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_forwards_timeout_to_rust", "status": "unmapped", "reason": "bridge-plumbing: asserts the Python call site forwards a timeout kwarg, Rust receives an already-constructed request"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_passes_default_request_timeout_to_rust", "status": "unmapped", "reason": "bridge-plumbing: asserts the Python call site supplies a default timeout kwarg, no Rust equivalent"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_provider_configs_expose_api_key_env_vars", "status": "unmapped", "reason": "asserts per-provider get_api_key_env_var() strings; the closest Rust test (ocr_dispatch_supports_migrated_providers) asserts provider dispatch/param resolution instead, not API key env var names"}, + + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_pdf_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection is Python-only preprocessing before the bridge call"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_png_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_jpg_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_jpeg_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_gif_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_webp_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_tiff_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_tif_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_bmp_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_be_case_insensitive", "status": "unmapped", "reason": "file-normalization: MIME detection case handling"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_fallback_for_unknown_extension", "status": "unmapped", "reason": "file-normalization: MIME detection fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pdf_pathlib_path_to_document_url", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion happens in Python"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_image_pathlib_path_to_image_url", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_reject_bare_str_path", "status": "unmapped", "reason": "file-normalization: arbitrary-file-read guard on bare str paths"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pathlib_path", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes", "status": "unmapped", "reason": "file-normalization: raw-bytes-to-data-URI conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_explicit_mime_type", "status": "unmapped", "reason": "file-normalization: explicit MIME override on raw bytes"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_image_mime_type", "status": "unmapped", "reason": "file-normalization: explicit MIME override on raw bytes"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object", "status": "unmapped", "reason": "file-normalization: file-like-object conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object_with_name", "status": "unmapped", "reason": "file-normalization: file-like-object name-based MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_missing_file_field", "status": "unmapped", "reason": "file-normalization: missing-field validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_nonexistent_pathlib_path", "status": "unmapped", "reason": "file-normalization: missing-file validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_empty_file", "status": "unmapped", "reason": "file-normalization: empty-file validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_unsupported_type", "status": "unmapped", "reason": "file-normalization: unsupported input type validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_invalid_mime_type", "status": "unmapped", "reason": "file-normalization: MIME-type injection validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_override_mime_type_for_pathlib_path", "status": "unmapped", "reason": "file-normalization: explicit MIME override precedence"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_document_url_for_pdf", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_png", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_jpeg", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_octet_stream", "status": "unmapped", "reason": "file-normalization: filename-based MIME fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_none", "status": "unmapped", "reason": "file-normalization: filename-based MIME fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_fallback_to_octet_stream_for_unknown", "status": "unmapped", "reason": "file-normalization: default MIME fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_preserve_base64_content_correctly", "status": "unmapped", "reason": "file-normalization: binary round-trip through base64"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_from_content_type", "status": "unmapped", "reason": "file-normalization: content-type parameter stripping"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_with_multiple_params", "status": "unmapped", "reason": "file-normalization: content-type parameter stripping"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_reject_file_type_document_in_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body file-type guard, a different mechanism than Rust's URL-fetch SSRF guard"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_accept_document_url_type_in_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body parsing"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_raise_on_invalid_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body parsing error path"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_ignore_document_form_field_injection", "status": "unmapped", "reason": "proxy-layer multipart form-field injection guard, a different mechanism than Rust's URL-fetch SSRF guard"}, + + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_extract_header_in_supported_params", "status": "unmapped", "reason": "Rust's fixed-list test checks the full list as one assertion, not this individual param"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_extract_footer_in_supported_params", "status": "unmapped", "reason": "Rust's fixed-list test checks the full list as one assertion, not this individual param"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_existing_params_still_present", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "supported_params_match_python_mistral_ocr_config", "justification": "both assert the full supported_ocr_params list matches the same fixed set of param names"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_header_passed_through", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_drops_unknown_params", "justification": "both assert extract_header survives map_ocr_params filtering unchanged"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_footer_passed_through", "status": "unmapped", "reason": "Rust's map_ocr_params test does not assert on extract_footer specifically"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_header_and_footer_together", "status": "unmapped", "reason": "combined extract_header+extract_footer passthrough is not asserted together in Rust"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_unknown_param_is_dropped", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_drops_unknown_params", "justification": "both assert an unrecognized param key is dropped while a known one is kept"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestNewSupportedParams::test_new_param_in_supported_list", "status": "unmapped", "reason": "OCR4-specific new params (table_format etc) are not individually verified against the Rust fixed-list test"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestNewParamsMapOcr::test_new_param_passed_through", "status": "unmapped", "reason": "OCR4-specific new params are not individually asserted in the Rust map_ocr_params test"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrRequest::test_param_included_in_request_body", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_builds_mistral_body", "justification": "both assert an optional param value ends up in the built request body alongside model/document"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrRequest::test_multiple_new_params_together", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_builds_mistral_body", "justification": "both assert multiple optional params (table_format/include_image_base64) land correctly in the same request body"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrResponseOcr4Fields::test_blocks_and_confidence_scores_preserved", "status": "unmapped", "reason": "OCR4 blocks/confidence_scores fields are not asserted by the Rust response test"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrResponseOcr4Fields::test_ocr4_fields_survive_model_dump", "status": "unmapped", "reason": "OCR4 tables/hyperlinks/header/footer fields are not asserted by the Rust response test"}, + + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_model_info_ocr4_price", "status": "unmapped", "reason": "cost-calc: pricing/model-info lookup is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr4_cost_scales_with_pages", "status": "unmapped", "reason": "cost-calc: per-page pricing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_pricing_entry", "status": "unmapped", "reason": "cost-calc: cost-map JSON entry validation is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_model_info_price", "status": "unmapped", "reason": "cost-calc: pricing/model-info lookup is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_cost_scales_with_pages", "status": "unmapped", "reason": "cost-calc: per-page pricing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_ocr_and_annotation_pages_at_their_own_rates", "status": "unmapped", "reason": "cost-calc: mixed-rate billing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_only_response", "status": "unmapped", "reason": "cost-calc: annotation-only billing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_pages_when_pages_processed_missing", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_azure_doc_ai_annotation_pages_fall_back_to_ocr_rate", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"}, + + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_serves_default_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_skipped_for_native_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_native_format_rejected_for_provider_without_support_as_bad_request", "status": "unmapped", "reason": "provider-support validation for req_format happens in Python"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_unknown_format_rejected_for_provider_without_support_as_bad_request", "status": "unmapped", "reason": "req_format validation error path is Python-only"}, + + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestHandlerDiscovery::test_handler_discovered_for_ocr", "status": "unmapped", "reason": "guardrail-translation handler discovery is a Python proxy-layer concern"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestHandlerDiscovery::test_handler_discovered_for_aocr", "status": "unmapped", "reason": "guardrail-translation handler discovery is a Python proxy-layer concern"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_document_url", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_image_url", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_no_document", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_invalid_document", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_input_blocking_guardrail", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_single_page", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_multiple_pages", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_empty_pages", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_page_with_empty_markdown", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_preserves_page_metadata", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_output_blocking_guardrail", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestPIIMaskingScenario::test_pii_masking_in_ocr_pages", "status": "unmapped", "reason": "PII redaction in the translation handler has no Rust equivalent"}, + + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_read_req_format_from_header", "status": "unmapped", "reason": "proxy-layer header parsing has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_prefer_body_req_format_over_header", "status": "unmapped", "reason": "proxy-layer body-vs-header precedence has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_omit_req_format_when_header_absent", "status": "unmapped", "reason": "proxy-layer parsing has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_reject_unknown_req_format", "status": "unmapped", "reason": "proxy-layer validation has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_return_native_payload_with_litellm_response_headers", "status": "unmapped", "reason": "proxy-layer response construction has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_return_normalized_response_when_no_native_payload", "status": "unmapped", "reason": "proxy-layer response construction has no Rust equivalent"} + ], + "rust_only_tests": [ + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "rust_custom_logger_reads_failure_payload_for_non_ocr_call_type", "reason": "exercises the non-OCR (acompletion) call-type branch of the logger; the OCR branch is covered separately by rust_custom_logger_reads_success_payload_for_ocr"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "no_callback_fast_path_dispatches_nothing", "reason": "Rust-only fast-path optimization test for when zero callbacks are registered; Python has no equivalent no-op dispatch path"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "with_standard_logging_payload_keeps_top_level_fields_in_sync", "reason": "Rust-internal builder-method invariant, Python has no equivalent internal builder"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "blocks_private_and_metadata_ips", "reason": "SSRF IP-blocking helper has no Python unit test; Python relies on the proxy-layer JSON/form guards instead"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "convert_document_url_rejects_loopback_fetch", "reason": "URL-fetch SSRF protection is Rust-gateway-only"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "convert_document_url_leaves_data_uri_untouched", "reason": "URL-fetch SSRF protection is Rust-gateway-only"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_passes_short_strings_through", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_caps_long_payloads", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_does_not_split_multibyte_chars", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_dispatch_supports_migrated_providers", "reason": "Rust-internal provider-config dispatch table has no equivalent Python unit test"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "string_headers_accepts_string_values", "reason": "Rust-gateway header-coercion helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "auth_header_detection_is_case_insensitive", "reason": "Rust-gateway header-detection helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_runs_pre_during_and_success_hooks", "reason": "full gateway-level guardrail-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_runs_failure_hook_on_provider_error", "reason": "full gateway-level failure-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_pre_call_block_skips_provider_socket", "reason": "full gateway-level pre-call-block-plus-socket-skip test with no Python equivalent at this integration scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_does_not_duplicate_authorization_header_when_header_is_supplied", "reason": "outgoing HTTP header dedup at the Rust gateway has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "document_intelligence_poll_uses_resolved_subscription_key", "reason": "full Azure DI poll-loop integration test with no Python equivalent at this scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "string_headers_rejects_non_string_values", "reason": "Rust-gateway header-coercion error path has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "azure_ai_reuses_mistral_body_transform", "reason": "Rust-internal delegation-to-Mistral-transform implementation detail, no Python test asserts this delegation"}, + {"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_request_uses_base64_source_for_data_uri", "reason": "no Python test asserts on the base64Source request body shape"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_mistral_url_uses_project_location_and_model", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_mistral_reuses_mistral_body_transform", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_deepseek_request_uses_ocr_endpoint_shape", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_deepseek_response_wraps_markdown_content", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_rejects_non_object_document", "reason": "non-object document rejection has no dedicated Python unit test"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_response_normalizes_mistral_json", "reason": "Python's response tests target OCR4-specific fields only, none asserts the same base normalization this Rust test checks"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "complete_url_defaults_and_dedupes_v1", "reason": "URL-building/defaulting for Mistral has no Python unit test"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "resolve_api_key_prefers_param_then_env", "reason": "API key resolution precedence at the Rust provider-config layer has no Python unit test"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "resolve_api_key_errors_when_absent", "reason": "API key resolution error path at the Rust provider-config layer has no Python unit test"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "rust_custom_logger_reads_success_payload_for_ocr", "reason": "Rust-internal custom-logger dispatch for OCR payloads has no Python unit test at this layer"} + ] +} diff --git a/tests/rust-python-harness/strategies/unit_tests/mapping_validator.py b/tests/rust-python-harness/strategies/unit_tests/mapping_validator.py new file mode 100644 index 00000000000..b8355e04266 --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/mapping_validator.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from ...shared.parity.ledger import TestLedger, load_ledger +from .python_runner import enumerate_python_tests +from .rust_runner import enumerate_rust_tests + +REPO_ROOT = Path(__file__).resolve().parents[4] +LEDGER_ROOT = Path(__file__).parent / "ledgers" + + +def ledger_path_for(sdk_function: str) -> Path: + return LEDGER_ROOT / sdk_function / f"{sdk_function}_test_ledger.json" + + +@dataclass(frozen=True, slots=True) +class AuditReport: + missing_python_tests: tuple[str, ...] + stale_python_tests: tuple[str, ...] + missing_rust_tests: tuple[str, ...] + stale_rust_tests: tuple[str, ...] + + @property + def is_clean(self) -> bool: + return not ( + self.missing_python_tests + or self.stale_python_tests + or self.missing_rust_tests + or self.stale_rust_tests + ) + + +def _ledger_python_tests_by_file(ledger: TestLedger) -> dict[str, set[str]]: + grouping: dict[str, set[str]] = {path: set() for path in ledger.python_scope} + for entry in ledger.entries: + grouping.setdefault(entry.python_file, set()).add(entry.python_test) + return grouping + + +def _ledger_rust_tests_by_file(ledger: TestLedger) -> dict[str, set[str]]: + grouping: dict[str, set[str]] = {path: set() for path in ledger.rust_scope} + for entry in ledger.entries: + if entry.status == "mapped": + grouping.setdefault(entry.rust_file, set()).add(entry.rust_test) + for rust_only in ledger.rust_only_tests: + grouping.setdefault(rust_only.rust_file, set()).add(rust_only.rust_test) + return grouping + + +def audit_ledger(ledger: TestLedger, repo_root: Path = REPO_ROOT) -> AuditReport: + missing_python: list[str] = [] + stale_python: list[str] = [] + for python_file, ledger_tests in _ledger_python_tests_by_file(ledger).items(): + actual_tests = enumerate_python_tests(repo_root, python_file) + for missing in sorted(ledger_tests - actual_tests): + missing_python.append(f"{python_file}:{missing}") + for stale in sorted(actual_tests - ledger_tests): + stale_python.append(f"{python_file}:{stale}") + + missing_rust: list[str] = [] + stale_rust: list[str] = [] + for rust_file, ledger_tests in _ledger_rust_tests_by_file(ledger).items(): + actual_tests = enumerate_rust_tests(repo_root, rust_file) + for missing in sorted(ledger_tests - actual_tests): + missing_rust.append(f"{rust_file}:{missing}") + for stale in sorted(actual_tests - ledger_tests): + stale_rust.append(f"{rust_file}:{stale}") + + return AuditReport( + missing_python_tests=tuple(missing_python), + stale_python_tests=tuple(stale_python), + missing_rust_tests=tuple(missing_rust), + stale_rust_tests=tuple(stale_rust), + ) + + +@dataclass(frozen=True, slots=True) +class FunctionReport: + sdk_function: str + ledger: TestLedger | None + audit: AuditReport | None + + @property + def has_ledger(self) -> bool: + return self.ledger is not None + + @property + def is_clean(self) -> bool: + return self.audit is None or self.audit.is_clean + + +def build_function_report(sdk_function: str, repo_root: Path = REPO_ROOT) -> FunctionReport: + path = ledger_path_for(sdk_function) + if not path.exists(): + return FunctionReport(sdk_function=sdk_function, ledger=None, audit=None) + ledger = load_ledger(path) + return FunctionReport( + sdk_function=sdk_function, ledger=ledger, audit=audit_ledger(ledger, repo_root) + ) diff --git a/tests/rust-python-harness/strategies/unit_tests/python_runner.py b/tests/rust-python-harness/strategies/unit_tests/python_runner.py new file mode 100644 index 00000000000..a7528d27756 --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/python_runner.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import ast +from pathlib import Path + + +def enumerate_python_tests(repo_root: Path, relative_path: str) -> frozenset[str]: + source = (repo_root / relative_path).read_text(encoding="utf-8") + tree = ast.parse(source, filename=relative_path) + + module_level: list[str] = [] + for node in ast.iter_child_nodes(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test_"): + module_level.append(node.name) + elif isinstance(node, ast.ClassDef): + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) and child.name.startswith( + "test_" + ): + module_level.append(f"{node.name}::{child.name}") + + return frozenset(module_level) diff --git a/tests/rust-python-harness/strategies/unit_tests/rust_runner.py b/tests/rust-python-harness/strategies/unit_tests/rust_runner.py new file mode 100644 index 00000000000..6b855adbc4b --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/rust_runner.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +import re +from pathlib import Path + +_RUST_TEST_PATTERN = re.compile( + r"#\[(?:test|tokio::test)\][^\n]*\n(?:[^\n]*\n)*?\s*(?:async\s+)?fn\s+(\w+)\s*\(" +) + + +def enumerate_rust_tests(repo_root: Path, relative_path: str) -> frozenset[str]: + source = (repo_root / relative_path).read_text(encoding="utf-8") + return frozenset(match.group(1) for match in _RUST_TEST_PATTERN.finditer(source)) diff --git a/tests/test_rust_python_harness.py b/tests/test_rust_python_harness.py index 90dc38663dd..6a8fa8d35cc 100644 --- a/tests/test_rust_python_harness.py +++ b/tests/test_rust_python_harness.py @@ -8,14 +8,24 @@ import pytest catalog = importlib.import_module("tests.rust-python-harness.catalog") cli = importlib.import_module("tests.rust-python-harness.cli") +ledger_module = importlib.import_module("tests.rust-python-harness.shared.parity.ledger") +mapping_validator = importlib.import_module( + "tests.rust-python-harness.strategies.unit_tests.mapping_validator" +) models = importlib.import_module("tests.rust-python-harness.models") runner = importlib.import_module("tests.rust-python-harness.runner") ui = importlib.import_module("tests.rust-python-harness.ui") load_catalog = catalog.load_catalog +load_ledger = ledger_module.load_ledger +ledger_path_for = mapping_validator.ledger_path_for +REPO_ROOT = mapping_validator.REPO_ROOT +audit_ledger = mapping_validator.audit_ledger +build_function_report = mapping_validator.build_function_report _pick_values = cli._pick_values _coverage_pytest_args = cli._coverage_pytest_args _select = cli._select +_validate_ledger = cli._validate_ledger CaseResult = models.CaseResult Coverage = models.Coverage HarnessCase = models.HarnessCase @@ -234,3 +244,48 @@ def test_should_report_confidence_for_each_sdk_section() -> None: assert scores["responses"].level.value == "MEDIUM" assert scores["count_tokens"].percentage == 0 assert scores["count_tokens"].level.value == "LOW" + + + +def test_should_report_no_ledger_for_a_function_without_one() -> None: + report = build_function_report("messages", repo_root=REPO_ROOT) + + assert report.has_ledger is False + assert report.is_clean is True + + +def test_should_report_ocr_ledger_stats_and_a_clean_audit() -> None: + ledger = load_ledger(ledger_path_for("ocr")) + + report = build_function_report("ocr", repo_root=REPO_ROOT) + + assert report.has_ledger is True + assert report.ledger.mapped_count == ledger.mapped_count + assert report.ledger.total_count == ledger.total_count + assert report.is_clean is True + + +def test_should_scope_validate_ledger_to_the_requested_function( + capsys: pytest.CaptureFixture[str], +) -> None: + exit_code = _validate_ledger({"messages"}) + + captured = capsys.readouterr() + assert exit_code == 0 + assert "messages" in captured.out + assert "no ledger yet" in captured.out + assert "ocr" not in captured.out + + +def test_should_have_every_python_and_rust_ocr_test_accounted_for_in_the_ledger() -> None: + ledger = load_ledger(ledger_path_for("ocr")) + + report = audit_ledger(ledger, repo_root=REPO_ROOT) + + assert report.is_clean, ( + "\nOCR test-parity ledger is out of sync with the live test files.\n" + f"Ledger references a Python test that no longer exists: {list(report.missing_python_tests)}\n" + f"Python test exists but is not tracked in the ledger: {list(report.stale_python_tests)}\n" + f"Ledger references a Rust test that no longer exists: {list(report.missing_rust_tests)}\n" + f"Rust test exists but is not tracked in the ledger: {list(report.stale_rust_tests)}\n" + ) From f77b3b2b5234b51aefc84d64dcb9a82fc3cc0f52 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:26:31 -0700 Subject: [PATCH 186/480] refactor(s3_vectors): embed search queries through the shared vector store executor S3 Vectors now subclasses BaseQueryEmbeddingVectorStoreConfig, so its query embedding runs through the Router executor with the request metadata instead of a private router lookup. embedding_model stays accepted as an alias of litellm_embedding_model. The router kwarg is gone from the search handler and every provider transform now that nothing but the executor fallback read it. --- .../azure_ai/vector_stores/transformation.py | 7 +- .../base_llm/vector_store/transformation.py | 13 +- .../bedrock/vector_stores/transformation.py | 2 - litellm/llms/custom_httpx/llm_http_handler.py | 8 - .../gemini/vector_stores/transformation.py | 2 - .../milvus/vector_stores/transformation.py | 7 +- .../openai/vector_stores/transformation.py | 2 - .../pg_vector/vector_stores/transformation.py | 2 - .../ragflow/vector_stores/transformation.py | 2 - .../vector_stores/transformation.py | 209 +++++-------- .../vector_stores/rag_api/transformation.py | 2 - .../search_api/transformation.py | 2 - litellm/vector_stores/main.py | 1 - .../test_s3_vectors_transformation.py | 282 +++++++++--------- tests/test_litellm/vector_stores/test_main.py | 25 +- 15 files changed, 241 insertions(+), 325 deletions(-) diff --git a/litellm/llms/azure_ai/vector_stores/transformation.py b/litellm/llms/azure_ai/vector_stores/transformation.py index db1a0fc89a3..7638229bc32 100644 --- a/litellm/llms/azure_ai/vector_stores/transformation.py +++ b/litellm/llms/azure_ai/vector_stores/transformation.py @@ -24,7 +24,6 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -121,11 +120,10 @@ class AzureAIVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, BaseAzureLLM litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], extra_body: Mapping[str, object] | None = None, - router: Router | None = None, embedding_executor: VectorStoreEmbeddingExecutor | None = None, ) -> tuple[str, dict[str, object]]: query_text: Final = self.query_text(query) - query_vector: Final = self.embed_query(query_text, litellm_params, embedding_executor, router) + query_vector: Final = self.embed_query(query_text, litellm_params, embedding_executor) return self._search_request( vector_store_id, query_text, @@ -145,11 +143,10 @@ class AzureAIVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, BaseAzureLLM litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], extra_body: Mapping[str, object] | None = None, - router: Router | None = None, embedding_executor: VectorStoreEmbeddingExecutor | None = None, ) -> tuple[str, dict[str, object]]: query_text: Final = self.query_text(query) - query_vector: Final = await self.aembed_query(query_text, litellm_params, embedding_executor, router) + query_vector: Final = await self.aembed_query(query_text, litellm_params, embedding_executor) return self._search_request( vector_store_id, query_text, diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 9624a721870..a3b8bcc499c 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -153,7 +153,6 @@ class BaseVectorStoreConfig: litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, - router: Router | None = None, ) -> tuple[str, dict]: pass @@ -166,7 +165,6 @@ class BaseVectorStoreConfig: litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, - router: Router | None = None, ) -> tuple[str, dict]: """ Optional async version of transform_search_vector_store_request. @@ -182,7 +180,6 @@ class BaseVectorStoreConfig: litellm_logging_obj=litellm_logging_obj, litellm_params=litellm_params, extra_body=extra_body, - router=router, ) @abstractmethod @@ -271,7 +268,6 @@ class BaseQueryEmbeddingVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], extra_body: Mapping[str, object] | None = None, - router: Router | None = None, embedding_executor: VectorStoreEmbeddingExecutor | None = None, ) -> tuple[str, dict[str, object]]: pass @@ -285,7 +281,6 @@ class BaseQueryEmbeddingVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], extra_body: Mapping[str, object] | None = None, - router: Router | None = None, embedding_executor: VectorStoreEmbeddingExecutor | None = None, ) -> tuple[str, dict[str, object]]: return self.transform_search_vector_store_request( @@ -296,7 +291,6 @@ class BaseQueryEmbeddingVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj=litellm_logging_obj, litellm_params=litellm_params, extra_body=extra_body, - router=router, embedding_executor=embedding_executor, ) @@ -338,11 +332,10 @@ class BaseQueryEmbeddingVectorStoreConfig(BaseVectorStoreConfig): query_text: str, litellm_params: Mapping[str, object], embedding_executor: VectorStoreEmbeddingExecutor | None, - router: Router | None = None, ) -> Sequence[float]: model: Final = self.query_embedding_model(litellm_params) configuration: Final = self.query_embedding_configuration(litellm_params) - executor: Final = self.query_embedding_executor(embedding_executor, router) + executor: Final = self.query_embedding_executor(embedding_executor, None) try: response: Final = executor.embed(model, query_text, configuration) except Exception as e: @@ -354,11 +347,10 @@ class BaseQueryEmbeddingVectorStoreConfig(BaseVectorStoreConfig): query_text: str, litellm_params: Mapping[str, object], embedding_executor: VectorStoreEmbeddingExecutor | None, - router: Router | None = None, ) -> Sequence[float]: model: Final = self.query_embedding_model(litellm_params) configuration: Final = self.query_embedding_configuration(litellm_params) - executor: Final = self.query_embedding_executor(embedding_executor, router) + executor: Final = self.query_embedding_executor(embedding_executor, None) try: response: Final = await executor.aembed(model, query_text, configuration) except Exception as e: @@ -408,7 +400,6 @@ class BaseDirectVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], extra_body: Mapping[str, object] | None = None, - router: Router | None = None, ) -> NoReturn: raise NotImplementedError("Direct vector store providers execute the search themselves; no HTTP request shape") diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index bad17a2181d..2d72db0cdba 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -27,7 +27,6 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -197,7 +196,6 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, - router: "Router | None" = None, ) -> tuple[str, dict]: if isinstance(query, list): query = " ".join(query) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 0f6966b0ae2..71a598a6fe7 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -184,7 +184,6 @@ if TYPE_CHECKING: AnthropicMessagesStreamingResponse, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig - from litellm.router import Router from litellm.types.llms.openai_evals import ( CancelEvalResponse, CancelRunResponse, @@ -9709,7 +9708,6 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, _is_async: bool = False, - router: "Router | None" = None, ) -> VectorStoreSearchResponse: if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig): self._pre_call_direct_vector_store_search( @@ -9760,7 +9758,6 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, - router=router, embedding_executor=embedding_executor, ) else: @@ -9775,7 +9772,6 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, - router=router, ) all_optional_params: Final[dict[str, object]] = dict(litellm_params) all_optional_params.update(vector_store_search_optional_params or {}) @@ -9828,7 +9824,6 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, _is_async: bool = False, - router: "Router | None" = None, ) -> VectorStoreSearchResponse | Coroutine[object, object, VectorStoreSearchResponse]: if _is_async: return self.async_vector_store_search_handler( @@ -9844,7 +9839,6 @@ class BaseLLMHTTPHandler: extra_body=extra_body, timeout=timeout, client=client, - router=router, ) if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig): @@ -9893,7 +9887,6 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, - router=router, embedding_executor=embedding_executor, ) else: @@ -9908,7 +9901,6 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, - router=router, ) all_optional_params: Final[dict[str, object]] = dict(litellm_params) diff --git a/litellm/llms/gemini/vector_stores/transformation.py b/litellm/llms/gemini/vector_stores/transformation.py index 82586b1f638..f6525a449b6 100644 --- a/litellm/llms/gemini/vector_stores/transformation.py +++ b/litellm/llms/gemini/vector_stores/transformation.py @@ -33,7 +33,6 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -169,7 +168,6 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Mapping[str, object] | None = None, - router: "Router | None" = None, ) -> tuple[str, dict]: """ Transform search request to Gemini's generateContent format. diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py index 4f3c366d8c1..70d3649debd 100644 --- a/litellm/llms/milvus/vector_stores/transformation.py +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -24,7 +24,6 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -129,11 +128,10 @@ class MilvusVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], extra_body: Mapping[str, object] | None = None, - router: Router | None = None, embedding_executor: VectorStoreEmbeddingExecutor | None = None, ) -> tuple[str, dict[str, object]]: query_text: Final = self.query_text(query) - query_vector: Final = self.embed_query(query_text, litellm_params, embedding_executor, router) + query_vector: Final = self.embed_query(query_text, litellm_params, embedding_executor) return self._search_request( vector_store_id, query_text, @@ -153,11 +151,10 @@ class MilvusVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], extra_body: Mapping[str, object] | None = None, - router: Router | None = None, embedding_executor: VectorStoreEmbeddingExecutor | None = None, ) -> tuple[str, dict[str, object]]: query_text: Final = self.query_text(query) - query_vector: Final = await self.aembed_query(query_text, litellm_params, embedding_executor, router) + query_vector: Final = await self.aembed_query(query_text, litellm_params, embedding_executor) return self._search_request( vector_store_id, query_text, diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py index 4e925494039..f6c093f2e2a 100644 --- a/litellm/llms/openai/vector_stores/transformation.py +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -21,7 +21,6 @@ from litellm.utils import add_openai_metadata if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -100,7 +99,6 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, - router: "Router | None" = None, ) -> tuple[str, dict]: encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url: Final = f"{api_base}/{encoded_vector_store_id}/search" diff --git a/litellm/llms/pg_vector/vector_stores/transformation.py b/litellm/llms/pg_vector/vector_stores/transformation.py index 9de1f589ae4..e4b06c36bf4 100644 --- a/litellm/llms/pg_vector/vector_stores/transformation.py +++ b/litellm/llms/pg_vector/vector_stores/transformation.py @@ -8,7 +8,6 @@ from litellm.types.vector_stores import VectorStoreSearchOptionalRequestParams if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -81,7 +80,6 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, - router: "Router | None" = None, ) -> tuple[str, dict]: encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url: Final = f"{api_base}/{encoded_vector_store_id}/search" diff --git a/litellm/llms/ragflow/vector_stores/transformation.py b/litellm/llms/ragflow/vector_stores/transformation.py index ffa6c9e1076..282cb7a92a7 100644 --- a/litellm/llms/ragflow/vector_stores/transformation.py +++ b/litellm/llms/ragflow/vector_stores/transformation.py @@ -17,7 +17,6 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -93,7 +92,6 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, - router: "Router | None" = None, ) -> tuple[str, dict]: """RAGFlow vector stores are management-only, search is not supported.""" raise NotImplementedError("RAGFlow vector stores support dataset management only, not search/retrieval") diff --git a/litellm/llms/s3_vectors/vector_stores/transformation.py b/litellm/llms/s3_vectors/vector_stores/transformation.py index 733358381fe..a9902a0d27c 100644 --- a/litellm/llms/s3_vectors/vector_stores/transformation.py +++ b/litellm/llms/s3_vectors/vector_stores/transformation.py @@ -1,9 +1,12 @@ +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx -from litellm.caching._embedding_router import resolve_embedding_router -from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig +from litellm.llms.base_llm.vector_store.transformation import ( + BaseQueryEmbeddingVectorStoreConfig, + VectorStoreEmbeddingExecutor, +) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( @@ -18,16 +21,18 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.router import Router else: LiteLLMLoggingObj = Any +_DEFAULT_QUERY_EMBEDDING_MODEL: Final = "text-embedding-3-small" +_DEFAULT_TOP_K: Final = 5 -class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): + +class S3VectorsVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, BaseAWSLLM): """Vector store configuration for AWS S3 Vectors.""" def __init__(self) -> None: - BaseVectorStoreConfig.__init__(self) + BaseQueryEmbeddingVectorStoreConfig.__init__(self) BaseAWSLLM.__init__(self) def get_auth_credentials(self, litellm_params: dict) -> BaseVectorStoreAuthCredentials: @@ -59,141 +64,94 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): return headers def get_complete_url(self, api_base: str | None, litellm_params: dict) -> str: - # Resolve region the same way the ingestion path does: - # dynamic param -> AWS_REGION_NAME -> AWS_REGION -> default (us-west-2) aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(litellm_params.get("aws_region_name")) return f"https://s3vectors.{aws_region_name}.api.aws" - def _resolve_query_embedding_router(self, embedding_model: str, router: "Router | None") -> "Router | None": - """Return the router iff it serves ``embedding_model`` as a deployment.""" - if router is None: - return None - model_list: Final = [ - dict(m) for m in (router.get_model_list() or ()) - ] # mutable-ok: resolve_embedding_router requires list[dict] - return resolve_embedding_router(embedding_model=embedding_model, llm_router=router, llm_model_list=model_list) + @staticmethod + def query_embedding_model(litellm_params: Mapping[str, object]) -> str: + configured: Final = litellm_params.get("litellm_embedding_model") or litellm_params.get("embedding_model") + return configured if isinstance(configured, str) and configured else _DEFAULT_QUERY_EMBEDDING_MODEL + + @staticmethod + def _query_target(vector_store_id: str, litellm_params: Mapping[str, object]) -> tuple[str, str]: + if ":" in vector_store_id: + bucket_name, index_name = vector_store_id.split(":", 1) + return bucket_name, index_name + bucket_name_from_params: Final = litellm_params.get("vector_bucket_name") + if not isinstance(bucket_name_from_params, str) or not bucket_name_from_params: + raise ValueError( + "vector_store_id must be in format 'bucket_name:index_name' for S3 Vectors, " + "or vector_bucket_name must be provided in litellm_params" + ) + return bucket_name_from_params, vector_store_id + + @staticmethod + def _query_request( + bucket_name: str, + index_name: str, + query_text: str, + query_vector: Sequence[float], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: LiteLLMLoggingObj, + ) -> tuple[str, dict[str, object]]: + litellm_logging_obj.model_call_details["query"] = query_text + return f"{api_base}/QueryVectors", { + "vectorBucketName": bucket_name, + "indexName": index_name, + "queryVector": {"float32": list(query_vector)}, + "topK": vector_store_search_optional_params.get("max_num_results", _DEFAULT_TOP_K), + "returnDistance": True, + "returnMetadata": True, + } def transform_search_vector_store_request( self, vector_store_id: str, - query: str | list[str], + query: str | Sequence[str], vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, api_base: str, litellm_logging_obj: LiteLLMLoggingObj, - litellm_params: dict, - extra_body: dict[str, Any] | None = None, - router: "Router | None" = None, - ) -> tuple[str, dict]: - """Sync version - generates embedding synchronously.""" - # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name - # If not in that format, try to construct it from litellm_params - bucket_name: str - index_name: str - - if ":" in vector_store_id: - bucket_name, index_name = vector_store_id.split(":", 1) - else: - # Try to get bucket_name from litellm_params - bucket_name_from_params: Final = litellm_params.get("vector_bucket_name") - if not bucket_name_from_params or not isinstance(bucket_name_from_params, str): - raise ValueError( - "vector_store_id must be in format 'bucket_name:index_name' for S3 Vectors, " - "or vector_bucket_name must be provided in litellm_params" - ) - bucket_name = bucket_name_from_params - index_name = vector_store_id - - if isinstance(query, list): - query = " ".join(query) - - # Generate embedding for the query - embedding_model: Final = litellm_params.get("embedding_model", "text-embedding-3-small") - embedding_router: Final = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router) - - import litellm as litellm_module - - embedding_input: Final = [query] # mutable-ok: the embedding API takes list input - embedding_response: Final = ( - embedding_router.embedding(model=embedding_model, input=embedding_input) - if embedding_router is not None - else litellm_module.embedding(model=embedding_model, input=embedding_input) + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: + bucket_name, index_name = self._query_target(vector_store_id, litellm_params) + query_text: Final = self.query_text(query) + query_vector: Final = self.embed_query(query_text, litellm_params, embedding_executor) + return self._query_request( + bucket_name, + index_name, + query_text, + query_vector, + vector_store_search_optional_params, + api_base, + litellm_logging_obj, ) - query_embedding: Final = embedding_response.data[0]["embedding"] - - url: Final = f"{api_base}/QueryVectors" - - request_body: Final[dict[str, Any]] = { - "vectorBucketName": bucket_name, - "indexName": index_name, - "queryVector": {"float32": query_embedding}, - "topK": vector_store_search_optional_params.get("max_num_results", 5), # Default to 5 - "returnDistance": True, - "returnMetadata": True, - } - - litellm_logging_obj.model_call_details["query"] = query - return url, request_body async def atransform_search_vector_store_request( self, vector_store_id: str, - query: str | list[str], + query: str | Sequence[str], vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, api_base: str, litellm_logging_obj: LiteLLMLoggingObj, - litellm_params: dict, - extra_body: dict[str, Any] | None = None, - router: "Router | None" = None, - ) -> tuple[str, dict]: - """Async version - generates embedding asynchronously.""" - # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name - # If not in that format, try to construct it from litellm_params - bucket_name: str - index_name: str - - if ":" in vector_store_id: - bucket_name, index_name = vector_store_id.split(":", 1) - else: - # Try to get bucket_name from litellm_params - bucket_name_from_params: Final = litellm_params.get("vector_bucket_name") - if not bucket_name_from_params or not isinstance(bucket_name_from_params, str): - raise ValueError( - "vector_store_id must be in format 'bucket_name:index_name' for S3 Vectors, " - "or vector_bucket_name must be provided in litellm_params" - ) - bucket_name = bucket_name_from_params - index_name = vector_store_id - - if isinstance(query, list): - query = " ".join(query) - - # Generate embedding for the query asynchronously - embedding_model: Final = litellm_params.get("embedding_model", "text-embedding-3-small") - embedding_router: Final = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router) - - import litellm as litellm_module - - embedding_input: Final = [query] # mutable-ok: the embedding API takes list input - embedding_response: Final = ( - await embedding_router.aembedding(model=embedding_model, input=embedding_input) - if embedding_router is not None - else await litellm_module.aembedding(model=embedding_model, input=embedding_input) + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: + bucket_name, index_name = self._query_target(vector_store_id, litellm_params) + query_text: Final = self.query_text(query) + query_vector: Final = await self.aembed_query(query_text, litellm_params, embedding_executor) + return self._query_request( + bucket_name, + index_name, + query_text, + query_vector, + vector_store_search_optional_params, + api_base, + litellm_logging_obj, ) - query_embedding: Final = embedding_response.data[0]["embedding"] - - url: Final = f"{api_base}/QueryVectors" - - request_body: Final[dict[str, Any]] = { - "vectorBucketName": bucket_name, - "indexName": index_name, - "queryVector": {"float32": query_embedding}, - "topK": vector_store_search_optional_params.get("max_num_results", 5), # Default to 5 - "returnDistance": True, - "returnMetadata": True, - } - - litellm_logging_obj.model_call_details["query"] = query - return url, request_body def sign_request( self, @@ -226,21 +184,13 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): if not source_text: continue - # Extract file information from metadata chunk_index = metadata.get("chunk_index", "0") file_id = f"s3-vectors-chunk-{chunk_index}" filename = metadata.get("filename", f"document-{chunk_index}") - # S3 Vectors returns distance, convert to similarity score (0-1) - # Lower distance = higher similarity - # We'll normalize using 1 / (1 + distance) to get a 0-1 score distance = item.get("distance") score = None if distance is not None: - # Convert distance to similarity score between 0 and 1 - # For cosine distance: similarity = 1 - distance - # For euclidean: use 1 / (1 + distance) - # Assuming cosine distance here score = max(0.0, min(1.0, 1.0 - float(distance))) results.append( @@ -265,7 +215,6 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): headers=response.headers, ) - # Vector store creation is not yet implemented def transform_create_vector_store_request( self, vector_store_create_optional_params, diff --git a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py index 36b57e7c995..5c250fc1a7e 100644 --- a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py @@ -21,7 +21,6 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -162,7 +161,6 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Mapping[str, object] | None = None, - router: "Router | None" = None, ) -> tuple[str, dict[str, object]]: """ Transform search request for Vertex AI RAG API diff --git a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py index f0812e3ed9f..0bcf16ee06f 100644 --- a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py @@ -25,7 +25,6 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -246,7 +245,6 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Mapping[str, object] | None = None, - router: "Router | None" = None, ) -> tuple[str, dict[str, object]]: """ Transform a search request for the Vertex AI Search (Discovery Engine) API. diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index 636bdd4b52e..2fe1965a192 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -482,7 +482,6 @@ def search( timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), - router=router, ) return response diff --git a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py index 4b58d220623..e2b02ea2151 100644 --- a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py +++ b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py @@ -1,40 +1,72 @@ +from collections.abc import Mapping from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest +from litellm.llms.base_llm.vector_store.transformation import ( + RouterVectorStoreEmbeddingExecutor, +) from litellm.llms.s3_vectors.vector_stores.transformation import ( S3VectorsVectorStoreConfig, ) +from litellm.types.utils import EmbeddingResponse from litellm.types.vector_stores import VectorStoreSearchResponse +QUERY_VECTOR = [0.1, 0.2, 0.3] -def _mock_router(model_names, sync=False): - """Router mock serving the given embedding model names.""" - router = MagicMock() - router.get_model_list.return_value = [{"model_name": name} for name in model_names] - embedding_response = Mock(data=[{"embedding": [0.1, 0.2, 0.3]}]) - if sync: - router.embedding = MagicMock(return_value=embedding_response) - else: - router.aembedding = AsyncMock(return_value=embedding_response) - return router + +def _embedding_response(vector): + return EmbeddingResponse(data=[{"embedding": vector, "index": 0, "object": "embedding"}]) + + +class _RecordingExecutor: + """Executor double recording every (model, query, configuration) it was asked to embed.""" + + def __init__(self, vector=QUERY_VECTOR): + self.vector = vector + self.calls = [] + + def embed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: + self.calls.append((model, query, dict(configuration))) + return _embedding_response(self.vector) + + async def aembed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: + self.calls.append((model, query, dict(configuration))) + return _embedding_response(self.vector) + + +def _logging_obj(): + logging_obj = Mock() + logging_obj.model_call_details = {} + return logging_obj + + +def _search_kwargs(**overrides): + kwargs = { + "vector_store_id": "test-bucket:test-index", + "query": "test query", + "vector_store_search_optional_params": {}, + "api_base": "https://s3vectors.us-west-2.api.aws", + "litellm_logging_obj": _logging_obj(), + "litellm_params": {}, + "extra_body": None, + } + kwargs.update(overrides) + return kwargs class TestS3VectorsVectorStoreConfig: def test_init(self): - """Test that S3VectorsVectorStoreConfig initializes correctly""" config = S3VectorsVectorStoreConfig() assert config is not None def test_get_supported_openai_params(self): - """Test that supported OpenAI params are returned""" config = S3VectorsVectorStoreConfig() params = config.get_supported_openai_params("test-model") assert "max_num_results" in params def test_get_complete_url(self): - """Test URL generation for S3 Vectors""" config = S3VectorsVectorStoreConfig() litellm_params = {"aws_region_name": "us-west-2"} url = config.get_complete_url(None, litellm_params) @@ -57,180 +89,149 @@ class TestS3VectorsVectorStoreConfig: assert url == "https://s3vectors.eu-west-1.api.aws" def test_get_complete_url_invalid_region_format(self): - """Invalid region format raises""" config = S3VectorsVectorStoreConfig() with pytest.raises(ValueError, match="Invalid AWS region format"): config.get_complete_url(None, {"aws_region_name": "Bad_Region!"}) def test_transform_search_request(self): - """Full request-body transformation with a router-injected embedding""" + """Full request-body transformation with the query embedded through the injected executor""" config = S3VectorsVectorStoreConfig() - mock_logging_obj = Mock() - mock_logging_obj.model_call_details = {} - router = _mock_router(["text-embedding-3-small"], sync=True) + logging_obj = _logging_obj() + executor = _RecordingExecutor() url, request_body = config.transform_search_vector_store_request( - vector_store_id="test-bucket:test-index", - query="test query", - vector_store_search_optional_params={"max_num_results": 7}, - api_base="https://s3vectors.us-west-2.api.aws", - litellm_logging_obj=mock_logging_obj, - litellm_params={}, - extra_body=None, - router=router, + **_search_kwargs( + vector_store_search_optional_params={"max_num_results": 7}, + litellm_logging_obj=logging_obj, + embedding_executor=executor, + ) ) assert url == "https://s3vectors.us-west-2.api.aws/QueryVectors" assert request_body == { "vectorBucketName": "test-bucket", "indexName": "test-index", - "queryVector": {"float32": [0.1, 0.2, 0.3]}, + "queryVector": {"float32": QUERY_VECTOR}, "topK": 7, "returnDistance": True, "returnMetadata": True, } - assert mock_logging_obj.model_call_details["query"] == "test query" + assert executor.calls == [("text-embedding-3-small", "test query", {})] + assert logging_obj.model_call_details["query"] == "test query" + + @pytest.mark.parametrize( + ("litellm_params", "expected_model"), + [ + ({}, "text-embedding-3-small"), + ({"embedding_model": ""}, "text-embedding-3-small"), + ({"embedding_model": "my-embedding-model"}, "my-embedding-model"), + ({"litellm_embedding_model": "shared-key-model"}, "shared-key-model"), + ( + {"litellm_embedding_model": "shared-key-model", "embedding_model": "legacy-alias"}, + "shared-key-model", + ), + ], + ) + def test_query_embedding_model_accepts_embedding_model_alias(self, litellm_params, expected_model): + assert S3VectorsVectorStoreConfig.query_embedding_model(litellm_params) == expected_model @pytest.mark.asyncio - async def test_atransform_search_uses_router_for_virtual_model(self): - """Regression: router-served embedding models must resolve via the router, - not a bare litellm.aembedding call (which has no deployment credentials).""" + async def test_atransform_search_embeds_alias_and_store_config_through_executor(self): + """The store's embedding_model alias and litellm_embedding_config reach the executor unchanged""" config = S3VectorsVectorStoreConfig() - mock_logging_obj = Mock() - mock_logging_obj.model_call_details = {} - router = _mock_router(["my-embedding-model"]) + executor = _RecordingExecutor(vector=[0.4, 0.5]) - with patch("litellm.aembedding", new=AsyncMock()) as mock_bare_aembedding: # test-quality-ok: guards that the bare-embedding path is not taken; dispatch seam is the behavior under test - url, request_body = await config.atransform_search_vector_store_request( - vector_store_id="test-bucket:test-index", - query="test query", - vector_store_search_optional_params={}, - api_base="https://s3vectors.us-west-2.api.aws", - litellm_logging_obj=mock_logging_obj, - litellm_params={"embedding_model": "my-embedding-model"}, - extra_body=None, - router=router, + _, request_body = await config.atransform_search_vector_store_request( + **_search_kwargs( + query=["test", "query"], + litellm_params={ + "embedding_model": "my-embedding-model", + "litellm_embedding_config": {"api_key": "store-key"}, + }, + embedding_executor=executor, ) + ) - router.aembedding.assert_awaited_once_with(model="my-embedding-model", input=["test query"]) - mock_bare_aembedding.assert_not_awaited() - assert request_body["queryVector"]["float32"] == [0.1, 0.2, 0.3] - assert request_body["topK"] == 5 # default - - @pytest.mark.asyncio - async def test_atransform_search_falls_back_when_router_does_not_serve_model(self): - """Router present but embedding_model is not a router deployment -> - bare litellm.aembedding keeps working (provider-prefixed + env creds stores).""" - config = S3VectorsVectorStoreConfig() - mock_logging_obj = Mock() - mock_logging_obj.model_call_details = {} - router = _mock_router(["some-other-model"]) - - mock_bare = AsyncMock(return_value=Mock(data=[{"embedding": [0.4, 0.5]}])) - with patch("litellm.aembedding", new=mock_bare): # test-quality-ok: stubs the bare-embedding fallback whose request body the test asserts on - _, request_body = await config.atransform_search_vector_store_request( - vector_store_id="test-bucket:test-index", - query="test query", - vector_store_search_optional_params={}, - api_base="https://s3vectors.us-west-2.api.aws", - litellm_logging_obj=mock_logging_obj, - litellm_params={"embedding_model": "azure/text-embedding-3-small"}, - extra_body=None, - router=router, - ) - - mock_bare.assert_awaited_once_with(model="azure/text-embedding-3-small", input=["test query"]) - router.aembedding.assert_not_awaited() + assert executor.calls == [("my-embedding-model", "test query", {"api_key": "store-key"})] assert request_body["queryVector"]["float32"] == [0.4, 0.5] + assert request_body["topK"] == 5 @pytest.mark.asyncio - async def test_atransform_search_without_router_uses_bare_embedding(self): - """Backward compat: no router -> bare litellm.aembedding as before""" + async def test_atransform_search_router_executor_carries_request_metadata(self): + """Regression (LIT-6750): a bare Router alias resolves through the Router with the request's + team metadata on the embedding call, so the embedding is attributed to the calling key and team.""" config = S3VectorsVectorStoreConfig() - mock_logging_obj = Mock() - mock_logging_obj.model_call_details = {} + router = MagicMock() + router.aembedding = AsyncMock(return_value=_embedding_response(QUERY_VECTOR)) + request_metadata = {"user_api_key_team_id": "team-a", "user_api_key": "hashed-key"} - mock_bare = AsyncMock(return_value=Mock(data=[{"embedding": [0.6, 0.7]}])) - with patch("litellm.aembedding", new=mock_bare): # test-quality-ok: stubs the bare-embedding fallback whose request body the test asserts on - _, request_body = await config.atransform_search_vector_store_request( - vector_store_id="test-bucket:test-index", - query="test query", - vector_store_search_optional_params={}, - api_base="https://s3vectors.us-west-2.api.aws", - litellm_logging_obj=mock_logging_obj, - litellm_params={}, - extra_body=None, + _, request_body = await config.atransform_search_vector_store_request( + **_search_kwargs( + litellm_params={"embedding_model": "team-embeddings"}, + embedding_executor=RouterVectorStoreEmbeddingExecutor(router=router, metadata=request_metadata), ) + ) + + router.aembedding.assert_awaited_once_with( + model="team-embeddings", input=["test query"], metadata=request_metadata + ) + assert request_body["queryVector"]["float32"] == QUERY_VECTOR + + @pytest.mark.asyncio + async def test_atransform_search_without_executor_uses_bare_embedding(self): + """Backward compat: SDK callers without an executor keep embedding through litellm.aembedding""" + config = S3VectorsVectorStoreConfig() + + mock_bare = AsyncMock(return_value=_embedding_response([0.6, 0.7])) + with patch("litellm.aembedding", new=mock_bare): # test-quality-ok: stubs the bare-embedding fallback whose request body the test asserts on + _, request_body = await config.atransform_search_vector_store_request(**_search_kwargs()) mock_bare.assert_awaited_once_with(model="text-embedding-3-small", input=["test query"]) assert request_body["queryVector"]["float32"] == [0.6, 0.7] - def test_transform_search_uses_router_for_virtual_model_sync(self): - """Sync twin: router-served embedding model resolves via router.embedding""" + def test_transform_search_without_executor_uses_bare_embedding_sync(self): + """Sync twin: no executor -> bare litellm.embedding as before""" config = S3VectorsVectorStoreConfig() - mock_logging_obj = Mock() - mock_logging_obj.model_call_details = {} - router = _mock_router(["my-embedding-model"], sync=True) - with patch("litellm.embedding", new=MagicMock()) as mock_bare_embedding: # test-quality-ok: guards that the bare-embedding path is not taken; dispatch seam is the behavior under test - _, request_body = config.transform_search_vector_store_request( - vector_store_id="test-bucket:test-index", - query="test query", - vector_store_search_optional_params={}, - api_base="https://s3vectors.us-west-2.api.aws", - litellm_logging_obj=mock_logging_obj, - litellm_params={"embedding_model": "my-embedding-model"}, - extra_body=None, - router=router, - ) - - router.embedding.assert_called_once_with(model="my-embedding-model", input=["test query"]) - mock_bare_embedding.assert_not_called() - assert request_body["queryVector"]["float32"] == [0.1, 0.2, 0.3] - - def test_transform_search_without_router_uses_bare_embedding_sync(self): - """Sync twin: no router -> bare litellm.embedding as before""" - config = S3VectorsVectorStoreConfig() - mock_logging_obj = Mock() - mock_logging_obj.model_call_details = {} - - mock_bare = MagicMock(return_value=Mock(data=[{"embedding": [0.8, 0.9]}])) + mock_bare = MagicMock(return_value=_embedding_response([0.8, 0.9])) with patch("litellm.embedding", new=mock_bare): # test-quality-ok: stubs the bare-embedding fallback whose request body the test asserts on _, request_body = config.transform_search_vector_store_request( - vector_store_id="test-bucket:test-index", - query="test query", - vector_store_search_optional_params={}, - api_base="https://s3vectors.us-west-2.api.aws", - litellm_logging_obj=mock_logging_obj, - litellm_params={}, - extra_body=None, + **_search_kwargs(litellm_params={"embedding_model": "my-embedding-model"}) ) - mock_bare.assert_called_once_with(model="text-embedding-3-small", input=["test query"]) + mock_bare.assert_called_once_with(model="my-embedding-model", input=["test query"]) assert request_body["queryVector"]["float32"] == [0.8, 0.9] def test_transform_search_request_invalid_vector_store_id(self): - """Test that invalid vector_store_id format raises error""" + """An unparseable vector_store_id raises before any embedding is generated""" config = S3VectorsVectorStoreConfig() - mock_logging_obj = Mock() - mock_logging_obj.model_call_details = {} + executor = _RecordingExecutor() with pytest.raises( ValueError, match="vector_store_id must be in format 'bucket_name:index_name'", ): config.transform_search_vector_store_request( - vector_store_id="invalid-format", - query="test query", - vector_store_search_optional_params={}, - api_base="https://s3vectors.us-west-2.api.aws", - litellm_logging_obj=mock_logging_obj, - litellm_params={}, - extra_body=None, + **_search_kwargs(vector_store_id="invalid-format", embedding_executor=executor) ) + assert executor.calls == [] + + def test_transform_search_request_bucket_from_litellm_params(self): + config = S3VectorsVectorStoreConfig() + + _, request_body = config.transform_search_vector_store_request( + **_search_kwargs( + vector_store_id="only-index", + litellm_params={"vector_bucket_name": "params-bucket"}, + embedding_executor=_RecordingExecutor(), + ) + ) + + assert request_body["vectorBucketName"] == "params-bucket" + assert request_body["indexName"] == "only-index" + def test_transform_search_response(self): - """Test search response transformation""" config = S3VectorsVectorStoreConfig() mock_logging_obj = Mock() mock_logging_obj.model_call_details = {"query": "test query"} @@ -239,7 +240,7 @@ class TestS3VectorsVectorStoreConfig: mock_response.json.return_value = { "vectors": [ { - "distance": 0.05, # S3 Vectors returns distance, not score + "distance": 0.05, "metadata": { "source_text": "This is test content", "chunk_index": "0", @@ -258,23 +259,18 @@ class TestS3VectorsVectorStoreConfig: mock_response.status_code = 200 mock_response.headers = {} - result = config.transform_search_vector_store_response( - mock_response, mock_logging_obj - ) + result = config.transform_search_vector_store_response(mock_response, mock_logging_obj) - # VectorStoreSearchResponse is a TypedDict, so check structure instead of isinstance assert result["object"] == "vector_store.search_results.page" assert result["search_query"] == "test query" assert len(result["data"]) == 2 - # Score should be 1 - distance (cosine similarity) - assert result["data"][0]["score"] == 0.95 # 1 - 0.05 + assert result["data"][0]["score"] == 0.95 assert result["data"][0]["content"][0]["text"] == "This is test content" assert result["data"][0]["filename"] == "test.pdf" - assert result["data"][1]["score"] == 0.85 # 1 - 0.15 + assert result["data"][1]["score"] == 0.85 assert result["data"][1]["content"][0]["text"] == "More test content" def test_map_openai_params(self): - """Test OpenAI parameter mapping""" config = S3VectorsVectorStoreConfig() non_default_params = {"max_num_results": 5} optional_params = {} diff --git a/tests/test_litellm/vector_stores/test_main.py b/tests/test_litellm/vector_stores/test_main.py index d01e696906a..234e0b01094 100644 --- a/tests/test_litellm/vector_stores/test_main.py +++ b/tests/test_litellm/vector_stores/test_main.py @@ -2,14 +2,17 @@ Tests for litellm/vector_stores/main.py. Pins the router threading contract for vector store search: the router is an -explicit named parameter that reaches the HTTP handler, and it must never leak -into litellm_params/kwargs where logging would model_dump() it (the #19550 -serialization trap). +explicit named parameter that reaches the HTTP handler wrapped in the embedding +executor, and it must never leak into litellm_params/kwargs where logging would +model_dump() it (the #19550 serialization trap). """ from unittest.mock import MagicMock, patch import litellm.vector_stores.main as vector_stores_main +from litellm.llms.base_llm.vector_store.transformation import ( + RouterVectorStoreEmbeddingExecutor, +) from litellm.vector_stores.main import search MOCK_SEARCH_RESPONSE = { @@ -19,17 +22,18 @@ MOCK_SEARCH_RESPONSE = { } -def test_search_threads_router_to_handler(): - """search() must pass its router param through to the HTTP handler""" +def test_search_wraps_router_into_the_handler_embedding_executor(): + """search() hands the HTTP handler a Router-backed embedding executor carrying the + request metadata, and no bare router kwarg (LIT-6750)""" mock_router = MagicMock() logger = MagicMock() with ( - patch( # test-quality-ok: stubs provider config resolution; the seam under test is the router kwarg threading + patch( # test-quality-ok: stubs provider config resolution; the seam under test is the executor threading "litellm.vector_stores.main.ProviderConfigManager.get_provider_vector_stores_config", return_value=MagicMock(), ), - patch.object( # test-quality-ok: the handler call is the observable boundary for the router kwarg contract + patch.object( # test-quality-ok: the handler call is the observable boundary for the executor contract vector_stores_main.base_llm_http_handler, "vector_store_search_handler", return_value=MOCK_SEARCH_RESPONSE, @@ -41,11 +45,16 @@ def test_search_threads_router_to_handler(): custom_llm_provider="s3_vectors", router=mock_router, litellm_logging_obj=logger, + litellm_metadata={"user_api_key_team_id": "team-a"}, ) assert response == MOCK_SEARCH_RESPONSE mock_handler.assert_called_once() - assert mock_handler.call_args.kwargs["router"] is mock_router + assert "router" not in mock_handler.call_args.kwargs + executor = mock_handler.call_args.kwargs["embedding_executor"] + assert isinstance(executor, RouterVectorStoreEmbeddingExecutor) + assert executor.router is mock_router + assert dict(executor.metadata) == {"user_api_key_team_id": "team-a"} def test_search_router_not_in_litellm_params(): From 64e45a069db6794577a94104a20d78e6b5803ffb Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 2 Sep 2026 19:36:09 -0700 Subject: [PATCH 187/480] feat(complexity_router): opt-in modality override of a kept session-affinity pin (#39454) A session pinned to a text-only model failed every image turn with a provider 400, because the modality gate exempts a kept pin by cause. Add modality_pin_override so that exemption is conditional: the image turn is re-placed on a capable model for that request only, reported as cause modality_pin_override, and the stored pin is left untouched so the next text turn replays it. The pin write on the replay path already happens upstream of the gate and stores the session's own model, so pin survival is structural rather than bookkeeping. The new cause joins the non-pinnable set. Default off at every layer. --- .../public_endpoints/autorouter_presets.json | 4 ++ .../complexity_router/README.md | 12 +++- .../complexity_router/complexity_router.py | 23 +++++-- .../complexity_router/config.py | 15 +++- litellm/types/utils.py | 4 ++ .../router_strategy/test_complexity_router.py | 68 ++++++++++++++++++- .../add_model/ComplexityRouterConfig.test.tsx | 38 +++++++++++ .../add_model/ComplexityRouterConfig.tsx | 1 + .../add_model/ModalityRoutingControls.tsx | 50 +++++++++----- .../add_model/add_auto_router_tab.test.tsx | 38 +++++++++++ .../add_model/add_auto_router_tab.tsx | 1 + .../build_complexity_router_config.test.ts | 9 +++ .../build_complexity_router_config.ts | 4 ++ ...d_updated_complexity_router_config.test.ts | 26 +++++++ .../edit_auto_router_modal.test.ts | 22 ++++++ .../edit_auto_router_modal.test.tsx | 44 ++++++++++++ .../edit_auto_router_modal.tsx | 5 ++ .../RoutingDecisionCard.test.tsx | 6 ++ .../LogDetailsDrawer/RoutingDecisionCard.tsx | 1 + .../src/lib/autorouter_presets.test.ts | 19 ++++++ .../src/lib/autorouter_presets.ts | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 ++- 22 files changed, 374 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/public_endpoints/autorouter_presets.json b/litellm/proxy/public_endpoints/autorouter_presets.json index 1e0d40567d7..c2b13b81542 100644 --- a/litellm/proxy/public_endpoints/autorouter_presets.json +++ b/litellm/proxy/public_endpoints/autorouter_presets.json @@ -17,6 +17,7 @@ "classification_mode": "every_request", "session_affinity": false, "modality_routing": false, + "modality_pin_override": false, "deployment_affinity": true } }, @@ -35,6 +36,7 @@ "classification_mode": "every_request", "session_affinity": false, "modality_routing": false, + "modality_pin_override": false, "deployment_affinity": true } }, @@ -63,6 +65,7 @@ "classification_mode": "every_request", "session_affinity": false, "modality_routing": false, + "modality_pin_override": false, "deployment_affinity": true } }, @@ -84,6 +87,7 @@ "classification_mode": "every_request", "session_affinity": false, "modality_routing": false, + "modality_pin_override": false, "deployment_affinity": true } } diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index ad8b67d5e8f..ee51add1ca1 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -187,6 +187,9 @@ model_list: # Replace a routed model that cannot take image input (default: false) modality_routing: true + + # Let that replacement also override a kept session pin, for image turns only (default: false) + modality_pin_override: true ``` ## Usage @@ -227,9 +230,16 @@ vision model sits below the decided tier gets the 400 and an actionable message A same-tier re-pick keeps the decision's cause and adds `modality:image` to `signals`; a tier change or default takeover records `cause: modality_escalation` with the displaced placement (`modality_escalated_from:` or `modality_displaced_default_model`). Escalations are never -pinned by session affinity, and a KEPT session pin bypasses the gate entirely: a session pinned +pinned by session affinity, and by default a KEPT session pin bypasses the gate: a session pinned to a text-only model keeps it even when an image arrives. +Add `modality_pin_override: true` to lift that last exemption. The image turn is then re-placed +the same way every other decision is, and records `cause: modality_pin_override` whether or not +the tier moved, since the model left the pin either way. The pin itself is untouched: the session +affinity write happens upstream of the gate and stores the session's own model, so the next text +turn replays the original pin and the override is never pinned in its place. It does nothing +unless `modality_routing` is also on. + ### Heuristic-first chaining `classifier_type: heuristic_first` runs the local scorer on every request and only calls the LLM diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 430efe339a2..a00ae6bee80 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -751,7 +751,8 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo A modality escalation is transient the same way: it describes what this one call carries (an image), not what the session's traffic looks like, and pinning it would hold every following - text turn on the vision-capable model the image forced. + text turn on the vision-capable model the image forced. A modality pin override is the same + fact on a session that already holds a pin, so it must not overwrite the pin it displaced. """ return decision is None or ( decision.get("cause") @@ -760,6 +761,7 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo "plan_mode", "housekeeping", "modality_escalation", + "modality_pin_override", ) and not decision.get("context_escalated") ) @@ -2393,8 +2395,11 @@ class ComplexityRouter(CustomLogger): """Replace a routed model that cannot accept this request's image input. The single modality owner, applied to the decided response at the hook's exits so every - routing path is covered uniformly. A KEPT session pin is exempt by design (its cause); - replacement picks and every other path are just responses. The re-placement walks + routing path is covered uniformly. A KEPT session pin is exempt by design (its cause) + unless modality_pin_override is set, in which case the image turn is re-placed and reported + as modality_pin_override while the stored pin, written upstream from the session's own + model, is left for the next text turn; replacement picks and every other path are just + responses. The re-placement walks UPWARD-ONLY from the decision's tier (so a plan-mode floor can never be undercut), picks through `_pick_model_for_tier` so routing plugins still apply, then falls to default_model (never on plugin routers, and never on a plan-floored decision, since @@ -2407,7 +2412,11 @@ class ComplexityRouter(CustomLogger): not self.config.modality_routing or not resolved_messages or response.model is None - or (decision is not None and decision.get("cause") == "session_affinity_pin") + or ( + decision is not None + and decision.get("cause") == "session_affinity_pin" + and not self.config.modality_pin_override + ) or not request_contains_image_content(resolved_messages) or self._model_accepts_image_input(response.model) ): @@ -2449,6 +2458,10 @@ class ComplexityRouter(CustomLogger): self._restamp_adaptive_choice(request_kwargs, response.model, new_model) same_tier: Final = capable is not None and decided == capable base_cause: Final = (decision.get("cause") if decision is not None else None) or "default_fallback" + # Reaching here on a kept pin means modality_pin_override is on, since the guard above + # returns otherwise. The model moved off the pin even on a same-tier repick, so reporting + # the pin's own cause would claim the session's model served a request it did not. + displaced_pin: Final = base_cause == "session_affinity_pin" displaced_default: Final = decided is None and response.model == self.config.default_model markers: Final = ( "modality:image", @@ -2458,7 +2471,7 @@ class ComplexityRouter(CustomLogger): old_signals: Final = tuple(decision.get("signals") or ()) if decision is not None else () new_decision: Final = self._build_routing_decision( routed_model=new_model, - cause=base_cause if same_tier else "modality_escalation", + cause="modality_pin_override" if displaced_pin else (base_cause if same_tier else "modality_escalation"), tier=new_tier, score=decision.get("score") if decision is not None else None, signals=(*old_signals, *markers), diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 0ae0db63fad..9f2054dda01 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -883,7 +883,20 @@ class ComplexityRouterConfig(BaseModel): "a routed model explicitly declared supports_vision false (deployment model_info " "or the model cost map; unmapped names stay routable) is replaced by the nearest " "HIGHER tier holding a capable model, then default_model, else a clear 400. A kept " - "session-affinity pin still wins even when an image arrives." + "session-affinity pin still wins even when an image arrives, unless " + "modality_pin_override is also enabled." + ), + ) + modality_pin_override: bool = Field( + default=False, + description=( + "Let modality_routing replace a kept session-affinity pin on the turns that carry an " + "image. Without this, a session pinned to a text-only model fails every image turn with " + "a provider 400, since the pin is exempt from the modality gate. When enabled, such a " + "turn routes to a capable model for that request only and the stored pin is left " + "untouched, so the next text turn replays the session's own model; the override is " + "reported as cause modality_pin_override and is never itself pinned. Inert unless " + "modality_routing is also enabled." ), ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ee6f09e05dc..569fce4f7b8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2879,6 +2879,10 @@ RoutingDecisionCause = Literal[ # routed model does not accept image input, so the nearest higher capable tier or # default_model served instead. The displaced placement rides in signals. "modality_escalation", + # modality_pin_override replaced a KEPT session-affinity pin for this request only: the turn + # carries an image the pinned model cannot accept. The stored pin is untouched, so the next + # text turn replays it. Distinct from "modality_escalation", which never displaces a pin. + "modality_pin_override", "session_affinity_pin", "session_affinity_escalation", # classification_mode 'user_turn': the request is an agent loop's continuation turn (no new diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index d7d02544efb..3ecccb673f3 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -10797,6 +10797,9 @@ class TestModalityRouting: ("custom_tiers_walk", "premium-model", "modality_escalation"), ("pin_kept_bypasses", "text-cheap", "session_affinity_pin"), ("pin_replacement_gated", "vision-big", "modality_escalation"), + ("pin_override_escalates", "vision-mid", "modality_pin_override"), + ("pin_override_same_tier", "vision-cheap", "modality_pin_override"), + ("pin_override_inert_without_modality_routing", "text-cheap", "session_affinity_pin"), ("adaptive_pick_rewritten", "vision-mid", "modality_escalation"), ], ) @@ -10847,7 +10850,7 @@ class TestModalityRouting: messages = [ {"role": "user", "content": [{"type": "text", "text": "quick lookup: what is this?"}, IMG_PART]} ] - elif path in ("pin_kept_bypasses", "pin_replacement_gated"): + elif path.startswith(("pin_kept", "pin_replacement", "pin_override")): cache = AsyncMock() cache.async_get_cache = AsyncMock(return_value={"model": "text-cheap", "tier": "SIMPLE"}) mock_router_instance.cache = cache @@ -10859,6 +10862,13 @@ class TestModalityRouting: messages = [ {"role": "user", "content": [{"type": "text", "text": "LITELLM ESCALATE describe this"}, IMG_PART]} ] + elif path == "pin_override_same_tier": + config["modality_pin_override"] = True + config["tiers"]["SIMPLE"] = ["text-cheap", "vision-cheap"] + vision["vision-cheap"] = True + elif path == "pin_override_inert_without_modality_routing": + config["modality_routing"] = False + config["modality_pin_override"] = path.startswith("pin_override") elif path == "adaptive_pick_rewritten": config["adaptive"] = True mock_router_instance.model_list = [] @@ -11025,4 +11035,60 @@ class TestModalityRouting: from litellm.router_strategy.complexity_router.complexity_router import _decision_is_pinnable assert _decision_is_pinnable({"cause": "modality_escalation"}) is False + assert _decision_is_pinnable({"cause": "modality_pin_override"}) is False assert _decision_is_pinnable({"cause": "heuristic_scorer"}) is True + + @pytest.mark.asyncio + async def test_pin_override_serves_the_image_turn_without_repinning(self, mock_router_instance): + """The override is for one request: the session keeps the model it was pinned to.""" + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value={"model": "text-cheap", "tier": "SIMPLE"}) + mock_router_instance.cache = cache + router = self._router( + mock_router_instance, + { + "tiers": dict(self.BASE_TIERS), + "modality_routing": True, + "modality_pin_override": True, + "session_affinity": True, + }, + dict(self.BASE_VISION), + ) + request_kwargs = {"metadata": {"session_id": "s1"}} + + image_turn = await router.async_pre_routing_hook( + model="m", request_kwargs=request_kwargs, messages=self.IMAGE_MESSAGE + ) + assert image_turn.model == "vision-mid" + assert image_turn.routing_decision["cause"] == "modality_pin_override" + assert "modality_escalated_from:SIMPLE" in image_turn.routing_decision["signals"] + + assert cache.async_set_cache.await_args.kwargs["value"] == {"model": "text-cheap", "tier": "SIMPLE"} + + text_turn = await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "s1"}}, messages=[{"role": "user", "content": "hi"}] + ) + assert text_turn.model == "text-cheap" + assert text_turn.routing_decision["cause"] == "session_affinity_pin" + + @pytest.mark.asyncio + async def test_pin_override_with_no_capable_model_rejects_and_keeps_the_pin(self, mock_router_instance): + """The clear 400 replaces the provider's, and a rejected turn must not cost the session its pin.""" + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value={"model": "text-cheap", "tier": "SIMPLE"}) + mock_router_instance.cache = cache + router = self._router( + mock_router_instance, + { + "tiers": {"SIMPLE": "text-cheap", "COMPLEX": "text-big"}, + "modality_routing": True, + "modality_pin_override": True, + "session_affinity": True, + }, + {"text-cheap": False, "text-big": False}, + ) + with pytest.raises(litellm.BadRequestError, match="no model"): + await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "s1"}}, messages=self.IMAGE_MESSAGE + ) + assert cache.async_set_cache.await_args.kwargs["value"] == {"model": "text-cheap", "tier": "SIMPLE"} diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 349d03356f8..9b647badf69 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -880,6 +880,44 @@ describe("ComplexityRouterConfig modality panel", () => { expect(screen.getByRole("switch", { name: "Route image requests to vision-capable models" })).toBeChecked(); }); + + // The backend ignores modality_pin_override unless modality_routing is on, so offering it while + // image routing is off would let an operator save a flag that does nothing. + it("disables the pin-override switch while image routing is off", () => { + const onChange = vi.fn(); + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Modality Routing")); + + const override = screen.getByRole("switch", { name: "Override session pin for image requests" }); + expect(override).toHaveAttribute("aria-disabled", "true"); + fireEvent.click(override); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("writes modality_pin_override through onChange once image routing is on", () => { + const onChange = vi.fn(); + const value = { ...defaultValue, modality_routing: true }; + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Modality Routing")); + + const override = screen.getByRole("switch", { name: "Override session pin for image requests" }); + expect(override).not.toBeChecked(); + fireEvent.click(override); + + expect(onChange).toHaveBeenCalledWith({ ...value, modality_pin_override: true }); + }); + + it("renders a stored modality_pin_override=true as on", () => { + renderWithProviders( + , + ); + fireEvent.click(screen.getByText("Advanced: Modality Routing")); + + expect(screen.getByRole("switch", { name: "Override session pin for image requests" })).toBeChecked(); + }); }); describe("ComplexityRouterConfig affinity panel", () => { diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 73ab4f0abef..29dd3d9ea83 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -410,6 +410,7 @@ export interface ComplexityRouterConfigValue { classification_mode?: ClassificationMode; session_affinity?: boolean; modality_routing?: boolean; + modality_pin_override?: boolean; deployment_affinity?: boolean; /** Plan-mode floor as a tier ROW ID, unset meaning off. The wire carries the row's name. */ plan_mode_min_tier?: string; diff --git a/ui/litellm-dashboard/src/components/add_model/ModalityRoutingControls.tsx b/ui/litellm-dashboard/src/components/add_model/ModalityRoutingControls.tsx index dd697b35239..8d6c4ed5d5e 100644 --- a/ui/litellm-dashboard/src/components/add_model/ModalityRoutingControls.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ModalityRoutingControls.tsx @@ -7,20 +7,36 @@ import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; export const ModalityRoutingControls: React.FC<{ value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; -}> = ({ value, onChange }) => ( - <> -
- onChange({ ...value, modality_routing: modalityRouting })} - aria-label="Route image requests to vision-capable models" - /> - Route image requests to vision-capable models -
- - Replaces a routed model that cannot take image input with the nearest higher tier that can, then the default - model, instead of failing with a provider 400. Only models explicitly declared supports_vision false are replaced, - and a kept session pin still wins. - - -); +}> = ({ value, onChange }) => { + const modalityRouting = value.modality_routing ?? false; + return ( + <> +
+ onChange({ ...value, modality_routing: nextModalityRouting })} + aria-label="Route image requests to vision-capable models" + /> + Route image requests to vision-capable models +
+ + Replaces a routed model that cannot take image input with the nearest higher tier that can, then the default + model, instead of failing with a provider 400. Only models explicitly declared supports_vision false are + replaced, and a kept session pin still wins unless you turn on the override below. + +
+ onChange({ ...value, modality_pin_override: modalityPinOverride })} + disabled={!modalityRouting} + aria-label="Override session pin for image requests" + /> + Override session pin for image requests +
+ + Route an image turn to a capable model even when the session is pinned to one that cannot take images. The pin + is kept, so the next text turn goes back to it. Needs image routing turned on. + + + ); +}; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 7769042a0c7..27a169524b9 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -590,6 +590,44 @@ describe("AddAutoRouterTab", () => { }); }); + it("writes both modality flags as false into the create payload when the panel stays untouched", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "modality-router" } }); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + modality_routing: false, + modality_pin_override: false, + }); + }); + + it("carries the pin override through to the create payload once image routing unlocks it", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "modality-router" } }); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Modality Routing")); + await user.click(await screen.findByRole("switch", { name: "Route image requests to vision-capable models" })); + await user.click(await screen.findByRole("switch", { name: "Override session pin for image requests" })); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + modality_routing: true, + modality_pin_override: true, + }); + }); + // Custom is the escape hatch, not the headline choice, so it's listed after every bundled preset // rather than first. it("lists Custom Configuration after the bundled presets", () => { diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index ab5ea4cbfc9..fc3fdaf39cd 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -358,6 +358,7 @@ const AddAutoRouterTab: React.FC = ({ classifierFallback: complexityRouterConfig.classifier_fallback, sessionAffinity: complexityRouterConfig.session_affinity ?? DEFAULT_SESSION_AFFINITY, modalityRouting: complexityRouterConfig.modality_routing ?? false, + modalityPinOverride: complexityRouterConfig.modality_pin_override ?? false, deploymentAffinity: complexityRouterConfig.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, customTechnicalKeywords, keywordTierRules, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index c6370541a45..3ef55b9295d 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -56,6 +56,7 @@ describe("buildComplexityRouterConfig", () => { session_affinity: false, deployment_affinity: true, modality_routing: false, + modality_pin_override: false, escalation_keywords: ["LITELLM ESCALATE"], }; expect(config).toEqual(expected); @@ -270,6 +271,14 @@ describe("buildComplexityRouterConfig", () => { expect(buildComplexityRouterConfig({ ...baseParams, modalityRouting: false }).modality_routing).toBe(false); }); + it("writes modality_pin_override explicitly both ways, so the stored config never relies on the backend default", () => { + expect(buildComplexityRouterConfig({ ...baseParams, modalityPinOverride: true }).modality_pin_override).toBe(true); + expect(buildComplexityRouterConfig(baseParams).modality_pin_override).toBe(false); + expect(buildComplexityRouterConfig({ ...baseParams, modalityPinOverride: false }).modality_pin_override).toBe( + false, + ); + }); + it("writes session_affinity=true so turning the toggle on overrides the backend's off-by-default", () => { const config = buildComplexityRouterConfig({ ...baseParams, sessionAffinity: true }); expect(config.session_affinity).toBe(true); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 94c5badf6d1..91e4fdb23b5 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -111,6 +111,7 @@ export interface BuildComplexityRouterConfigParams { classificationMode: ClassificationMode | undefined; sessionAffinity: boolean; modalityRouting?: boolean; + modalityPinOverride?: boolean; deploymentAffinity: boolean; customTechnicalKeywords: string[]; keywordTierRules: KeywordTierRule[]; @@ -169,6 +170,7 @@ export interface ComplexityRouterConfigPayload { session_affinity: boolean; deployment_affinity: boolean; modality_routing: boolean; + modality_pin_override: boolean; custom_technical_keywords?: string[]; keyword_tier_rules?: { keywords: string[]; tier: KeywordTierRule["tier"] }[]; semantic_keyword_matching?: boolean; @@ -409,6 +411,7 @@ export const buildComplexityRouterConfig = ({ classificationMode, sessionAffinity, modalityRouting, + modalityPinOverride, deploymentAffinity, customTechnicalKeywords, keywordTierRules, @@ -472,6 +475,7 @@ export const buildComplexityRouterConfig = ({ session_affinity: sessionAffinity, deployment_affinity: deploymentAffinity, modality_routing: modalityRouting ?? false, + modality_pin_override: modalityPinOverride ?? false, ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), ...(cleanedKeywordTierRules.length > 0 && { keyword_tier_rules: cleanedKeywordTierRules }), escalation_keywords: cleanedEscalationKeywords, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index fae6744d3ac..01198bcc118 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -257,6 +257,30 @@ describe("buildUpdatedComplexityRouterConfig session affinity", () => { }); }); +describe("buildUpdatedComplexityRouterConfig modality pin override", () => { + it("writes modality_pin_override explicitly both ways", () => { + expect( + buildUpdatedComplexityRouterConfig(STORED, { ...FORM_VALUE, modality_pin_override: true }).modality_pin_override, + ).toBe(true); + expect( + buildUpdatedComplexityRouterConfig(STORED, { ...FORM_VALUE, modality_pin_override: false }).modality_pin_override, + ).toBe(false); + }); + + it("re-asserts the backend's off-by-default when the form value is absent, rather than dropping the key", () => { + const result = buildUpdatedComplexityRouterConfig({ ...STORED, modality_pin_override: true }, FORM_VALUE); + expect(result.modality_pin_override).toBe(false); + }); + + it("round-trips a stored modality_pin_override=true through hydrate then save", () => { + const stored = { ...STORED, modality_routing: true, modality_pin_override: true }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + + expect(hydrated.modality_pin_override).toBe(true); + expect(buildUpdatedComplexityRouterConfig(stored, hydrated).modality_pin_override).toBe(true); + }); +}); + describe("buildUpdatedComplexityRouterConfig classification mode", () => { it("round-trips a stored user_turn through hydrate then save", () => { const stored = { ...STORED, classification_mode: "user_turn" }; @@ -505,6 +529,8 @@ describe("managed keys survive an untouched open-and-save", () => { classifier_fallback: "default_model", classification_mode: "user_turn", session_affinity: true, + modality_routing: true, + modality_pin_override: true, deployment_affinity: false, adaptive: true, adaptive_weights: { quality: 0.4, cost: 0.6 }, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts index 481ba2b6b00..651d5880db4 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts @@ -51,6 +51,7 @@ const expectedClassifiedTierConfig = { session_affinity: false, deployment_affinity: true, modality_routing: false, + modality_pin_override: false, adaptive: true, adaptive_weights: { quality: 0.4, cost: 0.6 }, adaptive_eligible: "classified_tier", @@ -74,6 +75,7 @@ const expectedAdaptiveDisabledConfig = { session_affinity: false, deployment_affinity: true, modality_routing: false, + modality_pin_override: false, }; describe("buildUpdatedComplexityRouterConfig", () => { @@ -109,6 +111,26 @@ describe("buildUpdatedComplexityRouterConfig", () => { expect(disabled.modality_routing).toBe(false); }); + it("hydrates a stored modality_pin_override into form state and defaults absent to off", () => { + expect( + hydrateComplexityRouterConfig({ ...storedConfig, modality_pin_override: true }, null).modality_pin_override, + ).toBe(true); + expect(hydrateComplexityRouterConfig(storedConfig, null).modality_pin_override).toBe(false); + }); + + it("round-trips modality_pin_override explicitly in both directions", () => { + const enabled = buildUpdatedComplexityRouterConfig(storedConfig, { + ...classifiedTierValue, + modality_pin_override: true, + }); + expect(enabled.modality_pin_override).toBe(true); + const disabled = buildUpdatedComplexityRouterConfig( + { ...storedConfig, modality_pin_override: true }, + { ...classifiedTierValue, modality_pin_override: false }, + ); + expect(disabled.modality_pin_override).toBe(false); + }); + it("includes return_raw_model_name only when enabled", () => { const updatedConfig = buildUpdatedComplexityRouterConfig(storedConfig, { ...classifiedTierValue, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx index 96e8549eac8..6067e72e547 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx @@ -549,6 +549,50 @@ describe("EditAutoRouterModal deployment affinity", () => { await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); expect(savedConfig().deployment_affinity).toBe(false); }); + + // modality_pin_override is a managed key, so the modal rewrites it from form state on save. A + // hydration gap would silently turn a stored override off on the next untouched save. + it("shows a stored modality_pin_override=true as on and preserves it through an untouched save", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, modality_routing: true, modality_pin_override: true }); + + await user.click(await screen.findByText("Advanced: Modality Routing")); + expect(await screen.findByRole("switch", { name: "Override session pin for image requests" })).toBeChecked(); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().modality_pin_override).toBe(true); + }); + + it("persists turning the modality pin override on", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, modality_routing: true }); + + await user.click(await screen.findByText("Advanced: Modality Routing")); + await user.click(await screen.findByRole("switch", { name: "Override session pin for image requests" })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().modality_pin_override).toBe(true); + }); + + it("writes modality_pin_override=false for a stored config that never carried the key", async () => { + const user = userEvent.setup(); + renderWithStoredConfig(STORED_CONFIG); + + await user.click(await screen.findByText("Advanced: Modality Routing")); + expect(await screen.findByRole("switch", { name: "Override session pin for image requests" })).toHaveAttribute( + "aria-disabled", + "true", + ); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().modality_pin_override).toBe(false); + }); }); describe("EditAutoRouterModal custom classifier prompt and fallback", () => { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 7aa5f4beb1f..442e1ec6039 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -104,6 +104,7 @@ export interface StoredComplexityRouterConfig { reasoning_override_min_score?: unknown; session_affinity?: unknown; modality_routing?: unknown; + modality_pin_override?: unknown; deployment_affinity?: unknown; adaptive?: boolean; adaptive_weights?: AdaptiveRouterWeights; @@ -181,6 +182,8 @@ export const hydrateComplexityRouterConfig = ( session_affinity: typeof parsedConfig.session_affinity === "boolean" ? parsedConfig.session_affinity : DEFAULT_SESSION_AFFINITY, modality_routing: typeof parsedConfig.modality_routing === "boolean" ? parsedConfig.modality_routing : false, + modality_pin_override: + typeof parsedConfig.modality_pin_override === "boolean" ? parsedConfig.modality_pin_override : false, deployment_affinity: typeof parsedConfig.deployment_affinity === "boolean" ? parsedConfig.deployment_affinity @@ -221,6 +224,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "classification_mode", "session_affinity", "modality_routing", + "modality_pin_override", "deployment_affinity", "adaptive", "adaptive_weights", @@ -318,6 +322,7 @@ export const buildUpdatedComplexityRouterConfig = ( classifierFallback: value.classifier_fallback, sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY, modalityRouting: value.modality_routing ?? false, + modalityPinOverride: value.modality_pin_override ?? false, deploymentAffinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, customTechnicalKeywords: customTechnicalKeywords ?? [], keywordTierRules: keywordMatching?.keywordTierRules ?? [], diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx index bf6a20a4af8..fd1777f802c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx @@ -186,6 +186,12 @@ describe("RoutingDecisionCard", () => { expect(screen.queryByText("housekeeping")).not.toBeInTheDocument(); }); + it("labels a modality pin override instead of showing the raw cause token", () => { + render(); + expect(screen.getByText("Overrode session pin for image input")).toBeInTheDocument(); + expect(screen.queryByText("modality_pin_override")).not.toBeInTheDocument(); + }); + it("labels a modality escalation instead of showing the raw cause token", () => { render(); expect(screen.getByText("Escalated for image input")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index ffef01010b7..cf2c71e64c6 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -93,6 +93,7 @@ const CONSTANT_CAUSE_LABELS: Record = { session_affinity_escalation: "Escalated from session pin", user_turn_continuation: "Continuation turn, classifier skipped", modality_escalation: "Escalated for image input", + modality_pin_override: "Overrode session pin for image input", quality_tier: "Quality tier mapping", bandit: "Adaptive bandit", default_fallback: "Default model, no route matched", diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index e3132067f4c..44ac6973a1c 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -148,6 +148,25 @@ describe("autorouter_presets", () => { expect(withoutFlag.complexityRouterConfig.modality_routing).toBe(false); }); + it("carries a preset's modality_pin_override into the prefilled form state", () => { + const preset = getPresetByKey("anthropic_family")!; + const withFlag = { ...preset.complexity_router_config, modality_routing: true, modality_pin_override: true }; + const prefill = buildPresetPrefill(withFlag, groupsOnly(getRequiredModelsInPreset(preset))); + expect(prefill.complexityRouterConfig.modality_pin_override).toBe(true); + const withoutFlag = buildPresetPrefill( + preset.complexity_router_config, + groupsOnly(getRequiredModelsInPreset(preset)), + ); + expect(withoutFlag.complexityRouterConfig.modality_pin_override).toBe(false); + }); + + it("ships every bundled preset with both modality flags written out, since the payload type requires them", () => { + for (const preset of getAllPresets()) { + expect(preset.complexity_router_config.modality_routing, preset.key).toBe(false); + expect(preset.complexity_router_config.modality_pin_override, preset.key).toBe(false); + } + }); + it("prefills the anthropic preset's effort through to tier_model_params", () => { const preset = getPresetByKey("anthropic_family")!; const prefill = buildPresetPrefill(preset.complexity_router_config, groupsOnly(getRequiredModelsInPreset(preset))); diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index 30f8ed99e74..ff482bd23b8 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -286,6 +286,7 @@ export const buildPresetPrefill = ( session_affinity: config.session_affinity ?? DEFAULT_SESSION_AFFINITY, deployment_affinity: config.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, modality_routing: config.modality_routing ?? false, + modality_pin_override: config.modality_pin_override ?? false, adaptive: config.adaptive, adaptive_weights: config.adaptive_weights, tier_distance_penalty: config.tier_distance_penalty, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 491c4fb6a44..b0f8e618645 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -34740,9 +34740,15 @@ export interface components { * @default 0.5 */ match_threshold: number; + /** + * Modality Pin Override + * @description Let modality_routing replace a kept session-affinity pin on the turns that carry an image. Without this, a session pinned to a text-only model fails every image turn with a provider 400, since the pin is exempt from the modality gate. When enabled, such a turn routes to a capable model for that request only and the stored pin is left untouched, so the next text turn replays the session's own model; the override is reported as cause modality_pin_override and is never itself pinned. Inert unless modality_routing is also enabled. + * @default false + */ + modality_pin_override: boolean; /** * Modality Routing - * @description Route image-bearing requests only to models that can accept image input. The classifier reads text alone, so an image request whose text classifies cheap otherwise lands on a text-only model and fails with a provider 400. When enabled, a routed model explicitly declared supports_vision false (deployment model_info or the model cost map; unmapped names stay routable) is replaced by the nearest HIGHER tier holding a capable model, then default_model, else a clear 400. A kept session-affinity pin still wins even when an image arrives. + * @description Route image-bearing requests only to models that can accept image input. The classifier reads text alone, so an image request whose text classifies cheap otherwise lands on a text-only model and fails with a provider 400. When enabled, a routed model explicitly declared supports_vision false (deployment model_info or the model cost map; unmapped names stay routable) is replaced by the nearest HIGHER tier holding a capable model, then default_model, else a clear 400. A kept session-affinity pin still wins even when an image arrives, unless modality_pin_override is also enabled. * @default false */ modality_routing: boolean; @@ -35945,7 +35951,7 @@ export interface components { * Cause * @enum {string} */ - cause?: "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; + cause?: "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "modality_pin_override" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; /** Classifier Cost */ classifier_cost?: number; /** Classifier Model */ From 291e84e565ca8a2f91981f32885970e7585d5ae6 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 2 Sep 2026 19:46:09 -0700 Subject: [PATCH 188/480] feat(datadog_llm_obs): cost tag dimensions, router decision fields, reasoning token metric, redaction gating (#39402) * feat(datadog_llm_obs): cost tag dimensions, router decision fields, reasoning token metric, redaction gating * test(datadog_llm_obs): satisfy test quality gate * fix: forward integer parent_id as its string form * fix(datadog): sanitize redacted message roles * fix(datadog): keep the A2A agent role on redacted spans * fix(datadog): merge current staging budget * style(datadog): format redaction tests * fix(datadog): handle malformed redacted roles * test(datadog): put the test quality suppression on the reported line Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- basedpyright-code-budget.json | 6 +- .../integrations/datadog/datadog_llm_obs.py | 217 ++++++++-- litellm/types/integrations/datadog_llm_obs.py | 1 + .../datadog/test_datadog_llm_obs.py | 390 +++++++++++++++++- type-discipline-budget.json | 4 +- 5 files changed, 577 insertions(+), 41 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 3f96531cf6f..2967fc2a505 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5601 }, "reportMissingTypeArgument": { - "limit": 15290 + "limit": 15288 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,7 +105,7 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38332 + "limit": 38324 }, "reportUnknownParameterType": { "limit": 19625 @@ -123,7 +123,7 @@ "limit": 4 }, "reportUnnecessaryIsInstance": { - "limit": 823 + "limit": 819 }, "reportUntypedBaseClass": { "limit": 0 diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 5e116b7301a..ec86c0ae1d9 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -19,11 +19,13 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm._uuid import uuid +from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.custom_batch_logger import CustomBatchLogger from litellm.integrations.datadog.datadog_handler import ( get_datadog_base_url_from_env, get_datadog_service, get_datadog_tags, + normalize_datadog_tag_value, ) from litellm.integrations.datadog.datadog_mock_client import ( create_mock_datadog_client, @@ -34,6 +36,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, handle_any_messages_to_chat_completion_str_messages_conversion, ) +from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.llms.custom_httpx.http_handler import ( @@ -43,6 +46,7 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.proxy.spend_tracking.savings import extract_cache_creation_tokens, extract_cache_read_tokens from litellm.types.integrations.datadog_llm_obs import * from litellm.types.utils import ( + PROMPT_QUOTING_ROUTING_DECISION_FIELDS, CallTypes, StandardLoggingGuardrailInformation, StandardLoggingPayload, @@ -52,6 +56,120 @@ from litellm.types.utils import ( _EMPTY_MAPPING: Final[Mapping[str, Any]] = MappingProxyType({}) _EMPTY_MESSAGE: Final[Message] = {"role": "", "content": ""} _MAX_PARSED_TOOL_ARGUMENT_CHARS: Final = 256 * 1024 +_SAFE_REDACTED_MESSAGE_ROLES: Final = frozenset( + {"agent", "assistant", "developer", "function", "model", "system", "tool", "user"} +) + +_PROMPT_CARRYING_METADATA_FIELDS: Final = frozenset( + { + "routing_decision", + "requester_metadata", + "prompt_management_metadata", + "mcp_tool_call_metadata", + "vector_store_request_metadata", + } +) + +_ROUTER_SPAN_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + { + "tier": "router_tier", + "cause": "router_cause", + "score": "router_score", + "escalated": "router_escalated", + "signals": "router_signals", + "routed_model": "routed_model", + } +) +_ROUTER_DIMENSIONS: Final[tuple[str, ...]] = ("router_tier", "router_cause", "router_escalated", "routed_model") +_COST_DIMENSIONS: Final[tuple[str, ...]] = ("team", "user", "key_alias", "model_group", *_ROUTER_DIMENSIONS) + + +def _metadata_of(standard_logging_payload: StandardLoggingPayload) -> Mapping[str, Any]: + metadata: Final = standard_logging_payload.get("metadata") + return metadata or _EMPTY_MAPPING + + +def _router_span_fields( + standard_logging_payload: StandardLoggingPayload, redact_prompt_text: bool +) -> Mapping[str, object]: + """Flatten the auto-router decision, omitting prompt-quoting fields when redaction is enabled.""" + routing_decision: Final = _mapping_field(_metadata_of(standard_logging_payload), "routing_decision") + if not routing_decision: + return _EMPTY_MAPPING + escalated: Final = bool(routing_decision.get("escalated") or routing_decision.get("context_escalated")) + return MappingProxyType( + { + _ROUTER_SPAN_FIELDS[record_field]: value + for record_field, value in (*routing_decision.items(), ("escalated", escalated)) + if record_field in _ROUTER_SPAN_FIELDS + and value is not None + and not (redact_prompt_text and record_field in PROMPT_QUOTING_ROUTING_DECISION_FIELDS) + } + ) + + +def _metadata_without_prompt_carriers(standard_logging_metadata: Mapping[str, Any]) -> Mapping[str, Any]: + """The metadata minus the records that quote prompts, tool arguments, tool results, or retrieved text.""" + return MappingProxyType( + { + field: value + for field, value in standard_logging_metadata.items() + if field not in _PROMPT_CARRYING_METADATA_FIELDS + } + ) + + +def _redact_messages(messages: Sequence[Message]) -> tuple[Message, ...]: + """Each message's shape with its content replaced and tool payloads dropped; no message is invented.""" + return tuple( + { + "role": role if isinstance(role, str) and role in _SAFE_REDACTED_MESSAGE_ROLES else "", + "content": REDACTED_BY_LITELLM, + } + for message in messages + for role in (message.get("role", ""),) + ) + + +def _cost_dimension_tags( + standard_logging_payload: StandardLoggingPayload, router_fields: Mapping[str, object] +) -> tuple[str, ...]: + """The dimensions LLM Obs breaks token and cost metrics down by, as span tags.""" + metadata: Final = _metadata_of(standard_logging_payload) + dimensions: Final = ( + ("user", metadata.get("user_api_key_user_id")), + ("key_alias", metadata.get("user_api_key_alias")), + ("model_group", standard_logging_payload.get("model_group")), + *((dimension, router_fields.get(dimension)) for dimension in _ROUTER_DIMENSIONS), + ) + return tuple( + f"{key}:{normalized}" + for key, value in dimensions + if value is not None and (normalized := normalize_datadog_tag_value(value)) != "" + ) + + +def _declared_cost_tags(span_tags: Sequence[str]) -> tuple[str, ...]: + """Declare only cost dimensions carrying a value on this span.""" + present: Final = frozenset(key for tag in span_tags if (key := tag.partition(":")[0]) and tag.partition(":")[2]) + return tuple(dimension for dimension in _COST_DIMENSIONS if dimension in present) + + +def _reasoning_output_tokens(usage_object: Mapping[str, Any] | None) -> float: + """The provider's reasoning-token count, from either the chat or the responses spelling.""" + if usage_object is None: + return 0.0 + return next( + ( + float(reasoning_tokens) + for details_field in ("completion_tokens_details", "output_tokens_details") + if isinstance( + reasoning_tokens := _mapping_field(usage_object, details_field).get("reasoning_tokens"), (int, float) + ) + and not isinstance(reasoning_tokens, bool) + ), + 0.0, + ) def _mapping_field(source: Mapping[str, Any], key: str) -> Mapping[str, Any]: @@ -316,12 +434,12 @@ class DataDogLLMObsLogger(CustomBatchLogger): dict_datadog_llm_obs_params: dict = {} if litellm.datadog_llm_observability_params is not None: if isinstance(litellm.datadog_llm_observability_params, DatadogLLMObsInitParams): - dict_datadog_llm_obs_params = litellm.datadog_llm_observability_params.model_dump() + dict_datadog_llm_obs_params = litellm.datadog_llm_observability_params.model_dump(exclude_unset=True) elif isinstance(litellm.datadog_llm_observability_params, dict): # only allow params that are of DatadogLLMObsInitParams dict_datadog_llm_obs_params = DatadogLLMObsInitParams( **litellm.datadog_llm_observability_params - ).model_dump() + ).model_dump(exclude_unset=True) return dict_datadog_llm_obs_params async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -410,25 +528,40 @@ class DataDogLLMObsLogger(CustomBatchLogger): if standard_logging_payload is None: raise Exception("DataDogLLMObs: standard_logging_object is not set") - metadata: Final = kwargs.get("litellm_params", {}).get("metadata", {}) + raw_metadata: Final = kwargs.get("litellm_params", {}).get("metadata", {}) + metadata: Final = raw_metadata if isinstance(raw_metadata, dict) else {} + redact_payload: Final = self._payload_logging_is_off(kwargs) - input_meta: Final = InputMeta(messages=_to_dd_messages(standard_logging_payload["messages"])) + input_messages: Final = _to_dd_messages(standard_logging_payload.get("messages")) + output_messages: Final = self._get_response_messages( + standard_logging_payload=standard_logging_payload, + call_type=standard_logging_payload.get("call_type"), + ) + input_meta: Final = InputMeta(messages=_redact_messages(input_messages) if redact_payload else input_messages) output_meta: Final = OutputMeta( - messages=self._get_response_messages( - standard_logging_payload=standard_logging_payload, - call_type=standard_logging_payload.get("call_type"), - ) + messages=_redact_messages(output_messages) if redact_payload else output_messages ) error_info: Final = self._assemble_error_info(standard_logging_payload) - metadata_parent_id: str | None = None - if isinstance(metadata, dict): - metadata_parent_id = metadata.get("parent_id") + raw_parent_id: Final = metadata.get("parent_id") + metadata_parent_id: Final[str | None] = str(raw_parent_id) if raw_parent_id else None - tool_definitions: Final = _to_dd_tool_definitions(standard_logging_payload.get("model_parameters")) + tool_definitions: Final = ( + () if redact_payload else _to_dd_tool_definitions(standard_logging_payload.get("model_parameters")) + ) span_kind: Final = self._get_datadog_span_kind(standard_logging_payload.get("call_type"), metadata_parent_id) - payload_metadata: Final = self._get_dd_llm_obs_payload_metadata(standard_logging_payload) + router_fields: Final = _router_span_fields(standard_logging_payload, redact_prompt_text=redact_payload) + span_tags: Final = [ + *get_datadog_tags(standard_logging_object=standard_logging_payload), + *_cost_dimension_tags(standard_logging_payload, router_fields), + ] + payload_metadata: Final = self._get_dd_llm_obs_payload_metadata( + standard_logging_payload, + router_fields=router_fields, + cost_tags=_declared_cost_tags(span_tags), + redact_prompt_text=redact_payload, + ) meta: Final[Meta] = { "kind": span_kind, @@ -451,7 +584,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): duration=int((end_time - start_time).total_seconds() * 1e9), metrics=metrics, status="error" if error_info else "ok", - tags=get_datadog_tags(standard_logging_object=standard_logging_payload), + tags=span_tags, ) apm_trace_id: Final = self._get_apm_trace_id() @@ -497,6 +630,13 @@ class DataDogLLMObsLogger(CustomBatchLogger): ) return error_info + def _payload_logging_is_off(self, kwargs: Mapping[str, Any]) -> bool: + return ( + bool(self.turn_off_message_logging) + or self.message_logging is not True + or should_redact_message_logging(dict(kwargs)) + ) + def _assemble_metrics(self, standard_logging_payload: StandardLoggingPayload) -> LLMMetrics: """ Build the span metrics, including the prompt-cache counts LLM Obs charts cache savings from. @@ -513,10 +653,11 @@ class DataDogLLMObsLogger(CustomBatchLogger): total_cost: Final = float(standard_logging_payload.get("response_cost", 0)) time_to_first_token: Final = self._get_time_to_first_token_seconds(standard_logging_payload) - raw_usage: Final = (standard_logging_payload.get("metadata") or {}).get("usage_object") + raw_usage: Final = _metadata_of(standard_logging_payload).get("usage_object") usage_object: Final = raw_usage if isinstance(raw_usage, dict) else None cache_read: Final = float(extract_cache_read_tokens(usage_object)) cache_write: Final = float(extract_cache_creation_tokens(usage_object)) + reasoning_output_tokens: Final = _reasoning_output_tokens(usage_object) metrics: Final[LLMMetrics] = { "input_tokens": prompt_tokens, @@ -533,6 +674,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): if cache_read or cache_write else {} ), + **({"reasoning_output_tokens": reasoning_output_tokens} if reasoning_output_tokens else {}), } return metrics @@ -707,11 +849,21 @@ class DataDogLLMObsLogger(CustomBatchLogger): # Default fallback for unknown or passthrough operations return "llm" - def _get_dd_llm_obs_payload_metadata(self, standard_logging_payload: StandardLoggingPayload) -> dict[str, object]: + def _get_dd_llm_obs_payload_metadata( + self, + standard_logging_payload: StandardLoggingPayload, + router_fields: Mapping[str, object] | None = None, + cost_tags: Sequence[str] = (), + redact_prompt_text: bool = False, + ) -> dict[str, object]: """ Fields to track in DD LLM Observability metadata from litellm standard logging payload """ - _metadata: Final[dict[str, object]] = { + raw_metadata: Final = _metadata_of(standard_logging_payload) + standard_logging_metadata: Final = ( + _metadata_without_prompt_carriers(raw_metadata) if redact_prompt_text else raw_metadata + ) + return { "model_name": standard_logging_payload.get("model", "unknown"), "model_provider": standard_logging_payload.get("custom_llm_provider", "unknown"), "id": standard_logging_payload.get("id", "unknown"), @@ -719,26 +871,21 @@ class DataDogLLMObsLogger(CustomBatchLogger): "cache_hit": standard_logging_payload.get("cache_hit", "unknown"), "cache_key": standard_logging_payload.get("cache_key", "unknown"), "saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0), - "guardrail_information": standard_logging_payload.get("guardrail_information", None), + "guardrail_information": ( + None if redact_prompt_text else standard_logging_payload.get("guardrail_information", None) + ), "is_streamed_request": self._get_stream_value_from_payload(standard_logging_payload), + "latency_metrics": dict(self._get_latency_metrics(standard_logging_payload)), + "spend_metrics": dict(self._get_spend_metrics(standard_logging_payload)), + **standard_logging_metadata, + **(router_fields or _EMPTY_MAPPING), + **( + {"_dd": {**_mapping_field(standard_logging_metadata, "_dd"), "cost_tags": list(cost_tags)}} + if cost_tags + else _EMPTY_MAPPING + ), } - ######################################################### - # Add latency metrics to metadata - ######################################################### - latency_metrics: Final = self._get_latency_metrics(standard_logging_payload) - _metadata.update({"latency_metrics": dict(latency_metrics)}) - - ######################################################### - # Add spend metrics to metadata - ######################################################### - spend_metrics: Final = self._get_spend_metrics(standard_logging_payload) - _metadata.update({"spend_metrics": dict(spend_metrics)}) - - _standard_logging_metadata: Final[dict] = dict(standard_logging_payload.get("metadata", {})) or {} - _metadata.update(_standard_logging_metadata) - return _metadata - def _get_latency_metrics(self, standard_logging_payload: StandardLoggingPayload) -> DDLLMObsLatencyMetrics: """ Get the latency metrics from the standard logging payload @@ -808,7 +955,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): spend_metrics["response_cost"] = standard_logging_payload.get("response_cost", 0.0) # Get budget information from metadata - metadata: Final = standard_logging_payload.get("metadata", {}) + metadata: Final = _metadata_of(standard_logging_payload) # API key max budget user_api_key_max_budget: Final = metadata.get("user_api_key_max_budget") diff --git a/litellm/types/integrations/datadog_llm_obs.py b/litellm/types/integrations/datadog_llm_obs.py index bae876dfdd9..17cf5831c96 100644 --- a/litellm/types/integrations/datadog_llm_obs.py +++ b/litellm/types/integrations/datadog_llm_obs.py @@ -86,6 +86,7 @@ class LLMMetrics(TypedDict, total=False): cache_read_input_tokens: ReadOnly[float] cache_write_input_tokens: ReadOnly[float] non_cached_input_tokens: ReadOnly[float] + reasoning_output_tokens: ReadOnly[float] class LLMObsPayload(TypedDict, total=False): diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py index 2d0605e3b7f..34c62864c4e 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py @@ -17,6 +17,7 @@ from unittest.mock import patch import pytest +import litellm from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -55,15 +56,22 @@ def build_payload( response_message: dict[str, Any] | None = None, usage_object: dict[str, Any] | None = None, model_parameters: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + model_group: str | None = None, prompt_tokens: int = 4447, ) -> dict[str, Any]: + standard_logging_metadata: dict[str, Any] = { + **(metadata or {}), + **({"usage_object": usage_object} if usage_object is not None else {}), + } return { "standard_logging_object": { "call_type": "acompletion", "messages": [{"role": "user", "content": "hi"}] if messages is NOT_GIVEN else messages, "response": {"choices": [{"message": response_message or {"role": "assistant", "content": "hello"}}]}, "model_parameters": model_parameters or {}, - "metadata": {"usage_object": usage_object} if usage_object is not None else {}, + "metadata": standard_logging_metadata, + "model_group": model_group, "prompt_tokens": prompt_tokens, "completion_tokens": 507, "total_tokens": prompt_tokens + 507, @@ -244,6 +252,43 @@ def test_no_cache_keys_when_the_provider_reports_no_caching(logger: DataDogLLMOb assert "non_cached_input_tokens" not in payload["metrics"] +def test_reasoning_tokens_are_reported_as_span_metrics(logger: DataDogLLMObsLogger) -> None: + payload = build(logger, usage_object={"completion_tokens_details": {"reasoning_tokens": 128}}) + + assert payload["metrics"]["reasoning_output_tokens"] == 128.0 + + +def test_responses_reasoning_tokens_are_reported_as_span_metrics(logger: DataDogLLMObsLogger) -> None: + payload = build(logger, usage_object={"output_tokens_details": {"reasoning_tokens": 64}}) + + assert payload["metrics"]["reasoning_output_tokens"] == 64.0 + + +def test_zero_reasoning_tokens_are_not_reported(logger: DataDogLLMObsLogger) -> None: + payload = build(logger, usage_object={"completion_tokens_details": {"reasoning_tokens": 0}}) + + assert "reasoning_output_tokens" not in payload["metrics"] + + +def test_reasoning_tokens_come_from_the_spelling_that_reports_them(logger: DataDogLLMObsLogger) -> None: + """A chat-details mapping without the count must not shadow the responses spelling that has it.""" + payload = build( + logger, + usage_object={ + "completion_tokens_details": {"accepted_prediction_tokens": 5}, + "output_tokens_details": {"reasoning_tokens": 64}, + }, + ) + + assert payload["metrics"]["reasoning_output_tokens"] == 64.0 + + +def test_boolean_reasoning_tokens_are_not_a_count(logger: DataDogLLMObsLogger) -> None: + payload = build(logger, usage_object={"completion_tokens_details": {"reasoning_tokens": True}}) + + assert "reasoning_output_tokens" not in payload["metrics"] + + def test_tool_definitions_are_sent_on_meta(logger: DataDogLLMObsLogger) -> None: payload = build(logger, model_parameters={"tools": [TOOL_DEFINITION]}) @@ -256,6 +301,340 @@ def test_tool_definitions_are_sent_on_meta(logger: DataDogLLMObsLogger) -> None: ] +def test_cost_tags_include_present_categories_and_dimensions(logger: DataDogLLMObsLogger) -> None: + payload = build( + logger, + metadata={ + "user_api_key_user_id": "User 42", + "user_api_key_alias": "Primary Key", + "team_alias": "Platform", + "routing_decision": { + "tier": "premium", + "cause": "high_complexity", + "score": 0.91, + "escalated": True, + "signals": ["long prompt"], + "routed_model": "openai/gpt-5", + }, + }, + model_group="premium-models", + ) + + assert payload["tags"][-8:] == [ + "team:platform", + "user:user_42", + "key_alias:primary_key", + "model_group:premium-models", + "router_tier:premium", + "router_cause:high_complexity", + "router_escalated:true", + "routed_model:openai/gpt-5", + ] + assert payload["meta"]["metadata"]["_dd"]["cost_tags"] == [ + "team", + "user", + "key_alias", + "model_group", + "router_tier", + "router_cause", + "router_escalated", + "routed_model", + ] + + +def test_missing_cost_tag_values_are_not_declared(logger: DataDogLLMObsLogger) -> None: + payload = build(logger, metadata={"team_alias": "Platform"}) + + assert payload["meta"]["metadata"]["_dd"]["cost_tags"] == ["team"] + assert not any(tag.startswith(("user:", "key_alias:", "model_group:")) for tag in payload["tags"]) + + +def test_values_that_normalize_to_empty_are_not_tagged_or_declared(logger: DataDogLLMObsLogger) -> None: + payload = build(logger, metadata={"user_api_key_user_id": "___", "user_api_key_alias": "!!!"}, model_group="tier-1") + + assert not any(tag in ("user:", "key_alias:") for tag in payload["tags"]) + assert payload["meta"]["metadata"]["_dd"]["cost_tags"] == ["model_group"] + + +def test_a_valueless_tag_from_the_shared_builder_is_not_declared(logger: DataDogLLMObsLogger) -> None: + """The team tag comes from the shared builder, which emits it bare when the alias normalizes away.""" + payload = build(logger, metadata={"team_alias": "!!!"}, model_group="tier-1") + + assert "team:" in payload["tags"] + assert payload["meta"]["metadata"]["_dd"]["cost_tags"] == ["model_group"] + + +def test_router_fields_are_flattened(logger: DataDogLLMObsLogger) -> None: + payload = build( + logger, + metadata={ + "routing_decision": { + "tier": "premium", + "cause": "high_complexity", + "score": 0.91, + "escalated": True, + "signals": ["secret prompt text"], + "routed_model": "openai/gpt-5", + } + }, + model_group="premium-models", + ) + + assert payload["meta"]["metadata"]["router_tier"] == "premium" + assert payload["meta"]["metadata"]["router_cause"] == "high_complexity" + assert payload["meta"]["metadata"]["router_score"] == 0.91 + assert payload["meta"]["metadata"]["router_escalated"] is True + assert payload["meta"]["metadata"]["router_signals"] == ["secret prompt text"] + assert payload["meta"]["metadata"]["routed_model"] == "openai/gpt-5" + + +def test_a_context_escalated_route_reports_as_escalated(logger: DataDogLLMObsLogger) -> None: + """The router records a size-driven escalation under its own key, and it is still an escalation.""" + payload = build(logger, metadata={"routing_decision": {"tier": "premium", "context_escalated": True}}) + + assert payload["meta"]["metadata"]["router_escalated"] is True + assert "router_escalated:true" in payload["tags"] + + +def test_a_routed_request_that_did_not_escalate_reports_false(logger: DataDogLLMObsLogger) -> None: + """Without this the escalation dimension is absent on ordinary traffic, so nothing can group by it.""" + payload = build(logger, metadata={"routing_decision": {"tier": "simple", "cause": "heuristic_scorer"}}) + + assert payload["meta"]["metadata"]["router_escalated"] is False + assert "router_escalated:false" in payload["tags"] + assert "router_escalated" in payload["meta"]["metadata"]["_dd"]["cost_tags"] + + +def test_a_request_that_never_reached_a_router_has_no_router_fields(logger: DataDogLLMObsLogger) -> None: + payload = build(logger, model_group="premium-models") + + assert "router_escalated" not in payload["meta"]["metadata"] + assert not any(tag.startswith("router_") for tag in payload["tags"]) + + +def test_redacted_payload_keeps_metrics_and_removes_sensitive_fields(logger: DataDogLLMObsLogger) -> None: + payload = build_payload( + messages=[{"role": "user", "content": "secret prompt"}], + response_message={"role": "assistant", "content": "secret response"}, + usage_object={"prompt_tokens_details": {"cached_tokens": 128}}, + metadata={"routing_decision": {"tier": "premium", "signals": ["secret prompt text"]}}, + model_parameters={"tools": [TOOL_DEFINITION]}, + ) + with patch.dict(os.environ, {"DD_API_KEY": "k", "DD_SITE": "us5.datadoghq.com"}, clear=True): + with patch("asyncio.create_task"): + redacted_logger = DataDogLLMObsLogger(turn_off_message_logging=True) + redacted_payload = redacted_logger.redact_standard_logging_payload_from_model_call_details(payload) + result = json.loads( + safe_dumps( + redacted_logger.create_llm_obs_payload( + redacted_payload, datetime(2026, 9, 1, 12, 0, 0), datetime(2026, 9, 1, 12, 0, 2) + ) + ) + ) + + assert result["meta"]["input"]["messages"][0]["content"] == "redacted-by-litellm" + assert result["meta"]["output"]["messages"][0]["content"] == "redacted-by-litellm" + assert result["meta"]["metadata"]["router_tier"] == "premium" + assert "router_signals" not in result["meta"]["metadata"] + assert "routing_decision" not in result["meta"]["metadata"] + assert "tool_definitions" not in result["meta"] + assert result["metrics"]["cache_read_input_tokens"] == 128.0 + assert result["metrics"]["total_cost"] == 0.02 + + +def test_redaction_drops_the_routing_record_carried_in_metadata(logger: DataDogLLMObsLogger) -> None: + """The whole routing record rides along in metadata, so dropping the flat copy alone leaks the prompt.""" + with patch.dict(os.environ, {"DD_API_KEY": "k", "DD_SITE": "us5.datadoghq.com"}, clear=True): + with patch("asyncio.create_task"): + redacted_logger = DataDogLLMObsLogger(turn_off_message_logging=True) + result = json.loads( + safe_dumps( + redacted_logger.create_llm_obs_payload( + build_payload( + metadata={ + "routing_decision": { + "tier": "premium", + "cause": "keyword_rule", + "signals": ["secret prompt text"], + "matched_keyword": "secret keyword", + "escalation_keyword": "secret escalation", + } + } + ), + datetime(2026, 9, 1, 12, 0, 0), + datetime(2026, 9, 1, 12, 0, 2), + ) + ) + ) + + assert "routing_decision" not in result["meta"]["metadata"] + assert result["meta"]["metadata"]["router_tier"] == "premium" + assert result["meta"]["metadata"]["router_cause"] == "keyword_rule" + assert "secret" not in safe_dumps(result["meta"]["metadata"]) + + +def test_a_failure_span_redacts_its_messages(logger: DataDogLLMObsLogger) -> None: + """The redaction hook only runs on success, so the failure span has to redact for itself.""" + failed = build_payload(messages=[{"role": "user", "content": "secret prompt"}]) + failed["standard_logging_object"]["status"] = "failure" + failed["standard_logging_object"]["response"] = None + failed["standard_logging_object"]["error_information"] = {"error_message": "boom", "error_class": "BadRequestError"} + with patch.dict(os.environ, {"DD_API_KEY": "k", "DD_SITE": "us5.datadoghq.com"}, clear=True): + with patch("asyncio.create_task"): + redacted_logger = DataDogLLMObsLogger(turn_off_message_logging=True) + result = json.loads( + safe_dumps( + redacted_logger.create_llm_obs_payload( + failed, datetime(2026, 9, 1, 12, 0, 0), datetime(2026, 9, 1, 12, 0, 2) + ) + ) + ) + + assert result["meta"]["input"]["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert result["meta"]["output"]["messages"] == [] + assert result["status"] == "error" + + +def test_excluding_messages_from_the_logging_payload_still_ships_the_span(logger: DataDogLLMObsLogger) -> None: + """`standard_logging_payload_excluded_fields` deletes the key, and a span with no prompt is still a span.""" + payload = build_payload() + del payload["standard_logging_object"]["messages"] + + span = json.loads( + safe_dumps( + logger.create_llm_obs_payload(payload, datetime(2026, 9, 1, 12, 0, 0), datetime(2026, 9, 1, 12, 0, 2)) + ) + ) + + assert span["meta"]["input"]["messages"] == [] + assert span["metrics"]["total_cost"] == 0.02 + + +def test_an_explicit_redaction_setting_survives_the_global_params(logger: DataDogLLMObsLogger) -> None: + """Global params carry defaults for keys the operator never set, and those must not win.""" + with patch.dict(os.environ, {"DD_API_KEY": "k", "DD_SITE": "us5.datadoghq.com"}, clear=True): + with patch("asyncio.create_task"): + with patch.object( # test-quality-ok: the ctor reads this module global with no injection seam + litellm, "datadog_llm_observability_params", {} + ): + configured_logger = DataDogLLMObsLogger( + turn_off_message_logging=True + ) # test-quality-ok: verifies ctor setting + + assert configured_logger.turn_off_message_logging is True + + +def _redacting_logger( + **kwargs: Any, +) -> DataDogLLMObsLogger: # test-quality-ok: shared test factory accepts init variants + with patch.dict(os.environ, {"DD_API_KEY": "k", "DD_SITE": "us5.datadoghq.com"}, clear=True): + with patch("asyncio.create_task"): + return DataDogLLMObsLogger(**kwargs) + + +def _span_json(logger_under_test: DataDogLLMObsLogger, payload: dict[str, Any]) -> dict[str, Any]: + span = logger_under_test.create_llm_obs_payload( + payload, datetime(2026, 9, 1, 12, 0, 0), datetime(2026, 9, 1, 12, 0, 2) + ) + return json.loads(safe_dumps(span)) + + +def test_redaction_keeps_the_conversation_shape_without_its_content() -> None: + """Roles and message count survive so the trace stays legible; contents and tool payloads do not.""" + result = _span_json( + _redacting_logger(turn_off_message_logging=True), + build_payload( + messages=[ + {"role": "user", "content": "secret prompt"}, + {"role": "assistant", "content": None, "tool_calls": [ASSISTANT_TOOL_CALL]}, + ], + response_message={"role": "assistant", "content": "secret response"}, + ), + ) + + assert result["meta"]["input"]["messages"] == [ + {"role": "user", "content": "redacted-by-litellm"}, + {"role": "assistant", "content": "redacted-by-litellm"}, + ] + assert result["meta"]["output"]["messages"] == [{"role": "assistant", "content": "redacted-by-litellm"}] + + +def test_redaction_drops_unrecognized_and_malformed_message_roles() -> None: + """Caller-controlled role values must not bypass redaction or crash span creation.""" + result = _span_json( + _redacting_logger(turn_off_message_logging=True), + build_payload( + messages=[ + {"role": "SECRET-39402", "content": "hello"}, + {"role": ["SECRET-39402"], "content": "hello"}, + {"role": {"secret": "SECRET-39402"}, "content": "hello"}, + {"role": "agent", "content": "hello"}, + ] + ), + ) + + assert result["meta"]["input"]["messages"] == [ + {"role": "", "content": "redacted-by-litellm"}, + {"role": "", "content": "redacted-by-litellm"}, + {"role": "", "content": "redacted-by-litellm"}, + {"role": "agent", "content": "redacted-by-litellm"}, + ] + assert "SECRET-39402" not in safe_dumps(result) + + +def test_the_deprecated_message_logging_flag_engages_the_same_redaction() -> None: + """The platform redacts for `message_logging is not True`, so this callback's own gate must agree.""" + result = _span_json( + _redacting_logger(message_logging=False), + build_payload( + messages=[{"role": "user", "content": "secret prompt"}], + model_parameters={"tools": [TOOL_DEFINITION]}, + metadata={"routing_decision": {"tier": "premium", "signals": ["secret prompt text"]}}, + ), + ) + + assert result["meta"]["input"]["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert "tool_definitions" not in result["meta"] + assert "routing_decision" not in result["meta"]["metadata"] + + +def test_a_truthy_redaction_setting_redacts_like_the_shared_hook() -> None: + """The shared hook redacts on truthiness, so a config-provided string must not half-redact the span.""" + result = _span_json( + _redacting_logger(turn_off_message_logging="yes"), + build_payload(messages=[{"role": "user", "content": "secret prompt"}]), + ) + + assert result["meta"]["input"]["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + + +def test_redaction_drops_every_prompt_carrying_metadata_record(logger: DataDogLLMObsLogger) -> None: + """Tool arguments, retrieved text, and the guardrail's copy of the request ride in metadata records too.""" + sensitive_metadata: dict[str, Any] = { + "requester_metadata": {"note": "secret prompt text"}, + "prompt_management_metadata": {"prompt_id": "p1", "prompt_variables": {"topic": "secret"}}, + "mcp_tool_call_metadata": {"name": "search", "arguments": {"query": "secret"}}, + "vector_store_request_metadata": [{"query": "secret"}], + } + + def sensitive_payload() -> dict[str, Any]: + payload = build_payload(metadata=sensitive_metadata) + payload["standard_logging_object"]["guardrail_information"] = [ + {"guardrail_name": "g", "guardrail_request": {"messages": [{"content": "secret prompt"}]}} + ] + return payload + + redacted = _span_json(_redacting_logger(turn_off_message_logging=True), sensitive_payload()) + unredacted = _span_json(logger, sensitive_payload()) + + assert "secret" not in safe_dumps(redacted["meta"]["metadata"]) + for record in sensitive_metadata: + assert record not in redacted["meta"]["metadata"] + assert record in unredacted["meta"]["metadata"] + assert redacted["meta"]["metadata"]["guardrail_information"] is None + assert unredacted["meta"]["metadata"]["guardrail_information"] is not None + + def test_tool_definitions_accept_the_bare_anthropic_shape(logger: DataDogLLMObsLogger) -> None: """The Anthropic surface declares tools unwrapped, with input_schema instead of parameters.""" payload = build( @@ -272,6 +651,15 @@ def test_meta_omits_tool_definitions_when_no_tools_were_offered(logger: DataDogL assert "tool_definitions" not in build(logger)["meta"] +def test_a_ddtrace_integer_parent_id_is_forwarded_as_its_string(logger: DataDogLLMObsLogger) -> None: + """ddtrace hands span ids as ints; dropping them detaches the span from its APM trace.""" + kwargs = build_payload() + kwargs["litellm_params"]["metadata"]["parent_id"] = 8675309 + start = datetime(2026, 9, 1, 12, 0, 0) + span = json.loads(safe_dumps(logger.create_llm_obs_payload(kwargs, start, start + timedelta(seconds=2)))) + assert span["parent_id"] == "8675309" + + def test_unparseable_tool_arguments_are_preserved_rather_than_dropped(logger: DataDogLLMObsLogger) -> None: """A truncated argument string is still the only record of what the model tried to call.""" payload = build( diff --git a/type-discipline-budget.json b/type-discipline-budget.json index f3c4c7760c6..5c25b8722ef 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22334 + "limit": 22330 }, "LIT002": { "limit": 26763 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16480 + "limit": 16478 }, "LIT011": { "limit": 5520 From bcd3e2d94d3faf06cdb70c4648d872e8ddd86f5b Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:50:08 -0700 Subject: [PATCH 189/480] feat(rust-python-harness): wire existing e2e SDK tests into the matrix (#39463) Adds chat_completions and transcription as SDK function columns, backed by the existing rust_bridge test files. Adds a fourth strategy folder, existing_e2e_test_sdk, that points at already-existing live-API SDK tests (tests/ocr_tests/ as a whole folder, plus chat completion and Whisper transcription tests) instead of writing new parity tests. Extends selector_matches_node with trailing-slash folder selectors so a whole test folder can back one matrix cell. --- tests/rust-python-harness/README.md | 9 ++++++--- tests/rust-python-harness/cli.py | 9 ++++----- .../e2e_fuzz_tests/strategy.json | 4 +++- .../existing_e2e_test_sdk/README.md | 3 +++ .../existing_e2e_test_sdk/strategy.json | 14 ++++++++++++++ tests/rust-python-harness/models.py | 2 +- tests/rust-python-harness/runner.py | 2 ++ tests/rust-python-harness/ui.py | 2 +- .../unit_tests_rust/strategy.json | 4 +++- .../validate_sub_methods/strategy.json | 4 +++- tests/test_rust_python_harness.py | 16 +++++++++++++--- 11 files changed, 53 insertions(+), 16 deletions(-) create mode 100644 tests/rust-python-harness/existing_e2e_test_sdk/README.md create mode 100644 tests/rust-python-harness/existing_e2e_test_sdk/strategy.json diff --git a/tests/rust-python-harness/README.md b/tests/rust-python-harness/README.md index e94ac87c3b3..a34e1a1ab73 100644 --- a/tests/rust-python-harness/README.md +++ b/tests/rust-python-harness/README.md @@ -8,14 +8,17 @@ The matrix always has these SDK columns: - `messages / amessages` - `responses / aresponses` - `count_tokens` +- `chat_completions / acompletion` +- `transcription / atranscription` -The harness has three deliberately broad test-strategy folders: +The harness has four deliberately broad test-strategy folders: | Strategy | Folder | | --- | --- | | Public SDK parity over generated and recorded inputs | [`e2e_fuzz_tests/`](e2e_fuzz_tests/) | | Focused tests of Rust-owned behavior | [`unit_tests_rust/`](unit_tests_rust/) | | Isolated transform and Python-to-Rust helper coverage | [`validate_sub_methods/`](validate_sub_methods/) | +| Already-existing live-API SDK tests | [`existing_e2e_test_sdk/`](existing_e2e_test_sdk/) | ## Run it @@ -112,7 +115,7 @@ The initial end-to-end entries deliberately show `◐`: the repository has Rust ## Attach parity tests -Each of the three folders contains a concise `README.md` and a `strategy.json`. Add a pytest file or node ID to the appropriate SDK function's `selectors` list: +Each of the four folders contains a concise `README.md` and a `strategy.json`. Add a pytest file or node ID to the appropriate SDK function's `selectors` list: ```json { @@ -123,7 +126,7 @@ Each of the three folders contains a concise `README.md` and a `strategy.json`. } ``` -Selectors use the same syntax as pytest. A file selector aggregates every test in the file; a node selector can target one test or parametrized family. The runner deduplicates selectors, so one test may intentionally prove more than one cell without executing twice. +Selectors use the same syntax as pytest. A file selector aggregates every test in the file; a node selector can target one test or parametrized family; a selector ending in `/` aggregates every test in that folder, recursively. The runner deduplicates selectors, so one test may intentionally prove more than one cell without executing twice. Use these coverage values: diff --git a/tests/rust-python-harness/cli.py b/tests/rust-python-harness/cli.py index f9e97d7ad43..c996b68846c 100644 --- a/tests/rust-python-harness/cli.py +++ b/tests/rust-python-harness/cli.py @@ -6,14 +6,13 @@ from collections.abc import Sequence from pathlib import Path from .catalog import load_catalog -from .models import HarnessCase, Strategy +from .models import SDK_FUNCTIONS, HarnessCase, Strategy from .runner import run_pytest from .ui import make_dashboard from .strategies.unit_tests.mapping_validator import FunctionReport, build_function_report REPO_ROOT = Path(__file__).resolve().parents[2] COVERAGE_ROOT = REPO_ROOT / "target" / "rust-python-harness" -SDK_FUNCTION_CHOICES = ("ocr", "messages", "responses", "count_tokens") def _parser() -> argparse.ArgumentParser: @@ -42,7 +41,7 @@ def _parser() -> argparse.ArgumentParser: action="append", default=[], dest="sdk_functions", - choices=SDK_FUNCTION_CHOICES, + choices=SDK_FUNCTIONS, help="run only this SDK function", ) parser.add_argument( @@ -110,7 +109,7 @@ def _interactive_filters(strategies: Sequence[Strategy]) -> tuple[set[str], set[ ) sdk_functions = _pick_values( "SDK functions", - [(name, name) for name in SDK_FUNCTION_CHOICES], + [(name, name) for name in SDK_FUNCTIONS], ) return strategy_ids, sdk_functions @@ -166,7 +165,7 @@ def _print_function_report(report: FunctionReport) -> None: def _validate_ledger(sdk_functions: set[str]) -> int: - functions = sdk_functions or set(SDK_FUNCTION_CHOICES) + functions = sdk_functions or set(SDK_FUNCTIONS) reports = tuple(build_function_report(function) for function in sorted(functions)) for report in reports: _print_function_report(report) diff --git a/tests/rust-python-harness/e2e_fuzz_tests/strategy.json b/tests/rust-python-harness/e2e_fuzz_tests/strategy.json index abeea01d9b5..d838486772d 100644 --- a/tests/rust-python-harness/e2e_fuzz_tests/strategy.json +++ b/tests/rust-python-harness/e2e_fuzz_tests/strategy.json @@ -7,6 +7,8 @@ "ocr": {"coverage": "partial", "selectors": ["tests/test_litellm/ocr/test_rust_bridge.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."}, "messages": {"coverage": "partial", "selectors": ["tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."}, "responses": {"coverage": "partial", "selectors": ["tests/test_litellm/responses/test_rust_bridge_websocket.py"], "note": "Covers the websocket bridge; full responses parity is still being added."}, - "count_tokens": {"coverage": "planned", "selectors": [], "note": "No Rust count_tokens parity test is present yet."} + "count_tokens": {"coverage": "planned", "selectors": [], "note": "No Rust count_tokens parity test is present yet."}, + "chat_completions": {"coverage": "partial", "selectors": ["tests/test_litellm/rust_bridge/test_chat_completions.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."}, + "transcription": {"coverage": "partial", "selectors": ["tests/test_litellm/test_audio_transcription_rust_bridge.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."} } } diff --git a/tests/rust-python-harness/existing_e2e_test_sdk/README.md b/tests/rust-python-harness/existing_e2e_test_sdk/README.md new file mode 100644 index 00000000000..fb84f170703 --- /dev/null +++ b/tests/rust-python-harness/existing_e2e_test_sdk/README.md @@ -0,0 +1,3 @@ +# Existing e2e SDK tests + +Wires already-existing live-API SDK tests into the matrix instead of writing new parity tests. Selectors point at real test files and folders, such as `tests/ocr_tests/`, rather than individual node IDs, so future tests added to those folders are picked up automatically. diff --git a/tests/rust-python-harness/existing_e2e_test_sdk/strategy.json b/tests/rust-python-harness/existing_e2e_test_sdk/strategy.json new file mode 100644 index 00000000000..eefceea1a75 --- /dev/null +++ b/tests/rust-python-harness/existing_e2e_test_sdk/strategy.json @@ -0,0 +1,14 @@ +{ + "order": 40, + "id": "existing_e2e_test_sdk", + "label": "Existing e2e SDK tests", + "description": "Wire already-existing live-API SDK tests into the matrix instead of writing new parity tests.", + "functions": { + "ocr": {"coverage": "partial", "selectors": ["tests/ocr_tests/"], "note": "Existing live OCR provider tests; not yet a frozen Rust/Python oracle comparison."}, + "messages": {"coverage": "planned", "selectors": []}, + "responses": {"coverage": "planned", "selectors": []}, + "count_tokens": {"coverage": "planned", "selectors": []}, + "chat_completions": {"coverage": "partial", "selectors": ["tests/llm_translation/test_anthropic_completion.py", "tests/llm_translation/test_bedrock_completion.py"], "note": "Existing live chat completion tests for providers with confirmed Rust bridge regressions."}, + "transcription": {"coverage": "partial", "selectors": ["tests/audio_tests/test_whisper.py"], "note": "Existing live Whisper transcription test."} + } +} diff --git a/tests/rust-python-harness/models.py b/tests/rust-python-harness/models.py index 21097e0f7d0..a02684dc63d 100644 --- a/tests/rust-python-harness/models.py +++ b/tests/rust-python-harness/models.py @@ -33,7 +33,7 @@ class ConfidenceLevel(str, Enum): LOW = "LOW" -SDK_FUNCTIONS = ("ocr", "messages", "responses", "count_tokens") +SDK_FUNCTIONS = ("ocr", "messages", "responses", "count_tokens", "chat_completions", "transcription") @dataclass(frozen=True) diff --git a/tests/rust-python-harness/runner.py b/tests/rust-python-harness/runner.py index 82393ef234e..286f2c4116d 100644 --- a/tests/rust-python-harness/runner.py +++ b/tests/rust-python-harness/runner.py @@ -15,6 +15,8 @@ UpdateCallback = Callable[[HarnessRun], None] def selector_matches_node(selector: str, nodeid: str) -> bool: normalized_selector = selector.replace("\\", "/") normalized_nodeid = nodeid.replace("\\", "/") + if normalized_selector.endswith("/"): + return normalized_nodeid.startswith(normalized_selector) if "::" in normalized_selector: return normalized_nodeid == normalized_selector or normalized_nodeid.startswith( f"{normalized_selector}[" diff --git a/tests/rust-python-harness/ui.py b/tests/rust-python-harness/ui.py index 57fedd17fa6..bf143a4e100 100644 --- a/tests/rust-python-harness/ui.py +++ b/tests/rust-python-harness/ui.py @@ -119,7 +119,7 @@ class RichDashboard(AbstractContextManager["RichDashboard"]): table = Table(box=box.ROUNDED, expand=True, title="Strategy × SDK function") table.add_column("Strategy", ratio=3) - for label in ("ocr/aocr", "messages", "responses", "count_tokens"): + for label in SDK_FUNCTIONS: table.add_column(label, justify="center", ratio=1) for strategy in self.strategies: cells = [] diff --git a/tests/rust-python-harness/unit_tests_rust/strategy.json b/tests/rust-python-harness/unit_tests_rust/strategy.json index 89e897c872d..bfb2bc0dad0 100644 --- a/tests/rust-python-harness/unit_tests_rust/strategy.json +++ b/tests/rust-python-harness/unit_tests_rust/strategy.json @@ -7,6 +7,8 @@ "ocr": {"coverage": "planned", "selectors": []}, "messages": {"coverage": "planned", "selectors": []}, "responses": {"coverage": "planned", "selectors": []}, - "count_tokens": {"coverage": "planned", "selectors": []} + "count_tokens": {"coverage": "planned", "selectors": []}, + "chat_completions": {"coverage": "planned", "selectors": []}, + "transcription": {"coverage": "planned", "selectors": []} } } diff --git a/tests/rust-python-harness/validate_sub_methods/strategy.json b/tests/rust-python-harness/validate_sub_methods/strategy.json index 6e6381678e0..a26bc2d70ed 100644 --- a/tests/rust-python-harness/validate_sub_methods/strategy.json +++ b/tests/rust-python-harness/validate_sub_methods/strategy.json @@ -7,6 +7,8 @@ "ocr": {"coverage": "planned", "selectors": []}, "messages": {"coverage": "planned", "selectors": []}, "responses": {"coverage": "planned", "selectors": []}, - "count_tokens": {"coverage": "planned", "selectors": []} + "count_tokens": {"coverage": "planned", "selectors": []}, + "chat_completions": {"coverage": "planned", "selectors": []}, + "transcription": {"coverage": "planned", "selectors": []} } } diff --git a/tests/test_rust_python_harness.py b/tests/test_rust_python_harness.py index 6a8fa8d35cc..514446577fd 100644 --- a/tests/test_rust_python_harness.py +++ b/tests/test_rust_python_harness.py @@ -66,13 +66,14 @@ def _manifest() -> dict[str, object]: } -def test_should_load_the_three_harness_strategies_in_order() -> None: +def test_should_load_the_four_harness_strategies_in_order() -> None: strategies = load_catalog() assert [strategy.id for strategy in strategies] == [ "e2e_fuzz_tests", "unit_tests_rust", "validate_sub_methods", + "existing_e2e_test_sdk", ] assert all( tuple(case.sdk_function for case in strategy.cases) == SDK_FUNCTIONS @@ -104,6 +105,8 @@ def test_should_reject_a_manifest_missing_an_sdk_function(tmp_path: Path) -> Non True, ), ("tests/test_parity.py::test_one", "tests/test_parity.py::test_two", False), + ("tests/ocr_tests/", "tests/ocr_tests/test_ocr_mistral.py::test_one", True), + ("tests/ocr_tests/", "tests/other_tests/test_ocr_mistral.py::test_one", False), ], ) def test_should_match_pytest_file_and_node_selectors( @@ -123,6 +126,13 @@ def test_should_only_return_selectors_whose_files_exist(tmp_path: Path) -> None: assert runnable_selectors((case,), tmp_path) == ("tests/test_parity.py",) +def test_should_treat_an_existing_folder_selector_as_runnable(tmp_path: Path) -> None: + (tmp_path / "tests" / "ocr_tests").mkdir(parents=True) + case = _case(selectors=("tests/ocr_tests/",)) + + assert runnable_selectors((case,), tmp_path) == ("tests/ocr_tests/",) + + def test_should_mark_planned_and_not_applicable_cases_without_running() -> None: planned = CaseResult(case=_case(coverage=Coverage.PLANNED)) not_applicable = CaseResult(case=_case(coverage=Coverage.NOT_APPLICABLE)) @@ -239,8 +249,8 @@ def test_should_report_confidence_for_each_sdk_section() -> None: } assert scores["responses"].verified_strategies == 1 - assert scores["responses"].required_strategies == 3 - assert scores["responses"].percentage == 33 + assert scores["responses"].required_strategies == 4 + assert scores["responses"].percentage == 25 assert scores["responses"].level.value == "MEDIUM" assert scores["count_tokens"].percentage == 0 assert scores["count_tokens"].level.value == "LOW" From 099d26204f1998f3c8181e7eb3581ac097a59d38 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Wed, 2 Sep 2026 19:54:09 -0700 Subject: [PATCH 190/480] fix(ui): read the preset catalog at runtime in the vitest mock The autoRouterPresets mock imported litellm/proxy/public_endpoints/autorouter_presets.json as a module. That path sits outside ui/litellm-dashboard, the only directory the UI Dockerfile copies, so `next build` type-checking inside the image failed with "Cannot find module" and the ui-image job went red on every PR that touched an image-scan path. Read the file with fs at runtime instead; vitest still derives expectations from the real bundled catalog. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HqEPCNLDrssxsuezhAaL4j --- .../tests/mocks/autoRouterPresets.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts b/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts index f73faaa70fe..7356d92d99e 100644 --- a/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts +++ b/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts @@ -1,10 +1,18 @@ +import { readFileSync } from "fs"; +import { resolve } from "path"; import { vi } from "vitest"; -import bundledPresets from "../../../../litellm/proxy/public_endpoints/autorouter_presets.json"; import { hydratePresets, type AutoRouterPresetsResponse } from "@/lib/autorouter_presets"; // Derived from the real bundled catalog so a preset edit there flows into test expectations -// instead of redding on a stale copy. Exported as vi.fn so a test can override the query state. -export const BUNDLED_PRESETS = hydratePresets(bundledPresets as AutoRouterPresetsResponse); +// instead of redding on a stale copy. Read at runtime rather than imported as a module: the +// catalog lives outside ui/litellm-dashboard, so a module import fails `next build`'s type +// check inside the UI Docker image, whose build context is only this package. +// Exported as vi.fn so a test can override the query state. +const CATALOG_PATH = resolve(process.cwd(), "../../litellm/proxy/public_endpoints/autorouter_presets.json"); + +export const BUNDLED_PRESETS = hydratePresets( + JSON.parse(readFileSync(CATALOG_PATH, "utf8")) as AutoRouterPresetsResponse, +); export const LOADED_PRESETS_QUERY = { data: BUNDLED_PRESETS, From fcc9b813afcc8faece3dd44d3acfc59b0f503456 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Wed, 2 Sep 2026 20:01:35 -0700 Subject: [PATCH 191/480] fix(ui): resolve the preset catalog relative to the mock, not cwd Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HqEPCNLDrssxsuezhAaL4j --- ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts b/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts index 7356d92d99e..cff417e6bfa 100644 --- a/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts +++ b/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts @@ -4,11 +4,8 @@ import { vi } from "vitest"; import { hydratePresets, type AutoRouterPresetsResponse } from "@/lib/autorouter_presets"; // Derived from the real bundled catalog so a preset edit there flows into test expectations -// instead of redding on a stale copy. Read at runtime rather than imported as a module: the -// catalog lives outside ui/litellm-dashboard, so a module import fails `next build`'s type -// check inside the UI Docker image, whose build context is only this package. -// Exported as vi.fn so a test can override the query state. -const CATALOG_PATH = resolve(process.cwd(), "../../litellm/proxy/public_endpoints/autorouter_presets.json"); +// instead of redding on a stale copy. Exported as vi.fn so a test can override the query state. +const CATALOG_PATH = resolve(__dirname, "../../../../litellm/proxy/public_endpoints/autorouter_presets.json"); export const BUNDLED_PRESETS = hydratePresets( JSON.parse(readFileSync(CATALOG_PATH, "utf8")) as AutoRouterPresetsResponse, From 291d02f8aa90e0ed2cc6a08b38c4976559f9cafe Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:32:43 -0700 Subject: [PATCH 192/480] fix(mcp): never exchange the LiteLLM virtual key as the upstream subject token (#39446) --- .../mcp_server/mcp_server_manager.py | 97 ++++-- .../proxy/_experimental/mcp_server/server.py | 9 + .../mcp_server/test_mcp_server.py | 6 +- .../mcp_server/test_mcp_server_manager.py | 311 +++++++++++++++++- 4 files changed, 396 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 1f552ff3e13..1434fa5bfea 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -820,20 +820,46 @@ def _should_strip_caller_authorization( if not (mcp_server.is_oauth_passthrough or mcp_server.is_oauth_delegate): return False - normalized_raw_headers: Final = {str(k).lower(): v for k, v in (raw_headers or {}).items() if isinstance(k, str)} - has_explicit_litellm_admission_header: Final = normalized_raw_headers.get("x-litellm-api-key") is not None + has_explicit_litellm_admission_header: Final = _has_explicit_litellm_admission_header(raw_headers) if mcp_server.is_oauth_delegate: return not has_explicit_litellm_admission_header - admission_consumed_authorization_as_litellm_key: Final = ( - user_api_key_auth is not None - and bool(getattr(user_api_key_auth, "api_key", None)) - and not has_explicit_litellm_admission_header - ) - return admission_consumed_authorization_as_litellm_key or ( + return _authorization_is_litellm_admission_credential(raw_headers, user_api_key_auth) or ( user_api_key_auth is None and not has_explicit_litellm_admission_header ) +LITELLM_VIRTUAL_KEY_PREFIX: Final = "sk-" + + +def _raw_header_value(raw_headers: Mapping[str, str] | None, name: str) -> str | None: + return next((v for k, v in (raw_headers or {}).items() if isinstance(k, str) and k.lower() == name), None) + + +def _has_explicit_litellm_admission_header(raw_headers: Mapping[str, str] | None) -> bool: + """Admission only consumes a non-empty ``x-litellm-api-key``; an empty one falls back to ``Authorization``.""" + return bool(_raw_header_value(raw_headers, "x-litellm-api-key")) + + +def _authorization_is_litellm_admission_credential( + raw_headers: Mapping[str, str] | None, + user_api_key_auth: UserAPIKeyAuth | None, +) -> bool: + """True when ``Authorization`` carries the LiteLLM key admission validated. + + That is the case when no usable ``x-litellm-api-key`` was sent, or when the client repeated the + same key in both headers. + """ + if user_api_key_auth is None or not user_api_key_auth.api_key: + return False + admission_header: Final = _raw_header_value(raw_headers, "x-litellm-api-key") + if not admission_header: + return True + authorization: Final = _raw_header_value(raw_headers, "authorization") + return authorization is not None and strip_auth_scheme(authorization, "Bearer") == strip_auth_scheme( + admission_header, "Bearer" + ) + + def _format_byok_openapi_auth_header(mcp_server: MCPServer, mcp_auth_header: str) -> str: """Format a raw BYOK credential for OpenAPI tool ``Authorization`` injection. @@ -3277,8 +3303,8 @@ class MCPServerManager: ######################################################### @staticmethod def _extract_bearer_token( - oauth2_headers: dict[str, str] | None, - raw_headers: dict[str, str] | None, + oauth2_headers: Mapping[str, str] | None, + raw_headers: Mapping[str, str] | None, ) -> str | None: """Extract the bare Bearer token from oauth2_headers or raw_headers. @@ -3298,10 +3324,29 @@ class MCPServerManager: return auth_value return None + @staticmethod + def _extract_subject_token( + oauth2_headers: Mapping[str, str] | None, + raw_headers: Mapping[str, str] | None, + user_api_key_auth: UserAPIKeyAuth | None, + ) -> str | None: + """The caller's upstream identity token, or ``None`` when the bearer is a LiteLLM key. + + Rejects the key admission validated and, because virtual keys always carry the ``sk-`` prefix, + any other LiteLLM key a client puts in ``Authorization`` next to ``x-litellm-api-key``. + """ + if _authorization_is_litellm_admission_credential(raw_headers, user_api_key_auth): + return None + bearer: Final = MCPServerManager._extract_bearer_token(oauth2_headers, raw_headers) + if bearer is not None and bearer.startswith(LITELLM_VIRTUAL_KEY_PREFIX): + return None + return bearer + def _obo_subject_token( self, server: MCPServer, - raw_headers: dict[str, str] | None, + raw_headers: Mapping[str, str] | None, + user_api_key_auth: UserAPIKeyAuth | None, ) -> str | None: """The caller's bearer as the token_exchange (OBO) subject token, for that mode only. @@ -3311,7 +3356,7 @@ class MCPServerManager: """ if server.auth_type != MCPAuth.oauth2_token_exchange: return None - return self._extract_bearer_token(None, raw_headers) + return self._extract_subject_token(None, raw_headers, user_api_key_auth) def _build_stdio_env( self, @@ -3566,6 +3611,7 @@ class MCPServerManager: server: MCPServer, oauth2_headers: dict[str, str] | None, user_api_key_auth: UserAPIKeyAuth | None, + raw_headers: Mapping[str, str] | None = None, ) -> None: """Run the OBO exchange for a caller-supplied subject at the transport edge. @@ -3577,13 +3623,15 @@ class MCPServerManager: """ if server.auth_type != MCPAuth.oauth2_token_exchange: return - subject_token: Final = self._extract_bearer_token(oauth2_headers, None) - if not subject_token: + if not self._extract_bearer_token(oauth2_headers, None): return resolved_server: Final = await self.ensure_oauth_metadata_discovered(server) spec: Final = to_server_spec(resolved_server) if spec is None or not isinstance(spec.config, TokenExchangeConfig): return + subject_token: Final = self._extract_subject_token(oauth2_headers, raw_headers, user_api_key_auth) + if subject_token is None: + raise_token_exchange_challenge(resolved_server, root_path=get_server_root_path()) match await self._cred_provider.resolve_credentials(to_subject(user_api_key_auth, subject_token), spec): case Ok(_): return @@ -3851,7 +3899,7 @@ class MCPServerManager: # token (mirrors the call path), not v1's deleted client_credentials fallback. Other modes # never read the inbound bearer, so leave subject_token None to avoid forwarding it. subject_token: Final = ( - self._extract_bearer_token(oauth2_headers, raw_headers) + self._extract_subject_token(oauth2_headers, raw_headers, user_api_key_auth) if server.auth_type == MCPAuth.oauth2_token_exchange else None ) @@ -3931,6 +3979,7 @@ class MCPServerManager: async def get_prompts_from_server( self, server: MCPServer, + user_api_key_auth: UserAPIKeyAuth | None, mcp_auth_header: str | dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, add_prefix: bool = True, @@ -3959,7 +4008,7 @@ class MCPServerManager: extra_headers.update(server.static_headers) stdio_env: Final = self._build_stdio_env(server, raw_headers) - subject_token: Final = self._obo_subject_token(server, raw_headers) + subject_token: Final = self._obo_subject_token(server, raw_headers, user_api_key_auth) client = await self._create_mcp_client( server=server, @@ -3982,6 +4031,7 @@ class MCPServerManager: async def get_resources_from_server( self, server: MCPServer, + user_api_key_auth: UserAPIKeyAuth | None, mcp_auth_header: str | dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, add_prefix: bool = True, @@ -4001,7 +4051,7 @@ class MCPServerManager: extra_headers.update(server.static_headers) stdio_env: Final = self._build_stdio_env(server, raw_headers) - subject_token: Final = self._obo_subject_token(server, raw_headers) + subject_token: Final = self._obo_subject_token(server, raw_headers, user_api_key_auth) client = await self._create_mcp_client( server=server, @@ -4024,6 +4074,7 @@ class MCPServerManager: async def get_resource_templates_from_server( self, server: MCPServer, + user_api_key_auth: UserAPIKeyAuth | None, mcp_auth_header: str | dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, add_prefix: bool = True, @@ -4043,7 +4094,7 @@ class MCPServerManager: extra_headers.update(server.static_headers) stdio_env: Final = self._build_stdio_env(server, raw_headers) - subject_token: Final = self._obo_subject_token(server, raw_headers) + subject_token: Final = self._obo_subject_token(server, raw_headers, user_api_key_auth) client = await self._create_mcp_client( server=server, @@ -4068,6 +4119,7 @@ class MCPServerManager: async def read_resource_from_server( self, server: MCPServer, + user_api_key_auth: UserAPIKeyAuth | None, url: AnyUrl, mcp_auth_header: str | dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, @@ -4084,7 +4136,7 @@ class MCPServerManager: extra_headers.update(server.static_headers) stdio_env: Final = self._build_stdio_env(server, raw_headers) - subject_token: Final = self._obo_subject_token(server, raw_headers) + subject_token: Final = self._obo_subject_token(server, raw_headers, user_api_key_auth) client: Final = await self._create_mcp_client( server=server, @@ -4099,6 +4151,7 @@ class MCPServerManager: async def get_prompt_from_server( self, server: MCPServer, + user_api_key_auth: UserAPIKeyAuth | None, prompt_name: str, arguments: dict[str, str] | None = None, mcp_auth_header: str | dict[str, str] | None = None, @@ -4116,7 +4169,7 @@ class MCPServerManager: extra_headers.update(server.static_headers) stdio_env: Final = self._build_stdio_env(server, raw_headers) - subject_token: Final = self._obo_subject_token(server, raw_headers) + subject_token: Final = self._obo_subject_token(server, raw_headers, user_api_key_auth) client: Final = await self._create_mcp_client( server=server, @@ -5290,7 +5343,7 @@ class MCPServerManager: MCPAuth.oauth2_token_exchange, MCPAuth.oauth2_id_jag, ): - subject_token = self._extract_bearer_token(oauth2_headers, raw_headers) + subject_token = self._extract_subject_token(oauth2_headers, raw_headers, user_api_key_auth) elif mcp_server.auth_type == MCPAuth.oauth2: if mcp_server.has_client_credentials: # For M2M OAuth servers, Authorization must come from token fetch. @@ -5638,7 +5691,7 @@ class MCPServerManager: subject_token: str | None = None if isinstance(spec.config, (TokenExchangeConfig, IdJagConfig)): - subject_token = self._extract_bearer_token(oauth2_headers, raw_headers) + subject_token = self._extract_subject_token(oauth2_headers, raw_headers, user_api_key_auth) elif isinstance(spec.config, PassthroughConfig): inbound_token, forwarded_headers = _take_forwarded_authorization(forwarded_headers) per_server_token: Final = _passthrough_token_from_mcp_auth_header(mcp_auth_header) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index af3ff6714af..bb075220530 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -2189,6 +2189,7 @@ if MCP_AVAILABLE: try: prompts = await global_mcp_server_manager.get_prompts_from_server( server=server, + user_api_key_auth=user_api_key_auth, mcp_auth_header=server_auth_header, extra_headers=extra_headers, add_prefix=True, # Always add server prefix @@ -2242,6 +2243,7 @@ if MCP_AVAILABLE: try: resources = await global_mcp_server_manager.get_resources_from_server( server=server, + user_api_key_auth=user_api_key_auth, mcp_auth_header=server_auth_header, extra_headers=extra_headers, add_prefix=True, # Always add server prefix @@ -2293,6 +2295,7 @@ if MCP_AVAILABLE: try: resource_templates = await global_mcp_server_manager.get_resource_templates_from_server( server=server, + user_api_key_auth=user_api_key_auth, mcp_auth_header=server_auth_header, extra_headers=extra_headers, add_prefix=True, # Always add server prefix @@ -3211,6 +3214,7 @@ if MCP_AVAILABLE: return await global_mcp_server_manager.get_prompt_from_server( server=server, + user_api_key_auth=user_api_key_auth, prompt_name=original_prompt_name, arguments=arguments, mcp_auth_header=server_auth_header, @@ -3261,6 +3265,7 @@ if MCP_AVAILABLE: return await global_mcp_server_manager.read_resource_from_server( server=server, + user_api_key_auth=user_api_key_auth, url=url, mcp_auth_header=server_auth_header, extra_headers=extra_headers, @@ -3723,6 +3728,7 @@ if MCP_AVAILABLE: user_api_key_auth: UserAPIKeyAuth | None, client_ip: str | None, allowed_server_ids: set[str] | None = None, + raw_headers: Mapping[str, str] | None = None, ) -> None: """Fail fast with HTTP 401 for MCP servers that need user auth but didn't receive it on this request. Covers both gateway-managed OAuth2 @@ -3867,6 +3873,7 @@ if MCP_AVAILABLE: server=server, oauth2_headers=oauth2_headers, user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, ) # Pass-through OAuth: when the admin has opted a server into @@ -4195,6 +4202,7 @@ if MCP_AVAILABLE: user_api_key_auth=user_api_key_auth, client_ip=_client_ip, allowed_server_ids=toolset_allowed_server_ids, + raw_headers=raw_headers, ) # Pre-flight auth check for pass-through servers. Must run after @@ -4518,6 +4526,7 @@ if MCP_AVAILABLE: user_api_key_auth=user_api_key_auth, client_ip=_sse_client_ip, allowed_server_ids=toolset_allowed_server_ids, + raw_headers=raw_headers, ) # Pre-flight auth check for pass-through servers: surface upstream diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 3f6d8f8837c..ff0007f47aa 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -914,6 +914,7 @@ async def test_mcp_get_prompt_success(): ) mock_manager.get_prompt_from_server.assert_awaited_once_with( server=server, + user_api_key_auth=user_api_key_auth, prompt_name="hello", arguments={"foo": "bar"}, mcp_auth_header={"Authorization": "token"}, @@ -976,6 +977,7 @@ async def test_mcp_read_resource_success(): ) mock_manager.read_resource_from_server.assert_awaited_once_with( server=server, + user_api_key_auth=user_api_key_auth, url="https://example.com/resource", mcp_auth_header={"Authorization": "token"}, extra_headers={"X-Test": "1"}, @@ -8268,7 +8270,9 @@ class TestOboPreflightScopedToAllowedServers: _, preflight = await self._run(requested, allowed=[requested], user_api_key_auth=key) - preflight.assert_awaited_once_with(server=requested, oauth2_headers=self.SUBJECT_HEADERS, user_api_key_auth=key) + preflight.assert_awaited_once_with( + server=requested, oauth2_headers=self.SUBJECT_HEADERS, user_api_key_auth=key, raw_headers=None + ) @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 5508259273d..5cde2e83f62 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -30,6 +30,7 @@ from mcp.types import ( TextResourceContents, ) from mcp.types import Tool as MCPTool +from pydantic import AnyUrl from litellm.constants import MCP_METADATA_TIMEOUT from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( @@ -2270,7 +2271,9 @@ class TestMCPServerManager: """prompts/list on an OBO server must exchange the caller's bearer, not connect with none.""" server = self._token_exchange_server("te-prompts") st = await self._capture_subject_token( - lambda m: m.get_prompts_from_server(server=server, raw_headers={"authorization": "Bearer subj-jwt"}) + lambda m: m.get_prompts_from_server( + server=server, user_api_key_auth=None, raw_headers={"authorization": "Bearer subj-jwt"} + ) ) assert st == "subj-jwt" @@ -2279,7 +2282,9 @@ class TestMCPServerManager: """resources/list on an OBO server must exchange the caller's bearer.""" server = self._token_exchange_server("te-resources") st = await self._capture_subject_token( - lambda m: m.get_resources_from_server(server=server, raw_headers={"authorization": "Bearer subj-jwt"}) + lambda m: m.get_resources_from_server( + server=server, user_api_key_auth=None, raw_headers={"authorization": "Bearer subj-jwt"} + ) ) assert st == "subj-jwt" @@ -2290,6 +2295,7 @@ class TestMCPServerManager: st = await self._capture_subject_token( lambda m: m.read_resource_from_server( server=server, + user_api_key_auth=None, url="https://up.example.com/r", raw_headers={"authorization": "Bearer subj-jwt"}, ) @@ -2307,7 +2313,9 @@ class TestMCPServerManager: auth_type=MCPAuth.none, ) st = await self._capture_subject_token( - lambda m: m.get_prompts_from_server(server=server, raw_headers={"authorization": "Bearer subj-jwt"}) + lambda m: m.get_prompts_from_server( + server=server, user_api_key_auth=None, raw_headers={"authorization": "Bearer subj-jwt"} + ) ) assert st is None @@ -3254,7 +3262,7 @@ class TestMCPServerManager: new_callable=AsyncMock, return_value=mock_client, ): - prompts = await manager.get_prompts_from_server(server, add_prefix=True) + prompts = await manager.get_prompts_from_server(server, user_api_key_auth=None, add_prefix=True) mock_client.list_prompts.assert_awaited_once() assert len(prompts) == 1 @@ -3289,6 +3297,7 @@ class TestMCPServerManager: ): result = await manager.get_prompt_from_server( server=server, + user_api_key_auth=None, prompt_name="hello", arguments={"tone": "casual"}, ) @@ -3334,6 +3343,7 @@ class TestMCPServerManager: ): result = await manager.get_resources_from_server( server=server, + user_api_key_auth=None, mcp_auth_header="auth", extra_headers={"X-Test": "1"}, add_prefix=True, @@ -3391,6 +3401,7 @@ class TestMCPServerManager: ): result = await manager.get_resource_templates_from_server( server=server, + user_api_key_auth=None, mcp_auth_header="auth", extra_headers=None, add_prefix=False, @@ -3441,6 +3452,7 @@ class TestMCPServerManager: ) as mock_create_client: result = await manager.read_resource_from_server( server=server, + user_api_key_auth=None, url="https://example.com/resource", mcp_auth_header="auth", extra_headers={"X-Test": "1"}, @@ -11006,3 +11018,294 @@ class TestOpenApiHandlerRelaysUpstreamAuth: assert result.isError is True assert "upstream returned HTTP 503" in result.content[0].text + + +class TestLitellmAdmissionKeyIsNeverTheSubjectToken: + """The bearer that admitted the request as a LiteLLM key must not be sent to the IdP as the + RFC 8693 subject_token (or ID-JAG assertion). Only ``x-litellm-api-key`` disambiguates: with it + present, ``Authorization`` is the caller's own identity token and is exchanged as before.""" + + _ADMISSION_KEY: Final = "sk-litellm-virtual-key" + _USER_TOKEN: Final = "user-idp-jwt" + + @staticmethod + def _token_exchange_server(server_id: str) -> MCPServer: + return MCPServer( + server_id=server_id, + name=f"{server_id}-server", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2_token_exchange, + token_exchange_endpoint="https://idp.example.com/token", + client_id="cid", + client_secret="csec", + ) + + @staticmethod + def _id_jag_server(server_id: str) -> MCPServer: + return MCPServer( + server_id=server_id, + name=f"{server_id}-server", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2_id_jag, + client_id="cid", + client_secret="csec", + token_exchange_endpoint="https://idp.example.com/token", + id_jag_resource_token_endpoint="https://resource-as.example.com/token", + ) + + @staticmethod + def _recording_provider() -> MagicMock: + from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth + from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok + + provider: Final = MagicMock() + provider.resolve_credentials = AsyncMock( + return_value=Ok(StaticHeaderAuth("Bearer MINTED", header_name="Authorization")) + ) + return provider + + @staticmethod + def _subjects_seen_by(provider: MagicMock) -> list[str | None]: + return [ + call.args[0].inbound_token.get_secret_value() if call.args[0].inbound_token else None + for call in provider.resolve_credentials.call_args_list + ] + + @staticmethod + def _manager_with_recording_client() -> MCPServerManager: + manager: Final = MCPServerManager() + client: Final = AsyncMock() + client.call_tool = AsyncMock(return_value=CallToolResult(content=[], isError=False)) + client.list_prompts = AsyncMock(return_value=[]) + client.read_resource = AsyncMock(return_value=ReadResourceResult(contents=[])) + manager._create_mcp_client = AsyncMock(return_value=client) + return manager + + @staticmethod + def _subject_token_given_to_client(manager: MCPServerManager) -> str | None: + return manager._create_mcp_client.call_args.kwargs["subject_token"] + + async def _call_tool_subject(self, server: MCPServer, oauth2_headers, raw_headers, user_api_key_auth): + manager: Final = self._manager_with_recording_client() + await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="tool", + arguments={}, + tasks=[], + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + proxy_logging_obj=None, + user_api_key_auth=user_api_key_auth, + ) + return self._subject_token_given_to_client(manager) + + @pytest.mark.asyncio + @pytest.mark.parametrize("auth_type", [MCPAuth.oauth2_token_exchange, MCPAuth.oauth2_id_jag]) + async def test_tools_call_with_only_the_litellm_key_has_no_subject(self, auth_type): + server = ( + self._token_exchange_server("te-call") + if auth_type == MCPAuth.oauth2_token_exchange + else self._id_jag_server("jag-call") + ) + subject_token = await self._call_tool_subject( + server, + oauth2_headers={"Authorization": f"Bearer {self._ADMISSION_KEY}"}, + raw_headers={"authorization": f"Bearer {self._ADMISSION_KEY}"}, + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + ) + assert subject_token is None + + @pytest.mark.asyncio + async def test_rest_tools_call_with_only_the_litellm_key_has_no_subject(self): + """The REST facade passes no oauth2_headers; the bearer is reached through raw_headers only.""" + subject_token = await self._call_tool_subject( + self._token_exchange_server("te-rest"), + oauth2_headers=None, + raw_headers={"Authorization": f"Bearer {self._ADMISSION_KEY}"}, + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + ) + assert subject_token is None + + @pytest.mark.asyncio + async def test_tools_call_exchanges_the_user_token_when_x_litellm_api_key_admits(self): + subject_token = await self._call_tool_subject( + self._token_exchange_server("te-split"), + oauth2_headers={"Authorization": f"Bearer {self._USER_TOKEN}"}, + raw_headers={ + "X-LiteLLM-API-Key": f"Bearer {self._ADMISSION_KEY}", + "authorization": f"Bearer {self._USER_TOKEN}", + }, + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + ) + assert subject_token == self._USER_TOKEN + + @pytest.mark.asyncio + async def test_tools_call_with_an_empty_x_litellm_api_key_has_no_subject(self): + """Admission ignores an empty ``x-litellm-api-key`` and validates ``Authorization`` instead.""" + subject_token = await self._call_tool_subject( + self._token_exchange_server("te-empty-header"), + oauth2_headers={"Authorization": f"Bearer {self._ADMISSION_KEY}"}, + raw_headers={"x-litellm-api-key": "", "authorization": f"Bearer {self._ADMISSION_KEY}"}, + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + ) + assert subject_token is None + + @pytest.mark.asyncio + async def test_tools_call_with_the_same_litellm_key_in_both_headers_has_no_subject(self): + subject_token = await self._call_tool_subject( + self._token_exchange_server("te-same-key"), + oauth2_headers={"Authorization": f"Bearer {self._ADMISSION_KEY}"}, + raw_headers={ + "x-litellm-api-key": self._ADMISSION_KEY, + "authorization": f"Bearer {self._ADMISSION_KEY}", + }, + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + ) + assert subject_token is None + + @pytest.mark.asyncio + async def test_tools_call_with_a_different_litellm_key_in_authorization_has_no_subject(self): + """A second ``sk-`` virtual key next to ``x-litellm-api-key`` is still a gateway credential.""" + subject_token = await self._call_tool_subject( + self._token_exchange_server("te-second-key"), + oauth2_headers={"Authorization": "Bearer sk-another-virtual-key"}, + raw_headers={ + "x-litellm-api-key": f"Bearer {self._ADMISSION_KEY}", + "authorization": "Bearer sk-another-virtual-key", + }, + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + ) + assert subject_token is None + + @pytest.mark.asyncio + async def test_tools_call_exchanges_the_bearer_when_jwt_admission_left_api_key_unset(self): + subject_token = await self._call_tool_subject( + self._token_exchange_server("te-jwt"), + oauth2_headers={"Authorization": f"Bearer {self._USER_TOKEN}"}, + raw_headers={"authorization": f"Bearer {self._USER_TOKEN}"}, + user_api_key_auth=UserAPIKeyAuth(api_key=None, user_id="alice"), + ) + assert subject_token == self._USER_TOKEN + + @pytest.mark.asyncio + async def test_tools_list_with_only_the_litellm_key_has_no_subject(self): + manager: Final = self._manager_with_recording_client() + manager._fetch_tools_with_timeout = AsyncMock(return_value=[]) + await manager._get_tools_from_server( + server=self._token_exchange_server("te-list-key"), + oauth2_headers={"Authorization": f"Bearer {self._ADMISSION_KEY}"}, + raw_headers={"authorization": f"Bearer {self._ADMISSION_KEY}"}, + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + ) + assert self._subject_token_given_to_client(manager) is None + + @pytest.mark.asyncio + async def test_prompts_list_with_only_the_litellm_key_has_no_subject(self): + manager: Final = self._manager_with_recording_client() + await manager.get_prompts_from_server( + server=self._token_exchange_server("te-prompts-key"), + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + raw_headers={"authorization": f"Bearer {self._ADMISSION_KEY}"}, + ) + assert self._subject_token_given_to_client(manager) is None + + @pytest.mark.asyncio + async def test_resource_read_with_only_the_litellm_key_has_no_subject(self): + manager: Final = self._manager_with_recording_client() + await manager.read_resource_from_server( + server=self._token_exchange_server("te-read-key"), + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + url=AnyUrl("file:///notes.txt"), + raw_headers={"authorization": f"Bearer {self._ADMISSION_KEY}"}, + ) + assert self._subject_token_given_to_client(manager) is None + + @pytest.mark.asyncio + async def test_resource_read_exchanges_the_user_token_when_x_litellm_api_key_admits(self): + manager: Final = self._manager_with_recording_client() + await manager.read_resource_from_server( + server=self._token_exchange_server("te-read-split"), + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + url=AnyUrl("file:///notes.txt"), + raw_headers={ + "x-litellm-api-key": f"Bearer {self._ADMISSION_KEY}", + "authorization": f"Bearer {self._USER_TOKEN}", + }, + ) + assert self._subject_token_given_to_client(manager) == self._USER_TOKEN + + @pytest.mark.asyncio + async def test_openapi_call_never_hands_the_litellm_key_to_the_exchanger(self): + provider: Final = self._recording_provider() + manager = MCPServerManager(cred_provider=provider) + server = MCPServer( + server_id="te-openapi", + name="te_openapi", + server_name="te_openapi", + url=None, + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2_token_exchange, + token_exchange_endpoint="https://idp.example.com/token", + client_id="cid", + client_secret="csec", + spec_path="https://api.example.com/openapi.json", + ) + user_auth = UserAPIKeyAuth(api_key="hashed-key", user_id="alice") + + await manager.resolve_openapi_upstream_auth( + mcp_server=server, + oauth2_headers={"Authorization": f"Bearer {self._ADMISSION_KEY}"}, + raw_headers={"authorization": f"Bearer {self._ADMISSION_KEY}"}, + mcp_auth_header=None, + user_api_key_auth=user_auth, + forwarded_headers=None, + ) + await manager.resolve_openapi_upstream_auth( + mcp_server=server, + oauth2_headers={"Authorization": f"Bearer {self._USER_TOKEN}"}, + raw_headers={ + "x-litellm-api-key": f"Bearer {self._ADMISSION_KEY}", + "authorization": f"Bearer {self._USER_TOKEN}", + }, + mcp_auth_header=None, + user_api_key_auth=user_auth, + forwarded_headers=None, + ) + assert self._subjects_seen_by(provider) == [None, self._USER_TOKEN] + + @pytest.mark.asyncio + async def test_preflight_challenges_instead_of_exchanging_the_litellm_key(self): + provider: Final = self._recording_provider() + manager = MCPServerManager(cred_provider=provider) + + with pytest.raises(HTTPException) as exc_info: + await manager.preflight_token_exchange( + server=self._token_exchange_server("te-preflight-key"), + oauth2_headers={"Authorization": f"Bearer {self._ADMISSION_KEY}"}, + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + raw_headers={"authorization": f"Bearer {self._ADMISSION_KEY}"}, + ) + assert exc_info.value.status_code == 401 + headers = exc_info.value.headers or {} + assert "resource_metadata" in (headers.get("WWW-Authenticate") or headers.get("www-authenticate") or "") + assert self._subjects_seen_by(provider) == [] + + @pytest.mark.asyncio + async def test_preflight_exchanges_the_user_token_when_x_litellm_api_key_admits(self): + provider: Final = self._recording_provider() + manager = MCPServerManager(cred_provider=provider) + + await manager.preflight_token_exchange( + server=self._token_exchange_server("te-preflight-split"), + oauth2_headers={"Authorization": f"Bearer {self._USER_TOKEN}"}, + user_api_key_auth=UserAPIKeyAuth(api_key="hashed-key", user_id="alice"), + raw_headers={ + "x-litellm-api-key": f"Bearer {self._ADMISSION_KEY}", + "authorization": f"Bearer {self._USER_TOKEN}", + }, + ) + assert self._subjects_seen_by(provider) == [self._USER_TOKEN] From e058aa68c4331ce7ffce2802528eacc2f2333642 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:40:00 -0700 Subject: [PATCH 193/480] test: add mistral ocr transformation parity coverage (#39482) * test: cover mistral ocr transformation parity Co-Authored-By: Claude Code * test: map mistral ocr parity contracts Co-Authored-By: Claude Code --------- Co-authored-by: Claude Code --- .../providers/mistral/ocr/transformation.rs | 188 ++- .../ledgers/ocr/ocr_test_ledger.json | 1240 ++++++++++++++--- 2 files changed, 1208 insertions(+), 220 deletions(-) diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs index 9648321d7ff..0125886aac1 100644 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs @@ -186,68 +186,180 @@ mod tests { use serde_json::json; #[test] - fn supported_params_match_python_mistral_ocr_config() { + fn extract_header_is_a_supported_ocr_param() { + assert!(supported_ocr_params().contains(&"extract_header")); + } + + #[test] + fn extract_footer_is_a_supported_ocr_param() { + assert!(supported_ocr_params().contains(&"extract_footer")); + } + + #[test] + fn existing_ocr_params_remain_supported() { + for param in [ + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + ] { + assert!(supported_ocr_params().contains(¶m)); + } + } + + #[test] + fn map_ocr_params_forwards_extract_header() { + let params = json!({"extract_header": true}); assert_eq!( - supported_ocr_params(), - &[ - "pages", - "include_image_base64", - "image_limit", - "image_min_size", - "bbox_annotation_format", - "document_annotation_format", - "document_annotation_prompt", - "extract_header", - "extract_footer", - "table_format", - "confidence_scores_granularity", - "include_blocks", - "id", - ] + map_ocr_params(params.as_object().unwrap()), + params.as_object().unwrap().clone() + ); + } + + #[test] + fn map_ocr_params_forwards_extract_footer() { + let params = json!({"extract_footer": true}); + assert_eq!( + map_ocr_params(params.as_object().unwrap()), + params.as_object().unwrap().clone() + ); + } + + #[test] + fn map_ocr_params_forwards_extract_header_and_footer() { + let params = json!({"extract_header": true, "extract_footer": false}); + assert_eq!( + map_ocr_params(params.as_object().unwrap()), + params.as_object().unwrap().clone() ); } #[test] fn map_ocr_params_drops_unknown_params() { - let params = json!({ - "extract_header": true, - "unsupported_param": "value", - "pages": [0, 1] - }); + let params = json!({"extract_header": true, "unsupported_param": "value"}); let mapped = map_ocr_params(params.as_object().unwrap()); - assert_eq!(mapped.get("extract_header"), Some(&json!(true))); - assert_eq!(mapped.get("pages"), Some(&json!([0, 1]))); assert!(!mapped.contains_key("unsupported_param")); } #[test] - fn transform_ocr_request_builds_mistral_body() { + fn new_ocr_params_are_supported() { + for param in [ + "table_format", + "confidence_scores_granularity", + "document_annotation_prompt", + "include_blocks", + "id", + ] { + assert!(supported_ocr_params().contains(¶m)); + } + } + + #[test] + fn map_ocr_params_forwards_new_ocr_params() { + for (param, value) in [ + ("table_format", json!("html")), + ("confidence_scores_granularity", json!("word")), + ( + "document_annotation_prompt", + json!("Extract all invoice line items"), + ), + ("include_blocks", json!(true)), + ("id", json!("req-123")), + ] { + let params = json!({param: value}); + assert_eq!( + map_ocr_params(params.as_object().unwrap()), + params.as_object().unwrap().clone() + ); + } + } + + #[test] + fn transform_ocr_request_includes_each_optional_param() { + let document = json!({ + "type": "document_url", + "document_url": "https://example.com/doc.pdf" + }); + for (param, value) in [ + ("table_format", json!("html")), + ("confidence_scores_granularity", json!("word")), + ( + "document_annotation_prompt", + json!("Extract all invoice line items"), + ), + ("id", json!("req-123")), + ("extract_header", json!(true)), + ("include_blocks", json!(true)), + ("pages", json!([0, 1])), + ] { + let result = transform_ocr_request( + "mistral-ocr-latest", + document.clone(), + json!({param: value}).as_object().unwrap().clone(), + ) + .expect("request should transform"); + assert_eq!(result.data.get(param), Some(&value)); + assert_eq!(result.data.get("model"), Some(&json!("mistral-ocr-latest"))); + assert_eq!(result.data.get("document"), Some(&document)); + assert_eq!(result.files, None); + } + } + + #[test] + fn transform_ocr_request_includes_multiple_new_params() { let document = json!({ "type": "document_url", "document_url": "https://example.com/doc.pdf" }); let optional_params = json!({ - "include_image_base64": true, - "table_format": "html" + "table_format": "html", + "confidence_scores_granularity": "page", + "extract_header": true }) .as_object() .unwrap() .clone(); - - let result = transform_ocr_request("mistral-ocr-latest", document.clone(), optional_params) + let result = transform_ocr_request("mistral-ocr-latest", document, optional_params) .expect("request should transform"); - + assert_eq!(result.data.get("table_format"), Some(&json!("html"))); assert_eq!( - result.data, - json!({ - "model": "mistral-ocr-latest", - "document": document, - "include_image_base64": true, - "table_format": "html" - }) + result.data.get("confidence_scores_granularity"), + Some(&json!("page")) ); - assert_eq!(result.files, None); + assert_eq!(result.data.get("extract_header"), Some(&json!(true))); + } + + #[test] + fn transform_ocr_response_preserves_blocks_and_confidence_scores() { + let blocks = json!([{"type": "title", "content": "Invoice"}]); + let confidence_scores = json!({"page": 0.98}); + let response = json!({ + "pages": [{"index": 0, "markdown": "# Invoice", "blocks": blocks, "confidence_scores": confidence_scores}], + "model": "mistral-ocr-4-0", + "usage_info": {"pages_processed": 1} + }); + let result = + transform_ocr_response("mistral-ocr-4-0", response).expect("response should transform"); + assert_eq!(result.pages[0].get("blocks"), Some(&blocks)); + assert_eq!( + result.pages[0].get("confidence_scores"), + Some(&confidence_scores) + ); + } + + #[test] + fn transform_ocr_response_preserves_ocr4_page_fields() { + let response = json!({ + "pages": [{"index": 0, "markdown": "table page", "tables": [{"rows": 2, "cols": 3}], "hyperlinks": ["https://example.com"], "header": "Acme Corp", "footer": "Page 1"}], + "model": "mistral-ocr-4-0", + "usage_info": {"pages_processed": 1} + }); + let result = transform_ocr_response("mistral-ocr-4-0", response.clone()) + .expect("response should transform"); + assert_eq!(result.pages[0], response["pages"][0]); } #[test] diff --git a/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json b/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json index 799a1320463..e617ddf8f94 100644 --- a/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json +++ b/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json @@ -20,189 +20,1065 @@ "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs" ], "entries": [ - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_should_encode_azure_document_intelligence_model_id", "status": "unmapped", "reason": "model-id URL percent-encoding has no Rust test; Rust only tests pages/features query building"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_should_reject_dot_segment_azure_document_intelligence_model_id", "status": "unmapped", "reason": "model-id dot-segment validation has no Rust test"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_preserves_azure_native_fields", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_response_normalizes_pages", "justification": "both assert page markdown, dimension (inch-to-pixel) normalization, and usage_info.pages_processed from the same Azure succeeded response shape"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_async_transform_ocr_response_preserves_azure_native_fields", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_response_normalizes_pages", "justification": "async twin of the sync case above, same underlying transform is exercised on the Rust side"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_tolerates_missing_native_fields", "status": "unmapped", "reason": "tables/keyValuePairs absence tolerance is not asserted by the Rust response test"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_non_succeeded_status_raises", "status": "unmapped", "reason": "no Rust test asserts on a non-succeeded Azure DI status"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_supported_ocr_params_includes_features", "status": "unmapped", "reason": "supported-params list content has no Rust equivalent for Azure"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_native_format_carries_raw_operation", "status": "unmapped", "reason": "native req_format raw-operation passthrough is not tested in Rust"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_async_transform_ocr_response_native_format_carries_raw_operation", "status": "unmapped", "reason": "native req_format raw-operation passthrough is not tested in Rust"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_default_format_omits_raw_operation", "status": "unmapped", "reason": "req_format gating of raw-operation output has no Rust test"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_passes_through_req_format", "status": "unmapped", "reason": "req_format passthrough in map_ocr_params has no Rust test"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_rejects_unknown_req_format_as_bad_request", "status": "unmapped", "reason": "req_format validation error path has no Rust test"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_omits_req_format_query_param", "status": "unmapped", "reason": "no Rust test asserts req_format is excluded from the built URL"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_features", "status": "unmapped", "reason": "features-string normalization in map_ocr_params has no Rust test"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_empty_features_list_omitted", "status": "unmapped", "reason": "empty-features omission has no Rust test"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_invalid_features_raises", "status": "unmapped", "reason": "features validation error path has no Rust test"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_appends_features_query", "status": "unmapped", "reason": "features query-param construction has no Rust test"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_combines_pages_and_features", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_url_normalizes_zero_based_pages", "justification": "both assert 0-based, duplicate page indices are deduped, sorted, and rewritten 1-based into the request URL"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_validate_environment_uses_subscription_key", "status": "unmapped", "reason": "Python-side header derivation from litellm_params; Rust's poll test only checks the header is present, not how it was resolved"}, - {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_validate_environment_falls_back_to_entra_token", "status": "unmapped", "reason": "Entra bearer-token fallback logic has no Rust test"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_doc_intelligence_route", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_documentintelligence_and_is_case_insensitive", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_does_not_match_mistral_route", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_does_not_hijack_doc_intelligence", "status": "unmapped", "reason": "api_base resolution from the secret manager runs before the Rust bridge is called, no Rust test exists for it"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_explicit_api_base_is_honoured_for_doc_intelligence", "status": "unmapped", "reason": "api_base precedence resolution is Python-only"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_still_applies_to_mistral_ocr", "status": "unmapped", "reason": "api_base precedence resolution is Python-only"}, - - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_use_litellm_rust_toggles_flag", "status": "unmapped", "reason": "bridge-plumbing: Python-side feature-flag toggle, no Rust equivalent"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_env_var_enables_rust_ocr", "status": "unmapped", "reason": "bridge-plumbing: Python-side env-var flag gating"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_returns_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: dependency-injection test hook, not provider behavior"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_loader_returns_none_when_extension_absent", "status": "unmapped", "reason": "bridge-plumbing: native-extension import/loader fallback"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_loader_caches_absent_extension", "status": "unmapped", "reason": "bridge-plumbing: loader caching behavior"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_available_reflects_loader", "status": "unmapped", "reason": "bridge-plumbing: loader availability check"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_aocr_returns_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: dependency-injection test hook"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_toggle_without_ocr_arg_preserves_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: injected-impl state retention regression"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_explicit_ocr_none_clears_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: injected-impl clearing behavior"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_none_when_extension_absent", "status": "unmapped", "reason": "bridge-plumbing: degrade path when the native extension is missing"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_uses_compiled_extension", "status": "unmapped", "reason": "bridge-plumbing: native module resolution"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_timeout_to_seconds_handles_float_timeout_and_none", "status": "unmapped", "reason": "bridge-plumbing: Python-side timeout normalization helper"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_bridge_wrapper_forwards_prepared_args_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: wrapper argument forwarding, asserted against a fake bridge not the real Rust code"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: async wrapper argument forwarding"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_prepares_request_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: request preparation and response wrapping in Python"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_resolves_key_via_secret_manager_when_missing", "status": "unmapped", "reason": "secret-manager: API key resolution happens in Python before the bridge is invoked"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_prefers_explicit_key_over_resolver", "status": "unmapped", "reason": "secret-manager: key precedence resolution"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_uses_provider_api_key_env_var", "status": "unmapped", "reason": "secret-manager: provider-specific env var name resolution"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_forwards_vertex_routing_metadata", "status": "unmapped", "reason": "secret-manager: vertex routing metadata merge happens in Python"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_manager", "status": "unmapped", "reason": "secret-manager: vertex project/location resolution"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager", "status": "unmapped", "reason": "secret-manager: azure_ai api_base resolution"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint", "status": "unmapped", "reason": "secret-manager: doc-intelligence endpoint resolution"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_runs_pre_call_logging", "status": "unmapped", "reason": "bridge-plumbing: Python logging-object pre_call invocation"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_routes_to_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: routing to a fake bridge, not the real Rust transform"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_routes_azure_ai_to_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: provider-prefix stripping before routing"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_rust_path_converts_file_document_before_bridge", "status": "unmapped", "reason": "file-normalization: raw-bytes-to-data-URI conversion happens in Python before the bridge call"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_exception_type_uses_resolved_provider_context", "status": "unmapped", "reason": "bridge-plumbing: Python exception-type mapping on bridge failure"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_aocr_routes_to_async_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: async routing to a fake bridge"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_aocr_exception_type_uses_resolved_provider_context", "status": "unmapped", "reason": "bridge-plumbing: async exception-type mapping on bridge failure"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_does_not_route_to_rust_when_disabled", "status": "unmapped", "reason": "bridge-plumbing: Python control flow for the toggle-disabled branch, no Rust-owned behavior runs"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_falls_back_to_python_when_bridge_unavailable", "status": "unmapped", "reason": "bridge-plumbing: Python-only fallback when the compiled Rust extension is absent, Rust cannot test its own absence"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_forwards_timeout_to_rust", "status": "unmapped", "reason": "bridge-plumbing: asserts the Python call site forwards a timeout kwarg, Rust receives an already-constructed request"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_passes_default_request_timeout_to_rust", "status": "unmapped", "reason": "bridge-plumbing: asserts the Python call site supplies a default timeout kwarg, no Rust equivalent"}, - {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_provider_configs_expose_api_key_env_vars", "status": "unmapped", "reason": "asserts per-provider get_api_key_env_var() strings; the closest Rust test (ocr_dispatch_supports_migrated_providers) asserts provider dispatch/param resolution instead, not API key env var names"}, - - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_pdf_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection is Python-only preprocessing before the bridge call"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_png_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_jpg_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_jpeg_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_gif_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_webp_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_tiff_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_tif_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_bmp_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_be_case_insensitive", "status": "unmapped", "reason": "file-normalization: MIME detection case handling"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_fallback_for_unknown_extension", "status": "unmapped", "reason": "file-normalization: MIME detection fallback"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pdf_pathlib_path_to_document_url", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion happens in Python"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_image_pathlib_path_to_image_url", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_reject_bare_str_path", "status": "unmapped", "reason": "file-normalization: arbitrary-file-read guard on bare str paths"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pathlib_path", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes", "status": "unmapped", "reason": "file-normalization: raw-bytes-to-data-URI conversion"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_explicit_mime_type", "status": "unmapped", "reason": "file-normalization: explicit MIME override on raw bytes"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_image_mime_type", "status": "unmapped", "reason": "file-normalization: explicit MIME override on raw bytes"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object", "status": "unmapped", "reason": "file-normalization: file-like-object conversion"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object_with_name", "status": "unmapped", "reason": "file-normalization: file-like-object name-based MIME detection"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_missing_file_field", "status": "unmapped", "reason": "file-normalization: missing-field validation"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_nonexistent_pathlib_path", "status": "unmapped", "reason": "file-normalization: missing-file validation"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_empty_file", "status": "unmapped", "reason": "file-normalization: empty-file validation"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_unsupported_type", "status": "unmapped", "reason": "file-normalization: unsupported input type validation"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_invalid_mime_type", "status": "unmapped", "reason": "file-normalization: MIME-type injection validation"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_override_mime_type_for_pathlib_path", "status": "unmapped", "reason": "file-normalization: explicit MIME override precedence"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_document_url_for_pdf", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_png", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_jpeg", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_octet_stream", "status": "unmapped", "reason": "file-normalization: filename-based MIME fallback"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_none", "status": "unmapped", "reason": "file-normalization: filename-based MIME fallback"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_fallback_to_octet_stream_for_unknown", "status": "unmapped", "reason": "file-normalization: default MIME fallback"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_preserve_base64_content_correctly", "status": "unmapped", "reason": "file-normalization: binary round-trip through base64"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_from_content_type", "status": "unmapped", "reason": "file-normalization: content-type parameter stripping"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_with_multiple_params", "status": "unmapped", "reason": "file-normalization: content-type parameter stripping"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_reject_file_type_document_in_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body file-type guard, a different mechanism than Rust's URL-fetch SSRF guard"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_accept_document_url_type_in_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body parsing"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_raise_on_invalid_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body parsing error path"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_ignore_document_form_field_injection", "status": "unmapped", "reason": "proxy-layer multipart form-field injection guard, a different mechanism than Rust's URL-fetch SSRF guard"}, - - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_extract_header_in_supported_params", "status": "unmapped", "reason": "Rust's fixed-list test checks the full list as one assertion, not this individual param"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_extract_footer_in_supported_params", "status": "unmapped", "reason": "Rust's fixed-list test checks the full list as one assertion, not this individual param"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_existing_params_still_present", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "supported_params_match_python_mistral_ocr_config", "justification": "both assert the full supported_ocr_params list matches the same fixed set of param names"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_header_passed_through", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_drops_unknown_params", "justification": "both assert extract_header survives map_ocr_params filtering unchanged"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_footer_passed_through", "status": "unmapped", "reason": "Rust's map_ocr_params test does not assert on extract_footer specifically"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_header_and_footer_together", "status": "unmapped", "reason": "combined extract_header+extract_footer passthrough is not asserted together in Rust"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_unknown_param_is_dropped", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_drops_unknown_params", "justification": "both assert an unrecognized param key is dropped while a known one is kept"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestNewSupportedParams::test_new_param_in_supported_list", "status": "unmapped", "reason": "OCR4-specific new params (table_format etc) are not individually verified against the Rust fixed-list test"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestNewParamsMapOcr::test_new_param_passed_through", "status": "unmapped", "reason": "OCR4-specific new params are not individually asserted in the Rust map_ocr_params test"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrRequest::test_param_included_in_request_body", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_builds_mistral_body", "justification": "both assert an optional param value ends up in the built request body alongside model/document"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrRequest::test_multiple_new_params_together", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_builds_mistral_body", "justification": "both assert multiple optional params (table_format/include_image_base64) land correctly in the same request body"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrResponseOcr4Fields::test_blocks_and_confidence_scores_preserved", "status": "unmapped", "reason": "OCR4 blocks/confidence_scores fields are not asserted by the Rust response test"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrResponseOcr4Fields::test_ocr4_fields_survive_model_dump", "status": "unmapped", "reason": "OCR4 tables/hyperlinks/header/footer fields are not asserted by the Rust response test"}, - - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_model_info_ocr4_price", "status": "unmapped", "reason": "cost-calc: pricing/model-info lookup is Python-only"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr4_cost_scales_with_pages", "status": "unmapped", "reason": "cost-calc: per-page pricing math is Python-only"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_pricing_entry", "status": "unmapped", "reason": "cost-calc: cost-map JSON entry validation is Python-only"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_model_info_price", "status": "unmapped", "reason": "cost-calc: pricing/model-info lookup is Python-only"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_cost_scales_with_pages", "status": "unmapped", "reason": "cost-calc: per-page pricing math is Python-only"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_ocr_and_annotation_pages_at_their_own_rates", "status": "unmapped", "reason": "cost-calc: mixed-rate billing math is Python-only"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_only_response", "status": "unmapped", "reason": "cost-calc: annotation-only billing math is Python-only"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_pages_when_pages_processed_missing", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"}, - {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_azure_doc_ai_annotation_pages_fall_back_to_ocr_rate", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"}, - - {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_serves_default_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_skipped_for_native_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_native_format_rejected_for_provider_without_support_as_bad_request", "status": "unmapped", "reason": "provider-support validation for req_format happens in Python"}, - {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_unknown_format_rejected_for_provider_without_support_as_bad_request", "status": "unmapped", "reason": "req_format validation error path is Python-only"}, - - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestHandlerDiscovery::test_handler_discovered_for_ocr", "status": "unmapped", "reason": "guardrail-translation handler discovery is a Python proxy-layer concern"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestHandlerDiscovery::test_handler_discovered_for_aocr", "status": "unmapped", "reason": "guardrail-translation handler discovery is a Python proxy-layer concern"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_document_url", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_image_url", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_no_document", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_invalid_document", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_input_blocking_guardrail", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_single_page", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_multiple_pages", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_empty_pages", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_page_with_empty_markdown", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_preserves_page_metadata", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_output_blocking_guardrail", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, - {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestPIIMaskingScenario::test_pii_masking_in_ocr_pages", "status": "unmapped", "reason": "PII redaction in the translation handler has no Rust equivalent"}, - - {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_read_req_format_from_header", "status": "unmapped", "reason": "proxy-layer header parsing has no Rust equivalent"}, - {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_prefer_body_req_format_over_header", "status": "unmapped", "reason": "proxy-layer body-vs-header precedence has no Rust equivalent"}, - {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_omit_req_format_when_header_absent", "status": "unmapped", "reason": "proxy-layer parsing has no Rust equivalent"}, - {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_reject_unknown_req_format", "status": "unmapped", "reason": "proxy-layer validation has no Rust equivalent"}, - {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_return_native_payload_with_litellm_response_headers", "status": "unmapped", "reason": "proxy-layer response construction has no Rust equivalent"}, - {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_return_normalized_response_when_no_native_payload", "status": "unmapped", "reason": "proxy-layer response construction has no Rust equivalent"} + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_should_encode_azure_document_intelligence_model_id", + "status": "unmapped", + "reason": "model-id URL percent-encoding has no Rust test; Rust only tests pages/features query building" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_should_reject_dot_segment_azure_document_intelligence_model_id", + "status": "unmapped", + "reason": "model-id dot-segment validation has no Rust test" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_transform_ocr_response_preserves_azure_native_fields", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", + "rust_test": "document_intelligence_response_normalizes_pages", + "justification": "both assert page markdown, dimension (inch-to-pixel) normalization, and usage_info.pages_processed from the same Azure succeeded response shape" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_async_transform_ocr_response_preserves_azure_native_fields", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", + "rust_test": "document_intelligence_response_normalizes_pages", + "justification": "async twin of the sync case above, same underlying transform is exercised on the Rust side" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_transform_ocr_response_tolerates_missing_native_fields", + "status": "unmapped", + "reason": "tables/keyValuePairs absence tolerance is not asserted by the Rust response test" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_transform_ocr_response_non_succeeded_status_raises", + "status": "unmapped", + "reason": "no Rust test asserts on a non-succeeded Azure DI status" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_get_supported_ocr_params_includes_features", + "status": "unmapped", + "reason": "supported-params list content has no Rust equivalent for Azure" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_transform_ocr_response_native_format_carries_raw_operation", + "status": "unmapped", + "reason": "native req_format raw-operation passthrough is not tested in Rust" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_async_transform_ocr_response_native_format_carries_raw_operation", + "status": "unmapped", + "reason": "native req_format raw-operation passthrough is not tested in Rust" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_transform_ocr_response_default_format_omits_raw_operation", + "status": "unmapped", + "reason": "req_format gating of raw-operation output has no Rust test" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_map_ocr_params_passes_through_req_format", + "status": "unmapped", + "reason": "req_format passthrough in map_ocr_params has no Rust test" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_map_ocr_params_rejects_unknown_req_format_as_bad_request", + "status": "unmapped", + "reason": "req_format validation error path has no Rust test" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_get_complete_url_omits_req_format_query_param", + "status": "unmapped", + "reason": "no Rust test asserts req_format is excluded from the built URL" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_map_ocr_params_features", + "status": "unmapped", + "reason": "features-string normalization in map_ocr_params has no Rust test" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_map_ocr_params_empty_features_list_omitted", + "status": "unmapped", + "reason": "empty-features omission has no Rust test" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_map_ocr_params_invalid_features_raises", + "status": "unmapped", + "reason": "features validation error path has no Rust test" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_get_complete_url_appends_features_query", + "status": "unmapped", + "reason": "features query-param construction has no Rust test" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_get_complete_url_combines_pages_and_features", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", + "rust_test": "document_intelligence_url_normalizes_zero_based_pages", + "justification": "both assert 0-based, duplicate page indices are deduped, sorted, and rewritten 1-based into the request URL" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_validate_environment_uses_subscription_key", + "status": "unmapped", + "reason": "Python-side header derivation from litellm_params; Rust's poll test only checks the header is present, not how it was resolved" + }, + { + "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "python_test": "test_validate_environment_falls_back_to_entra_token", + "status": "unmapped", + "reason": "Entra bearer-token fallback logic has no Rust test" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", + "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_doc_intelligence_route", + "status": "unmapped", + "reason": "model-route string matching is Python-only dispatch logic" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", + "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_documentintelligence_and_is_case_insensitive", + "status": "unmapped", + "reason": "model-route string matching is Python-only dispatch logic" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", + "python_test": "TestIsAzureDocumentIntelligenceModel::test_does_not_match_mistral_route", + "status": "unmapped", + "reason": "model-route string matching is Python-only dispatch logic" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", + "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_does_not_hijack_doc_intelligence", + "status": "unmapped", + "reason": "api_base resolution from the secret manager runs before the Rust bridge is called, no Rust test exists for it" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", + "python_test": "TestDocIntelligenceApiBaseResolution::test_explicit_api_base_is_honoured_for_doc_intelligence", + "status": "unmapped", + "reason": "api_base precedence resolution is Python-only" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", + "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_still_applies_to_mistral_ocr", + "status": "unmapped", + "reason": "api_base precedence resolution is Python-only" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_use_litellm_rust_toggles_flag", + "status": "unmapped", + "reason": "bridge-plumbing: Python-side feature-flag toggle, no Rust equivalent" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_env_var_enables_rust_ocr", + "status": "unmapped", + "reason": "bridge-plumbing: Python-side env-var flag gating" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_load_rust_ocr_returns_injected_impl", + "status": "unmapped", + "reason": "bridge-plumbing: dependency-injection test hook, not provider behavior" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_native_bridge_loader_returns_none_when_extension_absent", + "status": "unmapped", + "reason": "bridge-plumbing: native-extension import/loader fallback" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_native_bridge_loader_caches_absent_extension", + "status": "unmapped", + "reason": "bridge-plumbing: loader caching behavior" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_native_bridge_available_reflects_loader", + "status": "unmapped", + "reason": "bridge-plumbing: loader availability check" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_load_rust_aocr_returns_injected_impl", + "status": "unmapped", + "reason": "bridge-plumbing: dependency-injection test hook" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_toggle_without_ocr_arg_preserves_injected_impl", + "status": "unmapped", + "reason": "bridge-plumbing: injected-impl state retention regression" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_explicit_ocr_none_clears_injected_impl", + "status": "unmapped", + "reason": "bridge-plumbing: injected-impl clearing behavior" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_load_rust_ocr_none_when_extension_absent", + "status": "unmapped", + "reason": "bridge-plumbing: degrade path when the native extension is missing" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_load_rust_ocr_uses_compiled_extension", + "status": "unmapped", + "reason": "bridge-plumbing: native module resolution" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_timeout_to_seconds_handles_float_timeout_and_none", + "status": "unmapped", + "reason": "bridge-plumbing: Python-side timeout normalization helper" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_bridge_wrapper_forwards_prepared_args_and_wraps_response", + "status": "unmapped", + "reason": "bridge-plumbing: wrapper argument forwarding, asserted against a fake bridge not the real Rust code" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response", + "status": "unmapped", + "reason": "bridge-plumbing: async wrapper argument forwarding" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_run_rust_ocr_prepares_request_and_wraps_response", + "status": "unmapped", + "reason": "bridge-plumbing: request preparation and response wrapping in Python" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_run_rust_ocr_resolves_key_via_secret_manager_when_missing", + "status": "unmapped", + "reason": "secret-manager: API key resolution happens in Python before the bridge is invoked" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_run_rust_ocr_prefers_explicit_key_over_resolver", + "status": "unmapped", + "reason": "secret-manager: key precedence resolution" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_run_rust_ocr_uses_provider_api_key_env_var", + "status": "unmapped", + "reason": "secret-manager: provider-specific env var name resolution" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_prepare_rust_ocr_call_forwards_vertex_routing_metadata", + "status": "unmapped", + "reason": "secret-manager: vertex routing metadata merge happens in Python" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_manager", + "status": "unmapped", + "reason": "secret-manager: vertex project/location resolution" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager", + "status": "unmapped", + "reason": "secret-manager: azure_ai api_base resolution" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint", + "status": "unmapped", + "reason": "secret-manager: doc-intelligence endpoint resolution" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_run_rust_ocr_runs_pre_call_logging", + "status": "unmapped", + "reason": "bridge-plumbing: Python logging-object pre_call invocation" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_ocr_routes_to_rust_when_enabled", + "status": "unmapped", + "reason": "bridge-plumbing: routing to a fake bridge, not the real Rust transform" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_ocr_routes_azure_ai_to_rust_when_enabled", + "status": "unmapped", + "reason": "bridge-plumbing: provider-prefix stripping before routing" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_ocr_rust_path_converts_file_document_before_bridge", + "status": "unmapped", + "reason": "file-normalization: raw-bytes-to-data-URI conversion happens in Python before the bridge call" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_ocr_exception_type_uses_resolved_provider_context", + "status": "unmapped", + "reason": "bridge-plumbing: Python exception-type mapping on bridge failure" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_aocr_routes_to_async_rust_when_enabled", + "status": "unmapped", + "reason": "bridge-plumbing: async routing to a fake bridge" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_aocr_exception_type_uses_resolved_provider_context", + "status": "unmapped", + "reason": "bridge-plumbing: async exception-type mapping on bridge failure" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_ocr_does_not_route_to_rust_when_disabled", + "status": "unmapped", + "reason": "bridge-plumbing: Python control flow for the toggle-disabled branch, no Rust-owned behavior runs" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_ocr_falls_back_to_python_when_bridge_unavailable", + "status": "unmapped", + "reason": "bridge-plumbing: Python-only fallback when the compiled Rust extension is absent, Rust cannot test its own absence" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_ocr_forwards_timeout_to_rust", + "status": "unmapped", + "reason": "bridge-plumbing: asserts the Python call site forwards a timeout kwarg, Rust receives an already-constructed request" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_ocr_passes_default_request_timeout_to_rust", + "status": "unmapped", + "reason": "bridge-plumbing: asserts the Python call site supplies a default timeout kwarg, no Rust equivalent" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_ocr_provider_configs_expose_api_key_env_vars", + "status": "unmapped", + "reason": "asserts per-provider get_api_key_env_var() strings; the closest Rust test (ocr_dispatch_supports_migrated_providers) asserts provider dispatch/param resolution instead, not API key env var names" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestGetMimeType::test_should_detect_pdf_mime_type", + "status": "unmapped", + "reason": "file-normalization: MIME detection is Python-only preprocessing before the bridge call" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestGetMimeType::test_should_detect_png_mime_type", + "status": "unmapped", + "reason": "file-normalization: MIME detection" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestGetMimeType::test_should_detect_jpg_mime_type", + "status": "unmapped", + "reason": "file-normalization: MIME detection" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestGetMimeType::test_should_detect_jpeg_mime_type", + "status": "unmapped", + "reason": "file-normalization: MIME detection" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestGetMimeType::test_should_detect_gif_mime_type", + "status": "unmapped", + "reason": "file-normalization: MIME detection" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestGetMimeType::test_should_detect_webp_mime_type", + "status": "unmapped", + "reason": "file-normalization: MIME detection" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestGetMimeType::test_should_detect_tiff_mime_type", + "status": "unmapped", + "reason": "file-normalization: MIME detection" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestGetMimeType::test_should_detect_tif_mime_type", + "status": "unmapped", + "reason": "file-normalization: MIME detection" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestGetMimeType::test_should_detect_bmp_mime_type", + "status": "unmapped", + "reason": "file-normalization: MIME detection" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestGetMimeType::test_should_be_case_insensitive", + "status": "unmapped", + "reason": "file-normalization: MIME detection case handling" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestGetMimeType::test_should_fallback_for_unknown_extension", + "status": "unmapped", + "reason": "file-normalization: MIME detection fallback" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pdf_pathlib_path_to_document_url", + "status": "unmapped", + "reason": "file-normalization: local-path-to-data-URI conversion happens in Python" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_image_pathlib_path_to_image_url", + "status": "unmapped", + "reason": "file-normalization: local-path-to-data-URI conversion" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_reject_bare_str_path", + "status": "unmapped", + "reason": "file-normalization: arbitrary-file-read guard on bare str paths" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pathlib_path", + "status": "unmapped", + "reason": "file-normalization: local-path-to-data-URI conversion" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes", + "status": "unmapped", + "reason": "file-normalization: raw-bytes-to-data-URI conversion" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_explicit_mime_type", + "status": "unmapped", + "reason": "file-normalization: explicit MIME override on raw bytes" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_image_mime_type", + "status": "unmapped", + "reason": "file-normalization: explicit MIME override on raw bytes" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object", + "status": "unmapped", + "reason": "file-normalization: file-like-object conversion" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object_with_name", + "status": "unmapped", + "reason": "file-normalization: file-like-object name-based MIME detection" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_missing_file_field", + "status": "unmapped", + "reason": "file-normalization: missing-field validation" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_nonexistent_pathlib_path", + "status": "unmapped", + "reason": "file-normalization: missing-file validation" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_empty_file", + "status": "unmapped", + "reason": "file-normalization: empty-file validation" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_unsupported_type", + "status": "unmapped", + "reason": "file-normalization: unsupported input type validation" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_invalid_mime_type", + "status": "unmapped", + "reason": "file-normalization: MIME-type injection validation" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestConvertFileDocumentToUrlDocument::test_should_override_mime_type_for_pathlib_path", + "status": "unmapped", + "reason": "file-normalization: explicit MIME override precedence" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestBuildDocumentFromUpload::test_should_build_document_url_for_pdf", + "status": "unmapped", + "reason": "file-normalization: multipart upload conversion" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_png", + "status": "unmapped", + "reason": "file-normalization: multipart upload conversion" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_jpeg", + "status": "unmapped", + "reason": "file-normalization: multipart upload conversion" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_octet_stream", + "status": "unmapped", + "reason": "file-normalization: filename-based MIME fallback" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_none", + "status": "unmapped", + "reason": "file-normalization: filename-based MIME fallback" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestBuildDocumentFromUpload::test_should_fallback_to_octet_stream_for_unknown", + "status": "unmapped", + "reason": "file-normalization: default MIME fallback" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestBuildDocumentFromUpload::test_should_preserve_base64_content_correctly", + "status": "unmapped", + "reason": "file-normalization: binary round-trip through base64" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_from_content_type", + "status": "unmapped", + "reason": "file-normalization: content-type parameter stripping" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_with_multiple_params", + "status": "unmapped", + "reason": "file-normalization: content-type parameter stripping" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestProxySecurityGuard::test_should_reject_file_type_document_in_json_body", + "status": "unmapped", + "reason": "proxy-layer JSON-body file-type guard, a different mechanism than Rust's URL-fetch SSRF guard" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestProxySecurityGuard::test_should_accept_document_url_type_in_json_body", + "status": "unmapped", + "reason": "proxy-layer JSON-body parsing" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestProxySecurityGuard::test_should_raise_on_invalid_json_body", + "status": "unmapped", + "reason": "proxy-layer JSON-body parsing error path" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", + "python_test": "TestProxySecurityGuard::test_should_ignore_document_form_field_injection", + "status": "unmapped", + "reason": "proxy-layer multipart form-field injection guard, a different mechanism than Rust's URL-fetch SSRF guard" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestGetSupportedOcrParams::test_extract_header_in_supported_params", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "extract_header_is_a_supported_ocr_param", + "justification": "both assert extract_header appears in the Mistral supported OCR params list" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestGetSupportedOcrParams::test_extract_footer_in_supported_params", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "extract_footer_is_a_supported_ocr_param", + "justification": "both assert extract_footer appears in the Mistral supported OCR params list" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestGetSupportedOcrParams::test_existing_params_still_present", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "existing_ocr_params_remain_supported", + "justification": "both assert the previously supported params are still present in the supported list" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestMapOcrParams::test_extract_header_passed_through", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "map_ocr_params_forwards_extract_header", + "justification": "both assert extract_header alone survives map_ocr_params unchanged" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestMapOcrParams::test_extract_footer_passed_through", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "map_ocr_params_forwards_extract_footer", + "justification": "both assert extract_footer alone survives map_ocr_params unchanged" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestMapOcrParams::test_extract_header_and_footer_together", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "map_ocr_params_forwards_extract_header_and_footer", + "justification": "both assert header and footer passed together are both forwarded with their given values" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestMapOcrParams::test_unknown_param_is_dropped", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "map_ocr_params_drops_unknown_params", + "justification": "both assert an unrecognized param key is dropped while a known one is kept" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestNewSupportedParams::test_new_param_in_supported_list", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "new_ocr_params_are_supported", + "justification": "both assert each OCR4 param is in the supported list" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestNewParamsMapOcr::test_new_param_passed_through", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "map_ocr_params_forwards_new_ocr_params", + "justification": "both assert each OCR4 param/value pair survives map_ocr_params unchanged" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestTransformOcrRequest::test_param_included_in_request_body", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "transform_ocr_request_includes_each_optional_param", + "justification": "both assert each optional param value lands in the built request body alongside model/document with no files" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestTransformOcrRequest::test_multiple_new_params_together", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "transform_ocr_request_includes_multiple_new_params", + "justification": "both assert multiple OCR4 params passed together all land in the same request body" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestTransformOcrResponseOcr4Fields::test_blocks_and_confidence_scores_preserved", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "transform_ocr_response_preserves_blocks_and_confidence_scores", + "justification": "both assert blocks and confidence_scores survive the OCR response transform on the returned page" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "python_test": "TestTransformOcrResponseOcr4Fields::test_ocr4_fields_survive_model_dump", + "status": "mapped", + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "transform_ocr_response_preserves_ocr4_page_fields", + "justification": "both assert tables, hyperlinks, header and footer survive the OCR response transform on the returned page" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", + "python_test": "test_model_info_ocr4_price", + "status": "unmapped", + "reason": "cost-calc: pricing/model-info lookup is Python-only" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", + "python_test": "test_ocr4_cost_scales_with_pages", + "status": "unmapped", + "reason": "cost-calc: per-page pricing math is Python-only" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", + "python_test": "test_ocr3_pricing_entry", + "status": "unmapped", + "reason": "cost-calc: cost-map JSON entry validation is Python-only" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", + "python_test": "test_ocr3_model_info_price", + "status": "unmapped", + "reason": "cost-calc: pricing/model-info lookup is Python-only" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", + "python_test": "test_ocr3_cost_scales_with_pages", + "status": "unmapped", + "reason": "cost-calc: per-page pricing math is Python-only" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", + "python_test": "test_ocr3_bills_ocr_and_annotation_pages_at_their_own_rates", + "status": "unmapped", + "reason": "cost-calc: mixed-rate billing math is Python-only" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", + "python_test": "test_ocr3_bills_annotation_only_response", + "status": "unmapped", + "reason": "cost-calc: annotation-only billing math is Python-only" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", + "python_test": "test_ocr3_bills_annotation_pages_when_pages_processed_missing", + "status": "unmapped", + "reason": "cost-calc: fallback billing math is Python-only" + }, + { + "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", + "python_test": "test_azure_doc_ai_annotation_pages_fall_back_to_ocr_rate", + "status": "unmapped", + "reason": "cost-calc: fallback billing math is Python-only" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", + "python_test": "test_rust_ocr_serves_default_format", + "status": "unmapped", + "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", + "python_test": "test_rust_ocr_skipped_for_native_format", + "status": "unmapped", + "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", + "python_test": "test_native_format_rejected_for_provider_without_support_as_bad_request", + "status": "unmapped", + "reason": "provider-support validation for req_format happens in Python" + }, + { + "python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", + "python_test": "test_unknown_format_rejected_for_provider_without_support_as_bad_request", + "status": "unmapped", + "reason": "req_format validation error path is Python-only" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestHandlerDiscovery::test_handler_discovered_for_ocr", + "status": "unmapped", + "reason": "guardrail-translation handler discovery is a Python proxy-layer concern" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestHandlerDiscovery::test_handler_discovered_for_aocr", + "status": "unmapped", + "reason": "guardrail-translation handler discovery is a Python proxy-layer concern" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestInputProcessing::test_process_document_url", + "status": "unmapped", + "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestInputProcessing::test_process_image_url", + "status": "unmapped", + "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestInputProcessing::test_process_no_document", + "status": "unmapped", + "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestInputProcessing::test_process_invalid_document", + "status": "unmapped", + "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestInputProcessing::test_input_blocking_guardrail", + "status": "unmapped", + "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestOutputProcessing::test_process_single_page", + "status": "unmapped", + "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestOutputProcessing::test_process_multiple_pages", + "status": "unmapped", + "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestOutputProcessing::test_process_empty_pages", + "status": "unmapped", + "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestOutputProcessing::test_process_page_with_empty_markdown", + "status": "unmapped", + "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestOutputProcessing::test_process_preserves_page_metadata", + "status": "unmapped", + "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestOutputProcessing::test_output_blocking_guardrail", + "status": "unmapped", + "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" + }, + { + "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "python_test": "TestPIIMaskingScenario::test_pii_masking_in_ocr_pages", + "status": "unmapped", + "reason": "PII redaction in the translation handler has no Rust equivalent" + }, + { + "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", + "python_test": "test_should_read_req_format_from_header", + "status": "unmapped", + "reason": "proxy-layer header parsing has no Rust equivalent" + }, + { + "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", + "python_test": "test_should_prefer_body_req_format_over_header", + "status": "unmapped", + "reason": "proxy-layer body-vs-header precedence has no Rust equivalent" + }, + { + "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", + "python_test": "test_should_omit_req_format_when_header_absent", + "status": "unmapped", + "reason": "proxy-layer parsing has no Rust equivalent" + }, + { + "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", + "python_test": "test_should_reject_unknown_req_format", + "status": "unmapped", + "reason": "proxy-layer validation has no Rust equivalent" + }, + { + "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", + "python_test": "test_should_return_native_payload_with_litellm_response_headers", + "status": "unmapped", + "reason": "proxy-layer response construction has no Rust equivalent" + }, + { + "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", + "python_test": "test_should_return_normalized_response_when_no_native_payload", + "status": "unmapped", + "reason": "proxy-layer response construction has no Rust equivalent" + }, + { + "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", + "python_test": "test_explicit_false_overrides_process_enable", + "status": "unmapped", + "reason": "bridge-plumbing: asserts the explicit-per-request rust:False override wins over the Python process flag, no Rust-owned behavior runs" + } ], "rust_only_tests": [ - {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "rust_custom_logger_reads_failure_payload_for_non_ocr_call_type", "reason": "exercises the non-OCR (acompletion) call-type branch of the logger; the OCR branch is covered separately by rust_custom_logger_reads_success_payload_for_ocr"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "no_callback_fast_path_dispatches_nothing", "reason": "Rust-only fast-path optimization test for when zero callbacks are registered; Python has no equivalent no-op dispatch path"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "with_standard_logging_payload_keeps_top_level_fields_in_sync", "reason": "Rust-internal builder-method invariant, Python has no equivalent internal builder"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "blocks_private_and_metadata_ips", "reason": "SSRF IP-blocking helper has no Python unit test; Python relies on the proxy-layer JSON/form guards instead"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "convert_document_url_rejects_loopback_fetch", "reason": "URL-fetch SSRF protection is Rust-gateway-only"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "convert_document_url_leaves_data_uri_untouched", "reason": "URL-fetch SSRF protection is Rust-gateway-only"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_passes_short_strings_through", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_caps_long_payloads", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_does_not_split_multibyte_chars", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_dispatch_supports_migrated_providers", "reason": "Rust-internal provider-config dispatch table has no equivalent Python unit test"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "string_headers_accepts_string_values", "reason": "Rust-gateway header-coercion helper has no Python counterpart"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "auth_header_detection_is_case_insensitive", "reason": "Rust-gateway header-detection helper has no Python counterpart"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_runs_pre_during_and_success_hooks", "reason": "full gateway-level guardrail-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_runs_failure_hook_on_provider_error", "reason": "full gateway-level failure-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_pre_call_block_skips_provider_socket", "reason": "full gateway-level pre-call-block-plus-socket-skip test with no Python equivalent at this integration scope"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_does_not_duplicate_authorization_header_when_header_is_supplied", "reason": "outgoing HTTP header dedup at the Rust gateway has no Python counterpart"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "document_intelligence_poll_uses_resolved_subscription_key", "reason": "full Azure DI poll-loop integration test with no Python equivalent at this scope"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "string_headers_rejects_non_string_values", "reason": "Rust-gateway header-coercion error path has no Python counterpart"}, - {"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "azure_ai_reuses_mistral_body_transform", "reason": "Rust-internal delegation-to-Mistral-transform implementation detail, no Python test asserts this delegation"}, - {"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_request_uses_base64_source_for_data_uri", "reason": "no Python test asserts on the base64Source request body shape"}, - {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_mistral_url_uses_project_location_and_model", "reason": "vertex OCR support has no Python unit test coverage yet"}, - {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_mistral_reuses_mistral_body_transform", "reason": "vertex OCR support has no Python unit test coverage yet"}, - {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_deepseek_request_uses_ocr_endpoint_shape", "reason": "vertex OCR support has no Python unit test coverage yet"}, - {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_deepseek_response_wraps_markdown_content", "reason": "vertex OCR support has no Python unit test coverage yet"}, - {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_rejects_non_object_document", "reason": "non-object document rejection has no dedicated Python unit test"}, - {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_response_normalizes_mistral_json", "reason": "Python's response tests target OCR4-specific fields only, none asserts the same base normalization this Rust test checks"}, - {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "complete_url_defaults_and_dedupes_v1", "reason": "URL-building/defaulting for Mistral has no Python unit test"}, - {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "resolve_api_key_prefers_param_then_env", "reason": "API key resolution precedence at the Rust provider-config layer has no Python unit test"}, - {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "resolve_api_key_errors_when_absent", "reason": "API key resolution error path at the Rust provider-config layer has no Python unit test"}, - {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "rust_custom_logger_reads_success_payload_for_ocr", "reason": "Rust-internal custom-logger dispatch for OCR payloads has no Python unit test at this layer"} + { + "rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", + "rust_test": "rust_custom_logger_reads_failure_payload_for_non_ocr_call_type", + "reason": "exercises the non-OCR (acompletion) call-type branch of the logger; the OCR branch is covered separately by rust_custom_logger_reads_success_payload_for_ocr" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", + "rust_test": "no_callback_fast_path_dispatches_nothing", + "reason": "Rust-only fast-path optimization test for when zero callbacks are registered; Python has no equivalent no-op dispatch path" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", + "rust_test": "with_standard_logging_payload_keeps_top_level_fields_in_sync", + "reason": "Rust-internal builder-method invariant, Python has no equivalent internal builder" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", + "rust_test": "blocks_private_and_metadata_ips", + "reason": "SSRF IP-blocking helper has no Python unit test; Python relies on the proxy-layer JSON/form guards instead" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", + "rust_test": "convert_document_url_rejects_loopback_fetch", + "reason": "URL-fetch SSRF protection is Rust-gateway-only" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", + "rust_test": "convert_document_url_leaves_data_uri_untouched", + "reason": "URL-fetch SSRF protection is Rust-gateway-only" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "truncate_error_body_passes_short_strings_through", + "reason": "Rust-gateway error-body truncation helper has no Python counterpart" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "truncate_error_body_caps_long_payloads", + "reason": "Rust-gateway error-body truncation helper has no Python counterpart" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "truncate_error_body_does_not_split_multibyte_chars", + "reason": "Rust-gateway error-body truncation helper has no Python counterpart" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "ocr_dispatch_supports_migrated_providers", + "reason": "Rust-internal provider-config dispatch table has no equivalent Python unit test" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "string_headers_accepts_string_values", + "reason": "Rust-gateway header-coercion helper has no Python counterpart" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "auth_header_detection_is_case_insensitive", + "reason": "Rust-gateway header-detection helper has no Python counterpart" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "ocr_lifecycle_runs_pre_during_and_success_hooks", + "reason": "full gateway-level guardrail-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "ocr_lifecycle_runs_failure_hook_on_provider_error", + "reason": "full gateway-level failure-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "ocr_lifecycle_pre_call_block_skips_provider_socket", + "reason": "full gateway-level pre-call-block-plus-socket-skip test with no Python equivalent at this integration scope" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "ocr_does_not_duplicate_authorization_header_when_header_is_supplied", + "reason": "outgoing HTTP header dedup at the Rust gateway has no Python counterpart" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "document_intelligence_poll_uses_resolved_subscription_key", + "reason": "full Azure DI poll-loop integration test with no Python equivalent at this scope" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "rust_test": "string_headers_rejects_non_string_values", + "reason": "Rust-gateway header-coercion error path has no Python counterpart" + }, + { + "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", + "rust_test": "azure_ai_reuses_mistral_body_transform", + "reason": "Rust-internal delegation-to-Mistral-transform implementation detail, no Python test asserts this delegation" + }, + { + "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", + "rust_test": "document_intelligence_request_uses_base64_source_for_data_uri", + "reason": "no Python test asserts on the base64Source request body shape" + }, + { + "rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", + "rust_test": "vertex_mistral_url_uses_project_location_and_model", + "reason": "vertex OCR support has no Python unit test coverage yet" + }, + { + "rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", + "rust_test": "vertex_mistral_reuses_mistral_body_transform", + "reason": "vertex OCR support has no Python unit test coverage yet" + }, + { + "rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", + "rust_test": "vertex_deepseek_request_uses_ocr_endpoint_shape", + "reason": "vertex OCR support has no Python unit test coverage yet" + }, + { + "rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", + "rust_test": "vertex_deepseek_response_wraps_markdown_content", + "reason": "vertex OCR support has no Python unit test coverage yet" + }, + { + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "transform_ocr_request_rejects_non_object_document", + "reason": "non-object document rejection has no dedicated Python unit test" + }, + { + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "transform_ocr_response_normalizes_mistral_json", + "reason": "Python's response tests target OCR4-specific fields only, none asserts the same base normalization this Rust test checks" + }, + { + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "complete_url_defaults_and_dedupes_v1", + "reason": "URL-building/defaulting for Mistral has no Python unit test" + }, + { + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "resolve_api_key_prefers_param_then_env", + "reason": "API key resolution precedence at the Rust provider-config layer has no Python unit test" + }, + { + "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "rust_test": "resolve_api_key_errors_when_absent", + "reason": "API key resolution error path at the Rust provider-config layer has no Python unit test" + }, + { + "rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", + "rust_test": "rust_custom_logger_reads_success_payload_for_ocr", + "reason": "Rust-internal custom-logger dispatch for OCR payloads has no Python unit test at this layer" + } ] } From 7bc2d0b06e2fd8ab1226920cc45c7b6723abd6e0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:55:41 -0700 Subject: [PATCH 194/480] fix(router): keep retry breadcrumbs per request and out of the request snapshot Retry breadcrumbs were appended to one list owned by the Router and shared by every request, and each breadcrumb copied the whole kwargs including the proxy's snapshot of the inbound request. That snapshot's body aliases the live request metadata, breadcrumbs included, so every new breadcrumb nested all the earlier ones inside itself. Memory stayed small because these are shared references, but under --detailed_debug the repr of that structure expands, so one debug line grew from 10k to 219M characters over 14 failing requests and the proxy stopped answering. Breadcrumbs now accumulate in the metadata of the request that produced them, the request snapshot is excluded from a breadcrumb, and the cap of the last 4 failed attempts applies per request. --- litellm/router.py | 58 +++++++++--------- tests/test_litellm/test_router.py | 97 ++++++++++++++++++++++++++++--- 2 files changed, 118 insertions(+), 37 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 0af514fe8a2..69fe731bfbd 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -586,16 +586,20 @@ set_live_deployment_replay(_replay_live_router_model_cost) # Kwargs that carry no signal about the failed attempt, so log_retry drops them from a -# breadcrumb entirely: the request payload and the router-internal walk state. Credentials are -# handled separately by mask_credentials_in_payload, which scrubs credential-named values from -# whatever kwargs remain rather than trying to enumerate every credential-bearing key here. +# breadcrumb entirely: the request payload, the proxy's snapshot of the inbound request (its body +# aliases the live request metadata, earlier breadcrumbs included, so copying it would nest every +# breadcrumb inside the next one), and the router-internal walk state. Credentials are handled +# separately by mask_credentials_in_payload, which scrubs credential-named values from whatever +# kwargs remain rather than trying to enumerate every credential-bearing key here. RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset( ( "messages", "original_function", "attempted_targets", + "proxy_server_request", ) ) +RETRY_BREADCRUMB_LIMIT: Final = 4 class Router: @@ -954,7 +958,6 @@ class Router: self.total_calls: defaultdict = defaultdict(int) # dict to store total calls made to each model self.fail_calls: defaultdict = defaultdict(int) # dict to store fail_calls made to each model self.success_calls: defaultdict = defaultdict(int) # dict to store success_calls made to each model - self.previous_models: list = [] # list to store failed calls (passed in as metadata to next call) # make Router.chat.completions.create compatible for openai.chat.completions.create default_litellm_params = default_litellm_params or {} @@ -8048,35 +8051,30 @@ class Router: """ When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing """ - try: - _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" - # Log failed model as the previous model - previous_model: Final = { + _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" + request_metadata: Final[Mapping[str, object]] = kwargs[_metadata_var] + attempt_kwargs: Final = MappingProxyType( + {k: v for k, v in kwargs.items() if k != _metadata_var and k not in RETRY_BREADCRUMB_EXCLUDED_KWARGS} + ) + attempt_metadata: Final = MappingProxyType( + {k: v for k, v in request_metadata.items() if k != "previous_models"} + ) + previous_model: Final = MappingProxyType( + { "exception_type": type(e).__name__, "exception_string": str(e), + **attempt_kwargs, + _metadata_var: attempt_metadata, } - for ( - k, - v, - ) in kwargs.items(): # log everything in kwargs except the old previous_models value - prevent nesting - if k != _metadata_var and k not in RETRY_BREADCRUMB_EXCLUDED_KWARGS: - previous_model[k] = v - elif k == _metadata_var and isinstance(v, dict): - previous_model[_metadata_var] = {} - for metadata_k, metadata_v in kwargs[_metadata_var].items(): - if metadata_k != "previous_models": - previous_model[k][metadata_k] = metadata_v - - # check current size of self.previous_models, if it's larger than 3, remove the first element - if len(self.previous_models) > 3: - self.previous_models.pop(0) - - scrubbed_previous_model: Final = mask_credentials_in_payload(previous_model) - self.previous_models.append(scrubbed_previous_model) - kwargs[_metadata_var]["previous_models"] = self.previous_models - return kwargs - except Exception as e: - raise e + ) + earlier_breadcrumbs: Final = request_metadata.get("previous_models") + kept_breadcrumbs: Final[tuple[object, ...]] = ( + tuple(earlier_breadcrumbs)[-(RETRY_BREADCRUMB_LIMIT - 1) :] + if isinstance(earlier_breadcrumbs, (list, tuple)) + else () + ) + kwargs[_metadata_var]["previous_models"] = (*kept_breadcrumbs, mask_credentials_in_payload(previous_model)) + return kwargs def _update_usage(self, deployment_id: str, parent_otel_span: Span | None) -> int: """ diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c843a66a1c1..2b6d4694a61 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -9274,9 +9274,11 @@ class _FallbackAttemptRecorder(CustomLogger): def __init__(self): super().__init__() self.failed_targets = [] + self.breadcrumbs_per_target = [] async def log_failure_fallback_event(self, original_model_group, kwargs, original_exception): self.failed_targets.append(kwargs.get("model")) + self.breadcrumbs_per_target.append(kwargs.get("metadata", {}).get("previous_models", ())) def _cyclic_fallback_router(num_retries=0): @@ -9347,14 +9349,16 @@ async def test_retry_breadcrumbs_do_not_carry_the_walk_state(): A retry has to be configured for the walk state to reach log_retry at all.""" router = _cyclic_fallback_router(num_retries=1) capture = _LogCapture(logging.ERROR) + recorder = _FallbackAttemptRecorder() - await _drive_cyclic_fallback(router, capture) + await _drive_cyclic_fallback(router, capture, recorder) - assert router.previous_models, "no retry breadcrumbs were recorded" + breadcrumbs = [breadcrumb for hop in recorder.breadcrumbs_per_target for breadcrumb in hop] + assert breadcrumbs, "no retry breadcrumbs were recorded" assert any( - "fallback_depth" in breadcrumb for breadcrumb in router.previous_models + "fallback_depth" in breadcrumb for breadcrumb in breadcrumbs ), "no breadcrumb carried router walk state, so this test cannot see the leak" - for breadcrumb in router.previous_models: + for breadcrumb in breadcrumbs: assert "attempted_targets" not in breadcrumb @@ -9392,15 +9396,94 @@ async def test_retry_breadcrumbs_never_carry_a_forwarded_credential(container_ke container still reaches the breadcrumb, but the raw secret never does, whatever key holds it.""" router = _cyclic_fallback_router(num_retries=1) capture = _LogCapture(logging.ERROR) + metadata = {} - await _drive_cyclic_fallback(router, capture, **request_kwargs) + await _drive_cyclic_fallback(router, capture, metadata=metadata, **request_kwargs) - assert router.previous_models, "no retry breadcrumbs were recorded" - dumped = json.dumps(router.previous_models, default=str) + breadcrumbs = metadata["previous_models"] + assert breadcrumbs, "no retry breadcrumbs were recorded" + dumped = json.dumps(breadcrumbs, default=str) assert container_key in dumped, "the credential-bearing kwarg never reached the breadcrumb, so this test cannot see the leak" assert _BREADCRUMB_CREDENTIAL_CANARY not in dumped +def _always_failing_router(num_retries): + return litellm.Router( + model_list=[ + { + "model_name": "broken-group", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-fake", + "mock_response": "litellm.InternalServerError", + }, + } + ], + num_retries=num_retries, + ) + + +async def _fail_one_proxy_shaped_request(router, request_marker): + """The proxy hands the router a metadata dict and a proxy_server_request whose body is a + shallow copy of the request, so body["metadata"] is the very same dict the router later + stamps previous_models onto.""" + metadata = {"request_marker": request_marker} + with pytest.raises(litellm.InternalServerError): + await router.acompletion( + model="broken-group", + messages=[{"role": "user", "content": "hi"}], + metadata=metadata, + proxy_server_request={ + "url": "http://localhost:4000/v1/chat/completions", + "method": "POST", + "headers": {}, + "body": {"model": "broken-group", "metadata": metadata}, + }, + ) + return metadata["previous_models"] + + +def _nested_breadcrumb_lists(node): + if isinstance(node, dict): + return [v for k, v in node.items() if k == "previous_models"] + [ + found for v in node.values() for found in _nested_breadcrumb_lists(v) + ] + if isinstance(node, (list, tuple)): + return [found for item in node for found in _nested_breadcrumb_lists(item)] + return [] + + +@pytest.mark.asyncio +async def test_retry_breadcrumbs_stay_per_request_and_flat_across_failing_requests(): + """Every failed attempt appends a breadcrumb to metadata["previous_models"], and the proxy's + request snapshot aliases that same metadata dict. Kept on the Router and copied wholesale, + each breadcrumb embedded every earlier one from every earlier request, so the breadcrumb + tree, and with it the debug repr of the kwargs, roughly doubled on each failed attempt until + a single-worker proxy spent minutes in the redaction regex and stopped answering.""" + router = _always_failing_router(num_retries=2) + + breadcrumbs_per_request = [ + await _fail_one_proxy_shaped_request(router, f"request-{request_number}") for request_number in range(1, 7) + ] + + for request_number, breadcrumbs in enumerate(breadcrumbs_per_request, start=1): + assert len(breadcrumbs) == 3, "one initial attempt plus two retries failed, each leaving one breadcrumb" + assert {breadcrumb["metadata"]["request_marker"] for breadcrumb in breadcrumbs} == {f"request-{request_number}"} + for breadcrumb in breadcrumbs: + assert _nested_breadcrumb_lists(breadcrumb) == [] + assert len({len(repr(breadcrumbs)) for breadcrumbs in breadcrumbs_per_request}) == 1 + + +@pytest.mark.asyncio +async def test_retry_breadcrumbs_keep_only_the_last_four_attempts(): + router = _always_failing_router(num_retries=6) + + breadcrumbs = await _fail_one_proxy_shaped_request(router, "request-1") + + assert len(breadcrumbs) == 4 + assert [breadcrumb["metadata"]["attempted_retries"] for breadcrumb in breadcrumbs] == [3, 4, 5, 6] + + @pytest.mark.asyncio async def test_fallback_traceback_stays_available_at_debug_level(): """Dropping the stack from the ERROR line is only safe because the fallback path still From f9f32b49a655b31b72014a5bf3e382065221bd38 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:13:15 -0700 Subject: [PATCH 195/480] fix(tests): qualify traced function names on Python 3.10 --- tests/sdk_function_trace/profiler.py | 57 ++++++++++++++++++++--- tests/sdk_function_trace/test_profiler.py | 31 +++++++++++- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/tests/sdk_function_trace/profiler.py b/tests/sdk_function_trace/profiler.py index c71c74ab0d3..44697e7b9f6 100644 --- a/tests/sdk_function_trace/profiler.py +++ b/tests/sdk_function_trace/profiler.py @@ -2,11 +2,12 @@ from __future__ import annotations import sys import threading -from collections.abc import Generator, Sequence +from collections.abc import Generator, Iterator, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass +from functools import lru_cache from pathlib import Path -from types import CodeType, FrameType, FunctionType +from types import CodeType, FrameType, FunctionType, MappingProxyType from typing import Final @@ -27,11 +28,11 @@ class PythonProfiler: def __call__(self, frame: FrameType, event: str, _arg: object) -> None: if event != "call" or frame in self._seen_frames: return - function_name: Final = self.function_name(frame.f_code) + function_name: Final = self.function_name(frame) if function_name is None: return ancestors: Final = tuple( - name for ancestor in _frame_ancestors(frame) if (name := self.function_name(ancestor.f_code)) is not None + name for ancestor in _frame_ancestors(frame) if (name := self.function_name(ancestor)) is not None ) self._seen_frames.add(frame) self.events.append( @@ -42,13 +43,57 @@ class PythonProfiler: ) ) - def function_name(self, code: CodeType) -> str | None: + def function_name(self, frame: FrameType) -> str | None: + code: Final = frame.f_code if self._source_root is None: return self._names_by_code.get(code) if not code.co_filename.startswith(self._source_root): return None relative: Final = code.co_filename.removeprefix(self._source_root) - return f"{relative}:{code.co_firstlineno} {getattr(code, 'co_qualname', code.co_name)}" + return f"{relative}:{code.co_firstlineno} {_qualified_name(frame)}" + + +def _qualified_name(frame: FrameType) -> str: + code: Final = frame.f_code + native: Final = getattr(code, "co_qualname", None) + if isinstance(native, str): + return native + module_name: Final = frame.f_globals.get("__name__") + if not isinstance(module_name, str): + return code.co_name + return _module_qualnames(module_name).get(code, code.co_name) + + +@lru_cache(maxsize=None) +def _module_qualnames(module_name: str) -> Mapping[CodeType, str]: + module: Final = sys.modules.get(module_name) + if module is None: + return MappingProxyType({}) + return MappingProxyType(dict(_declared_functions(vars(module), frozenset()))) + + +def _declared_functions(namespace: Mapping[str, object], visited: frozenset[int]) -> Iterator[tuple[CodeType, str]]: + for attribute in tuple(namespace.values()): + for value in _accessors(attribute): + if isinstance(value, FunctionType): + yield from ((wrapped.__code__, wrapped.__qualname__) for wrapped in _unwrapped(value)) + elif isinstance(value, type) and id(value) not in visited: + yield from _declared_functions(dict(vars(value)), visited | {id(value)}) + + +def _unwrapped(function: FunctionType) -> Iterator[FunctionType]: + yield function + inner: Final = getattr(function, "__wrapped__", None) + if isinstance(inner, FunctionType): + yield from _unwrapped(inner) + + +def _accessors(value: object) -> tuple[object, ...]: + if isinstance(value, (staticmethod, classmethod)): + return (value.__func__,) + if isinstance(value, property): + return tuple(accessor for accessor in (value.fget, value.fset, value.fdel) if accessor is not None) + return (value,) def _frame_ancestors(frame: FrameType) -> Generator[FrameType]: diff --git a/tests/sdk_function_trace/test_profiler.py b/tests/sdk_function_trace/test_profiler.py index 10a266fb1e8..21a1ab134b2 100644 --- a/tests/sdk_function_trace/test_profiler.py +++ b/tests/sdk_function_trace/test_profiler.py @@ -2,9 +2,11 @@ from __future__ import annotations import asyncio import sys +from collections.abc import Callable +from functools import wraps from pathlib import Path from types import FunctionType -from typing import Final, cast +from typing import Final, ParamSpec, TypeVar, cast import pytest @@ -14,7 +16,18 @@ from tests.sdk_function_trace import ( TraceStep, assert_function_trace_parity, ) -from tests.sdk_function_trace.profiler import profile_python +from tests.sdk_function_trace.profiler import _module_qualnames, profile_python + +_P = ParamSpec("_P") +_T = TypeVar("_T") + + +def _passthrough(function: Callable[_P, _T]) -> Callable[_P, _T]: + @wraps(function) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T: + return function(*args, **kwargs) + + return wrapper class First: @@ -29,6 +42,20 @@ class Second: return None +class Decorated: + @_passthrough + def call(self) -> None: + return None + + +def test_source_profiler_qualifies_decorated_methods_by_class() -> None: + with profile_python(source_root=Path(__file__).parent) as profiler: + Decorated().call() + + assert any(event.function.endswith(" Decorated.call") for event in profiler.events) + assert _module_qualnames(__name__)[cast(FunctionType, Decorated.call.__wrapped__).__code__] == "Decorated.call" + + def test_profiler_matches_code_objects_and_keeps_repeated_calls() -> None: with profile_python((First.run,)) as profiler: Second.run() From 425c8d37fc6a60c43bc215127361efe9ea81b1c3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:26:34 -0700 Subject: [PATCH 196/480] fix(containers): page upstream until a non-admin container list fills its limit Forwarding limit to OpenAI made the ownership filter cut the page down after the fact, so a key that owned an older container got an empty first page and its cursor never moved. Non-admin lists now walk upstream pages of 100 until they have enough owned containers (or five pages), trim to the requested limit, and report first_id, last_id and has_more off what the caller keeps. Also assigns tests/test_litellm/proxy/container_endpoints to a CI shard. --- .github/workflows/test-unit.yml | 1 + .../proxy/container_endpoints/endpoints.py | 90 +++--- .../proxy/container_endpoints/ownership.py | 134 +++++---- .../test_container_proxy_ownership.py | 259 ++++++++++-------- .../container_endpoints/test_endpoints.py | 122 +++++++-- .../test_handler_factory.py | 35 +-- 6 files changed, 389 insertions(+), 252 deletions(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 6da5fc07e80..6bc44995804 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -151,6 +151,7 @@ jobs: tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/batches_endpoints + tests/test_litellm/proxy/container_endpoints tests/test_litellm/proxy/fine_tuning_endpoints tests/test_litellm/proxy/vector_store_files_endpoints tests/test_litellm/proxy/video_endpoints diff --git a/litellm/proxy/container_endpoints/endpoints.py b/litellm/proxy/container_endpoints/endpoints.py index 9e0f6fa741f..f3a2abc1225 100644 --- a/litellm/proxy/container_endpoints/endpoints.py +++ b/litellm/proxy/container_endpoints/endpoints.py @@ -15,10 +15,11 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( get_custom_llm_provider_from_request_headers, get_custom_llm_provider_from_request_query, ) +from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.proxy.container_endpoints.ownership import ( assert_user_can_access_container, - filter_container_list_response, get_container_forwarding_params, + list_owned_containers, record_container_owner, ) @@ -209,61 +210,54 @@ async def list_containers( version, ) - # Read query parameters query_params: Final = dict(request.query_params) - data: Final[dict[str, Any]] = { - "query_params": query_params, - "model": query_params.get("model"), - "after": after, - "limit": limit, - "order": order, - } - - # Extract custom_llm_provider using priority chain custom_llm_provider: Final = ( get_custom_llm_provider_from_request_headers(request=request) or get_custom_llm_provider_from_request_query(request=request) or "openai" ) + data: Final[dict[str, Any]] = { + "query_params": query_params, + "model": query_params.get("model"), + "order": order, + "custom_llm_provider": custom_llm_provider, + } - # Add custom_llm_provider to data - data["custom_llm_provider"] = custom_llm_provider + async def fetch_page(page_after: str | None, page_limit: int | None) -> object: + processor: Final = ProxyBaseLLMRequestProcessing(data={**data, "after": page_after, "limit": page_limit}) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="alist_containers", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) - # Process request using ProxyBaseLLMRequestProcessing - processor: Final = ProxyBaseLLMRequestProcessing(data=data) - try: - response: Final = await processor.base_process_llm_request( - request=request, - fastapi_response=fastapi_response, - user_api_key_dict=user_api_key_dict, - route_type="alist_containers", - proxy_logging_obj=proxy_logging_obj, - llm_router=llm_router, - general_settings=general_settings, - proxy_config=proxy_config, - select_data_generator=select_data_generator, - model=None, - user_model=user_model, - user_temperature=user_temperature, - user_request_timeout=user_request_timeout, - user_max_tokens=user_max_tokens, - user_api_base=user_api_base, - version=version, - ) - except Exception as e: - raise await processor._handle_llm_api_exception( - e=e, - user_api_key_dict=user_api_key_dict, - proxy_logging_obj=proxy_logging_obj, - version=version, - ) - - # Ownership filtering runs OUTSIDE the LLM-exception scope: a DB error - # in the ownership lookup is not an LLM-API error and shouldn't be - # translated to a provider-shaped failure (which would also fire the - # post_call_failure_hook for what is in fact a successful upstream call). - return await filter_container_list_response( - response=response, + if is_proxy_admin(user_api_key_dict): + return await fetch_page(after, limit) + return await list_owned_containers( + fetch_page=fetch_page, + after=after, + limit=limit, user_api_key_dict=user_api_key_dict, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/proxy/container_endpoints/ownership.py b/litellm/proxy/container_endpoints/ownership.py index e3088771c82..14480232a4a 100644 --- a/litellm/proxy/container_endpoints/ownership.py +++ b/litellm/proxy/container_endpoints/ownership.py @@ -1,9 +1,10 @@ import json -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, TypeAlias from fastapi import HTTPException +from pydantic import BaseModel from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache @@ -46,6 +47,12 @@ _CONTAINER_STORED_ID_CACHE: Final = InMemoryCache(max_size_in_memory=10000, defa # different users with different scopes get disjoint cache entries. _ALLOWED_CONTAINER_IDS_CACHE: Final = InMemoryCache(max_size_in_memory=2048, default_ttl=60) +DEFAULT_CONTAINER_LIST_LIMIT: Final = 20 +OWNED_CONTAINER_LIST_PAGE_SIZE: Final = 100 +OWNED_CONTAINER_LIST_MAX_PAGES: Final = 5 + +FetchContainerListPage: TypeAlias = Callable[[str | None, int | None], Awaitable[object]] + def _allowed_container_ids_cache_key(owner_scopes: Sequence[str]) -> str: """JSON-encode the sorted scope list — using a separator like ``|`` @@ -337,27 +344,23 @@ def _get_container_list_data(response: object) -> Sequence[object] | None: return data if isinstance(data, list) else None -def _set_container_list_data(response: Any, data: list[object], removed_filtered_items: bool = False) -> object: +def _get_has_more(response: object) -> bool: if isinstance(response, dict): - response["data"] = data - if data: - response["first_id"] = _get_response_id(data[0]) - response["last_id"] = _get_response_id(data[-1]) - else: - response["first_id"] = None - response["last_id"] = None - response["has_more"] = False - if removed_filtered_items: - response["has_more"] = False - return response + return response.get("has_more") is True + return getattr(response, "has_more", None) is True - response.data = data - response.first_id = _get_response_id(data[0]) if data else None - response.last_id = _get_response_id(data[-1]) if data else None - if not data and hasattr(response, "has_more"): - response.has_more = False - if removed_filtered_items and hasattr(response, "has_more"): - response.has_more = False + +def _with_container_list_page(response: object, data: Sequence[object], has_more: bool) -> object: + page: Final = { + "data": list(data), + "first_id": _get_response_id(data[0]) if data else None, + "last_id": _get_response_id(data[-1]) if data else None, + "has_more": has_more, + } + if isinstance(response, dict): + return {**response, **page} + if isinstance(response, BaseModel): + return response.model_copy(update=page) return response @@ -366,16 +369,16 @@ async def _get_allowed_container_ids( ) -> AbstractSet[str]: owner_scopes: Final = get_resource_owner_scopes(user_api_key_dict) if not owner_scopes: - return set() + return frozenset() cache_key: Final = _allowed_container_ids_cache_key(owner_scopes) cached: Final = _ALLOWED_CONTAINER_IDS_CACHE.get_cache(cache_key) if cached is not None: - return set(cached) + return frozenset(cached) prisma_client: Final = await _get_prisma_client() if prisma_client is None: - return set() + return frozenset() table: Final = ManagedObjectRepository(prisma_client).table rows: Final[Sequence[prisma_models.LiteLLM_ManagedObjectTable]] = await table.find_many( @@ -384,34 +387,69 @@ async def _get_allowed_container_ids( "created_by": {"in": owner_scopes}, } ) - allowed_ids: Final = {row.model_object_id for row in rows if getattr(row, "model_object_id", None) is not None} - # ``InMemoryCache.get_cache`` attempts ``json.loads`` on the stored - # value; passing a set would round-trip through that path - # unnecessarily. Store as a list and rehydrate above. - _ALLOWED_CONTAINER_IDS_CACHE.set_cache(cache_key, list(allowed_ids)) + allowed_ids: Final = frozenset( + row.model_object_id for row in rows if getattr(row, "model_object_id", None) is not None + ) + _ALLOWED_CONTAINER_IDS_CACHE.set_cache(cache_key, tuple(allowed_ids)) return allowed_ids -async def filter_container_list_response( - response: object, +def _is_owned_container(item: object, allowed_container_ids: AbstractSet[str], custom_llm_provider: str) -> bool: + container_id: Final = _get_response_id(item) + if container_id is None: + return False + original_container_id, resolved_provider = decode_container_id_for_ownership(container_id, custom_llm_provider) + return _container_model_object_id(original_container_id, resolved_provider) in allowed_container_ids + + +async def _collect_owned_containers( + fetch_page: FetchContainerListPage, + after: str | None, + needed: int, + allowed_container_ids: AbstractSet[str], + custom_llm_provider: str, + pages_left: int, + collected: tuple[object, ...], +) -> tuple[object, tuple[object, ...]]: + page: Final = await fetch_page(after, OWNED_CONTAINER_LIST_PAGE_SIZE) + page_data: Final = _get_container_list_data(page) or () + owned: Final = collected + tuple( + item for item in page_data if _is_owned_container(item, allowed_container_ids, custom_llm_provider) + ) + upstream_last_id: Final = _get_response_id(page_data[-1]) if page_data else None + if len(owned) >= needed or upstream_last_id is None or pages_left <= 1 or not _get_has_more(page): + return page, owned + return await _collect_owned_containers( + fetch_page=fetch_page, + after=upstream_last_id, + needed=needed, + allowed_container_ids=allowed_container_ids, + custom_llm_provider=custom_llm_provider, + pages_left=pages_left - 1, + collected=owned, + ) + + +async def list_owned_containers( + fetch_page: FetchContainerListPage, + after: str | None, + limit: int | None, user_api_key_dict: UserAPIKeyAuth, custom_llm_provider: str, ) -> object: - if is_proxy_admin(user_api_key_dict): - return response - - data: Final = _get_container_list_data(response) - if data is None: - return response - allowed_container_ids: Final = await _get_allowed_container_ids(user_api_key_dict) - filtered: Final[list[object]] = [] - for item in data: - container_id = _get_response_id(item) - if container_id is None: - continue - original_container_id, resolved_provider = decode_container_id_for_ownership(container_id, custom_llm_provider) - if _container_model_object_id(original_container_id, resolved_provider) in allowed_container_ids: - filtered.append(item) - - return _set_container_list_data(response, filtered, removed_filtered_items=len(filtered) != len(data)) + page_limit: Final = limit if limit is not None else DEFAULT_CONTAINER_LIST_LIMIT + last_page, owned = await _collect_owned_containers( + fetch_page=fetch_page, + after=after, + needed=page_limit + 1, + allowed_container_ids=allowed_container_ids, + custom_llm_provider=custom_llm_provider, + pages_left=OWNED_CONTAINER_LIST_MAX_PAGES, + collected=(), + ) + return _with_container_list_page( + last_page, + owned[:page_limit], + has_more=len(owned) > page_limit or _get_has_more(last_page), + ) diff --git a/tests/test_litellm/containers/test_container_proxy_ownership.py b/tests/test_litellm/containers/test_container_proxy_ownership.py index 176405bb9ca..38988d65c04 100644 --- a/tests/test_litellm/containers/test_container_proxy_ownership.py +++ b/tests/test_litellm/containers/test_container_proxy_ownership.py @@ -1,7 +1,7 @@ import json import sys from types import SimpleNamespace -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException @@ -242,110 +242,156 @@ async def test_should_not_reassign_existing_container_to_different_owner(monkeyp table.update.assert_not_awaited() -@pytest.mark.asyncio -async def test_should_filter_container_list_to_owned_records(monkeypatch): +def _owned_containers_in_db(monkeypatch, *model_object_ids: str) -> AsyncMock: table = AsyncMock() - table.find_many.return_value = [ - SimpleNamespace(model_object_id="container:openai:cntr_owned"), - ] - prisma_client = SimpleNamespace( - db=SimpleNamespace(litellm_managedobjecttable=table) - ) + table.find_many.return_value = [SimpleNamespace(model_object_id=object_id) for object_id in model_object_ids] monkeypatch.setattr( ownership, "_get_prisma_client", - AsyncMock(return_value=prisma_client), + AsyncMock(return_value=SimpleNamespace(db=SimpleNamespace(litellm_managedobjecttable=table))), ) - auth = UserAPIKeyAuth(user_id="user-1") - response = ContainerListResponse( + return table + + +def _upstream(pages_by_after): + calls = [] + + async def fetch_page(after, limit): + calls.append((after, limit)) + return pages_by_after[after] + + return fetch_page, calls + + +def _page(*container_ids: str, has_more: bool) -> ContainerListResponse: + return ContainerListResponse( object="list", - data=[_container("cntr_owned"), _container("cntr_other")], - has_more=True, + data=[_container(container_id) for container_id in container_ids], + has_more=has_more, ) - filtered = await ownership.filter_container_list_response( - response=response, - user_api_key_dict=auth, + +async def _list_owned(fetch_page, after=None, limit=None): + return await ownership.list_owned_containers( + fetch_page=fetch_page, + after=after, + limit=limit, + user_api_key_dict=UserAPIKeyAuth(user_id="user-1"), custom_llm_provider="openai", ) - assert [item.id for item in filtered.data] == ["cntr_owned"] - assert filtered.first_id == "cntr_owned" - assert filtered.last_id == "cntr_owned" - assert filtered.has_more is False + +@pytest.mark.asyncio +async def test_should_page_upstream_until_owned_containers_fill_the_limit(monkeypatch): + table = _owned_containers_in_db(monkeypatch, "container:openai:cntr_owned") + fetch_page, calls = _upstream( + { + None: _page("cntr_other_1", "cntr_other_2", has_more=True), + "cntr_other_2": _page("cntr_owned", has_more=False), + } + ) + + listed = await _list_owned(fetch_page, limit=1) + + assert [item.id for item in listed.data] == ["cntr_owned"] + assert listed.first_id == "cntr_owned" + assert listed.last_id == "cntr_owned" + assert listed.has_more is False + assert calls == [(None, 100), ("cntr_other_2", 100)] where = table.find_many.await_args.kwargs["where"] assert where["file_purpose"] == ownership.CONTAINER_OBJECT_PURPOSE assert where["created_by"]["in"] == ["user-1", "user:user-1"] @pytest.mark.asyncio -async def test_should_clear_has_more_when_filtered_container_list_is_empty( - monkeypatch, -): - table = AsyncMock() - table.find_many.return_value = [ - SimpleNamespace(model_object_id="container:openai:cntr_owned"), - ] - prisma_client = SimpleNamespace( - db=SimpleNamespace(litellm_managedobjecttable=table) - ) - monkeypatch.setattr( - ownership, - "_get_prisma_client", - AsyncMock(return_value=prisma_client), - ) - auth = UserAPIKeyAuth(user_id="user-1") - response = ContainerListResponse( - object="list", - data=[_container("cntr_other")], - has_more=True, - ) +async def test_should_trim_owned_containers_to_the_limit_without_mutating_the_upstream_page(monkeypatch): + _owned_containers_in_db(monkeypatch, "container:openai:cntr_owned_1", "container:openai:cntr_owned_2") + upstream_page = _page("cntr_owned_1", "cntr_other", "cntr_owned_2", has_more=False) + fetch_page, calls = _upstream({None: upstream_page}) - filtered = await ownership.filter_container_list_response( - response=response, - user_api_key_dict=auth, - custom_llm_provider="openai", - ) + listed = await _list_owned(fetch_page, limit=1) - assert filtered.data == [] - assert filtered.first_id is None - assert filtered.last_id is None - assert filtered.has_more is False + assert [item.id for item in listed.data] == ["cntr_owned_1"] + assert listed.first_id == "cntr_owned_1" + assert listed.last_id == "cntr_owned_1" + assert listed.has_more is True + assert calls == [(None, 100)] + assert [item.id for item in upstream_page.data] == ["cntr_owned_1", "cntr_other", "cntr_owned_2"] + assert upstream_page.has_more is False @pytest.mark.asyncio -async def test_should_clear_dict_has_more_when_filtered_container_list_is_empty( - monkeypatch, -): - table = AsyncMock() - table.find_many.return_value = [ - SimpleNamespace(model_object_id="container:openai:cntr_owned"), - ] - prisma_client = SimpleNamespace( - db=SimpleNamespace(litellm_managedobjecttable=table) +async def test_should_start_paging_from_the_requested_cursor(monkeypatch): + _owned_containers_in_db(monkeypatch, "container:openai:cntr_owned_2") + fetch_page, calls = _upstream({"cntr_owned_1": _page("cntr_other", "cntr_owned_2", has_more=False)}) + + listed = await _list_owned(fetch_page, after="cntr_owned_1", limit=1) + + assert [item.id for item in listed.data] == ["cntr_owned_2"] + assert listed.has_more is False + assert calls == [("cntr_owned_1", 100)] + + +@pytest.mark.asyncio +async def test_should_default_to_twenty_owned_containers_per_page(monkeypatch): + owned_ids = tuple(f"cntr_owned_{index}" for index in range(21)) + _owned_containers_in_db(monkeypatch, *(f"container:openai:{container_id}" for container_id in owned_ids)) + fetch_page, _ = _upstream({None: _page(*owned_ids, has_more=False)}) + + listed = await _list_owned(fetch_page) + + assert [item.id for item in listed.data] == list(owned_ids[:20]) + assert listed.last_id == "cntr_owned_19" + assert listed.has_more is True + + +@pytest.mark.asyncio +async def test_should_stop_after_five_upstream_pages_and_keep_has_more(monkeypatch): + _owned_containers_in_db(monkeypatch, "container:openai:cntr_owned") + fetch_page, calls = _upstream( + { + None: _page("cntr_other_0", has_more=True), + **{f"cntr_other_{index}": _page(f"cntr_other_{index + 1}", has_more=True) for index in range(6)}, + } ) - monkeypatch.setattr( - ownership, - "_get_prisma_client", - AsyncMock(return_value=prisma_client), - ) - auth = UserAPIKeyAuth(user_id="user-1") - response = { + + listed = await _list_owned(fetch_page, limit=1) + + assert listed.data == [] + assert listed.first_id is None + assert listed.last_id is None + assert listed.has_more is True + assert len(calls) == 5 + + +@pytest.mark.asyncio +async def test_should_stop_when_upstream_has_no_more_pages(monkeypatch): + _owned_containers_in_db(monkeypatch, "container:openai:cntr_owned") + fetch_page, calls = _upstream({None: _page("cntr_other", has_more=False)}) + + listed = await _list_owned(fetch_page, limit=1) + + assert listed.data == [] + assert listed.has_more is False + assert calls == [(None, 100)] + + +@pytest.mark.asyncio +async def test_should_build_dict_pages_without_mutating_the_upstream_page(monkeypatch): + _owned_containers_in_db(monkeypatch, "container:openai:cntr_owned") + upstream_page = {"object": "list", "data": [{"id": "cntr_other"}, {"id": "cntr_owned"}], "has_more": False} + fetch_page, _ = _upstream({None: upstream_page}) + + listed = await _list_owned(fetch_page, limit=1) + + assert listed == { "object": "list", - "data": [{"id": "cntr_other"}], - "has_more": True, + "data": [{"id": "cntr_owned"}], + "first_id": "cntr_owned", + "last_id": "cntr_owned", + "has_more": False, } - - filtered = await ownership.filter_container_list_response( - response=response, - user_api_key_dict=auth, - custom_llm_provider="openai", - ) - - assert filtered["data"] == [] - assert filtered["first_id"] is None - assert filtered["last_id"] is None - assert filtered["has_more"] is False + assert [item["id"] for item in upstream_page["data"]] == ["cntr_other", "cntr_owned"] @pytest.mark.asyncio @@ -647,7 +693,7 @@ async def test_should_return_response_when_owner_recording_raises_unexpected( @pytest.mark.asyncio -async def test_should_filter_container_list_inside_list_endpoint(monkeypatch): +async def test_should_list_owned_containers_inside_list_endpoint(monkeypatch): from litellm.proxy.container_endpoints import endpoints proxy_server_stub = SimpleNamespace( @@ -665,42 +711,37 @@ async def test_should_filter_container_list_inside_list_endpoint(monkeypatch): ) monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", proxy_server_stub) - response = ContainerListResponse( - object="list", - data=[_container("cntr_provider")], - has_more=False, - ) - - class FakeProcessor: - def __init__(self, data): - pass - - async def base_process_llm_request(self, **kwargs): - return response - - async def _handle_llm_api_exception(self, **kwargs): - raise kwargs["e"] - - filter_response = AsyncMock(return_value=response) - monkeypatch.setattr(endpoints, "ProxyBaseLLMRequestProcessing", FakeProcessor) - monkeypatch.setattr( - endpoints, - "filter_container_list_response", - filter_response, + upstream_page = _page("cntr_provider", has_more=False) + processor_cls = MagicMock( + side_effect=lambda data: SimpleNamespace(base_process_llm_request=AsyncMock(return_value=upstream_page)) ) + monkeypatch.setattr(endpoints, "ProxyBaseLLMRequestProcessing", processor_cls) + list_owned = AsyncMock(return_value=upstream_page) + monkeypatch.setattr(endpoints, "list_owned_containers", list_owned) result = await endpoints.list_containers( request=SimpleNamespace(query_params={}, headers={}), fastapi_response=SimpleNamespace(), user_api_key_dict=UserAPIKeyAuth(user_id="user-1"), + after="cntr_prev", + limit=2, + order="desc", ) - assert result == response - filter_response.assert_awaited_once_with( - response=response, - user_api_key_dict=UserAPIKeyAuth(user_id="user-1"), - custom_llm_provider="openai", - ) + assert result == upstream_page + kwargs = list_owned.await_args.kwargs + assert kwargs["after"] == "cntr_prev" + assert kwargs["limit"] == 2 + assert kwargs["user_api_key_dict"] == UserAPIKeyAuth(user_id="user-1") + assert kwargs["custom_llm_provider"] == "openai" + processor_cls.assert_not_called() + + assert await kwargs["fetch_page"]("cntr_page_cursor", 100) == upstream_page + forwarded = processor_cls.call_args.kwargs["data"] + assert forwarded["after"] == "cntr_page_cursor" + assert forwarded["limit"] == 100 + assert forwarded["order"] == "desc" + assert forwarded["custom_llm_provider"] == "openai" @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/container_endpoints/test_endpoints.py b/tests/test_litellm/proxy/container_endpoints/test_endpoints.py index 3604da65258..1beff4c82ba 100644 --- a/tests/test_litellm/proxy/container_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/container_endpoints/test_endpoints.py @@ -1,14 +1,15 @@ import sys from types import SimpleNamespace -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock +import pytest from fastapi import FastAPI from fastapi.testclient import TestClient -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.container_endpoints import endpoints -from litellm.types.containers.main import ContainerListResponse +from litellm.proxy.container_endpoints import endpoints, ownership +from litellm.types.containers.main import ContainerListResponse, ContainerObject PROXY_SERVER_STUB = SimpleNamespace( general_settings={}, @@ -24,41 +25,110 @@ PROXY_SERVER_STUB = SimpleNamespace( user_temperature=None, version="test", ) +ADMIN = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) +NON_ADMIN = UserAPIKeyAuth(user_id="user-1") -def _client() -> TestClient: +@pytest.fixture(autouse=True) +def clear_allowed_container_ids_cache(): + ownership._ALLOWED_CONTAINER_IDS_CACHE.cache_dict.clear() + ownership._ALLOWED_CONTAINER_IDS_CACHE.ttl_dict.clear() + yield + ownership._ALLOWED_CONTAINER_IDS_CACHE.cache_dict.clear() + ownership._ALLOWED_CONTAINER_IDS_CACHE.ttl_dict.clear() + + +def _client(auth: UserAPIKeyAuth) -> TestClient: app = FastAPI() app.include_router(endpoints.router) - app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="user-1") + app.dependency_overrides[user_api_key_auth] = lambda: auth return TestClient(app) -def test_list_containers_forwards_typed_pagination_params(monkeypatch): +def _container(container_id: str) -> ContainerObject: + return ContainerObject(id=container_id, object="container", created_at=1, status="active") + + +def _page(*container_ids: str, has_more: bool) -> ContainerListResponse: + return ContainerListResponse( + object="list", + data=[_container(container_id) for container_id in container_ids], + has_more=has_more, + ) + + +def _upstream_pages(monkeypatch, pages_by_after) -> MagicMock: + processor_cls = MagicMock( + side_effect=lambda data: SimpleNamespace( + base_process_llm_request=AsyncMock(return_value=pages_by_after[data["after"]]) + ) + ) + monkeypatch.setattr(endpoints, "ProxyBaseLLMRequestProcessing", processor_cls) + return processor_cls + + +def _forwarded_pages(processor_cls: MagicMock): + return [(call.kwargs["data"]["after"], call.kwargs["data"]["limit"]) for call in processor_cls.call_args_list] + + +def test_list_containers_forwards_typed_pagination_params_for_admins(monkeypatch): monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", PROXY_SERVER_STUB) - upstream = ContainerListResponse(object="list", data=[], has_more=True) - captured = {} + processor_cls = _upstream_pages(monkeypatch, {"cntr_prev": _page("cntr_next", has_more=True)}) - class FakeProcessor: - def __init__(self, data): - captured["data"] = data - - async def base_process_llm_request(self, **kwargs): - return upstream - - async def _handle_llm_api_exception(self, **kwargs): - raise kwargs["e"] - - monkeypatch.setattr(endpoints, "ProxyBaseLLMRequestProcessing", FakeProcessor) - monkeypatch.setattr(endpoints, "filter_container_list_response", AsyncMock(return_value=upstream)) - - response = _client().get( + response = _client(ADMIN).get( "/v1/containers", params={"limit": "1", "order": "desc", "after": "cntr_prev"}, headers={"Authorization": "Bearer sk-test"}, ) assert response.status_code == 200 + assert [item["id"] for item in response.json()["data"]] == ["cntr_next"] assert response.json()["has_more"] is True - assert captured["data"]["limit"] == 1 - assert captured["data"]["order"] == "desc" - assert captured["data"]["after"] == "cntr_prev" + assert _forwarded_pages(processor_cls) == [("cntr_prev", 1)] + assert processor_cls.call_args.kwargs["data"]["order"] == "desc" + + +def test_list_containers_rejects_a_non_integer_limit(monkeypatch): + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", PROXY_SERVER_STUB) + processor_cls = _upstream_pages(monkeypatch, {}) + + response = _client(ADMIN).get( + "/v1/containers", + params={"limit": "abc"}, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 422 + processor_cls.assert_not_called() + + +def test_list_containers_pages_upstream_until_non_admin_keys_see_their_containers(monkeypatch): + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", PROXY_SERVER_STUB) + table = AsyncMock() + table.find_many.return_value = [SimpleNamespace(model_object_id="container:openai:cntr_owned")] + monkeypatch.setattr( + ownership, + "_get_prisma_client", + AsyncMock(return_value=SimpleNamespace(db=SimpleNamespace(litellm_managedobjecttable=table))), + ) + processor_cls = _upstream_pages( + monkeypatch, + { + None: _page("cntr_other", has_more=True), + "cntr_other": _page("cntr_owned", has_more=False), + }, + ) + + response = _client(NON_ADMIN).get( + "/v1/containers", + params={"limit": "1"}, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200 + body = response.json() + assert [item["id"] for item in body["data"]] == ["cntr_owned"] + assert body["first_id"] == "cntr_owned" + assert body["last_id"] == "cntr_owned" + assert body["has_more"] is False + assert _forwarded_pages(processor_cls) == [(None, 100), ("cntr_other", 100)] diff --git a/tests/test_litellm/proxy/container_endpoints/test_handler_factory.py b/tests/test_litellm/proxy/container_endpoints/test_handler_factory.py index 01a03f6ef03..a471f915071 100644 --- a/tests/test_litellm/proxy/container_endpoints/test_handler_factory.py +++ b/tests/test_litellm/proxy/container_endpoints/test_handler_factory.py @@ -1,6 +1,6 @@ import sys from types import SimpleNamespace -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock from fastapi import FastAPI from fastapi.testclient import TestClient @@ -39,20 +39,11 @@ def test_list_container_files_forwards_declared_query_params(monkeypatch): "assert_user_can_access_container", AsyncMock(return_value=("cntr_123", "openai")), ) - captured = {} - - class FakeProcessor: - def __init__(self, data): - captured["data"] = data - - async def base_process_llm_request(self, **kwargs): - captured["route_type"] = kwargs["route_type"] - return {"object": "list", "data": [], "has_more": True} - - async def _handle_llm_api_exception(self, **kwargs): - raise kwargs["e"] - - monkeypatch.setattr(handler_factory, "ProxyBaseLLMRequestProcessing", FakeProcessor) + processor_cls = MagicMock() + processor_cls.return_value.base_process_llm_request = AsyncMock( + return_value={"object": "list", "data": [], "has_more": True} + ) + monkeypatch.setattr(handler_factory, "ProxyBaseLLMRequestProcessing", processor_cls) response = _client().get( "/v1/containers/cntr_123/files", @@ -61,9 +52,11 @@ def test_list_container_files_forwards_declared_query_params(monkeypatch): ) assert response.status_code == 200 - assert captured["route_type"] == "alist_container_files" - assert captured["data"]["container_id"] == "cntr_123" - assert captured["data"]["limit"] == "1" - assert captured["data"]["order"] == "desc" - assert captured["data"]["after"] == "cfile_prev" - assert "unknown" not in captured["data"] + assert response.json()["has_more"] is True + assert processor_cls.return_value.base_process_llm_request.await_args.kwargs["route_type"] == "alist_container_files" + forwarded = processor_cls.call_args.kwargs["data"] + assert forwarded["container_id"] == "cntr_123" + assert forwarded["limit"] == "1" + assert forwarded["order"] == "desc" + assert forwarded["after"] == "cfile_prev" + assert "unknown" not in forwarded From 7f7e0d55178c38a5800462c91425a8240b28ed69 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:32:00 -0700 Subject: [PATCH 197/480] fix(vector-store): embed through the SDK when the Router does not serve the query embedding model The Router executor only routed a query embedding when the vector store carried extra embedding configuration, so a store registered with no embedding model at all always went to the Router and 500'd on the s3_vectors default text-embedding-3-small when no deployment served it. Route on whether the Router serves the model, which is the rule the executor had before, and keep the request metadata on the SDK fallback so the embedding stays attributed either way. --- .../base_llm/vector_store/transformation.py | 7 ++-- .../test_bedrock_knowledgebase_hook.py | 2 +- .../test_router_embedding_integration.py | 13 ++++---- .../test_s3_vectors_transformation.py | 33 +++++++++++++++---- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index a3b8bcc499c..c8d2b7fe522 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -99,12 +99,9 @@ class RouterVectorStoreEmbeddingExecutor: ) return bool(resolved) or model in deployment_models - def _embeds_through_sdk(self, model: str, configuration: Mapping[str, object]) -> bool: - return bool(configuration) and not self._router_serves(model) - def embed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: embedding_kwargs: Final = self._embedding_kwargs(configuration) - if self._embeds_through_sdk(model, configuration): + if not self._router_serves(model): return LiteLLMVectorStoreEmbeddingExecutor().embed(model, query, embedding_kwargs) return self.router.embedding( # pyright: ignore[reportUnknownMemberType] # Router embedding input retains a legacy untyped list model=model, @@ -114,7 +111,7 @@ class RouterVectorStoreEmbeddingExecutor: async def aembed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: embedding_kwargs: Final = self._embedding_kwargs(configuration) - if self._embeds_through_sdk(model, configuration): + if not self._router_serves(model): return await LiteLLMVectorStoreEmbeddingExecutor().aembed(model, query, embedding_kwargs) return await self.router.aembedding( # pyright: ignore[reportUnknownMemberType] # Router embedding input retains a legacy untyped list model=model, diff --git a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py index 98045725177..044e1e0de8e 100644 --- a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py +++ b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py @@ -375,7 +375,7 @@ async def test_bedrock_kb_request_body_has_transformed_filters( timeout=None, client=None, _is_async=False, - router: "litellm.Router | None" = None, + embedding_executor=None, ): litellm_params_dict = ( litellm_params.model_dump(exclude_none=False) diff --git a/tests/router_unit_tests/test_router_embedding_integration.py b/tests/router_unit_tests/test_router_embedding_integration.py index 2cc9914c9b3..e10ba0f0962 100644 --- a/tests/router_unit_tests/test_router_embedding_integration.py +++ b/tests/router_unit_tests/test_router_embedding_integration.py @@ -187,7 +187,7 @@ class TestRouterEmbeddingIntegration: assert _sent(store_route, 1) == ("Bearer store-key", "text-embedding-3-large", ["async query"]) @pytest.mark.asyncio - async def test_router_executor_rejects_unserved_models_without_explicit_config( + async def test_router_executor_embeds_unserved_models_through_the_sdk( self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) @@ -198,12 +198,13 @@ class TestRouterEmbeddingIntegration: metadata={"user_api_key_team_id": "team-a"}, ) - with pytest.raises(litellm.BadRequestError): - executor.embed("openai/text-embedding-3-large", "sync query", {}) - with pytest.raises(litellm.BadRequestError): - await executor.aembed("openai/text-embedding-3-large", "async query", {}) + sync_response = executor.embed("text-embedding-3-large", "sync query", {}) + async_response = await executor.aembed("text-embedding-3-large", "async query", {}) - assert openai_route.call_count == 0 + assert sync_response.data[0]["embedding"] == QUERY_VECTOR + assert async_response.data[0]["embedding"] == QUERY_VECTOR + assert _sent(openai_route, 0) == ("Bearer env-key", "text-embedding-3-large", ["sync query"]) + assert _sent(openai_route, 1) == ("Bearer env-key", "text-embedding-3-large", ["async query"]) def test_router_executor_routes_deployment_model_names_through_the_router( self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch diff --git a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py index e2b02ea2151..e313b749d06 100644 --- a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py +++ b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py @@ -21,8 +21,6 @@ def _embedding_response(vector): class _RecordingExecutor: - """Executor double recording every (model, query, configuration) it was asked to embed.""" - def __init__(self, vector=QUERY_VECTOR): self.vector = vector self.calls = [] @@ -94,7 +92,6 @@ class TestS3VectorsVectorStoreConfig: config.get_complete_url(None, {"aws_region_name": "Bad_Region!"}) def test_transform_search_request(self): - """Full request-body transformation with the query embedded through the injected executor""" config = S3VectorsVectorStoreConfig() logging_obj = _logging_obj() executor = _RecordingExecutor() @@ -137,7 +134,6 @@ class TestS3VectorsVectorStoreConfig: @pytest.mark.asyncio async def test_atransform_search_embeds_alias_and_store_config_through_executor(self): - """The store's embedding_model alias and litellm_embedding_config reach the executor unchanged""" config = S3VectorsVectorStoreConfig() executor = _RecordingExecutor(vector=[0.4, 0.5]) @@ -177,9 +173,34 @@ class TestS3VectorsVectorStoreConfig: ) assert request_body["queryVector"]["float32"] == QUERY_VECTOR + @pytest.mark.asyncio + async def test_atransform_search_default_model_falls_back_to_the_sdk(self): + """Regression (LIT-6750): a store that never named an embedding model keeps working on a proxy + whose model list has no text-embedding-3-small, embedding through the SDK instead of erroring.""" + config = S3VectorsVectorStoreConfig() + router = MagicMock() + router.get_model_list.return_value = [ + {"model_name": "team-embeddings", "litellm_params": {"model": "openai/text-embedding-3-small"}} + ] + router.resolved_litellm_models.return_value = [] + router.aembedding = AsyncMock(side_effect=AssertionError("unserved model must not reach the Router")) + request_metadata = {"user_api_key_team_id": "team-a"} + + mock_bare = AsyncMock(return_value=_embedding_response(QUERY_VECTOR)) + with patch("litellm.aembedding", new=mock_bare): # test-quality-ok: stubs the bare-embedding fallback whose call the test asserts on + _, request_body = await config.atransform_search_vector_store_request( + **_search_kwargs( + embedding_executor=RouterVectorStoreEmbeddingExecutor(router=router, metadata=request_metadata) + ) + ) + + mock_bare.assert_awaited_once_with( + model="text-embedding-3-small", input=["test query"], metadata=request_metadata + ) + assert request_body["queryVector"]["float32"] == QUERY_VECTOR + @pytest.mark.asyncio async def test_atransform_search_without_executor_uses_bare_embedding(self): - """Backward compat: SDK callers without an executor keep embedding through litellm.aembedding""" config = S3VectorsVectorStoreConfig() mock_bare = AsyncMock(return_value=_embedding_response([0.6, 0.7])) @@ -190,7 +211,6 @@ class TestS3VectorsVectorStoreConfig: assert request_body["queryVector"]["float32"] == [0.6, 0.7] def test_transform_search_without_executor_uses_bare_embedding_sync(self): - """Sync twin: no executor -> bare litellm.embedding as before""" config = S3VectorsVectorStoreConfig() mock_bare = MagicMock(return_value=_embedding_response([0.8, 0.9])) @@ -203,7 +223,6 @@ class TestS3VectorsVectorStoreConfig: assert request_body["queryVector"]["float32"] == [0.8, 0.9] def test_transform_search_request_invalid_vector_store_id(self): - """An unparseable vector_store_id raises before any embedding is generated""" config = S3VectorsVectorStoreConfig() executor = _RecordingExecutor() From 7c87451eadffa7720536937dd2a47c714554788c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:40:25 -0700 Subject: [PATCH 198/480] chore(router): document the breadcrumb write-back and ratchet lint budgets --- basedpyright-code-budget.json | 8 ++++---- litellm/router.py | 3 ++- ruff-strict-budget.json | 8 ++++---- type-discipline-budget.json | 6 +++--- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 3f96531cf6f..5872a6a7257 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -3,7 +3,7 @@ "limit": 14074 }, "reportArgumentType": { - "limit": 2215 + "limit": 2214 }, "reportAssignmentType": { "limit": 319 @@ -42,7 +42,7 @@ "limit": 12 }, "reportIndexIssue": { - "limit": 25 + "limit": 24 }, "reportInvalidTypeForm": { "limit": 34 @@ -57,7 +57,7 @@ "limit": 5601 }, "reportMissingTypeArgument": { - "limit": 15290 + "limit": 15289 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,7 +105,7 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38332 + "limit": 38328 }, "reportUnknownParameterType": { "limit": 19625 diff --git a/litellm/router.py b/litellm/router.py index 69fe731bfbd..11a042861f8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8073,7 +8073,8 @@ class Router: if isinstance(earlier_breadcrumbs, (list, tuple)) else () ) - kwargs[_metadata_var]["previous_models"] = (*kept_breadcrumbs, mask_credentials_in_payload(previous_model)) + breadcrumbs: Final = (*kept_breadcrumbs, mask_credentials_in_payload(previous_model)) + kwargs[_metadata_var]["previous_models"] = breadcrumbs # rebind-ok: the logging object already holds this dict return kwargs def _update_usage(self, deployment_id: str, parent_otel_span: Span | None) -> int: diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 4fcf650a8bc..d7cf9e039bc 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -144,7 +144,7 @@ "limit": 1 }, "PLR1704": { - "limit": 3 + "limit": 2 }, "PLR1714": { "limit": 253 @@ -240,13 +240,13 @@ "limit": 96 }, "TRY201": { - "limit": 403 + "limit": 402 }, "TRY203": { - "limit": 111 + "limit": 110 }, "TRY300": { - "limit": 854 + "limit": 853 }, "UP028": { "limit": 2 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index f3c4c7760c6..d73b1b2e277 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 22334 + "limit": 22333 }, "LIT002": { - "limit": 26763 + "limit": 26760 }, "LIT003": { "limit": 261 @@ -30,7 +30,7 @@ "limit": 16480 }, "LIT011": { - "limit": 5520 + "limit": 5518 }, "LIT012": { "limit": 4489 From f62e87d28c0bd82da95479db188aa2309384583d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:42:47 -0700 Subject: [PATCH 199/480] fix(ui): read the preset catalog through the shared mock in the lib test --- ui/litellm-dashboard/src/lib/autorouter_presets.test.ts | 5 ++--- ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 44ac6973a1c..76996447ac8 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -1,9 +1,8 @@ import { describe, it, expect } from "vitest"; -import bundledPresets from "../../../../litellm/proxy/public_endpoints/autorouter_presets.json"; +import { BUNDLED_PRESETS_RESPONSE } from "../../tests/mocks/autoRouterPresets"; import { hydratePresets, AutoRouterPreset, - AutoRouterPresetsResponse, getRequiredModelsInPreset, getMissingModelsInPreset, getRequiredModels, @@ -21,7 +20,7 @@ import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKe const groupsOnly = (models: Iterable) => buildModelAvailability(models, []); // Hydrated from the real bundled catalog so a catalog edit flows into these expectations. -const PRESETS = hydratePresets(bundledPresets as AutoRouterPresetsResponse); +const PRESETS = hydratePresets(BUNDLED_PRESETS_RESPONSE); const getAllPresets = (): AutoRouterPreset[] => PRESETS; const getPresetByKey = (key: string): AutoRouterPreset | undefined => PRESETS.find((p) => p.key === key); diff --git a/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts b/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts index cff417e6bfa..ac6da9bceba 100644 --- a/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts +++ b/ui/litellm-dashboard/tests/mocks/autoRouterPresets.ts @@ -7,9 +7,9 @@ import { hydratePresets, type AutoRouterPresetsResponse } from "@/lib/autorouter // instead of redding on a stale copy. Exported as vi.fn so a test can override the query state. const CATALOG_PATH = resolve(__dirname, "../../../../litellm/proxy/public_endpoints/autorouter_presets.json"); -export const BUNDLED_PRESETS = hydratePresets( - JSON.parse(readFileSync(CATALOG_PATH, "utf8")) as AutoRouterPresetsResponse, -); +export const BUNDLED_PRESETS_RESPONSE = JSON.parse(readFileSync(CATALOG_PATH, "utf8")) as AutoRouterPresetsResponse; + +export const BUNDLED_PRESETS = hydratePresets(BUNDLED_PRESETS_RESPONSE); export const LOADED_PRESETS_QUERY = { data: BUNDLED_PRESETS, From 3ea61c23c749b7a2c4a87393976b7fa20f6b5207 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:55:16 -0700 Subject: [PATCH 200/480] fix(vector-stores): survive a failing vector store search in the chat completions hook One unreachable vector store used to wipe out every store's context on a chat completion carrying vector_store_ids: the search raised, the blanket handler returned the original messages, and the request answered with no retrieved context at all. Each store's search now has its own handler that warns with the vector store id and moves on to the next store. The same loop appended every store's results to the original messages instead of the running copy, so with two healthy stores only the last one reached the model. It now chains through modified_messages. The Router is injected through a ProxyRuntime protocol instead of an in-function litellm.proxy.proxy_server import, so the hook's routing can be driven in tests without touching proxy globals. --- .../vector_store_pre_call_hook.py | 76 +++--- .../test_vector_store_pre_call_hook.py | 220 ++++++++++++++++++ 2 files changed, 269 insertions(+), 27 deletions(-) create mode 100644 tests/test_litellm/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index e012d35b8f3..12ff38ce4ba 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -6,7 +6,8 @@ It searches the vector store for relevant context and appends it to the messages """ from collections.abc import Awaitable, Callable -from typing import TYPE_CHECKING, Any, Final, cast +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Final, Protocol, cast import litellm import litellm.vector_stores @@ -24,10 +25,35 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.proxy.utils import PrismaClient + from litellm.router import Router else: LiteLLMLoggingObj = Any +class ProxyRuntime(Protocol): + def llm_router(self) -> "Router | None": ... + + def prisma_client(self) -> "PrismaClient | None": ... + + +@dataclass(frozen=True, slots=True) +class ProxyServerRuntime: + def llm_router(self) -> "Router | None": + try: + from litellm.proxy.proxy_server import llm_router + except ImportError: + return None + return llm_router + + def prisma_client(self) -> "PrismaClient | None": + try: + from litellm.proxy.proxy_server import prisma_client + except ImportError: + return None + return prisma_client + + class VectorStorePreCallHook(CustomLogger): CONTENT_PREFIX_STRING = "Context:\n\n" """ @@ -39,8 +65,9 @@ class VectorStorePreCallHook(CustomLogger): 3. Appends the search results as context to the messages """ - def __init__(self): + def __init__(self, proxy_runtime: ProxyRuntime | None = None): super().__init__() + self.proxy_runtime: Final[ProxyRuntime] = proxy_runtime or ProxyServerRuntime() async def async_get_chat_completion_prompt( self, @@ -79,21 +106,8 @@ class VectorStorePreCallHook(CustomLogger): if litellm.vector_store_registry is None: return model, messages, non_default_params - # Get prisma_client for database fallback - prisma_client = None - llm_router = None - try: - from litellm.proxy.proxy_server import ( - llm_router as _llm_router, - ) - from litellm.proxy.proxy_server import ( - prisma_client as _prisma_client, - ) - - prisma_client = _prisma_client - llm_router = _llm_router - except ImportError: - pass + prisma_client: Final = self.proxy_runtime.prisma_client() + llm_router: Final = self.proxy_runtime.llm_router() # Use database fallback to ensure synchronization across instances vector_stores_to_run: list[ @@ -136,15 +150,23 @@ class VectorStorePreCallHook(CustomLogger): Callable[..., Awaitable[VectorStoreSearchResponse]], litellm.vector_stores.asearch, ) - search_response = await search_function( - **{ - "vector_store_id": vector_store_id, - "query": query, - "custom_llm_provider": custom_llm_provider, - "metadata": request_metadata, - **litellm_params_for_vector_store, - }, - ) + try: + search_response = await search_function( + **{ + "vector_store_id": vector_store_id, + "query": query, + "custom_llm_provider": custom_llm_provider, + "metadata": request_metadata, + **litellm_params_for_vector_store, + }, + ) + except Exception as search_error: + verbose_logger.warning( + "Vector store search failed for vector_store_id=%s, continuing without its context: %s", + vector_store_id, + search_error, + ) + continue verbose_logger.debug("search_response: %s", search_response) @@ -153,7 +175,7 @@ class VectorStorePreCallHook(CustomLogger): # Process search results and append as context modified_messages = self._append_search_results_to_messages( - messages=messages, search_response=search_response + messages=modified_messages, search_response=search_response ) # Get the number of results for logging diff --git a/tests/test_litellm/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py b/tests/test_litellm/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py new file mode 100644 index 00000000000..9c0ed38f1a3 --- /dev/null +++ b/tests/test_litellm/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py @@ -0,0 +1,220 @@ +import logging +from dataclasses import dataclass, field +from typing import Any + +import pytest + +import litellm +from litellm._logging import verbose_logger +from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + VectorStorePreCallHook, +) +from litellm.types.vector_stores import ( + VectorStoreResultContent, + VectorStoreSearchResponse, + VectorStoreSearchResult, +) +from litellm.vector_stores.vector_store_registry import ( + LiteLLM_ManagedVectorStore, + VectorStoreRegistry, +) + + +def _search_response(text: str) -> VectorStoreSearchResponse: + return VectorStoreSearchResponse( + object="vector_store.search_results.page", + search_query="what is litellm?", + data=[ + VectorStoreSearchResult( + score=1.0, + content=[VectorStoreResultContent(text=text, type="text")], + ) + ], + ) + + +@dataclass +class RecordingRouter: + failing_vector_store_ids: frozenset[str] = frozenset() + calls: list[dict[str, Any]] = field(default_factory=list) + + async def avector_store_search(self, **kwargs: Any) -> VectorStoreSearchResponse: + self.calls.append(kwargs) + vector_store_id = kwargs["vector_store_id"] + if vector_store_id in self.failing_vector_store_ids: + raise litellm.BadRequestError( + message=f"no healthy deployments for {vector_store_id}", + model="text-embedding-3-small", + llm_provider="openai", + ) + return _search_response(f"context from {vector_store_id}") + + +@dataclass(frozen=True) +class FakeProxyRuntime: + router: RecordingRouter | None + + def llm_router(self) -> RecordingRouter | None: + return self.router + + def prisma_client(self) -> None: + return None + + +class RecordingHandler(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.WARNING) + self.records: list[logging.LogRecord] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.records.append(record) + + +@pytest.fixture +def registry_with(monkeypatch: pytest.MonkeyPatch): + def _register(*vector_store_ids: str, custom_llm_provider: str = "bedrock") -> None: + monkeypatch.setattr( + litellm, + "vector_store_registry", + VectorStoreRegistry( + vector_stores=[ + LiteLLM_ManagedVectorStore(vector_store_id=vector_store_id, custom_llm_provider=custom_llm_provider) + for vector_store_id in vector_store_ids + ], + ), + ) + + return _register + + +@pytest.fixture +def warnings(): + handler = RecordingHandler() + verbose_logger.addHandler(handler) + yield handler.records + verbose_logger.removeHandler(handler) + + +class FakeLoggingObj: + def __init__(self, metadata: dict[str, Any]) -> None: + self.model_call_details: dict[str, Any] = {"litellm_params": {"metadata": metadata}} + + +async def _run_hook(hook: VectorStorePreCallHook, vector_store_ids: list[str], logging_obj: FakeLoggingObj): + return await hook.async_get_chat_completion_prompt( + model="chat-model", + messages=[{"role": "user", "content": "what is litellm?"}], + non_default_params={"vector_store_ids": vector_store_ids}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + litellm_logging_obj=logging_obj, + ) + + +@pytest.mark.asyncio +async def test_hook_searches_through_the_injected_router_with_the_request_metadata(registry_with): + """Regression (LIT-6752): the hook must reach the Router through its injected runtime, not a proxy_server import.""" + registry_with("vs-router") + router = RecordingRouter() + logging_obj = FakeLoggingObj({"user_api_key_team_id": "team-a"}) + + _, messages, _ = await _run_hook( + VectorStorePreCallHook(proxy_runtime=FakeProxyRuntime(router=router)), + ["vs-router"], + logging_obj, + ) + + assert router.calls == [ + { + "vector_store_id": "vs-router", + "query": "what is litellm?", + "custom_llm_provider": "bedrock", + "metadata": {"user_api_key_team_id": "team-a"}, + } + ] + assert messages[0]["content"] == "Context:\n\ncontext from vs-router\n\n" + + +@pytest.mark.asyncio +async def test_hook_falls_back_to_the_sdk_when_the_runtime_has_no_router(registry_with, warnings): + registry_with("vs-sdk", custom_llm_provider="lit6752-not-a-provider") + + _, messages, _ = await _run_hook( + VectorStorePreCallHook(proxy_runtime=FakeProxyRuntime(router=None)), + ["vs-sdk"], + FakeLoggingObj({"user_api_key_team_id": "team-a"}), + ) + + assert messages == [{"role": "user", "content": "what is litellm?"}] + assert len(warnings) == 1 + assert ( + warnings[0] + .getMessage() + .startswith("Vector store search failed for vector_store_id=vs-sdk, continuing without its context: ") + ) + assert "is not a valid LlmProviders" in warnings[0].getMessage() + + +@pytest.mark.asyncio +async def test_every_healthy_vector_store_contributes_its_own_context(registry_with): + """Regression (LIT-6752): each store appended its context to the original messages, so only the last one survived.""" + registry_with("vs-one", "vs-two") + router = RecordingRouter() + + _, messages, _ = await _run_hook( + VectorStorePreCallHook(proxy_runtime=FakeProxyRuntime(router=router)), + ["vs-one", "vs-two"], + FakeLoggingObj({}), + ) + + assert [message["content"] for message in messages] == [ + "Context:\n\ncontext from vs-one\n\n", + "Context:\n\ncontext from vs-two\n\n", + "what is litellm?", + ] + + +@pytest.mark.asyncio +async def test_a_failing_vector_store_warns_with_its_id_and_the_other_stores_still_answer(registry_with, warnings): + """Regression (LIT-6752): one unreachable store must not silently drop every other store's context.""" + registry_with("vs-broken", "vs-healthy") + router = RecordingRouter(failing_vector_store_ids=frozenset({"vs-broken"})) + logging_obj = FakeLoggingObj({"user_api_key_team_id": "team-a"}) + + _, messages, _ = await _run_hook( + VectorStorePreCallHook(proxy_runtime=FakeProxyRuntime(router=router)), + ["vs-broken", "vs-healthy"], + logging_obj, + ) + + assert [call["vector_store_id"] for call in router.calls] == ["vs-broken", "vs-healthy"] + assert messages[0]["content"] == "Context:\n\ncontext from vs-healthy\n\n" + assert len(logging_obj.model_call_details["search_results"]) == 1 + assert [record.getMessage() for record in warnings] == [ + "Vector store search failed for vector_store_id=vs-broken, continuing without its context: " + "litellm.BadRequestError: no healthy deployments for vs-broken" + ] + + +@pytest.mark.asyncio +async def test_the_only_vector_store_failing_leaves_the_messages_untouched(registry_with, warnings): + registry_with("vs-broken") + original_messages = [{"role": "user", "content": "what is litellm?"}] + + _, messages, _ = await _run_hook( + VectorStorePreCallHook( + proxy_runtime=FakeProxyRuntime(router=RecordingRouter(failing_vector_store_ids=frozenset({"vs-broken"}))) + ), + ["vs-broken"], + FakeLoggingObj({}), + ) + + assert messages == original_messages + assert [(record.levelname, record.getMessage()) for record in warnings] == [ + ( + "WARNING", + "Vector store search failed for vector_store_id=vs-broken, continuing without its context: " + "litellm.BadRequestError: no healthy deployments for vs-broken", + ) + ] From cf958c0e6f9ffe98493e3f4fcd20154d5eab16f9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:56:23 -0700 Subject: [PATCH 201/480] ci(rust): build and test the ai-gateway server feature litellm-ai-gateway's server feature is off by default and nothing in the workspace turns it on, so the workspace clippy and test steps never compiled src/auth, src/routes, src/state, src/realtime or the gateway binary. 43 tests ran instead of 57. Adds the two steps CLAUDE.md already documents as the local gate, and fixes the three collapsible_if violations that had accumulated behind the flag. --- .github/workflows/test-rust.yml | 6 +++++ litellm-rust/CLAUDE.md | 2 ++ .../ai-gateway/src/realtime/streaming.rs | 18 +++++++-------- .../ai-gateway/src/routes/realtime/service.rs | 23 +++++++++---------- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 1b71232bc2e..ae80155305a 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -74,12 +74,18 @@ jobs: - name: Run Clippy with Bedrock auth run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings + - name: Run Clippy with the gateway server + run: cargo clippy -p litellm-ai-gateway --all-targets --features server --locked -- -D warnings + - name: Run Rust tests run: cargo test --workspace --locked - name: Run core tests with Bedrock auth run: cargo test -p litellm-core --features bedrock-auth --locked + - name: Run gateway tests with the server feature + run: cargo test -p litellm-ai-gateway --features server --locked + release-wheel: name: release wheel runs-on: ubuntu-latest diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md index 3dcf1853efc..be0fcdd1474 100644 --- a/litellm-rust/CLAUDE.md +++ b/litellm-rust/CLAUDE.md @@ -178,6 +178,8 @@ cargo fmt --check cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings cargo test --workspace +# the `auth`, `routes`, `state` and `realtime` tests only exist under `server` +cargo test -p litellm-ai-gateway --features server ``` When a Rust path is exposed through Python, add Python parity tests that compare diff --git a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs b/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs index c32e727de54..edd9338b4f2 100644 --- a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs +++ b/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs @@ -106,16 +106,16 @@ impl RealTimeStreaming { /// `litellm_call_id`, replacing the gateway-generated fallback. fn on_session(&mut self, event: &RealtimeEvent) { let session = event.data.get("session").and_then(Value::as_object); - if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str) { - if !id.is_empty() { - self.id = id.to_string(); - self.litellm_call_id = id.to_string(); - } + if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str) + && !id.is_empty() + { + self.id = id.to_string(); + self.litellm_call_id = id.to_string(); } - if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str) { - if !model.is_empty() { - self.model = model.to_string(); - } + if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str) + && !model.is_empty() + { + self.model = model.to_string(); } } diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs index b8ee77c4269..f7bbb37dff4 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs @@ -50,18 +50,17 @@ where provider_model, params.api_key.as_deref(), params.api_base.as_deref(), - ) { - if let Some(handoff) = pool.take(&key) { - return crate::io::realtime::realtime_warm( - provider_model, - handoff, - idle_timeout, - observe, - client_in, - client_out, - ) - .await; - } + ) && let Some(handoff) = pool.take(&key) + { + return crate::io::realtime::realtime_warm( + provider_model, + handoff, + idle_timeout, + observe, + client_in, + client_out, + ) + .await; } // Cold path: fresh dial (the original behavior). From 06e60e08d28e0ec8a8c0539b11101e2ca8639403 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:09:55 -0700 Subject: [PATCH 202/480] ci(ui): run the UI build check through the image's ui-builder stage The build-ui check compiled the dashboard from a full checkout, so any import reaching above ui/litellm-dashboard/ resolved there and only broke inside the images, where the stage copies the dashboard tree alone. Building the stage itself puts the check on the same file boundary the shipped images use. --- .github/workflows/test-litellm-ui-build.yml | 24 ++++++--------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index b3a07a6e0ff..4eb6b272c43 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -19,9 +19,6 @@ jobs: build-ui: runs-on: ubuntu-latest timeout-minutes: 10 - defaults: - run: - working-directory: ui/litellm-dashboard steps: - name: Checkout repository @@ -35,18 +32,11 @@ jobs: with: category: ui - - name: Setup Node.js + # Built through the image stage rather than the checkout, because the + # stage copies ui/litellm-dashboard/ alone: an import reaching above the + # dashboard root resolves in a checkout and fails in every image we ship. + # Dockerfile, docker/Dockerfile.non_root and ui/Dockerfile share this + # stage verbatim, so building one covers all three. + - name: Build the dashboard as the shipped images build it if: steps.changes.outputs.decision != 'skip' - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 - with: - node-version-file: ui/litellm-dashboard/.nvmrc - cache: "npm" - cache-dependency-path: ui/litellm-dashboard/package-lock.json - - - name: Install dependencies - if: steps.changes.outputs.decision != 'skip' - run: npm ci - - - name: Build - if: steps.changes.outputs.decision != 'skip' - run: npm run build + run: docker build --target ui-builder -f Dockerfile . From 6966a331507bdea2dbd27fc1065571c5ecfff018 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:09:58 -0700 Subject: [PATCH 203/480] test(vector-stores): type the pre-call hook regression tests without Any --- .../test_vector_store_pre_call_hook.py | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/tests/test_litellm/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py b/tests/test_litellm/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py index 9c0ed38f1a3..4dd97d22822 100644 --- a/tests/test_litellm/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py +++ b/tests/test_litellm/integrations/vector_store_integrations/test_vector_store_pre_call_hook.py @@ -1,6 +1,7 @@ import logging +from collections.abc import Iterator from dataclasses import dataclass, field -from typing import Any +from typing import Protocol import pytest @@ -9,6 +10,7 @@ from litellm._logging import verbose_logger from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( VectorStorePreCallHook, ) +from litellm.types.llms.openai import AllMessageValues from litellm.types.vector_stores import ( VectorStoreResultContent, VectorStoreSearchResponse, @@ -36,11 +38,11 @@ def _search_response(text: str) -> VectorStoreSearchResponse: @dataclass class RecordingRouter: failing_vector_store_ids: frozenset[str] = frozenset() - calls: list[dict[str, Any]] = field(default_factory=list) + calls: list[dict[str, object]] = field(default_factory=list) - async def avector_store_search(self, **kwargs: Any) -> VectorStoreSearchResponse: + async def avector_store_search(self, **kwargs: object) -> VectorStoreSearchResponse: self.calls.append(kwargs) - vector_store_id = kwargs["vector_store_id"] + vector_store_id = str(kwargs["vector_store_id"]) if vector_store_id in self.failing_vector_store_ids: raise litellm.BadRequestError( message=f"no healthy deployments for {vector_store_id}", @@ -70,8 +72,12 @@ class RecordingHandler(logging.Handler): self.records.append(record) +class RegisterStores(Protocol): + def __call__(self, *vector_store_ids: str, custom_llm_provider: str = "bedrock") -> None: ... + + @pytest.fixture -def registry_with(monkeypatch: pytest.MonkeyPatch): +def registry_with(monkeypatch: pytest.MonkeyPatch) -> RegisterStores: def _register(*vector_store_ids: str, custom_llm_provider: str = "bedrock") -> None: monkeypatch.setattr( litellm, @@ -88,7 +94,7 @@ def registry_with(monkeypatch: pytest.MonkeyPatch): @pytest.fixture -def warnings(): +def warnings() -> Iterator[list[logging.LogRecord]]: handler = RecordingHandler() verbose_logger.addHandler(handler) yield handler.records @@ -96,11 +102,15 @@ def warnings(): class FakeLoggingObj: - def __init__(self, metadata: dict[str, Any]) -> None: - self.model_call_details: dict[str, Any] = {"litellm_params": {"metadata": metadata}} + def __init__(self, metadata: dict[str, str]) -> None: + self.model_call_details: dict[str, object] = {"litellm_params": {"metadata": metadata}} -async def _run_hook(hook: VectorStorePreCallHook, vector_store_ids: list[str], logging_obj: FakeLoggingObj): +async def _run_hook( + hook: VectorStorePreCallHook, + vector_store_ids: list[str], + logging_obj: FakeLoggingObj, +) -> tuple[str, list[AllMessageValues], dict[str, object]]: return await hook.async_get_chat_completion_prompt( model="chat-model", messages=[{"role": "user", "content": "what is litellm?"}], @@ -113,7 +123,9 @@ async def _run_hook(hook: VectorStorePreCallHook, vector_store_ids: list[str], l @pytest.mark.asyncio -async def test_hook_searches_through_the_injected_router_with_the_request_metadata(registry_with): +async def test_hook_searches_through_the_injected_router_with_the_request_metadata( + registry_with: RegisterStores, +) -> None: """Regression (LIT-6752): the hook must reach the Router through its injected runtime, not a proxy_server import.""" registry_with("vs-router") router = RecordingRouter() @@ -137,7 +149,10 @@ async def test_hook_searches_through_the_injected_router_with_the_request_metada @pytest.mark.asyncio -async def test_hook_falls_back_to_the_sdk_when_the_runtime_has_no_router(registry_with, warnings): +async def test_hook_falls_back_to_the_sdk_when_the_runtime_has_no_router( + registry_with: RegisterStores, + warnings: list[logging.LogRecord], +) -> None: registry_with("vs-sdk", custom_llm_provider="lit6752-not-a-provider") _, messages, _ = await _run_hook( @@ -157,7 +172,7 @@ async def test_hook_falls_back_to_the_sdk_when_the_runtime_has_no_router(registr @pytest.mark.asyncio -async def test_every_healthy_vector_store_contributes_its_own_context(registry_with): +async def test_every_healthy_vector_store_contributes_its_own_context(registry_with: RegisterStores) -> None: """Regression (LIT-6752): each store appended its context to the original messages, so only the last one survived.""" registry_with("vs-one", "vs-two") router = RecordingRouter() @@ -176,7 +191,10 @@ async def test_every_healthy_vector_store_contributes_its_own_context(registry_w @pytest.mark.asyncio -async def test_a_failing_vector_store_warns_with_its_id_and_the_other_stores_still_answer(registry_with, warnings): +async def test_a_failing_vector_store_warns_with_its_id_and_the_other_stores_still_answer( + registry_with: RegisterStores, + warnings: list[logging.LogRecord], +) -> None: """Regression (LIT-6752): one unreachable store must not silently drop every other store's context.""" registry_with("vs-broken", "vs-healthy") router = RecordingRouter(failing_vector_store_ids=frozenset({"vs-broken"})) @@ -188,9 +206,12 @@ async def test_a_failing_vector_store_warns_with_its_id_and_the_other_stores_sti logging_obj, ) + search_results = logging_obj.model_call_details["search_results"] + assert [call["vector_store_id"] for call in router.calls] == ["vs-broken", "vs-healthy"] assert messages[0]["content"] == "Context:\n\ncontext from vs-healthy\n\n" - assert len(logging_obj.model_call_details["search_results"]) == 1 + assert isinstance(search_results, list) + assert len(search_results) == 1 assert [record.getMessage() for record in warnings] == [ "Vector store search failed for vector_store_id=vs-broken, continuing without its context: " "litellm.BadRequestError: no healthy deployments for vs-broken" @@ -198,7 +219,10 @@ async def test_a_failing_vector_store_warns_with_its_id_and_the_other_stores_sti @pytest.mark.asyncio -async def test_the_only_vector_store_failing_leaves_the_messages_untouched(registry_with, warnings): +async def test_the_only_vector_store_failing_leaves_the_messages_untouched( + registry_with: RegisterStores, + warnings: list[logging.LogRecord], +) -> None: registry_with("vs-broken") original_messages = [{"role": "user", "content": "what is litellm?"}] From 6fdd3128d9700bf52094d74ab7f874476a50c091 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:16:39 -0700 Subject: [PATCH 204/480] fix(ai-gateway): build the gateway binary in the release image The Dockerfile asked cargo for --features python-config, which cannot select the litellm-ai-gateway bin target: that target carries required-features = ["server"], so cargo silently built nothing and the later COPY of /build/litellm-rust/target/release/litellm-ai-gateway had no file to copy. Turn the server feature on and name the bin explicitly so a future required-features drift fails at the cargo step instead of silently producing an empty release dir. --- litellm-rust/crates/ai-gateway/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm-rust/crates/ai-gateway/Dockerfile b/litellm-rust/crates/ai-gateway/Dockerfile index adf6fca0741..2bc3c05ad7e 100644 --- a/litellm-rust/crates/ai-gateway/Dockerfile +++ b/litellm-rust/crates/ai-gateway/Dockerfile @@ -36,12 +36,12 @@ FROM chef AS builder # whenever only gateway source changes. COPY --from=planner /build/litellm-rust/recipe.json recipe.json RUN cargo chef cook --locked --release \ - -p litellm-ai-gateway --features python-config \ + -p litellm-ai-gateway --features server,python-config \ --recipe-path recipe.json # Now copy the real sources and build the gateway binary. Deps are already cooked # above, so this step only recompiles the gateway crate. COPY litellm-rust/ . -RUN cargo build --locked --release -p litellm-ai-gateway --features python-config +RUN cargo build --locked --release -p litellm-ai-gateway --bin litellm-ai-gateway --features server,python-config # ---- Runtime ---------------------------------------------------------------- # python:3.11-slim-bookworm ships libpython3.11, matching the builder's PyO3 From f986fc52f03f60220e5d421fb30864b34bd1de45 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:16:40 -0700 Subject: [PATCH 205/480] ci(rust): lint every gateway feature and keep one checks runbook Clippy never links, so python-config's pyo3/auto-initialize needs no libpython and the gateway clippy step can cover every feature at once. The test step stays on --features server because cargo test does link and this job installs no Python. The check list existed in three places that had already drifted apart; CLAUDE.md is now the only copy and the other two point at it. --- .github/workflows/test-rust.yml | 5 +++-- litellm-rust/CLAUDE.md | 6 ++++-- litellm-rust/README.md | 11 +++-------- .../CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md | 10 ++-------- 4 files changed, 12 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index ae80155305a..9b8b132df62 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -74,8 +74,8 @@ jobs: - name: Run Clippy with Bedrock auth run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings - - name: Run Clippy with the gateway server - run: cargo clippy -p litellm-ai-gateway --all-targets --features server --locked -- -D warnings + - name: Run Clippy with all gateway features + run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings - name: Run Rust tests run: cargo test --workspace --locked @@ -83,6 +83,7 @@ jobs: - name: Run core tests with Bedrock auth run: cargo test -p litellm-core --features bedrock-auth --locked + # Not --all-features: python-config links libpython, which this job does not install. - name: Run gateway tests with the server feature run: cargo test -p litellm-ai-gateway --features server --locked diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md index be0fcdd1474..d9c944529df 100644 --- a/litellm-rust/CLAUDE.md +++ b/litellm-rust/CLAUDE.md @@ -174,10 +174,12 @@ for changes under `litellm-rust/`. ```bash cd litellm-rust cargo fmt --check +cargo clippy --workspace --all-targets -- -D warnings +cargo clippy -p litellm-core --all-targets --features bedrock-auth -- -D warnings # the ai-gateway binary + server code is behind the `server` feature -cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings -cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings +cargo clippy -p litellm-ai-gateway --all-targets --all-features -- -D warnings cargo test --workspace +cargo test -p litellm-core --features bedrock-auth # the `auth`, `routes`, `state` and `realtime` tests only exist under `server` cargo test -p litellm-ai-gateway --features server ``` diff --git a/litellm-rust/README.md b/litellm-rust/README.md index a0d79c6f0a5..e43dc7ea6ad 100644 --- a/litellm-rust/README.md +++ b/litellm-rust/README.md @@ -49,11 +49,6 @@ function per top-level route, mirroring the core entrypoints. ## Checks -Run these before pushing Rust changes. GitHub Actions runs the same checks for -changes under `litellm-rust/`. - -```bash -cargo fmt --check -cargo clippy --workspace --all-targets -- -D warnings -cargo test --workspace -``` +Run the commands under "Checks" in [CLAUDE.md](CLAUDE.md) before pushing Rust +changes. That list is the single source of truth and matches what GitHub Actions +runs for changes under `litellm-rust/`. diff --git a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md index 4a689cb9579..c0a29ab14bc 100644 --- a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md +++ b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md @@ -49,11 +49,5 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages` ## Checks before push -25. Run, and keep green: - ```bash - cd litellm-rust - cargo fmt --check - cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings - cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings - cargo test --workspace - ``` +25. Run, and keep green, the commands under "Checks" in `litellm-rust/CLAUDE.md`. + That list is the single source of truth and matches what GitHub Actions runs. From 70dc0a69a4bbd9d1c7e23c72d516dba28ab39d6c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:16:41 -0700 Subject: [PATCH 206/480] test(ai-gateway): cover the blank session id and model fallbacks The emptiness guards in on_session had no test, so the let-chain rewrite could have dropped them unnoticed. --- .../ai-gateway/src/realtime/streaming.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs b/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs index edd9338b4f2..c0d72e90b77 100644 --- a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs +++ b/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs @@ -323,6 +323,32 @@ mod tests { assert_eq!(streaming.dropped(), 0); } + #[test] + fn blank_session_id_and_model_keep_the_gateway_fallbacks() { + let mut streaming = RealTimeStreaming::new( + Vec::new(), + "call_fallback".to_string(), + "gpt-realtime".to_string(), + RequestMetadata::default(), + ); + + streaming.observe(&event( + r#"{"type":"session.created","session":{"id":"","model":""}}"#, + )); + let payload = streaming.build_payload(); + assert_eq!(payload.id, "call_fallback"); + assert_eq!(payload.litellm_call_id, "call_fallback"); + assert_eq!(payload.model, "gpt-realtime"); + + streaming.observe(&event( + r#"{"type":"session.updated","session":{"id":"sess_002","model":""}}"#, + )); + let payload = streaming.build_payload(); + assert_eq!(payload.id, "sess_002"); + assert_eq!(payload.litellm_call_id, "sess_002"); + assert_eq!(payload.model, "gpt-realtime"); + } + #[test] fn payload_serializes_with_camelcase_times_and_realtime_call_type() { let mut streaming = RealTimeStreaming::new( From 534003da03a90a9f9bab5f7ed319ac4b5016fcb4 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 2 Sep 2026 22:33:14 -0700 Subject: [PATCH 207/480] feat(ui): add 1M context auto-router preset (#39490) * feat(ui): add 1M context auto-router preset * feat(ui): use heuristic v2 for 1M preset * fix(ui): keep 1M preset test within lint budget --- .../public_endpoints/autorouter_presets.json | 22 +++++++++++++++ .../public_endpoints/test_public_endpoints.py | 10 +++++++ .../add_model/add_auto_router_tab.test.tsx | 18 +++++++++++-- .../src/lib/autorouter_presets.test.ts | 27 ++++++++++++++++++- 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/public_endpoints/autorouter_presets.json b/litellm/proxy/public_endpoints/autorouter_presets.json index c2b13b81542..6a6642d4211 100644 --- a/litellm/proxy/public_endpoints/autorouter_presets.json +++ b/litellm/proxy/public_endpoints/autorouter_presets.json @@ -1,4 +1,26 @@ { + "1m_context": { + "label": "1M Context", + "description": "Routes across models with 1M-token context windows: Luna for simple queries, Terra for medium, Opus 5 for complex, Opus 5 at high thinking for reasoning.", + "complexity_router_config": { + "tiers": { + "SIMPLE": ["gpt-5.6-luna"], + "MEDIUM": ["gpt-5.6-terra"], + "COMPLEX": ["claude-opus-5"], + "REASONING": ["claude-opus-5"] + }, + "tier_model_configs": { + "REASONING": [{ "model_name": "claude-opus-5", "litellm_params": { "reasoning_effort": "high" } }] + }, + "classifier_type": "heuristic_v2", + "escalation_keywords": ["LITELLM ESCALATE"], + "classification_mode": "every_request", + "session_affinity": false, + "modality_routing": false, + "modality_pin_override": false, + "deployment_affinity": true + } + }, "anthropic_family": { "label": "Anthropic Family", "description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex, Opus at high thinking for reasoning.", diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 551e27a18f5..41439f28638 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -1104,6 +1104,16 @@ def test_get_autorouter_presets_local_mode_serves_bundled_catalog( assert response.status_code == 200 payload = response.json() assert "anthropic_family" in payload + assert payload["1m_context"]["complexity_router_config"]["classifier_type"] == "heuristic_v2" + assert payload["1m_context"]["complexity_router_config"]["tiers"] == { + "SIMPLE": ["gpt-5.6-luna"], + "MEDIUM": ["gpt-5.6-terra"], + "COMPLEX": ["claude-opus-5"], + "REASONING": ["claude-opus-5"], + } + assert payload["1m_context"]["complexity_router_config"]["tier_model_configs"] == { + "REASONING": [{"model_name": "claude-opus-5", "litellm_params": {"reasoning_effort": "high"}}] + } for preset in payload.values(): assert isinstance(preset["label"], str) assert isinstance(preset["description"], str) diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 27a169524b9..836b4c6ff3d 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -636,7 +636,14 @@ describe("AddAutoRouterTab", () => { const labels = visibleOptions().map((option) => option.querySelector(".font-medium")?.textContent); - expect(labels).toEqual(["Anthropic Family", "Gemini Family", "Lite", "OpenAI Family", "Custom Configuration"]); + expect(labels).toEqual([ + "1M Context", + "Anthropic Family", + "Gemini Family", + "Lite", + "OpenAI Family", + "Custom Configuration", + ]); }); describe("routing test", () => { @@ -1060,7 +1067,14 @@ describe("AddAutoRouterTab", () => { expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); }); const labels = visibleOptions().map((option) => option.querySelector(".font-medium")?.textContent); - expect(labels).toEqual(["Anthropic Family", "Gemini Family", "Lite", "OpenAI Family", "Custom Configuration"]); + expect(labels).toEqual([ + "Anthropic Family", + "1M Context", + "Gemini Family", + "Lite", + "OpenAI Family", + "Custom Configuration", + ]); }); it.each([ diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 44ac6973a1c..a7333e4704b 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -28,7 +28,13 @@ const getPresetByKey = (key: string): AutoRouterPreset | undefined => PRESETS.fi describe("autorouter_presets", () => { it("hydrates exactly the bundled presets", () => { const presets = getAllPresets(); - expect(presets.map((p) => p.label).sort()).toEqual(["Anthropic Family", "Gemini Family", "Lite", "OpenAI Family"]); + expect(presets.map((p) => p.label).sort()).toEqual([ + "1M Context", + "Anthropic Family", + "Gemini Family", + "Lite", + "OpenAI Family", + ]); // Every preset carries all four fields the UI relies on; a JSON typo dropping one fails here. for (const p of presets) { expect(p).toMatchObject({ key: expect.any(String), label: expect.any(String), description: expect.any(String) }); @@ -202,6 +208,25 @@ describe("autorouter_presets", () => { }); }); + it("pins the 1M context preset to Luna, Terra, and Opus at high thinking", () => { + const preset = getPresetByKey("1m_context")!; + const expectedTiers = { + SIMPLE: ["gpt-5.6-luna"], + MEDIUM: ["gpt-5.6-terra"], + COMPLEX: ["claude-opus-5"], + REASONING: ["claude-opus-5"], + }; + expect(preset.complexity_router_config.classifier_type).toBe("heuristic_v2"); + expect(preset.complexity_router_config.tiers).toEqual(expectedTiers); + expect(preset.complexity_router_config.tier_model_configs).toEqual({ + REASONING: [{ model_name: "claude-opus-5", litellm_params: { reasoning_effort: "high" } }], + }); + const prefill = buildPresetPrefill(preset.complexity_router_config, groupsOnly(getRequiredModelsInPreset(preset))); + expect(prefill.complexityRouterConfig.tier_model_params).toEqual({ + REASONING: { "claude-opus-5": { reasoning_effort: "high" } }, + }); + }); + it("pins the gemini preset to concrete model ids, never Google's hot-swapping -latest aliases", () => { const gemini = getPresetByKey("gemini_family")!; const config = gemini.complexity_router_config; From c841a56e9ab930e86ba6c484444f4e56eb0d7bab Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 2 Sep 2026 22:34:15 -0700 Subject: [PATCH 208/480] fix(ui): stop the create team form resetting organization and models (#39476) * fix(ui): stop the create team form resetting organization and models The organization preselect ran in an effect keyed on the organizations query, so any refetch of that list while the Create Team modal was open overwrote the user's organization pick, which in turn cleared their models pick. The models field was also cleared whenever the available models fetch resolved. Preselect the organization when the modal opens instead, and clear the models only when the user picks a different organization. An org admin whose admin orgs narrow to one while the form is open can still pick, rather than facing a locked empty field. * fix(ui): block team create when the picked organization is no longer available An organization picked in the Create Team form now survives a refetch of the organization list, so it can outlive the admin's access to it. Refuse the create with a message on the field rather than letting the request fail authorization at the proxy. * fix(ui): keep the team create organization field usable when the pick goes stale Locking the field on a single admin organization also locked it while it held a rejected organization, so an admin who lost access could not pick the one organization left. Lock it only while it holds that organization. * test(ui): hoist the created team fixture out of the mock call The inline object pushed the repo past its no-large-inline-object-arg lint budget, which has no headroom. --- ui/litellm-dashboard/eslint-suppressions.json | 3 - .../src/components/Teams.test.tsx | 202 ++++++++++++++++++ ui/litellm-dashboard/src/components/Teams.tsx | 74 ++++--- 3 files changed, 243 insertions(+), 36 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 7de7373b20b..7d7066addcf 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1399,9 +1399,6 @@ }, "prefer-const": { "count": 2 - }, - "react-hooks/set-state-in-effect": { - "count": 1 } }, "src/components/TeamsPage/teamTableColumns.tsx": { diff --git a/ui/litellm-dashboard/src/components/Teams.test.tsx b/ui/litellm-dashboard/src/components/Teams.test.tsx index bee35ac5c5d..2bceb00aae1 100644 --- a/ui/litellm-dashboard/src/components/Teams.test.tsx +++ b/ui/litellm-dashboard/src/components/Teams.test.tsx @@ -15,6 +15,7 @@ import { teamCreateCall, } from "./networking"; import Teams from "./Teams"; +import { chooseSelectOption } from "../../tests/test-utils"; const can = vi.fn(); vi.mock("@/app/(dashboard)/hooks/useCan", () => ({ @@ -1488,3 +1489,204 @@ describe("Teams - the exact bytes the create call sends", () => { expect(teamCreateCall).not.toHaveBeenCalled(); }); }); + +describe("Teams - the create form keeps the organization and models picks while it is open", () => { + const ORGS = [ + { organization_id: "org-1", organization_alias: "Org 1", models: [], members: [] }, + { organization_id: "org-2", organization_alias: "Org 2", models: [], members: [] }, + ]; + + const orgField = () => screen.getByRole("combobox", { name: /organization/i }); + const modelsField = () => screen.getByTestId("create-team-models-select"); + + const openCreateModal = async () => { + act(() => { + fireEvent.click(screen.getAllByRole("button", { name: /create team/i })[0]); + }); + await screen.findByLabelText(/team name/i); + }; + + beforeEach(() => { + vi.clearAllMocks(); + mockTeamInfoView.mockClear(); + vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue(["gpt-4", "gpt-3.5-turbo"]); + vi.mocked(fetchMCPAccessGroups).mockResolvedValue([]); + vi.mocked(getGuardrailsList).mockResolvedValue({ guardrails: [] }); + vi.mocked(getDefaultTeamSettings).mockResolvedValue({ values: {} }); + mockUseOrganizations.mockReturnValue({ data: ORGS }); + }); + + it("keeps both picks when the organizations list comes back changed from a refetch", async () => { + const user = userEvent.setup(); + renderWithQueryClient(); + await openCreateModal(); + + await chooseSelectOption(user, orgField(), /Org 1/); + fireEvent.change(modelsField(), { target: { value: "gpt-4" } }); + + mockUseOrganizations.mockReturnValue({ data: ORGS.map((org) => ({ ...org, spend: 1 })) }); + fireEvent.click(screen.getByText("Additional Settings")); + + expect(orgField()).toHaveValue("Org 1"); + expect(modelsField()).toHaveValue("gpt-4"); + }); + + it("keeps models picked before the available models finish loading", async () => { + let resolveModels: (models: string[]) => void = () => {}; + vi.mocked(fetchAvailableModelsForTeamOrKey).mockReturnValue( + new Promise((resolve) => { + resolveModels = resolve; + }), + ); + renderWithQueryClient(); + await openCreateModal(); + + fireEvent.change(modelsField(), { target: { value: "gpt-4" } }); + await act(async () => { + resolveModels(["gpt-4", "gpt-3.5-turbo"]); + }); + + expect(modelsField()).toHaveValue("gpt-4"); + }); + + it("clears the models pick when the organization is changed, since models are org scoped", async () => { + const user = userEvent.setup(); + renderWithQueryClient(); + await openCreateModal(); + + await chooseSelectOption(user, orgField(), /Org 1/); + fireEvent.change(modelsField(), { target: { value: "gpt-4" } }); + await chooseSelectOption(user, orgField(), /Org 2/); + + await waitFor(() => expect(orgField()).toHaveValue("Org 2")); + expect(modelsField()).toHaveValue(""); + }); + + it("keeps the models pick when the same organization is chosen again", async () => { + const user = userEvent.setup(); + renderWithQueryClient(); + await openCreateModal(); + + await chooseSelectOption(user, orgField(), /Org 1/); + fireEvent.change(modelsField(), { target: { value: "gpt-4" } }); + await chooseSelectOption(user, orgField(), /Org 1/); + + expect(orgField()).toHaveValue("Org 1"); + expect(modelsField()).toHaveValue("gpt-4"); + }); + + it("still preselects the only organization an org admin can create teams in", async () => { + mockUseOrganizations.mockReturnValue({ + data: [ + { + organization_id: "org-1", + organization_alias: "Org 1", + models: [], + members: [{ user_id: "user-123", user_role: "org_admin" }], + }, + ], + }); + renderWithQueryClient(); + await openCreateModal(); + + expect(orgField()).toHaveValue("Org 1"); + expect(orgField()).toBeDisabled(); + }); + + it("leaves an org admin able to pick when their admin orgs narrow to one while the form is open", async () => { + const orgAdminOrgs = [ + { + organization_id: "org-1", + organization_alias: "Org 1", + models: [], + members: [{ user_id: "user-123", user_role: "org_admin" }], + }, + { + organization_id: "org-2", + organization_alias: "Org 2", + models: [], + members: [{ user_id: "user-123", user_role: "org_admin" }], + }, + ]; + mockUseOrganizations.mockReturnValue({ data: orgAdminOrgs }); + renderWithQueryClient(); + await openCreateModal(); + expect(orgField()).toHaveValue(""); + + mockUseOrganizations.mockReturnValue({ data: [orgAdminOrgs[0]] }); + fireEvent.click(screen.getByText("Additional Settings")); + + expect(orgField()).toBeEnabled(); + }); + + it("refuses to create the team in an organization the admin has lost access to", async () => { + const user = userEvent.setup(); + const orgAdminOrgs = ORGS.map((org) => ({ ...org, members: [{ user_id: "user-123", user_role: "org_admin" }] })); + mockUseOrganizations.mockReturnValue({ data: orgAdminOrgs }); + renderWithQueryClient(); + await openCreateModal(); + + fireEvent.change(screen.getByTestId("team-name-input"), { target: { value: "Revoked Team" } }); + await chooseSelectOption(user, orgField(), /Org 1/); + + mockUseOrganizations.mockReturnValue({ data: [orgAdminOrgs[1]] }); + fireEvent.click(screen.getByText("Additional Settings")); + + const submitButtons = screen.getAllByRole("button", { name: /create team/i }); + fireEvent.click(submitButtons[submitButtons.length - 1]); + + await screen.findByText(/no longer create teams in this organization/i); + expect(teamCreateCall).not.toHaveBeenCalled(); + }); + + it("lets the admin switch to the one organization left after losing access to their pick", async () => { + const user = userEvent.setup(); + const orgAdminOrgs = ORGS.map((org) => ({ ...org, members: [{ user_id: "user-123", user_role: "org_admin" }] })); + mockUseOrganizations.mockReturnValue({ data: orgAdminOrgs }); + const createdTeam = { + team_id: "new-team-1", + team_alias: "Recovered Team", + models: [], + organization_id: "org-2", + keys: [], + members_with_roles: [], + spend: 0, + }; + vi.mocked(teamCreateCall).mockResolvedValue(createdTeam); + renderWithQueryClient(); + await openCreateModal(); + + fireEvent.change(screen.getByTestId("team-name-input"), { target: { value: "Recovered Team" } }); + await chooseSelectOption(user, orgField(), /Org 1/); + + mockUseOrganizations.mockReturnValue({ data: [orgAdminOrgs[1]] }); + fireEvent.click(screen.getByText("Additional Settings")); + + expect(orgField()).toBeEnabled(); + await chooseSelectOption(user, orgField(), /Org 2/); + const submitButtons = screen.getAllByRole("button", { name: /create team/i }); + fireEvent.click(submitButtons[submitButtons.length - 1]); + + await waitFor(() => + expect(teamCreateCall).toHaveBeenCalledWith( + "test-token", + expect.objectContaining({ team_alias: "Recovered Team", organization_id: "org-2" }), + ), + ); + }); + + it("starts the form clean again when the modal is closed and reopened", async () => { + const user = userEvent.setup(); + renderWithQueryClient(); + await openCreateModal(); + + await chooseSelectOption(user, orgField(), /Org 1/); + fireEvent.change(modelsField(), { target: { value: "gpt-4" } }); + fireEvent.click(screen.getByRole("button", { name: /^close$/i })); + await waitFor(() => expect(screen.queryByLabelText(/team name/i)).not.toBeInTheDocument()); + + await openCreateModal(); + expect(orgField()).toHaveValue(""); + expect(modelsField()).toHaveValue(""); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index e2eda4adb23..ef58237a6aa 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -208,7 +208,6 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser const queryClient = useQueryClient(); const refreshTeams = () => queryClient.invalidateQueries({ queryKey: teamsTableKeys.all }); const [currentOrg] = useState(null); - const [currentOrgForCreateTeam, setCurrentOrgForCreateTeam] = useState(null); const isOrgAdmin = userRole !== "Admin"; const [additionalSettingsOpen, setAdditionalSettingsOpen] = useState(false); @@ -216,17 +215,33 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser const [agentSettingsOpen, setAgentSettingsOpen] = useState(false); const [searchToolSettingsOpen, setSearchToolSettingsOpen] = useState(false); + const adminOrgs = useMemo( + () => getAdminOrganizations(userRole, userID, organizations), + [userRole, userID, organizations], + ); + const teamCreateSchema = useMemo( () => teamCreateFieldsSchema.superRefine((values, ctx) => { if (isOrgAdmin && !values.organization_id) { ctx.addIssue({ code: "custom", message: SUPPRESSED_BY_DESCRIPTION, path: ["organization_id"] }); } + const organizationIsStillPickable = + values.organization_id == null || + organizations == null || + adminOrgs.some((org) => org.organization_id === values.organization_id); + if (!organizationIsStillPickable) { + ctx.addIssue({ + code: "custom", + message: "You can no longer create teams in this organization", + path: ["organization_id"], + }); + } if (additionalSettingsOpen && !isParsableJson(values.secret_manager_settings)) { ctx.addIssue({ code: "custom", message: SUPPRESSED_BY_DESCRIPTION, path: ["secret_manager_settings"] }); } }), - [isOrgAdmin, additionalSettingsOpen], + [isOrgAdmin, additionalSettingsOpen, adminOrgs, organizations], ); const form = useZodForm(teamCreateSchema, { defaultValues: EMPTY_TEAM_CREATE_VALUES }); @@ -264,28 +279,6 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser ? `Default: ${getBudgetDurationLabel(defaultBudgetDuration)} (${defaultBudgetDuration})` : "n/a"; - useEffect(() => { - form.setValue("models", []); - }, [currentOrgForCreateTeam, userModels]); - - // Handle organization preselection when modal opens - useEffect(() => { - if (isTeamModalVisible) { - const adminOrgs = getAdminOrganizations(userRole, userID, organizations); - - // Org admins must scope a team to an org, so with exactly one we preselect it. - // Proxy admins can create org-less teams, so the field stays optional regardless of org count. - if (isOrgAdmin && adminOrgs.length === 1) { - const org = adminOrgs[0]; - form.setValue("organization_id", org.organization_id); - setCurrentOrgForCreateTeam(org); - } else { - form.setValue("organization_id", currentOrg?.organization_id || null); - setCurrentOrgForCreateTeam(currentOrg); - } - } - }, [isTeamModalVisible, isOrgAdmin, userRole, userID, organizations, currentOrg]); - // Add this useEffect to fetch guardrails useEffect(() => { const fetchGuardrails = async () => { @@ -320,6 +313,26 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser if (canViewPolicies) fetchPolicies(); }, [accessToken, canViewPolicies]); + const openCreateTeamModal = () => { + // Org admins must scope a team to an org, so with exactly one we preselect it. + // Proxy admins can create org-less teams, so the field stays optional regardless of org count. + if (isOrgAdmin && adminOrgs.length === 1) { + form.setValue("organization_id", adminOrgs[0].organization_id); + } + setIsTeamModalVisible(true); + }; + + const selectCreateTeamOrganization = ( + next: string, + currentOrganizationId: string | null, + onChange: (organizationId: string | null) => void, + ) => { + const nextOrganizationId = next === "" ? null : next; + if (nextOrganizationId === currentOrganizationId) return; + onChange(nextOrganizationId); + form.setValue("models", []); + }; + const resetCreateForm = () => { form.reset(EMPTY_TEAM_CREATE_VALUES); setAdditionalSettingsOpen(false); @@ -636,7 +649,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser subtitle="Manage teams, members, and their access to models and budgets" primaryAction={ canCreateOrManageTeams(userRole, userID, organizations) ? ( - setIsTeamModalVisible(true)} data-testid="create-team-button"> + Create Team @@ -683,9 +696,9 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser )} {(() => { - const adminOrgs = getAdminOrganizations(userRole, userID, organizations); const isSingleOrg = adminOrgs.length === 1; const hasNoOrgs = adminOrgs.length === 0; + const soleOrganizationId = isSingleOrg ? adminOrgs[0].organization_id ?? null : null; return ( <> @@ -715,18 +728,13 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser label: org.organization_alias ?? "", sublabel: org.organization_id ?? "", }))} - disabled={isOrgAdmin && isSingleOrg} + disabled={isOrgAdmin && soleOrganizationId !== null && value === soleOrganizationId} allowClear={!isOrgAdmin} placeholder={ hasNoOrgs ? "No organizations available" : "Search or select an Organization" } emptyText="No organizations available" - onValueChange={(next) => { - onChange(next === "" ? null : next); - setCurrentOrgForCreateTeam( - adminOrgs.find((org) => org.organization_id === next) ?? null, - ); - }} + onValueChange={(next) => selectCreateTeamOrganization(next, value ?? null, onChange)} /> )} From 3cac5e5cd4c12a782e0afe96218aaff986ef3f60 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 2 Sep 2026 22:44:17 -0700 Subject: [PATCH 209/480] fix(sso): resolve multi-valued role claims to the highest privilege role (#39480) * fix(sso): resolve multi-valued role claims to the highest privilege role A role claim carrying several roles used to resolve to whichever one the IdP listed first, so a user holding both proxy_admin_viewer and internal_user lost org-level spend visibility depending on claim ordering alone. get_litellm_user_role now picks the highest privilege role out of a list-valued claim, and the Entra app_roles path shares that same resolution instead of keeping its own copy of the hierarchy. SAML assertions carrying several role values go through the same path rather than taking the first value. * test(sso): lock ranked-over-unranked role resolution for mixed claims org_admin, team and customer sit outside the privilege ladder. Pin the resolution for a claim that mixes one of them with a ranked role so the asymmetry is covered rather than implicit. * fix(sso): label the claim-sequence cast for the type-discipline gate * fix(sso): resolve claim entries without recursing The repo's recursive-function gate rejects self-recursion here, and a role claim is flat anyway. Pull the single-value lookup into its own helper so the list branch maps over it instead of calling back into itself. --- .../management_endpoints/sso/saml_sso.py | 6 +- litellm/proxy/management_endpoints/types.py | 62 +++++++-- litellm/proxy/management_endpoints/ui_sso.py | 19 +-- ruff-strict-budget.json | 4 +- .../management_endpoints/test_saml_sso.py | 33 +++++ .../proxy/management_endpoints/test_ui_sso.py | 129 +++++++++++++++++- type-discipline-budget.json | 4 +- 7 files changed, 218 insertions(+), 39 deletions(-) diff --git a/litellm/proxy/management_endpoints/sso/saml_sso.py b/litellm/proxy/management_endpoints/sso/saml_sso.py index 466b100ea1f..12e1f1a03f3 100644 --- a/litellm/proxy/management_endpoints/sso/saml_sso.py +++ b/litellm/proxy/management_endpoints/sso/saml_sso.py @@ -443,7 +443,9 @@ class SAMLAuthHandler: last_name: Final = SAMLAuthHandler._attribute_value( attributes, "SAML_ATTRIBUTE_LAST_NAME", _LAST_NAME_ATTRIBUTE_CANDIDATES ) - role_value = SAMLAuthHandler._attribute_value(attributes, "SAML_ATTRIBUTE_ROLE", _ROLE_ATTRIBUTE_CANDIDATES) + role_values: Final = SAMLAuthHandler._attribute_values( + attributes, "SAML_ATTRIBUTE_ROLE", _ROLE_ATTRIBUTE_CANDIDATES + ) team_ids: Final = SAMLAuthHandler._attribute_values( attributes, "SAML_ATTRIBUTE_TEAM_IDS", _TEAM_IDS_ATTRIBUTE_CANDIDATES ) @@ -464,7 +466,7 @@ class SAMLAuthHandler: picture=None, provider="saml", team_ids=team_ids, - user_role=get_litellm_user_role(role_value) if role_value else None, + user_role=get_litellm_user_role(role_values), ) except ValidationError as e: raise HTTPException( diff --git a/litellm/proxy/management_endpoints/types.py b/litellm/proxy/management_endpoints/types.py index 070df97d09a..4414eed97b2 100644 --- a/litellm/proxy/management_endpoints/types.py +++ b/litellm/proxy/management_endpoints/types.py @@ -4,12 +4,44 @@ Types for the management endpoints Might include fastapi/proxy requirements.txt related imports """ +from collections.abc import Iterable, Sequence from typing import Any, Final, cast from fastapi_sso.sso.base import OpenID from litellm.proxy._types import LitellmUserRoles +# Ordered highest to lowest privilege +LITELLM_USER_ROLE_HIERARCHY: Final = ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, +) + + +def highest_privilege_role(roles: Iterable[LitellmUserRoles]) -> LitellmUserRoles | None: + """ + Pick the highest privilege role out of the roles an IdP asserted for one user. + + IdPs do not guarantee ordering within a multi-valued role claim, so a user holding + several roles resolves to the most privileged one rather than whichever came first. + Roles the hierarchy does not rank (org_admin, team, customer) resolve by name to stay + deterministic. + + Args: + roles: The roles resolved from the claim + + Returns: + The highest privilege role, or None if `roles` is empty + """ + resolved: Final = frozenset(roles) + if not resolved: + return None + + ranked: Final = next((role for role in LITELLM_USER_ROLE_HIERARCHY if role in resolved), None) + return ranked if ranked is not None else min(resolved, key=lambda role: role.value) + def is_valid_litellm_user_role(role_str: str) -> bool: """ @@ -28,12 +60,22 @@ def is_valid_litellm_user_role(role_str: str) -> bool: return False -def get_litellm_user_role(role_str) -> LitellmUserRoles | None: +def _role_from_claim_value(role_str: object) -> LitellmUserRoles | None: + if not isinstance(role_str, str): + return None + # Use _value2member_map_ for O(1) lookup, case-insensitive + result: Final = LitellmUserRoles._value2member_map_.get(role_str.lower()) + return cast(LitellmUserRoles | None, result) + + +def get_litellm_user_role(role_str: object) -> LitellmUserRoles | None: """ Convert a string (or list of strings) to a LitellmUserRoles enum if valid (case-insensitive). Handles list inputs since some SSO providers (e.g., Keycloak) return roles - as arrays like ["proxy_admin"] instead of plain strings. + as arrays like ["proxy_admin"] instead of plain strings. A claim carrying several + roles resolves to the highest privilege one, so a user does not lose access just + because the IdP listed a weaker role first. Args: role_str: String or list to convert (e.g., "proxy_admin", ["proxy_admin"]) @@ -41,16 +83,12 @@ def get_litellm_user_role(role_str) -> LitellmUserRoles | None: Returns: LitellmUserRoles enum if valid, None otherwise """ - try: - if isinstance(role_str, list): - if len(role_str) == 0: - return None - role_str = role_str[0] - # Use _value2member_map_ for O(1) lookup, case-insensitive - result: Final = LitellmUserRoles._value2member_map_.get(role_str.lower()) - return cast(LitellmUserRoles | None, result) - except Exception: - return None + if isinstance(role_str, (list, tuple)): + entries: Final = cast(Sequence[object], role_str) # cast-ok: isinstance narrows the claim, not its elements + return highest_privilege_role( + role for role in (_role_from_claim_value(entry) for entry in entries) if role is not None + ) + return _role_from_claim_value(role_str) class CustomOpenID(OpenID): diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 1feefa5725d..84150ef7935 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -112,6 +112,7 @@ from litellm.proxy.management_endpoints.sso_helper_utils import ( ) from litellm.proxy.management_endpoints.team_endpoints import new_team, team_member_add from litellm.proxy.management_endpoints.types import ( + LITELLM_USER_ROLE_HIERARCHY, CustomOpenID, get_litellm_user_role, is_valid_litellm_user_role, @@ -809,15 +810,6 @@ def normalize_email(email: str | None) -> str | None: return email.lower() if isinstance(email, str) else email -# Ordered highest to lowest privilege -LITELLM_USER_ROLE_HIERARCHY: Final = ( - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - LitellmUserRoles.INTERNAL_USER, - LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, -) - - def determine_role_from_groups( user_groups: list[str], role_mappings: "RoleMappings", @@ -4312,14 +4304,7 @@ class MicrosoftSSOHandler: listed first. Roles the hierarchy does not rank (org_admin, team, customer) resolve by name to stay deterministic """ - resolved: Final = frozenset( - role for role in (get_litellm_user_role(role_str) for role_str in app_roles or ()) if role is not None - ) - if not resolved: - return None - - ranked: Final = next((role for role in LITELLM_USER_ROLE_HIERARCHY if role in resolved), None) - return ranked if ranked is not None else min(resolved, key=lambda role: role.value) + return get_litellm_user_role(tuple(app_roles or ())) @staticmethod def get_app_roles_from_id_token(id_token: str | None) -> list[str]: diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 4fcf650a8bc..be2b30fc189 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,6 +1,6 @@ { "ANN001": { - "limit": 2985 + "limit": 2984 }, "ANN002": { "limit": 71 @@ -57,7 +57,7 @@ "limit": 3 }, "BLE001": { - "limit": 2917 + "limit": 2916 }, "C401": { "limit": 8 diff --git a/tests/test_litellm/proxy/management_endpoints/test_saml_sso.py b/tests/test_litellm/proxy/management_endpoints/test_saml_sso.py index 36d6414ba9f..635e6332958 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_saml_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_saml_sso.py @@ -516,6 +516,39 @@ async def test_team_ids_extracted_from_groups_attribute(saml_env_idp_initiated): assert result.team_ids == ["team-a", "team-b"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "roles", + [ + ["internal_user", "proxy_admin_viewer"], + ["proxy_admin_viewer", "internal_user"], + ], +) +async def test_multi_valued_role_attribute_resolves_to_highest_privilege(saml_env_idp_initiated, roles): + """An assertion carrying several roles must not depend on the order the IdP emitted them in.""" + key_pem, cert_pem = saml_env_idp_initiated + resp = _build_signed_response( + key_pem, + cert_pem, + attributes={ + "email": ["dave@example.com"], + "role": roles, + }, + ) + + result = await _acs(_b64(resp), _shared_cache()) + assert result.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + + +@pytest.mark.asyncio +async def test_assertion_without_role_attribute_has_no_user_role(saml_env_idp_initiated): + key_pem, cert_pem = saml_env_idp_initiated + resp = _build_signed_response(key_pem, cert_pem, attributes={"email": ["erin@example.com"]}) + + result = await _acs(_b64(resp), _shared_cache()) + assert result.user_role is None + + @pytest.mark.asyncio async def test_build_login_redirect_targets_idp_and_caches_request_id(saml_env): cache = DualCache() diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index dd8c752a868..5dfff53f7c3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -6598,13 +6598,94 @@ def test_get_litellm_user_role_with_invalid_role(): assert result is None -def test_get_litellm_user_role_with_list_multiple_roles(): - """Test that get_litellm_user_role takes the first element from a multi-element list.""" +@pytest.mark.parametrize( + "role_claim", + [ + ["proxy_admin", "internal_user"], + ["internal_user", "proxy_admin"], + ], +) +def test_get_litellm_user_role_picks_highest_privilege_regardless_of_order(role_claim): + """A multi-valued role claim resolves to the most privileged role, not the first one listed.""" from litellm.proxy._types import LitellmUserRoles from litellm.proxy.management_endpoints.types import get_litellm_user_role - result = get_litellm_user_role(["proxy_admin", "internal_user"]) - assert result == LitellmUserRoles.PROXY_ADMIN + assert get_litellm_user_role(role_claim) == LitellmUserRoles.PROXY_ADMIN + + +@pytest.mark.parametrize( + "role_claim", + [ + ["proxy_admin_viewer", "internal_user"], + ["internal_user", "proxy_admin_viewer"], + ], +) +def test_get_litellm_user_role_keeps_org_spend_visibility_for_mixed_roles(role_claim): + """ + Regression for LIT-6077: a user holding both proxy_admin_viewer and internal_user kept + losing org-level spend visibility whenever the IdP happened to list internal_user first. + """ + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.management_endpoints.types import get_litellm_user_role + + assert get_litellm_user_role(role_claim) == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + + +def test_get_litellm_user_role_ignores_unrecognised_entries(): + """Roles LiteLLM does not know about are skipped rather than swallowing the whole claim.""" + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.management_endpoints.types import get_litellm_user_role + + assert get_litellm_user_role(["some_idp_group", "internal_user"]) == LitellmUserRoles.INTERNAL_USER + assert get_litellm_user_role(["some_idp_group", "another_group"]) is None + + +def test_get_litellm_user_role_list_lookup_is_case_insensitive(): + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.management_endpoints.types import get_litellm_user_role + + assert get_litellm_user_role(["INTERNAL_USER", "Proxy_Admin"]) == LitellmUserRoles.PROXY_ADMIN + + +@pytest.mark.parametrize( + "role_claim", + [ + ["org_admin", "team"], + ["team", "org_admin"], + ], +) +def test_get_litellm_user_role_is_deterministic_for_unranked_roles(role_claim): + """Roles outside the privilege hierarchy still resolve the same way in either claim order.""" + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.management_endpoints.types import get_litellm_user_role + + assert get_litellm_user_role(role_claim) == LitellmUserRoles.ORG_ADMIN + + +@pytest.mark.parametrize( + "role_claim", + [ + ["org_admin", "internal_user"], + ["internal_user", "org_admin"], + ], +) +def test_get_litellm_user_role_prefers_a_ranked_role_over_an_unranked_one(role_claim): + """ + org_admin, team and customer sit outside the privilege ladder, so a claim mixing one of + them with a ranked role settles on the ranked role in either order. Same rule the Entra + app_roles and role_mappings paths already follow. + """ + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.management_endpoints.types import get_litellm_user_role + + assert get_litellm_user_role(role_claim) == LitellmUserRoles.INTERNAL_USER + + +def test_get_litellm_user_role_returns_none_for_non_string_claims(): + from litellm.proxy.management_endpoints.types import get_litellm_user_role + + assert get_litellm_user_role(None) is None + assert get_litellm_user_role({"role": "proxy_admin"}) is None # ============================================================================ @@ -6654,6 +6735,46 @@ def test_process_sso_jwt_access_token_extracts_role_from_access_token(): assert result.user_role == LitellmUserRoles.PROXY_ADMIN +@pytest.mark.parametrize( + "role_claim", + [ + ["internal_user", "proxy_admin_viewer"], + ["proxy_admin_viewer", "internal_user"], + ], +) +def test_process_sso_jwt_access_token_resolves_highest_privilege_role(role_claim): + """ + The generic SSO access-token path must land on the same role for a user whose role + claim holds several roles, whichever order the IdP emitted them in. + """ + import jwt as pyjwt + + from litellm.proxy._types import LitellmUserRoles + + access_token_str = pyjwt.encode( + {"sub": "user-123", "email": "mixed@test.com", "litellm_role": role_claim}, + "secret", + algorithm="HS256", + ) + result = CustomOpenID( + id="user-123", + email="mixed@test.com", + display_name="Mixed Role User", + team_ids=[], + user_role=None, + ) + + with patch.dict(os.environ, {"GENERIC_USER_ROLE_ATTRIBUTE": "litellm_role"}): + process_sso_jwt_access_token( + access_token_str=access_token_str, + sso_jwt_handler=None, + result=result, + role_mappings=None, + ) + + assert result.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + + def test_process_sso_jwt_access_token_does_not_override_existing_role(): """ Test that process_sso_jwt_access_token does NOT override a role that was diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 5c25b8722ef..d5bf3883be4 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -27,10 +27,10 @@ "limit": 0 }, "LIT010": { - "limit": 16478 + "limit": 16477 }, "LIT011": { - "limit": 5520 + "limit": 5519 }, "LIT012": { "limit": 4489 From 47611fa207f020f521d82e248df24c6c97df0bb1 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 23:22:40 -0700 Subject: [PATCH 210/480] fix(test): drop the duplicate embedding_executor arg in the Bedrock KB fake handler Two branches independently added embedding_executor to the same fake search handler in this file, #39472 in the middle of the signature and #39474 at the end. Neither conflicted with the other, so both edits merged and the function ended up declaring the parameter twice. Python rejects that at compile time, so the whole module fails to import and every test in the file is uncollectable, taking the logging_testing job down on staging. Keep the earlier of the two, which sits where the real handler declares the parameter. --- tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py index 63bad6d0e22..9faaaf492e8 100644 --- a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py +++ b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py @@ -376,7 +376,6 @@ async def test_bedrock_kb_request_body_has_transformed_filters( timeout=None, client=None, _is_async=False, - embedding_executor=None, ): litellm_params_dict = ( litellm_params.model_dump(exclude_none=False) From 9c795e52f3a9bc012872bc5f05ff2af4758a4d4f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:55:32 -0700 Subject: [PATCH 211/480] chore(lint): drop budget limits back to the values on the merged base The staging merge resolved three budget conflicts by keeping this branch's older, higher numbers, which turned budget-ratchet red. Nothing on the branch adds violations for those rules, so the base's limits hold. --- basedpyright-code-budget.json | 2 +- type-discipline-budget.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 4cbe9661d02..60136bb0650 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -123,7 +123,7 @@ "limit": 4 }, "reportUnnecessaryIsInstance": { - "limit": 823 + "limit": 819 }, "reportUntypedBaseClass": { "limit": 0 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 4837cc9e75f..1474171717f 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22332 + "limit": 22330 }, "LIT002": { "limit": 26760 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16480 + "limit": 16478 }, "LIT011": { "limit": 5518 From 7b8cc0319e70d90dd096fbe86830b987a67ef491 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:56:09 -0700 Subject: [PATCH 212/480] fix(proxy-extras): only spend a migrate-deploy attempt when a pass made no progress The v2 migration resolver gave `prisma migrate deploy` four attempts, and every recovery path ended in a bare `continue`, so each one burned an attempt. A database first brought up with `--use_prisma_db_push` has a full schema and no migrations ledger, so the baseline spent attempt one and the first three migrations whose objects already existed spent the rest. The proxy then exited before binding its port, and that database could never be moved onto the resolver. The retry budget now counts only attempts that got nowhere. Creating the baseline, and each migration newly marked applied, leaves the budget alone, so a push-created database works through its pre-existing objects one pass at a time. Timeouts, deadlock rollbacks, advisory-lock waits, and a repeat of a recovery that already ran still spend an attempt, so a run that stops making progress gives up exactly as before. --- .../litellm_proxy_extras/utils.py | 67 ++++++- .../test_litellm_proxy_extras_utils.py | 167 ++++++++++++++++++ 2 files changed, 226 insertions(+), 8 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index d22484bc0e8..21f255b0666 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -6,6 +6,7 @@ import shutil import subprocess import tempfile import time +from dataclasses import dataclass, replace from pathlib import Path from typing import Optional @@ -45,6 +46,38 @@ _MIGRATION_TS_RE = re.compile(r"^(\d{14})_") _MIGRATION_DEADLOCK_MARKER = "deadlock detected" +MAX_MIGRATE_DEPLOY_ATTEMPTS = 4 + + +@dataclass(frozen=True) +class _MigrateAttemptBudget: + """Retries left, and the recoveries already run. + + A recovery that lands something new costs nothing, so a database full of + objects `prisma db push` created works through them one per pass. Anything + that made no progress spends an attempt, so a stuck run still gives up. + """ + + attempts_left: int + recoveries: frozenset[str] = frozenset() + + @property + def exhausted(self) -> bool: + return self.attempts_left <= 0 + + @property + def attempt_number(self) -> int: + return MAX_MIGRATE_DEPLOY_ATTEMPTS - self.attempts_left + 1 + + def spend(self) -> "_MigrateAttemptBudget": + return replace(self, attempts_left=self.attempts_left - 1) + + def after_recovery(self, recovery: str) -> "_MigrateAttemptBudget": + if recovery in self.recoveries: + return self.spend() + return replace(self, recoveries=self.recoveries | {recovery}) + + _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE) _SPEND_LOGS_ARTIFACT_DROP_RE = re.compile( r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE @@ -716,6 +749,9 @@ class ProxyExtrasDBManager: Ahead-of-HEAD state (DB has migrations newer than this build ships) is logged as a warning, not a fatal error — users whose DBs got into weird shapes from the old thrashing should still be able to start. + + The retry budget only counts attempts that made no progress: see + _MigrateAttemptBudget. """ schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma" migrations_dir = ProxyExtrasDBManager._get_prisma_dir() @@ -749,8 +785,9 @@ class ProxyExtrasDBManager: original_dir = os.getcwd() os.chdir(migrations_dir) deploy_timeout = prisma_migrate_deploy_timeout() + budget = _MigrateAttemptBudget(attempts_left=MAX_MIGRATE_DEPLOY_ATTEMPTS) try: - for attempt in range(4): + while not budget.exhausted: try: result = subprocess.run( [_get_prisma_command(), "migrate", "deploy"], @@ -767,10 +804,11 @@ class ProxyExtrasDBManager: logger.warning( "prisma migrate deploy attempt %s timed out after %ss, retrying. " "Raise %s if this database needs longer to apply its pending migrations.", - attempt + 1, + budget.attempt_number, deploy_timeout, PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, ) + budget = budget.spend() time.sleep(random.randrange(5, 15)) continue @@ -781,7 +819,14 @@ class ProxyExtrasDBManager: logger.info( "Schema exists but no migrations ledger — creating baseline" ) - ProxyExtrasDBManager._create_baseline_migration(schema_path) + baselined = ProxyExtrasDBManager._create_baseline_migration( + schema_path + ) + budget = ( + budget.after_recovery("baseline") + if baselined + else budget.spend() + ) continue if "P3009" in stderr: @@ -818,6 +863,7 @@ class ProxyExtrasDBManager: f"intervention may be required.\n\n" f"Detail: {resolve_err}" ) from resolve_err + budget = budget.after_recovery(f"resolved:{name}") continue if migration_match: migration_name = migration_match.group(1) @@ -831,6 +877,7 @@ class ProxyExtrasDBManager: migration_name, ) ProxyExtrasDBManager._roll_back_migration_best_effort(migration_name) + budget = budget.spend() time.sleep(random.randrange(5, 15)) continue raise RuntimeError( @@ -876,6 +923,7 @@ class ProxyExtrasDBManager: f"intervention may be required.\n\n" f"Detail: {resolve_err}" ) from resolve_err + budget = budget.after_recovery(f"resolved:{name}") continue if migration_match and _MIGRATION_DEADLOCK_MARKER in stderr: @@ -888,6 +936,7 @@ class ProxyExtrasDBManager: ProxyExtrasDBManager._roll_back_migration_best_effort( migration_match.group(1) ) + budget = budget.spend() time.sleep(random.randrange(5, 15)) continue @@ -900,8 +949,9 @@ class ProxyExtrasDBManager: logger.info( "prisma migrate deploy attempt %s deadlocked against " "a concurrent migrate deploy, retrying", - attempt + 1, + budget.attempt_number, ) + budget = budget.spend() time.sleep(random.randrange(5, 15)) continue @@ -909,8 +959,9 @@ class ProxyExtrasDBManager: logger.info( "prisma migrate deploy attempt %s timed out waiting for " "the advisory lock a concurrent migrate deploy holds, retrying", - attempt + 1, + budget.attempt_number, ) + budget = budget.spend() time.sleep(random.randrange(5, 15)) continue @@ -920,9 +971,9 @@ class ProxyExtrasDBManager: ) from e raise RuntimeError( - "Database migration failed after 4 attempts (retry loop " - "exhausted by timeouts, deadlock retries, or repeated " - "idempotent-recovery continues). Check database connectivity, " + f"Database migration failed after {MAX_MIGRATE_DEPLOY_ATTEMPTS} " + "attempts that made no progress (timeouts, deadlock retries, or a " + "recovery that had already run once). Check database connectivity, " "load, and _prisma_migrations ledger state, and raise " f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out." ) diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index b3d457707b8..22558faedad 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -703,3 +703,170 @@ class TestSpendLogsPartitionDetectionMissingPsycopg: assert any( "psycopg is not installed" in record.message for record in caplog.records ) + + +_ATTEMPT_BUDGET = 4 + +_P3005_STDERR = """Error: P3005 + +The database schema is not empty. Read more about how to baseline an existing production database: https://pris.ly/d/migrate-baseline +""" + + +def _p3018_stderr(migration_name): + return f"""Error: P3018 + +A migration failed to apply. New migrations cannot be applied before the error is recovered from. + +Migration name: {migration_name} + +Database error code: 42P07 + +Database error: +ERROR: relation "SomeTable" already exists +""" + + +class _MigrateDeployHarness: + """Drives _setup_database_v2 with a scripted sequence of + `prisma migrate deploy` outcomes, with every recovery command faked out so + nothing touches a database or the packaged migrations directory.""" + + def __init__(self, monkeypatch, tmp_path, outcomes, repeat_last=False): + import subprocess as subprocess_module + + import litellm_proxy_extras.utils as utils_module + + self.deploy_calls = [] + self.resolved = [] + self.baselines = 0 + self._outcomes = list(outcomes) + self._repeat_last = repeat_last + self._subprocess_module = subprocess_module + + monkeypatch.delenv("DATABASE_URL", raising=False) + monkeypatch.setattr( + ProxyExtrasDBManager, "_get_prisma_dir", staticmethod(lambda: str(tmp_path)) + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_create_baseline_migration", + staticmethod(self._fake_baseline), + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_roll_back_migration", + staticmethod(lambda name: None), + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_resolve_specific_migration", + staticmethod(self.resolved.append), + ) + monkeypatch.setattr(utils_module.subprocess, "run", self._fake_run) + monkeypatch.setattr(utils_module.time, "sleep", lambda seconds: None) + + self.baseline_succeeds = True + + def _fake_baseline(self, *args, **kwargs): + self.baselines += 1 + return self.baseline_succeeds + + def _next_outcome(self): + if self._outcomes: + if self._repeat_last and len(self._outcomes) == 1: + return self._outcomes[0] + return self._outcomes.pop(0) + raise AssertionError("prisma migrate deploy called more times than scripted") + + def _fake_run(self, cmd, **kwargs): + assert cmd[1:] == ["migrate", "deploy"], f"unexpected prisma command: {cmd}" + self.deploy_calls.append(cmd) + outcome = self._next_outcome() + if outcome == "ok": + return _FakeCompleted() + if outcome == "timeout": + raise self._subprocess_module.TimeoutExpired(cmd, 1) + raise self._subprocess_module.CalledProcessError(1, cmd, stderr=outcome) + + def run(self): + return ProxyExtrasDBManager._setup_database_v2(use_migrate=True) + + +class TestMigrateDeployAttemptAccounting: + """A `prisma db push` database has a full schema and no ledger, so the v2 + resolver baselines it and then works through every migration whose objects + already exist. Those recoveries make progress, so they must not spend the + retry budget, which is there to stop a run that is getting nowhere.""" + + def test_a_push_created_database_finishes_bootstrapping( + self, monkeypatch, tmp_path + ): + already_there = [ + "20250329084805_new_cron_job_table", + "20250806095134_rename_alias_to_server_name_mcp_table", + "20260224203854_add_agent_object_permissions_table", + "20260301120000_fourth_table", + "20260302120000_fifth_table", + "20260303120000_sixth_table", + ] + harness = _MigrateDeployHarness( + monkeypatch, + tmp_path, + [_P3005_STDERR] + + [_p3018_stderr(name) for name in already_there] + + ["ok"], + ) + + assert harness.run() is True + assert harness.baselines == 1 + assert harness.resolved == already_there + assert len(harness.deploy_calls) == len(already_there) + 2 + + def test_repeated_recovery_of_one_migration_still_gives_up( + self, monkeypatch, tmp_path + ): + harness = _MigrateDeployHarness( + monkeypatch, + tmp_path, + [_p3018_stderr("20250329084805_new_cron_job_table")], + repeat_last=True, + ) + + with pytest.raises(RuntimeError): + harness.run() + assert len(harness.deploy_calls) <= _ATTEMPT_BUDGET + 1 + + def test_timeouts_still_spend_the_budget(self, monkeypatch, tmp_path): + harness = _MigrateDeployHarness( + monkeypatch, tmp_path, ["timeout"], repeat_last=True + ) + + with pytest.raises(RuntimeError): + harness.run() + assert len(harness.deploy_calls) == _ATTEMPT_BUDGET + + def test_a_baseline_that_never_lands_stops_after_the_budget( + self, monkeypatch, tmp_path + ): + harness = _MigrateDeployHarness( + monkeypatch, tmp_path, [_P3005_STDERR], repeat_last=True + ) + harness.baseline_succeeds = False + + with pytest.raises(RuntimeError): + harness.run() + assert len(harness.deploy_calls) == _ATTEMPT_BUDGET + + def test_an_unrecoverable_error_is_not_retried(self, monkeypatch, tmp_path): + harness = _MigrateDeployHarness( + monkeypatch, + tmp_path, + ["Error: P3018\n\nMigration name: 20260101000000_x\n\nERROR: syntax error at or near \"SLECT\"\n"], + repeat_last=True, + ) + + with pytest.raises(RuntimeError): + harness.run() + assert len(harness.deploy_calls) == 1 + assert harness.resolved == [] From ecabfbd5af446ce5d4e52562c27eff845a4394e2 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Thu, 3 Sep 2026 00:01:03 -0700 Subject: [PATCH 213/480] fix(guardrail): hide-secrets playground redaction and guardrail telemetry (#39398) * Fix hide-secrets guardrail: playground redaction, UI dropdown entry, spend-log telemetry The hide-secrets guardrail never implemented apply_guardrail, so the UI test playground echoed secrets verbatim; it was missing from the Add Guardrail dropdown; and it recorded no guardrail_information, so Spend Logs could not distinguish a redacted request from a clean one. - implement apply_guardrail (unified interface) with use_native_lifecycle_hooks so proxied traffic stays on async_pre_call_hook (per-key opt-out and data["prompt"] handling live only there) - record standard_logging_guardrail_information (allow/mask + masked_entity_count) via _process_response/_process_error; opted-out keys and legacy nameless callback instances record nothing - advertise hide-secrets in /guardrails/ui/add_guardrail_settings (pre_call only) and /guardrails/ui/provider_specific_params with a config model Resolves LIT-3548 * Fix hide-secrets passthrough telemetry and JSON config input * fix(guardrails): validate hide-secrets object config before submit - apply_guardrail treats empty-string-only texts as no input, so no false allow is recorded - the UI object field keeps raw text while editing and blocks submission until it parses to a JSON object, instead of posting a string to an object-only API - supported_modes_by_provider keeps its dict[str, list[str]] value type * fix(guardrails): record no hide-secrets telemetry when nothing was inspected walk_user_text and the prompt redaction now report how many non-empty strings they visited; when neither inspected anything (image-only content, empty strings), the run records no guardrail entry instead of an 'allow' row that counts a check which never saw any text. --- .../enterprise_callbacks/secret_detection.py | 201 ++++++++++--- litellm/integrations/custom_guardrail.py | 4 +- .../proxy/guardrails/guardrail_endpoints.py | 17 +- .../guardrail_hooks/hide_secrets.py | 20 ++ .../test_secret_detection.py | 274 ++++++++++++++++++ .../guardrails/test_guardrail_endpoints.py | 31 ++ .../_components/guardrail_info_helpers.tsx | 1 + .../guardrail_provider_fields.test.tsx | 99 +++++++ .../_components/guardrail_provider_fields.tsx | 62 +++- 9 files changed, 659 insertions(+), 50 deletions(-) create mode 100644 litellm/types/proxy/guardrails/guardrail_hooks/hide_secrets.py create mode 100644 tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_provider_fields.test.tsx diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py b/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py index f441ce71ab9..1fddc527ec8 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py @@ -11,17 +11,35 @@ import sys sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path +import functools import tempfile -from typing import Optional +from contextvars import ContextVar +from typing import TYPE_CHECKING, ClassVar, Literal, Optional from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails._content_utils import walk_user_text +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import GenericGuardrailAPIInputs + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj GUARDRAIL_NAME = "hide_secrets" +GUARDRAIL_PROVIDER = "hide-secrets" + +# Per-invocation tally of redacted secrets by detect-secrets plugin type; None +# means the guardrail did not run, so _process_response records nothing. +_masked_entity_count: ContextVar[Optional[dict]] = ContextVar( + "hide_secrets_masked_entity_count", default=None +) + _custom_plugins_path = "file://" + os.path.join( os.path.dirname(os.path.abspath(__file__)), "secrets_plugins" ) @@ -422,6 +440,10 @@ _default_detect_secrets_config = { class _ENTERPRISE_SecretDetection(CustomGuardrail): + # Keeps proxied traffic on async_pre_call_hook (the unified apply_guardrail + # path skips should_run_check and never sees data["prompt"]). + use_native_lifecycle_hooks: ClassVar[bool] = True + def __init__(self, detect_secrets_config: Optional[dict] = None, **kwargs): self.user_defined_detect_secrets_config = detect_secrets_config super().__init__(**kwargs) @@ -455,6 +477,26 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail): return detected_secrets + def redact_text(self, text: str, source: str = "message") -> str: + """Replace every detected secret in ``text`` with ``[REDACTED]`` and + tally the detected types into the per-invocation masked-entity count.""" + detected_secrets = self.scan_message_for_secrets(text) + if not detected_secrets: + return text + counts = _masked_entity_count.get() + if counts is not None: + for secret in detected_secrets: + counts[secret["type"]] = counts.get(secret["type"], 0) + 1 + secret_types = [secret["type"] for secret in detected_secrets] + verbose_proxy_logger.warning( + f"Detected and redacted secrets in {source}: {secret_types}" + ) + return functools.reduce( + lambda redacted, secret: redacted.replace(secret["value"], "[REDACTED]"), + detected_secrets, + text, + ) + async def should_run_check(self, user_api_key_dict: UserAPIKeyAuth) -> bool: if user_api_key_dict.permissions is not None: if GUARDRAIL_NAME in user_api_key_dict.permissions: @@ -463,7 +505,45 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail): return True + @log_guardrail_information + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional["LiteLLMLoggingObj"] = None, + ) -> GenericGuardrailAPIInputs: + """Unified-interface entrypoint, used by /guardrails/apply_guardrail + (the UI test playground). Proxied traffic keeps using + ``async_pre_call_hook``, see ``use_native_lifecycle_hooks``.""" + texts = inputs.get("texts") + if not texts or not any(texts): + return inputs + _masked_entity_count.set({}) + return {**inputs, "texts": [self.redact_text(text) for text in texts]} + + def _redact_prompt(self, data: dict) -> int: + """Redact ``data["prompt"]`` (the text-completion shape, which + ``walk_user_text`` does not cover) and return how many non-empty + strings were inspected.""" + prompt = data.get("prompt") + if isinstance(prompt, str): + if not prompt: + return 0 + data["prompt"] = self.redact_text(prompt, source="prompt") + return 1 + if isinstance(prompt, list): + data["prompt"] = [ # mutable-ok: data["prompt"] is a list on the wire + self.redact_text(item, source="prompt") + if isinstance(item, str) and item + else item + for item in prompt + ] + return sum(1 for item in prompt if isinstance(item, str) and item) + return 0 + #### CALL HOOKS - proxy only #### + @log_guardrail_information async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -471,53 +551,84 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail): data: dict, call_type: str, # "completion", "embeddings", "image_generation", "moderation" ): + _masked_entity_count.set(None) if await self.should_run_check(user_api_key_dict) is False: return + _masked_entity_count.set({}) + # Covers multimodal list content + Responses-API input. - def _redact_message_text(text: str) -> str: - detected_secrets = self.scan_message_for_secrets(text) - for secret in detected_secrets: - text = text.replace(secret["value"], "[REDACTED]") - if detected_secrets: - secret_types = [secret["type"] for secret in detected_secrets] - verbose_proxy_logger.warning( - f"Detected and redacted secrets in message: {secret_types}" - ) - return text + inspected = walk_user_text(data, self.redact_text) + self._redact_prompt(data) - walk_user_text(data, _redact_message_text) + if inspected == 0: + # Image-only, empty-text, and unsupported payloads inspected + # nothing, so recording "allow" would count a run that never + # looked at any content. + _masked_entity_count.set(None) - if "prompt" in data: - if isinstance(data["prompt"], str): - detected_secrets = self.scan_message_for_secrets(data["prompt"]) - for secret in detected_secrets: - data["prompt"] = data["prompt"].replace( - secret["value"], "[REDACTED]" - ) - if len(detected_secrets) > 0: - secret_types = [secret["type"] for secret in detected_secrets] - verbose_proxy_logger.warning( - f"Detected and redacted secrets in prompt: {secret_types}" - ) - elif isinstance(data["prompt"], list): - # Index back into the list — assigning to ``item`` would only - # rebind the loop variable and leave ``data["prompt"]`` - # carrying the unredacted secret. - for idx, item in enumerate(data["prompt"]): - if isinstance(item, str): - detected_secrets = self.scan_message_for_secrets(item) - for secret in detected_secrets: - item = item.replace(secret["value"], "[REDACTED]") - data["prompt"][idx] = item - if len(detected_secrets) > 0: - secret_types = [ - secret["type"] for secret in detected_secrets - ] - verbose_proxy_logger.warning( - f"Detected and redacted secrets in prompt: {secret_types}" - ) - - # ``data["input"]`` (Responses API and embeddings/moderation) is - # already covered by ``walk_user_text`` above. return + + def _process_response( + self, + response: Optional[dict], + request_data: dict, + start_time: Optional[float] = None, + end_time: Optional[float] = None, + duration: Optional[float] = None, + event_type: Optional[GuardrailEventHooks] = None, + original_inputs: Optional[dict] = None, + ): + """Record allow/mask plus the masked-entity tally for a completed run. + + Records nothing when the guardrail inspected nothing (opted-out key, + empty inputs) or when the instance has no guardrail_name (legacy + ``litellm_settings.callbacks`` deployments, which predate guardrail + telemetry and stay without it). + """ + counts = _masked_entity_count.get() + _masked_entity_count.set(None) + if counts is None or self.guardrail_name is None: + return response + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response="mask" if counts else "allow", + request_data=request_data, + guardrail_status="success", + duration=duration, + start_time=start_time, + end_time=end_time, + event_type=event_type, + guardrail_provider=GUARDRAIL_PROVIDER, + masked_entity_count=counts, + ) + return response + + def _process_error( + self, + e: Exception, + request_data: dict, + start_time: Optional[float] = None, + end_time: Optional[float] = None, + duration: Optional[float] = None, + event_type: Optional[GuardrailEventHooks] = None, + ): + """Label the failed run with this guardrail's provider so error rows + group with the successful ones in the monitor. Nameless legacy + instances record nothing, matching ``_process_response``.""" + _masked_entity_count.set(None) + if self.guardrail_name is None: + raise e + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=e, + request_data=request_data, + guardrail_status=( + "guardrail_intervened" + if self._is_guardrail_intervention(e) + else "guardrail_failed_to_respond" + ), + duration=duration, + start_time=start_time, + end_time=end_time, + event_type=event_type, + guardrail_provider=GUARDRAIL_PROVIDER, + ) + raise e diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 372c9bf6b91..c8e5610b4a9 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1485,7 +1485,7 @@ def log_guardrail_information(func): if func.__name__ == "apply_guardrail" and "inputs" in kwargs: original_inputs = kwargs.get("inputs") - logging_obj: Final = kwargs.get("logging_obj") + logging_obj: Final = kwargs.get("logging_obj") or request_data.get("litellm_logging_obj") self_recorded_token: Final = _guardrail_self_recorded.set(False) try: response: Final = await func(*args, **kwargs) @@ -1527,7 +1527,7 @@ def log_guardrail_information(func): if func.__name__ == "apply_guardrail" and "inputs" in kwargs: original_inputs = kwargs.get("inputs") - logging_obj: Final = kwargs.get("logging_obj") + logging_obj: Final = kwargs.get("logging_obj") or request_data.get("litellm_logging_obj") self_recorded_token: Final = _guardrail_self_recorded.set(False) try: response: Final = func(*args, **kwargs) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 3d2ed641a30..744b2959c73 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -8,7 +8,7 @@ import json import os from collections.abc import Awaitable, Callable, Mapping, Sequence from datetime import datetime, timezone -from types import UnionType +from types import MappingProxyType, UnionType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, Union, cast, get_args, get_origin from urllib.parse import urlparse @@ -51,6 +51,9 @@ from litellm.types.guardrails import ( SupportedGuardrailIntegrations, ToolPermissionGuardrailConfigModel, ) +from litellm.types.proxy.guardrails.guardrail_hooks.hide_secrets import ( + HideSecretsGuardrailConfigModel, +) if TYPE_CHECKING: from types import CodeType @@ -1401,7 +1404,11 @@ async def get_guardrail_ui_settings(): provider: [hook.value for hook in hooks] for provider, guardrail_class in guardrail_class_registry.items() if (hooks := guardrail_class.get_supported_event_hooks()) is not None - } + } | MappingProxyType( + # hide-secrets lives in the enterprise package, not in the registry + # above; it only runs on pre_call. + {SupportedGuardrailIntegrations.HIDE_SECRETS.value: [GuardrailEventHooks.pre_call.value]} + ) return GuardrailUIAddGuardrailSettings( supported_entities=[entity.value for entity in PiiEntityType], @@ -1953,12 +1960,18 @@ async def get_provider_specific_params(): tool_permission_fields["ui_friendly_name"] = ToolPermissionGuardrailConfigModel.ui_friendly_name() + # hide-secrets lives in the enterprise package, not in the registry loop below. + hide_secrets_fields: Final = _get_fields_from_model(HideSecretsGuardrailConfigModel) + + hide_secrets_fields["ui_friendly_name"] = HideSecretsGuardrailConfigModel.ui_friendly_name() + # Return the provider-specific parameters provider_params: Final = { SupportedGuardrailIntegrations.BEDROCK.value: bedrock_fields, SupportedGuardrailIntegrations.PRESIDIO.value: presidio_fields, SupportedGuardrailIntegrations.LAKERA_V2.value: lakera_v2_fields, SupportedGuardrailIntegrations.TOOL_PERMISSION.value: tool_permission_fields, + SupportedGuardrailIntegrations.HIDE_SECRETS.value: hide_secrets_fields, } ### get the config model for the guardrail - go through the registry and get the config model for the guardrail diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/hide_secrets.py b/litellm/types/proxy/guardrails/guardrail_hooks/hide_secrets.py new file mode 100644 index 00000000000..3100968e8b1 --- /dev/null +++ b/litellm/types/proxy/guardrails/guardrail_hooks/hide_secrets.py @@ -0,0 +1,20 @@ +"""Types for the Hide Secrets guardrail.""" + +from pydantic import Field + +from .base import GuardrailConfigModel + + +class HideSecretsGuardrailConfigModel(GuardrailConfigModel): + """Configuration for the Hide Secrets guardrail. Detection runs in-process + on the detect-secrets library; ``detect_secrets_config`` overrides the + bundled plugin set.""" + + detect_secrets_config: dict | None = Field( # mutable-ok: UI type derivation maps dict to "object" + default=None, + description="Optional detect-secrets configuration (plugins_used, filters_used) overriding the bundled plugin set", + ) + + @staticmethod + def ui_friendly_name() -> str: + return "Hide Secrets" diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py b/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py new file mode 100644 index 00000000000..dc1cbb9983e --- /dev/null +++ b/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py @@ -0,0 +1,274 @@ +"""Tests for the hide-secrets guardrail (LIT-3548). + +Covers the three defects from the ticket: +- ``apply_guardrail`` (the UI test playground path) must redact, not echo. +- Guardrail runs must record ``standard_logging_guardrail_information`` so + Spend Logs / the guardrails monitor show activity, with hits ("mask" + + masked_entity_count) distinguishable from clean requests ("allow"). +- Defining ``apply_guardrail`` must NOT reroute proxied traffic off the + native ``async_pre_call_hook`` (per-key opt-out and ``data["prompt"]`` + handling live only on the native path). +""" + +import pytest + +from litellm_enterprise.enterprise_callbacks.secret_detection import ( + _ENTERPRISE_SecretDetection, +) +from litellm.caching.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth + +AWS_KEY = "AKIAIOSFODNN7EXAMPLE" + + +def _guardrail() -> _ENTERPRISE_SecretDetection: + return _ENTERPRISE_SecretDetection( + guardrail_name="hide-secrets", event_hook="pre_call", default_on=True + ) + + +def _recorded(request_data: dict) -> dict: + entries = request_data["metadata"]["standard_logging_guardrail_information"] + assert len(entries) == 1 + return entries[0] + + +@pytest.mark.asyncio +async def test_apply_guardrail_redacts_secrets(): + """Playground path: the returned texts must carry [REDACTED], not the secret.""" + guardrail = _guardrail() + request_data: dict = {"metadata": {}} + + result = await guardrail.apply_guardrail( + inputs={"texts": [f"my key is {AWS_KEY}, keep it safe"]}, + request_data=request_data, + input_type="request", + ) + + assert result["texts"] == ["my key is [REDACTED], keep it safe"] + + recorded = _recorded(request_data) + assert recorded["guardrail_status"] == "success" + assert recorded["guardrail_response"] == "mask" + assert recorded["guardrail_provider"] == "hide-secrets" + assert recorded["masked_entity_count"] == {"AWS Access Key": 1} + + +@pytest.mark.asyncio +async def test_apply_guardrail_clean_text_records_allow(): + guardrail = _guardrail() + request_data: dict = {"metadata": {}} + + result = await guardrail.apply_guardrail( + inputs={"texts": ["nothing sensitive here"]}, + request_data=request_data, + input_type="request", + ) + + assert result["texts"] == ["nothing sensitive here"] + + recorded = _recorded(request_data) + assert recorded["guardrail_status"] == "success" + assert recorded["guardrail_response"] == "allow" + assert recorded["masked_entity_count"] == {} + + +@pytest.mark.asyncio +async def test_pre_call_hook_records_mask_with_entity_count(): + """Live-traffic path: a redaction must be visible in spend-log telemetry.""" + guardrail = _guardrail() + data = { + "messages": [{"role": "user", "content": f"use {AWS_KEY} for auth"}], + "metadata": {}, + } + + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + assert data["messages"][0]["content"] == "use [REDACTED] for auth" + + recorded = _recorded(data) + assert recorded["guardrail_status"] == "success" + assert recorded["guardrail_response"] == "mask" + assert recorded["guardrail_provider"] == "hide-secrets" + assert recorded["masked_entity_count"] == {"AWS Access Key": 1} + + +@pytest.mark.asyncio +async def test_pre_call_hook_clean_request_records_allow(): + """A request with no secrets must be distinguishable from a redacted one.""" + guardrail = _guardrail() + data = { + "messages": [{"role": "user", "content": "what's the weather"}], + "metadata": {}, + } + + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + recorded = _recorded(data) + assert recorded["guardrail_status"] == "success" + assert recorded["guardrail_response"] == "allow" + assert recorded["masked_entity_count"] == {} + + +@pytest.mark.asyncio +async def test_pre_call_hook_opt_out_records_nothing(): + """A key with permissions={"hide_secrets": False} skips redaction, so no + telemetry is recorded: every reader of a recorded entry (guardrail usage + tracking, compliance checks, the spend-log viewer) counts it as a run.""" + guardrail = _guardrail() + content = f"my key is {AWS_KEY}" + data = {"messages": [{"role": "user", "content": content}], "metadata": {}} + + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(permissions={"hide_secrets": False}), + cache=DualCache(), + data=data, + call_type="completion", + ) + + assert data["messages"][0]["content"] == content # untouched + assert "standard_logging_guardrail_information" not in data["metadata"] + + +@pytest.mark.asyncio +async def test_pre_call_hook_still_redacts_text_completion_prompt(): + """data["prompt"] (str and list) is a native-hook-only surface; it must + keep redacting now that the class also implements apply_guardrail.""" + guardrail = _guardrail() + data = {"prompt": f"key {AWS_KEY} end", "metadata": {}} + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + assert data["prompt"] == "key [REDACTED] end" + + guardrail = _guardrail() + data = {"prompt": [f"key {AWS_KEY}", "clean"], "metadata": {}} + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + assert data["prompt"] == ["key [REDACTED]", "clean"] + + +def test_proxied_traffic_stays_on_native_hooks(): + """Implementing apply_guardrail must not reroute proxied requests onto the + unified path: that path skips ``should_run_check`` (per-key opt-out) and + never sees ``data["prompt"]``.""" + guardrail = _guardrail() + assert guardrail.uses_apply_guardrail_interface() is True + assert guardrail._deployment_pre_call_target() is guardrail + + +@pytest.mark.asyncio +async def test_apply_guardrail_without_texts_records_nothing(): + """No inputs means nothing was inspected, so no "allow" row is recorded. + Empty strings count as no input: there is no content to inspect.""" + guardrail = _guardrail() + + empty_variants: list[list[str]] = [[], ["", ""]] + for texts in empty_variants: + request_data: dict = {"metadata": {}} + result = await guardrail.apply_guardrail( + inputs={"texts": texts}, request_data=request_data, input_type="request" + ) + assert result == {"texts": texts} + assert "standard_logging_guardrail_information" not in request_data["metadata"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "data", + [ + pytest.param( + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "https://x/y.png"}} + ], + } + ], + "metadata": {}, + }, + id="image_only", + ), + pytest.param( + {"messages": [{"role": "user", "content": ""}], "metadata": {}}, + id="empty_message", + ), + pytest.param({"prompt": "", "metadata": {}}, id="empty_prompt"), + pytest.param({"prompt": ["", ""], "metadata": {}}, id="empty_prompt_list"), + ], +) +async def test_pre_call_hook_without_inspectable_text_records_nothing(data: dict): + """A payload the guardrail could not inspect (image-only content, empty + strings) must not record an "allow" run: monitoring would count a check + that never looked at any text.""" + guardrail = _guardrail() + + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + assert "standard_logging_guardrail_information" not in data["metadata"] + + +@pytest.mark.asyncio +async def test_pre_call_hook_mixed_prompt_list_still_redacts_and_records(): + """A prompt list mixing empty and real strings is inspected, so the run is + recorded and the non-empty entry is still redacted.""" + guardrail = _guardrail() + data = {"prompt": ["", f"key {AWS_KEY}"], "metadata": {}} + + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + assert data["prompt"] == ["", "key [REDACTED]"] + recorded = _recorded(data) + assert recorded["guardrail_response"] == "mask" + assert recorded["masked_entity_count"] == {"AWS Access Key": 1} + + +@pytest.mark.asyncio +async def test_legacy_nameless_instance_records_nothing(): + """``litellm_settings.callbacks: ["hide_secrets"]`` builds an arg-less + instance with no guardrail_name. It still redacts, but recording a nameless + entry would flip every spend row's guardrail status with nothing to join on.""" + guardrail = _ENTERPRISE_SecretDetection() + data = { + "messages": [{"role": "user", "content": f"use {AWS_KEY} for auth"}], + "metadata": {}, + } + + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + assert data["messages"][0]["content"] == "use [REDACTED] for auth" + assert "standard_logging_guardrail_information" not in data["metadata"] diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index d85c6da659b..b99dcb062b4 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -670,6 +670,37 @@ def test_get_provider_specific_params(): ) # Literal type should be select +@pytest.mark.asyncio +async def test_provider_specific_params_includes_hide_secrets(): + """hide-secrets lives in the enterprise package so it is not in + guardrail_class_registry; the endpoint must still advertise it or the + Add Guardrail UI dropdown never offers it (LIT-3548).""" + from litellm.proxy.guardrails.guardrail_endpoints import ( + get_provider_specific_params, + ) + + provider_params = await get_provider_specific_params() + + assert "hide-secrets" in provider_params + # populateGuardrailProviders() in the dashboard only lists providers whose + # entry carries a ui_friendly_name. + assert provider_params["hide-secrets"]["ui_friendly_name"] == "Hide Secrets" + assert provider_params["hide-secrets"]["detect_secrets_config"]["required"] is False + + +@pytest.mark.asyncio +async def test_add_guardrail_settings_restricts_hide_secrets_to_pre_call(): + """hide-secrets only implements async_pre_call_hook, so offering the other + modes in the UI would create configs that boot clean and never run.""" + from litellm.proxy.guardrails.guardrail_endpoints import ( + get_guardrail_ui_settings, + ) + + settings = await get_guardrail_ui_settings() + + assert settings.supported_modes_by_provider["hide-secrets"] == ["pre_call"] + + def test_optional_params_not_returned_when_not_overridden(): """Test that optional_params is not returned when the config model doesn't override it""" from typing import Optional diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx index c1f2ddcf51c..f686ff5644a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx @@ -201,6 +201,7 @@ export const guardrailLogoMap = { XecGuard: xecguardLogo.src, "LiteLLM Content Filter": litellmLogo.src, "LiteLLM LLM as a Judge": litellmLogo.src, + "Hide Secrets": litellmLogo.src, Akto: aktoLogo.src, "DeepKeep AI Firewall": deepkeepLogo.src, "Qostodian Nexus": qohashLogo.src, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_provider_fields.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_provider_fields.test.tsx new file mode 100644 index 00000000000..ab63b7637b1 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_provider_fields.test.tsx @@ -0,0 +1,99 @@ +import React from "react"; +import { fireEvent, screen, waitFor } from "@testing-library/react"; +import { useForm } from "react-hook-form"; +import { renderWithProviders } from "@/../tests/test-utils"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import GuardrailProviderFields from "./guardrail_provider_fields"; +import { populateGuardrailProviderMap } from "./guardrail_info_helpers"; +import type { GuardrailFormValues } from "./GuardrailFormField"; + +vi.mock("@/lib/toast", () => ({ toast: { error: vi.fn() } })); + +const HIDE_SECRETS_PARAMS = { + "hide-secrets": { + ui_friendly_name: "Hide Secrets", + detect_secrets_config: { + param: "detect_secrets_config", + description: "Optional detect-secrets configuration", + required: false, + type: "object", + }, + }, +}; + +const Harness: React.FC<{ onValid: (values: GuardrailFormValues) => void }> = ({ onValid }) => { + const form = useForm(); + return ( +
+ + + + ); +}; + +const renderHarness = () => { + populateGuardrailProviderMap(HIDE_SECRETS_PARAMS); + const onValid = vi.fn(); + renderWithProviders(); + const textarea = screen.getByLabelText(/detect_secrets_config/) as HTMLTextAreaElement; + return { onValid, textarea }; +}; + +describe("GuardrailProviderFields object field", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("commits a valid JSON object as a parsed dict", async () => { + const { onValid, textarea } = renderHarness(); + + fireEvent.change(textarea, { target: { value: '{"plugins_used": [{"name": "AWSKeyDetector"}]}' } }); + fireEvent.blur(textarea); + fireEvent.click(screen.getByRole("button", { name: "save" })); + + await waitFor(() => expect(onValid).toHaveBeenCalledTimes(1)); + expect(onValid.mock.calls[0][0].detect_secrets_config).toEqual({ + plugins_used: [{ name: "AWSKeyDetector" }], + }); + }); + + it("blocks submission while the field holds malformed JSON", async () => { + const { onValid, textarea } = renderHarness(); + + fireEvent.change(textarea, { target: { value: "{not json" } }); + fireEvent.blur(textarea); + fireEvent.click(screen.getByRole("button", { name: "save" })); + + await screen.findByText("detect_secrets_config must be a valid JSON object"); + expect(onValid).not.toHaveBeenCalled(); + expect(textarea.value).toBe("{not json"); + }); + + it.each(['["array"]', '"scalar"', "null", "42"])("blocks non-object JSON %s", async (raw) => { + const { onValid, textarea } = renderHarness(); + + fireEvent.change(textarea, { target: { value: raw } }); + fireEvent.blur(textarea); + fireEvent.click(screen.getByRole("button", { name: "save" })); + + await screen.findByText("detect_secrets_config must be a valid JSON object"); + expect(onValid).not.toHaveBeenCalled(); + }); + + it("treats a cleared field as unset and submits", async () => { + const { onValid, textarea } = renderHarness(); + + fireEvent.change(textarea, { target: { value: '{"a": 1}' } }); + fireEvent.blur(textarea); + fireEvent.change(textarea, { target: { value: "" } }); + fireEvent.blur(textarea); + fireEvent.click(screen.getByRole("button", { name: "save" })); + + await waitFor(() => expect(onValid).toHaveBeenCalledTimes(1)); + expect(onValid.mock.calls[0][0].detect_secrets_config).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_provider_fields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_provider_fields.tsx index 76c91d4c176..7d48da89ebf 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_provider_fields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_provider_fields.tsx @@ -13,6 +13,8 @@ import { FieldGroup } from "@/components/ui/field"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Slider } from "@/components/ui/slider"; +import { Textarea } from "@/components/ui/textarea"; +import { toast } from "@/lib/toast"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { asStringArray, @@ -22,6 +24,7 @@ import { readRecord, requiredRule, type GuardrailFieldControlProps, + type GuardrailFieldRules, type GuardrailFormControl, } from "./GuardrailFormField"; @@ -60,6 +63,44 @@ const BOOLEAN_ITEMS = [ const isSecretKey = (fieldKey: string): boolean => fieldKey.includes("password") || fieldKey.includes("secret") || fieldKey.includes("key"); +const isPlainObject = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +// Object fields hold the raw text while the user types, so submission must be +// blocked until the value parses to a plain JSON object (or is cleared). +const jsonObjectRule = (fieldKey: string): GuardrailFieldRules => ({ + validate: (value: unknown) => + value === undefined || isPlainObject(value) ? true : `${fieldKey} must be a valid JSON object`, +}); + +// Commits a parsed object (or undefined for a cleared field) to the form on +// blur; anything else stays as raw text so jsonObjectRule blocks submission. +const commitObjectField = (raw: string, onChange: (value: unknown) => void): void => { + const next = raw.trim(); + if (next === "") { + onChange(undefined); + return; + } + let parsed: unknown; + try { + parsed = JSON.parse(next); + } catch { + parsed = next; + } + if (isPlainObject(parsed)) { + onChange(parsed); + } else { + toast.error("Enter a valid JSON object for this configuration"); + } +}; + +const fieldRules = (field: ProviderParam, fieldKey: string): GuardrailFieldRules | undefined => { + if (field.type === "object") { + return jsonObjectRule(fieldKey); + } + return field.required ? requiredRule(`${fieldKey} is required`) : undefined; +}; + interface ProviderFieldInputProps { descriptor: ProviderParam; fieldKey: string; @@ -141,6 +182,25 @@ const ProviderFieldInput: React.FC = ({ descriptor, fie ); } + if (descriptor.type === "object") { + const objectValue = typeof value === "object" && value !== null ? JSON.stringify(value, null, 2) : asText(value); + return ( +