From f07ea5921b361471198aced3c941d83fe7727ffb Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 16 Jul 2026 08:36:46 -0700 Subject: [PATCH 001/132] feat(proxy): enforce team isolation for provider-format batch ids and output files Managed batch ownership rows were already written for every create path, but retrieve/cancel/file-content checks only fired for unified (litellm_proxy-prefixed) ids, so model-encoded and raw provider batch ids bypassed isolation entirely. - enforce can_access_resource on retrieve/cancel for provider-format batch ids when an ownership row exists; ids with no row stay accessible so pass-through reads keep working - make batch sync operations update-only so a retrieve/cancel can never mint an ownership row attributed to the first caller - write ownership rows for a synced batch's provider-format output/error file ids, inherited from the owning batch row, and enforce them on file content/retrieve/delete --- .../proxy/hooks/managed_files.py | 142 ++++++ .../proxy/hooks/test_managed_files.py | 412 ++++++++++++++++++ 2 files changed, 554 insertions(+) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 3f42867d90e..27363f45505 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -30,10 +30,12 @@ from litellm.proxy._types import ( ) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, + decode_model_from_file_id, get_batch_id_from_unified_batch_id, get_content_type_from_file_object, get_model_id_from_unified_batch_id, get_models_from_unified_file_id, + get_original_file_id, normalize_mime_type_for_provider, ) from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue] @@ -165,7 +167,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_object_id: str, file_purpose: Literal["batch", "fine-tune", "response"], user_api_key_dict: UserAPIKeyAuth, + update_only: bool = False, ) -> None: + """Persist a managed object row. + + With ``update_only=True`` an existing row is refreshed but a missing + row is NOT created: sync operations (retrieve/cancel) must never mint + an ownership row attributed to whoever happened to call them first. + """ verbose_logger.info( f"Storing LiteLLM Managed {file_purpose} object with id={unified_object_id} in cache" ) @@ -175,6 +184,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): file_purpose=file_purpose, file_object=file_object, ) + + if update_only: + updated_count = ( + await self.prisma_client.db.litellm_managedobjecttable.update_many( + where={"unified_object_id": unified_object_id}, + data={ + "file_object": file_object.model_dump_json(), + "status": file_object.status, + "updated_by": user_api_key_dict.user_id, + }, + ) + ) + if updated_count == 0: + return + await self.internal_usage_cache.async_set_cache( + key=unified_object_id, + value=litellm_managed_object.model_dump(), + litellm_parent_otel_span=litellm_parent_otel_span, + ) + return + await self.internal_usage_cache.async_set_cache( key=unified_object_id, value=litellm_managed_object.model_dump(), @@ -284,6 +314,103 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): detail=f"Object not found: {unified_object_id}", ) + async def enforce_batch_object_access( + self, object_id: str, user_api_key_dict: UserAPIKeyAuth + ) -> None: + """Deny access to a provider-format batch id owned by another caller. + + Ids with no ownership row (batches created before ownership tracking, + or directly on the provider account) stay accessible so pass-through + reads keep working. + """ + if self.prisma_client is None: + return + managed_object = ( + await self.prisma_client.db.litellm_managedobjecttable.find_first( + where={"unified_object_id": object_id} + ) + ) + if managed_object is None: + return + if not can_access_resource( + user_api_key_dict=user_api_key_dict, + created_by=managed_object.created_by, + resource_team_id=managed_object.team_id, + ): + raise HTTPException( + status_code=403, + detail=f"User {user_api_key_dict.user_id} does not have access to the object {object_id}", + ) + + async def enforce_provider_file_access( + self, file_id: str, user_api_key_dict: UserAPIKeyAuth + ) -> None: + """Deny access to a provider-format file id owned by another caller. + + Ownership rows for provider-format ids are written when a managed + batch's output/error files are first synced; ids with no row stay + accessible so pass-through reads keep working. + """ + if self.prisma_client is None: + return + managed_file = ( + await self.prisma_client.db.litellm_managedfiletable.find_first( + where={"unified_file_id": file_id} + ) + ) + if managed_file is None: + return + if not can_access_resource( + user_api_key_dict=user_api_key_dict, + created_by=managed_file.created_by, + resource_team_id=managed_file.team_id, + ): + raise HTTPException( + status_code=403, + detail=f"User {user_api_key_dict.user_id} does not have access to the file {file_id}", + ) + + async def store_batch_output_file_ownership( + self, response: LiteLLMBatch, litellm_parent_otel_span: Optional[Span] + ) -> None: + """Record ownership rows for a batch's provider-format output/error + file ids, inherited from the owning batch row (never the caller), so + file reads can be isolation-checked.""" + provider_file_ids = tuple( + file_id + for file_id in ( + getattr(response, "output_file_id", None), + getattr(response, "error_file_id", None), + ) + if file_id and not _is_base64_encoded_unified_file_id(file_id) + ) + if not provider_file_ids: + return + if self.prisma_client is None: + return + batch_row = ( + await self.prisma_client.db.litellm_managedobjecttable.find_first( + where={"unified_object_id": response.id} + ) + ) + if batch_row is None or ( + batch_row.created_by is None and batch_row.team_id is None + ): + return + owner_identity = UserAPIKeyAuth( + user_id=batch_row.created_by, team_id=batch_row.team_id + ) + for file_id in provider_file_ids: + model_name = decode_model_from_file_id(file_id) + raw_file_id = get_original_file_id(file_id) + await self.store_unified_file_id( + file_id=file_id, + file_object=None, + litellm_parent_otel_span=litellm_parent_otel_span, + model_mappings={model_name: raw_file_id} if model_name else {}, + user_api_key_dict=owner_identity, + ) + async def list_user_batches( self, user_api_key_dict: UserAPIKeyAuth, @@ -397,6 +524,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): status_code=403, detail=f"User {user_api_key_dict.user_id} does not have access to the file {retrieve_file_id}", ) + if retrieve_file_id: + await self.enforce_provider_file_access( + retrieve_file_id, user_api_key_dict + ) return False async def check_file_ids_access( @@ -580,6 +711,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): data[accessor_key] = get_batch_id_from_unified_batch_id( potential_llm_object_id ) + elif retrieve_object_id and accessor_key == "batch_id": + await self.enforce_batch_object_access( + retrieve_object_id, user_api_key_dict + ) elif call_type == CallTypes.acreate_fine_tuning_job.value: input_file_id = cast(Optional[str], data.get("training_file")) if input_file_id: @@ -1183,6 +1318,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_mappings={model_id: provider_file_id}, user_api_key_dict=user_api_key_dict, ) + is_batch_create = "completion_window" in data or "input_file_id" in data await self.store_unified_object_id( unified_object_id=response.id, file_object=response, @@ -1190,7 +1326,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_object_id=original_response_id, file_purpose="batch", user_api_key_dict=user_api_key_dict, + update_only=not is_batch_create, ) + if not is_batch_create: + await self.store_batch_output_file_ownership( + response=response, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + ) # Only record batch creation metric on actual create (not retrieve/cancel). # unified_file_id in _hidden_params is only set by the create_batch endpoint. diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index 68dc3269f34..49d7a53d40a 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -11,6 +11,7 @@ from litellm.caching import DualCache from litellm.proxy._types import CallTypes from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, + encode_file_id_with_model, ) @@ -2367,3 +2368,414 @@ async def test_same_user_different_keys_can_access_batch(): assert "batch_id" in result2 # Both keys should get the same result assert result1["batch_id"] == result2["batch_id"] + + +MODEL_ENCODED_BATCH_ID = encode_file_id_with_model( + "batch_provider123", "gpt-4o-team-alias", id_type="batch" +) +MODEL_ENCODED_OUTPUT_FILE_ID = encode_file_id_with_model( + "file-output456", "gpt-4o-team-alias", id_type="file" +) +RAW_PROVIDER_BATCH_ID = "batch_provider123" +RAW_PROVIDER_FILE_ID = "file-output456" + + +def _owned_record(created_by, team_id): + record = MagicMock() + record.created_by = created_by + record.team_id = team_id + return record + + +def _batch_response(batch_id, output_file_id=None): + from litellm.types.utils import LiteLLMBatch + + return LiteLLMBatch( + id=batch_id, + completion_window="24h", + created_at=1700000000, + endpoint="/v1/chat/completions", + input_file_id="file-input789", + object="batch", + status="completed", + output_file_id=output_file_id, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["aretrieve_batch", "acancel_batch"]) +@pytest.mark.parametrize( + "batch_id", [MODEL_ENCODED_BATCH_ID, RAW_PROVIDER_BATCH_ID] +) +async def test_team_b_cannot_access_team_a_provider_format_batch( + call_type, batch_id +): + """ + Cross-team retrieve/cancel of a model-encoded or raw provider batch id + must 403 when an ownership row exists for another team. + + Regression test: before this check only unified (litellm_proxy-prefixed) + batch ids were enforced, so any key could read any model-encoded or raw + provider batch. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + with pytest.raises(HTTPException) as exc_info: + await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"batch_id": batch_id}, + call_type=call_type, + ) + + assert exc_info.value.status_code == 403 + prisma_client.db.litellm_managedobjecttable.find_first.assert_awaited_once_with( + where={"unified_object_id": batch_id} + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "caller_kwargs", + [ + {"user_id": "user_a", "team_id": "team_a"}, + {"user_id": "teammate_of_a", "team_id": "team_a"}, + {"user_id": "admin_user", "user_role": "proxy_admin"}, + ], +) +async def test_authorized_callers_can_access_provider_format_batch(caller_kwargs): + """ + The creator, a same-team member, and a proxy admin can all retrieve a + model-encoded batch owned by team_a. Data must pass through unmodified so + the endpoint's own routing still applies. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + parent_otel_span=MagicMock(), **caller_kwargs + ), + cache=MagicMock(), + data={"batch_id": MODEL_ENCODED_BATCH_ID}, + call_type="aretrieve_batch", + ) + + assert result["batch_id"] == MODEL_ENCODED_BATCH_ID + assert "model" not in result + + +@pytest.mark.asyncio +async def test_provider_format_batch_without_ownership_row_stays_accessible(): + """ + A provider-format batch id with no ownership row (created before ownership + tracking, or directly on the provider account) must stay retrievable. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.find_first.return_value = None + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"batch_id": RAW_PROVIDER_BATCH_ID}, + call_type="aretrieve_batch", + ) + + assert result["batch_id"] == RAW_PROVIDER_BATCH_ID + + +@pytest.mark.asyncio +async def test_fine_tuning_provider_format_id_not_enforced(): + """ + Provider-format fine-tuning job ids are deliberately out of scope for + ownership enforcement; only unified fine-tuning ids are checked. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"fine_tuning_job_id": "ftjob-abc123"}, + call_type="aretrieve_fine_tuning_job", + ) + + assert result["fine_tuning_job_id"] == "ftjob-abc123" + prisma_client.db.litellm_managedobjecttable.find_first.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_type", ["afile_content", "afile_retrieve", "afile_delete"] +) +@pytest.mark.parametrize( + "file_id", [MODEL_ENCODED_OUTPUT_FILE_ID, RAW_PROVIDER_FILE_ID] +) +async def test_team_b_cannot_access_team_a_provider_format_file( + call_type, file_id +): + """ + Cross-team content/retrieve/delete of a model-encoded or raw provider + file id must 403 when an ownership row exists for another team. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedfiletable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(), prisma_client=prisma_client + ) + + with pytest.raises(HTTPException) as exc_info: + await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"file_id": file_id}, + call_type=call_type, + ) + + assert exc_info.value.status_code == 403 + prisma_client.db.litellm_managedfiletable.find_first.assert_awaited_once_with( + where={"unified_file_id": file_id} + ) + + +@pytest.mark.asyncio +async def test_same_team_can_access_provider_format_file(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedfiletable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="teammate_of_a", team_id="team_a", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"file_id": MODEL_ENCODED_OUTPUT_FILE_ID}, + call_type="afile_content", + ) + + assert result["file_id"] == MODEL_ENCODED_OUTPUT_FILE_ID + + +@pytest.mark.asyncio +async def test_provider_format_file_without_ownership_row_stays_accessible(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedfiletable.find_first.return_value = None + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(), prisma_client=prisma_client + ) + + result = await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"file_id": RAW_PROVIDER_FILE_ID}, + call_type="afile_content", + ) + + assert result["file_id"] == RAW_PROVIDER_FILE_ID + + +@pytest.mark.asyncio +async def test_post_call_batch_create_stores_ownership_row(): + """ + Batch creation (request data carries completion_window/input_file_id) + must write an ownership row attributed to the creating key. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client + ) + + await proxy_managed_files.async_post_call_success_hook( + data={ + "input_file_id": "file-input789", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + user_api_key_dict=UserAPIKeyAuth( + user_id="user_a", team_id="team_a", parent_otel_span=MagicMock() + ), + response=_batch_response(MODEL_ENCODED_BATCH_ID), + ) + + upsert_call = prisma_client.db.litellm_managedobjecttable.upsert.await_args + assert upsert_call.kwargs["where"] == { + "unified_object_id": MODEL_ENCODED_BATCH_ID + } + create_data = upsert_call.kwargs["data"]["create"] + assert create_data["created_by"] == "user_a" + assert create_data["team_id"] == "team_a" + prisma_client.db.litellm_managedobjecttable.update_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_post_call_batch_sync_does_not_claim_ownership(): + """ + Retrieve/cancel of a batch with no ownership row must NOT create one: + otherwise the first foreign key to touch a legacy batch would become its + owner and lock out the real creator once enforcement is on. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.update_many.return_value = 0 + internal_usage_cache = MagicMock(async_set_cache=AsyncMock()) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + internal_usage_cache, prisma_client=prisma_client + ) + + await proxy_managed_files.async_post_call_success_hook( + data={"batch_id": MODEL_ENCODED_BATCH_ID}, + user_api_key_dict=UserAPIKeyAuth( + user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() + ), + response=_batch_response(MODEL_ENCODED_BATCH_ID), + ) + + prisma_client.db.litellm_managedobjecttable.upsert.assert_not_awaited() + internal_usage_cache.async_set_cache.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_post_call_batch_sync_updates_existing_row(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.update_many.return_value = 1 + prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client + ) + + await proxy_managed_files.async_post_call_success_hook( + data={"batch_id": MODEL_ENCODED_BATCH_ID}, + user_api_key_dict=UserAPIKeyAuth( + user_id="user_a", team_id="team_a", parent_otel_span=MagicMock() + ), + response=_batch_response(MODEL_ENCODED_BATCH_ID), + ) + + update_call = prisma_client.db.litellm_managedobjecttable.update_many.await_args + assert update_call.kwargs["where"] == { + "unified_object_id": MODEL_ENCODED_BATCH_ID + } + assert update_call.kwargs["data"]["status"] == "completed" + prisma_client.db.litellm_managedobjecttable.upsert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_post_call_batch_sync_stores_output_file_ownership_from_batch_row(): + """ + When a synced batch reports a provider-format output file id, an + ownership row for that file must be written with the BATCH row's + created_by/team_id, not the caller's identity. + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + prisma_client.db.litellm_managedobjecttable.update_many.return_value = 1 + prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( + _owned_record(created_by="user_a", team_id="team_a") + ) + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client + ) + + await proxy_managed_files.async_post_call_success_hook( + data={"batch_id": MODEL_ENCODED_BATCH_ID}, + user_api_key_dict=UserAPIKeyAuth( + user_id="admin_user", + user_role="proxy_admin", + parent_otel_span=MagicMock(), + ), + response=_batch_response( + MODEL_ENCODED_BATCH_ID, output_file_id=MODEL_ENCODED_OUTPUT_FILE_ID + ), + ) + + file_upsert = prisma_client.db.litellm_managedfiletable.upsert.await_args + assert file_upsert.kwargs["where"] == { + "unified_file_id": MODEL_ENCODED_OUTPUT_FILE_ID + } + create_data = file_upsert.kwargs["data"]["create"] + assert create_data["created_by"] == "user_a" + assert create_data["team_id"] == "team_a" + assert create_data["flat_model_file_ids"] == ["file-output456"] + + +@pytest.mark.asyncio +async def test_post_call_batch_create_does_not_store_output_file_ownership(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client + ) + + await proxy_managed_files.async_post_call_success_hook( + data={ + "input_file_id": "file-input789", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + user_api_key_dict=UserAPIKeyAuth( + user_id="user_a", team_id="team_a", parent_otel_span=MagicMock() + ), + response=_batch_response( + MODEL_ENCODED_BATCH_ID, output_file_id=MODEL_ENCODED_OUTPUT_FILE_ID + ), + ) + + prisma_client.db.litellm_managedfiletable.upsert.assert_not_awaited() From 7bac4a41af549070d7084174f9e6e52fea282300 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:17:22 -0700 Subject: [PATCH 002/132] fix(prompts): key the in-memory prompt registry by environment LiteLLM_PromptTable is unique on (prompt_id, version, environment) and version numbering restarts at 1 per environment, but the in-memory registry keyed prompts as {prompt_id}.v{version} with no environment, so environments sharing a prompt id shadowed each other and only one environment's template ever served. Registry entries are now keyed {versioned_id}::{environment}, and serve time resolution goes through resolve_prompt_spec(base_id, version, environment): production > staging > development when no environment is requested, latest version within the chosen environment when no version is requested. Chat requests can pin an environment with a new optional prompt_environment body param, filtered from provider-bound params like prompt_id and prompt_version. The newest-updated_at dedupe in _init_prompts_in_db is dropped since registry keys can no longer collide, and the key-parsing serve helpers plus dead registry getters are removed --- litellm/proxy/prompts/prompt_endpoints.py | 247 +++--------------- litellm/proxy/prompts/prompt_registry.py | 174 +++++++++--- litellm/proxy/proxy_server.py | 11 +- litellm/proxy/utils.py | 28 +- litellm/types/utils.py | 1 + .../proxy/prompts/test_prompt_endpoints.py | 96 +------ .../prompts/test_prompt_endpoints_crud.py | 68 ++--- .../proxy/prompts/test_prompt_environment.py | 12 +- .../proxy/prompts/test_prompt_registry.py | 127 ++++++++- tests/test_litellm/proxy/test_proxy_server.py | 63 +++-- .../proxy_logging/test_guardrail_pipeline.py | 54 +++- 11 files changed, 413 insertions(+), 468 deletions(-) diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index 425ff7572d0..32fd1d53777 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -29,6 +29,12 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_utils import is_request_body_safe from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.path_utils import safe_filename +from litellm.proxy.prompts.prompt_registry import ( + DEFAULT_PROMPT_ENVIRONMENT, + get_base_prompt_id, + get_version_number, + prompt_environment_or_default, +) from litellm.repositories.table_repositories import PromptRepository from litellm.types.prompts.init_prompts import ( ListPromptsResponse, @@ -102,165 +108,20 @@ def _prompt_table(prisma_client: "PrismaClient") -> _PromptTableActions: return PromptRepository(prisma_client).table -def get_base_prompt_id(prompt_id: str) -> str: - """ - Extract the base prompt ID by stripping the version suffix if present. - - Args: - prompt_id: Prompt ID that may include version suffix (e.g., "jack_success.v1" or "jack_success_v1") - - Returns: - Base prompt ID without version suffix (e.g., "jack_success") - - Examples: - >>> get_base_prompt_id("jack_success.v1") - "jack_success" - >>> get_base_prompt_id("jack_success_v1") - "jack_success" - >>> get_base_prompt_id("jack_success") - "jack_success" - """ - # Try dot separator first (.v) - if ".v" in prompt_id: - return prompt_id.split(".v")[0] - # Try underscore separator (_v) - if "_v" in prompt_id: - return prompt_id.split("_v")[0] - return prompt_id - - -def get_version_number(prompt_id: str) -> int: - """ - Extract the version number from a versioned prompt ID. - - Args: - prompt_id: Prompt ID that may include version suffix (e.g., "jack_success.v2" or "jack_success_v2") - - Returns: - Version number (defaults to 1 if no version suffix or invalid format) - - Examples: - >>> get_version_number("jack_success.v2") - 2 - >>> get_version_number("jack_success_v2") - 2 - >>> get_version_number("jack_success") - 1 - """ - # Try dot separator first (.v) - if ".v" in prompt_id: - version_str = prompt_id.split(".v")[1] - try: - return int(version_str) - except ValueError: - pass - - # Try underscore separator (_v) - if "_v" in prompt_id: - version_str = prompt_id.split("_v")[1] - try: - return int(version_str) - except ValueError: - pass - - return 1 - - -def construct_versioned_prompt_id(prompt_id: str, version: int | None = None) -> str: - """ - Construct a versioned prompt ID from a base prompt_id and version number. - - Args: - prompt_id: Base prompt ID (e.g., "jack_success") - version: Version number (if None, returns the base prompt_id unchanged) - - Returns: - Versioned prompt ID (e.g., "jack_success.v4") - - Examples: - >>> construct_versioned_prompt_id("jack_success", 4) - "jack_success.v4" - >>> construct_versioned_prompt_id("jack_success", None) - "jack_success" - >>> construct_versioned_prompt_id("jack_success.v2", 4) - "jack_success.v4" - """ - if version is None: - return prompt_id - - # Strip any existing version suffix first - base_id: Final = get_base_prompt_id(prompt_id) - return f"{base_id}.v{version}" - - -def get_latest_version_prompt_id(prompt_id: str, all_prompt_ids: Mapping[str, object]) -> str: - """ - Find the latest version of a prompt from available prompt IDs. - - Args: - prompt_id: Base prompt ID or versioned prompt ID (e.g., "jack_success" or "jack_success.v2") - all_prompt_ids: Dictionary of all available prompt IDs (keys are prompt IDs) - - Returns: - The prompt ID with the highest version number, or the original prompt_id if no versions exist - - Examples: - >>> all_ids = {"jack.v1": {}, "jack.v2": {}, "jack.v3": {}} - >>> get_latest_version_prompt_id("jack", all_ids) - "jack.v3" - >>> get_latest_version_prompt_id("jack.v1", all_ids) - "jack.v3" - >>> all_ids = {"simple": {}} - >>> get_latest_version_prompt_id("simple", all_ids) - "simple" - """ - base_id: Final = get_base_prompt_id(prompt_id=prompt_id) - - # Find all versions of this prompt - matching_versions: Final = [] - for stored_prompt_id in all_prompt_ids: - if get_base_prompt_id(prompt_id=stored_prompt_id) == base_id: - version_num = get_version_number(prompt_id=stored_prompt_id) - matching_versions.append((version_num, stored_prompt_id)) - - # Use the highest version number - if matching_versions: - matching_versions.sort(reverse=True) - return matching_versions[0][1] - else: - # No versioned prompts found, use the base ID as-is - return prompt_id - - def get_latest_prompt_versions(prompts: list[PromptSpec]) -> list[PromptSpec]: """ - Filter a list of prompts to return only the latest version of each unique prompt. - - Args: - prompts: List of PromptSpec objects - - Returns: - List of PromptSpec objects with only the latest version of each prompt + Filter prompts down to the latest version per (base prompt id, environment). """ - latest_prompts: Final[dict[str, PromptSpec]] = {} - - for prompt in prompts: - base_id = get_base_prompt_id(prompt_id=prompt.prompt_id) - version = get_version_number(prompt_id=prompt.prompt_id) - - # Keep the prompt with the highest version number - if base_id not in latest_prompts: - latest_prompts[base_id] = prompt - else: - existing_version = get_version_number(prompt_id=latest_prompts[base_id].prompt_id) - if version > existing_version: - latest_prompts[base_id] = prompt - + sorted_prompts: Final = sorted(prompts, key=lambda prompt: get_version_number(prompt_id=prompt.prompt_id)) + latest_prompts: Final = { + (get_base_prompt_id(prompt_id=prompt.prompt_id), prompt_environment_or_default(prompt.environment)): prompt + for prompt in sorted_prompts + } return list(latest_prompts.values()) async def get_next_version_for_prompt( - prisma_client: "PrismaClient", prompt_id: str, environment: str = "development" + prisma_client: "PrismaClient", prompt_id: str, environment: str = DEFAULT_PROMPT_ENVIRONMENT ) -> int: """ Get the next version number for a prompt in a specific environment. @@ -403,11 +264,14 @@ async def list_prompts( if key_metadata is not None: prompts: Final = cast(list[str] | None, key_metadata.get("prompts", None)) if prompts is not None: - all_prompts = [ - IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[prompt_id] - for prompt_id in prompts - if prompt_id in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS + allowed_prompt_ids: Final = frozenset(prompts) + allowed_prompts: Final = [ + spec + for spec in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values() + if spec.prompt_id in allowed_prompt_ids + or get_base_prompt_id(prompt_id=spec.prompt_id) in allowed_prompt_ids ] + all_prompts = get_latest_prompt_versions(prompts=allowed_prompts) if environment: all_prompts = [p for p in all_prompts if p.environment == environment] prompt_list: Final = [] @@ -576,7 +440,7 @@ def _get_prompt_template(prompt_spec: PromptSpec, base_prompt_id: str) -> Prompt metadata=parsed.get("metadata"), ) else: - prompt_callback: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id(prompt_spec.prompt_id) + prompt_callback: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=prompt_spec) if prompt_callback is not None: integration_name: Final = prompt_callback.integration_name if integration_name == "dotprompt": @@ -690,15 +554,8 @@ async def get_prompt_info( if env_prompts: prompt_spec = create_versioned_prompt_spec(db_prompt=env_prompts[0]) - # Fallback: use in-memory registry (no environment filter) if prompt_spec is None and environment is None: - prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) - if prompt_spec is None: - latest_prompt_id: Final = get_latest_version_prompt_id( - prompt_id=prompt_id, - all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS, - ) - prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id) + prompt_spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec(prompt_id, version=requested_version) if prompt_spec is None: raise HTTPException( @@ -785,7 +642,7 @@ async def create_prompt( environment: Final = ( request.prompt_info.environment if request.prompt_info and request.prompt_info.environment - else "development" + else DEFAULT_PROMPT_ENVIRONMENT ) # Get next version number @@ -885,7 +742,7 @@ async def update_prompt( environment: Final = ( request.prompt_info.environment if request.prompt_info and request.prompt_info.environment - else "development" + else DEFAULT_PROMPT_ENVIRONMENT ) # Check if any version of this prompt exists (in any environment) @@ -897,9 +754,7 @@ async def update_prompt( detail=f"Prompt with ID {base_prompt_id} not found", ) - # Check if it's a config prompt - existing_in_memory: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) - if existing_in_memory and existing_in_memory.prompt_info.prompt_type == "config": + if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id): raise HTTPException( status_code=400, detail="Cannot update config prompts.", @@ -988,52 +843,24 @@ async def delete_prompt( raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) try: - # Try to get prompt directly first - existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) - - # If not found, try to find the latest version - if existing_prompt is None: - latest_prompt_id: Final = get_latest_version_prompt_id( - prompt_id=prompt_id, - all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS, - ) - existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id) - # Use the resolved prompt_id for deletion - prompt_id = latest_prompt_id + base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) + existing_prompt: Final = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec(prompt_id, environment=environment) if existing_prompt is None: raise HTTPException(status_code=404, detail=f"Prompt with ID {prompt_id} not found") - if existing_prompt.prompt_info.prompt_type == "config": + if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id): raise HTTPException( status_code=400, detail="Cannot delete config prompts.", ) - # Get the base prompt ID (without version suffix) for database deletion - base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) - - # Build delete filter; scope to environment if provided - delete_where: Final[dict[str, str]] = {"prompt_id": base_prompt_id} - if environment: - delete_where["environment"] = environment - - # Delete versions from the database (scoped to environment if provided) + delete_where: Final[dict[str, str]] = { + "prompt_id": base_prompt_id, + **({"environment": environment} if environment else {}), + } await _prompt_table(prisma_client).delete_many(where=delete_where) - - # Remove matching prompts from memory — scope to environment if provided - if environment: - prompts_to_delete: Final = [ - pid - for pid, prompt in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.items() - if get_base_prompt_id(prompt_id=pid) == base_prompt_id and prompt.environment == environment - ] - for pid in prompts_to_delete: - del IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[pid] - if pid in IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt: - del IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt[pid] - else: - IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id) + IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id=base_prompt_id, environment=environment) env_msg: Final = f" from {environment}" if environment else "" return {"message": f"Prompt {base_prompt_id} deleted successfully{env_msg}"} @@ -1105,7 +932,7 @@ async def patch_prompt( try: # Resolve the target row: find the latest version in the given environment base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) - env: Final = environment or "development" + env: Final = prompt_environment_or_default(environment) requested_version: Final = get_version_number(prompt_id=prompt_id) if prompt_id != base_prompt_id else None # Build query to find the exact row by composite unique key @@ -1129,11 +956,7 @@ async def patch_prompt( target_row: Final = db_rows[0] - # Check if prompt exists in memory - versioned_id: Final = f"{base_prompt_id}.v{target_row.version}" - existing_prompt: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(versioned_id) - - if existing_prompt and existing_prompt.prompt_info.prompt_type == "config": + if IN_MEMORY_PROMPT_REGISTRY.has_config_prompt(base_prompt_id=base_prompt_id): raise HTTPException( status_code=400, detail="Cannot update config prompts.", diff --git a/litellm/proxy/prompts/prompt_registry.py b/litellm/proxy/prompts/prompt_registry.py index d4342773a85..b575184a229 100644 --- a/litellm/proxy/prompts/prompt_registry.py +++ b/litellm/proxy/prompts/prompt_registry.py @@ -1,6 +1,6 @@ import importlib import os -from collections.abc import Callable +from collections.abc import Callable, Sequence from pathlib import Path from typing import Final @@ -14,6 +14,77 @@ from litellm.types.prompts.init_prompts import ( prompt_initializer_registry = {} +DEFAULT_PROMPT_ENVIRONMENT: Final = "development" +PROMPT_ENVIRONMENT_SERVE_PRECEDENCE: Final = ("production", "staging", "development") + + +def get_base_prompt_id(prompt_id: str) -> str: + """ + Extract the base prompt ID by stripping the version suffix if present. + + Examples: + >>> get_base_prompt_id("jack_success.v1") + "jack_success" + >>> get_base_prompt_id("jack_success_v1") + "jack_success" + >>> get_base_prompt_id("jack_success") + "jack_success" + """ + if ".v" in prompt_id: + return prompt_id.split(".v")[0] + if "_v" in prompt_id: + return prompt_id.split("_v")[0] + return prompt_id + + +def get_version_number(prompt_id: str) -> int: + """ + Extract the version number from a versioned prompt ID (defaults to 1). + + Examples: + >>> get_version_number("jack_success.v2") + 2 + >>> get_version_number("jack_success_v2") + 2 + >>> get_version_number("jack_success") + 1 + """ + if ".v" in prompt_id: + version_str = prompt_id.split(".v")[1] + try: + return int(version_str) + except ValueError: + pass + + if "_v" in prompt_id: + version_str = prompt_id.split("_v")[1] + try: + return int(version_str) + except ValueError: + pass + + return 1 + + +def prompt_environment_or_default(environment: str | None) -> str: + return environment or DEFAULT_PROMPT_ENVIRONMENT + + +def registry_key_for_prompt(prompt: PromptSpec) -> str: + return f"{prompt.prompt_id}::{prompt_environment_or_default(prompt.environment)}" + + +def _spec_version(prompt: PromptSpec) -> int: + return prompt.version if prompt.version is not None else get_version_number(prompt_id=prompt.prompt_id) + + +def _default_serve_environment(prompts: Sequence[PromptSpec]) -> str: + present: Final = frozenset(prompt_environment_or_default(prompt.environment) for prompt in prompts) + ladder_pick: Final = next((env for env in PROMPT_ENVIRONMENT_SERVE_PRECEDENCE if env in present), None) + if ladder_pick is not None: + return ladder_pick + return min(present) if present else DEFAULT_PROMPT_ENVIRONMENT + def get_prompt_initializer_from_integrations(): """ @@ -113,17 +184,16 @@ class InMemoryPromptRegistry: """ import litellm - prompt_id: Final = prompt.prompt_id - if prompt_id in self.IN_MEMORY_PROMPTS: - verbose_proxy_logger.debug("prompt_id already exists in IN_MEMORY_PROMPTS") - return self.IN_MEMORY_PROMPTS[prompt_id] + registry_key: Final = registry_key_for_prompt(prompt) + if registry_key in self.IN_MEMORY_PROMPTS: + verbose_proxy_logger.debug("prompt already exists in IN_MEMORY_PROMPTS") + return self.IN_MEMORY_PROMPTS[registry_key] parsed_prompt, custom_prompt_callback = self._build_prompt_callback(prompt=prompt) litellm.logging_callback_manager.add_litellm_callback(custom_prompt_callback) - # store references to the prompt in memory - self.IN_MEMORY_PROMPTS[prompt_id] = parsed_prompt - self.prompt_id_to_custom_prompt[prompt_id] = custom_prompt_callback + self.IN_MEMORY_PROMPTS[registry_key] = parsed_prompt + self.prompt_id_to_custom_prompt[registry_key] = custom_prompt_callback return parsed_prompt @@ -166,57 +236,85 @@ class InMemoryPromptRegistry: import litellm parsed_prompt, new_callback = self._build_prompt_callback(prompt=prompt) - stale_callback: Final = self.prompt_id_to_custom_prompt.pop(prompt.prompt_id, None) - self.IN_MEMORY_PROMPTS.pop(prompt.prompt_id, None) + registry_key: Final = registry_key_for_prompt(parsed_prompt) + stale_callback: Final = self.prompt_id_to_custom_prompt.pop(registry_key, None) + self.IN_MEMORY_PROMPTS.pop(registry_key, None) if stale_callback is not None: litellm.logging_callback_manager.remove_callback_from_all_lists(stale_callback) litellm.logging_callback_manager.add_litellm_callback(new_callback) - self.IN_MEMORY_PROMPTS[prompt.prompt_id] = parsed_prompt - self.prompt_id_to_custom_prompt[prompt.prompt_id] = new_callback + self.IN_MEMORY_PROMPTS[registry_key] = parsed_prompt + self.prompt_id_to_custom_prompt[registry_key] = new_callback return parsed_prompt def sync_prompt_from_db(self, prompt: PromptSpec) -> PromptSpec | None: - existing: Final = self.IN_MEMORY_PROMPTS.get(prompt.prompt_id) + existing: Final = self.IN_MEMORY_PROMPTS.get(registry_key_for_prompt(prompt)) if existing is None: return self.initialize_prompt(prompt=prompt) if existing.litellm_params == prompt.litellm_params and existing.prompt_info == prompt.prompt_info: return existing return self.reload_prompt(prompt=prompt) - def get_prompt_by_id(self, prompt_id: str) -> PromptSpec | None: + def resolve_prompt_spec( + self, + prompt_id: str, + version: int | None = None, + environment: str | None = None, + ) -> PromptSpec | None: """ - Get a prompt by its ID from memory - """ - return self.IN_MEMORY_PROMPTS.get(prompt_id) + Resolve a prompt spec by base prompt id, optional version, and optional environment. - def get_prompt_callback_by_id(self, prompt_id: str) -> CustomPromptManagement | None: + With no environment, resolves within the default serve environment + (production > staging > development > alphabetical first present). + With no version, resolves to the highest version in the chosen environment. """ - Get a prompt callback by its ID from memory + base_prompt_id: Final = get_base_prompt_id(prompt_id=prompt_id) + base_matches: Final = tuple( + spec + for spec in self.IN_MEMORY_PROMPTS.values() + if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id + ) + if not base_matches: + return None + resolved_environment: Final = ( + environment if environment is not None else _default_serve_environment(base_matches) + ) + env_matches: Final = tuple( + spec for spec in base_matches if prompt_environment_or_default(spec.environment) == resolved_environment + ) + if not env_matches: + return None + if version is not None: + return next((spec for spec in env_matches if _spec_version(spec) == version), None) + return max(env_matches, key=_spec_version) + + def get_prompt_callback_for_prompt(self, prompt: PromptSpec) -> CustomPromptManagement | None: + return self.prompt_id_to_custom_prompt.get(registry_key_for_prompt(prompt)) + + def has_config_prompt(self, base_prompt_id: str) -> bool: + return any( + spec.prompt_info.prompt_type == "config" + for spec in self.IN_MEMORY_PROMPTS.values() + if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id + ) + + def delete_prompts_by_base_id(self, base_prompt_id: str, environment: str | None = None) -> list[str]: """ - return self.prompt_id_to_custom_prompt.get(prompt_id) + Delete matching prompts from memory, scoped to one environment when given. - def delete_prompts_by_base_id(self, base_prompt_id: str) -> list[str]: + Returns the registry keys that were deleted. """ - Delete all prompts matching the given base prompt ID from memory. - - Args: - base_prompt_id: The base prompt ID (without version suffix) - - Returns: - List of prompt IDs that were deleted - """ - from litellm.proxy.prompts.prompt_endpoints import get_base_prompt_id - - prompts_to_delete: Final = [ - pid for pid in self.IN_MEMORY_PROMPTS if get_base_prompt_id(prompt_id=pid) == base_prompt_id + keys_to_delete: Final = [ + key + for key, spec in self.IN_MEMORY_PROMPTS.items() + if get_base_prompt_id(prompt_id=spec.prompt_id) == base_prompt_id + and (environment is None or prompt_environment_or_default(spec.environment) == environment) ] - for pid in prompts_to_delete: - del self.IN_MEMORY_PROMPTS[pid] - if pid in self.prompt_id_to_custom_prompt: - del self.prompt_id_to_custom_prompt[pid] + for key in keys_to_delete: + del self.IN_MEMORY_PROMPTS[key] + self.prompt_id_to_custom_prompt.pop(key, None) - return prompts_to_delete + return keys_to_delete IN_MEMORY_PROMPT_REGISTRY: Final = InMemoryPromptRegistry() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2cfe08fe332..bc4eaaab761 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7273,16 +7273,7 @@ class ProxyConfig: parsed_specs: Final[tuple[PromptSpec, ...]] = tuple( spec for row in prompts_in_db if (spec := parse_row(row)) is not None ) - newest_spec_per_id: Final[Mapping[str, PromptSpec]] = MappingProxyType( - { - spec.prompt_id: spec - for spec in sorted( - parsed_specs, - key=lambda s: s.updated_at.timestamp() if s.updated_at else float("-inf"), - ) - } - ) - for prompt_spec in newest_spec_per_id.values(): + for prompt_spec in parsed_specs: try: IN_MEMORY_PROMPT_REGISTRY.sync_prompt_from_db(prompt=prompt_spec) except Exception as prompt_sync_error: # noqa: BLE001 # one poisoned row must not block syncing the remaining prompts diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8cbf5b685fd..95f02279f19 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1397,27 +1397,26 @@ class ProxyLogging: ) -> None: """Process prompt template if applicable.""" - from litellm.proxy.prompts.prompt_endpoints import ( - construct_versioned_prompt_id, - get_latest_version_prompt_id, - ) from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY from litellm.utils import get_non_default_completion_params - if prompt_version is None: - lookup_prompt_id = get_latest_version_prompt_id( - prompt_id=prompt_id, - all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS, - ) - else: - lookup_prompt_id = construct_versioned_prompt_id(prompt_id=prompt_id, version=prompt_version) - - custom_logger: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id(lookup_prompt_id) - prompt_spec: Final = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(lookup_prompt_id) + raw_prompt_environment: Final = data.get("prompt_environment", None) + prompt_environment: Final = raw_prompt_environment if isinstance(raw_prompt_environment, str) else None + prompt_spec: Final = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec( + prompt_id, + version=prompt_version, + environment=prompt_environment, + ) + custom_logger: Final = ( + IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=prompt_spec) + if prompt_spec is not None + else None + ) litellm_prompt_id: str | None = None if prompt_spec is not None: litellm_prompt_id = prompt_spec.litellm_params.prompt_id data.pop("prompt_id", None) + data.pop("prompt_environment", None) if custom_logger and prompt_spec is not None: ( @@ -1444,6 +1443,7 @@ class ProxyLogging: data.pop("prompt_variables", None) data.pop("prompt_label", None) data.pop("prompt_version", None) + data.pop("prompt_environment", None) def _process_guardrail_metadata(self, data: dict) -> None: """Process guardrails from metadata and add to applied_guardrails.""" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ef3586f2559..27437b74251 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3522,6 +3522,7 @@ all_litellm_params = ( "litellm_system_prompt", "provider_specific_header", "prompt_version", + "prompt_environment", "api_base", "force_timeout", "logger_fn", diff --git a/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py b/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py index 6ebb10eff76..35402db219f 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py @@ -104,97 +104,6 @@ class TestPromptVersioning: assert get_base_prompt_id(prompt_id="jack") == "jack" assert get_base_prompt_id(prompt_id="my_prompt.v10") == "my_prompt" - def test_get_latest_version_prompt_id(self): - """ - Test that get_latest_version_prompt_id returns the highest version - """ - from litellm.proxy.prompts.prompt_endpoints import get_latest_version_prompt_id - - # Mock prompt IDs dictionary - all_prompt_ids = { - "jack.v1": {}, - "jack.v2": {}, - "jack.v3": {}, - "jane.v1": {}, - "simple_prompt": {}, - } - - # Test with base prompt ID - should return latest version - assert ( - get_latest_version_prompt_id( - prompt_id="jack", all_prompt_ids=all_prompt_ids - ) - == "jack.v3" - ) - - # Test with versioned prompt ID - should still return latest version - assert ( - get_latest_version_prompt_id( - prompt_id="jack.v1", all_prompt_ids=all_prompt_ids - ) - == "jack.v3" - ) - - # Test with single version - assert ( - get_latest_version_prompt_id( - prompt_id="jane", all_prompt_ids=all_prompt_ids - ) - == "jane.v1" - ) - - # Test with non-versioned prompt - assert ( - get_latest_version_prompt_id( - prompt_id="simple_prompt", all_prompt_ids=all_prompt_ids - ) - == "simple_prompt" - ) - - # Test with non-existent prompt - assert ( - get_latest_version_prompt_id( - prompt_id="nonexistent", all_prompt_ids=all_prompt_ids - ) - == "nonexistent" - ) - - def test_construct_versioned_prompt_id(self): - """ - Test that construct_versioned_prompt_id correctly builds versioned IDs - """ - from litellm.proxy.prompts.prompt_endpoints import construct_versioned_prompt_id - - # Test with base prompt ID and version - assert ( - construct_versioned_prompt_id(prompt_id="jack_success", version=4) - == "jack_success.v4" - ) - - # Test with None version - should return base ID unchanged - assert ( - construct_versioned_prompt_id(prompt_id="jack_success", version=None) - == "jack_success" - ) - - # Test with existing versioned ID - should replace version - assert ( - construct_versioned_prompt_id(prompt_id="jack_success.v2", version=4) - == "jack_success.v4" - ) - - # Test with hyphenated prompt ID - assert ( - construct_versioned_prompt_id(prompt_id="my-prompt", version=1) - == "my-prompt.v1" - ) - - # Test with double-digit version - assert ( - construct_versioned_prompt_id(prompt_id="test_prompt", version=10) - == "test_prompt.v10" - ) - class TestPromptVersionsEndpoint: """ @@ -444,7 +353,7 @@ class TestAdminViewerReadAccess: "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry, ): - mock_registry.get_prompt_by_id.return_value = PromptSpec( + mock_registry.resolve_prompt_spec.return_value = PromptSpec( prompt_id="jack.v2", litellm_params=PromptLiteLLMParams( prompt_id="jack", @@ -453,8 +362,7 @@ class TestAdminViewerReadAccess: ), prompt_info=PromptInfo(prompt_type="db"), ) - mock_registry.IN_MEMORY_PROMPTS = {"jack.v1": {}, "jack.v2": {}} - mock_registry.get_prompt_callback_by_id.return_value = None + mock_registry.get_prompt_callback_for_prompt.return_value = None response = await get_prompt_info(prompt_id="jack", user_api_key_dict=viewer) diff --git a/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py b/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py index 688b739fb5a..6916a7163f8 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py @@ -52,8 +52,6 @@ async def test_delete_prompt_success(): with patch( "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry: - # User passes "test_prompt.v2" - # We simulate that get_prompt_by_id returns the prompt spec for v2 prompt_spec = PromptSpec( prompt_id="test_prompt.v2", litellm_params=PromptLiteLLMParams( @@ -61,7 +59,8 @@ async def test_delete_prompt_success(): ), prompt_info=PromptInfo(prompt_type="db"), ) - mock_registry.get_prompt_by_id.return_value = prompt_spec + mock_registry.resolve_prompt_spec.return_value = prompt_spec + mock_registry.has_config_prompt.return_value = False # Patch the prisma client in the endpoint module with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): @@ -79,7 +78,7 @@ async def test_delete_prompt_success(): # 2. Memory deletion should use base ID mock_registry.delete_prompts_by_base_id.assert_called_once_with( - expected_base_id + base_prompt_id=expected_base_id, environment=None ) assert response == { @@ -108,31 +107,14 @@ async def test_delete_prompt_by_base_id_success(): with patch( "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry: - # User passes "test_prompt" (base ID) - # 1. get_prompt_by_id("test_prompt") -> None (if it's not registered as base) - # 2. It calls get_latest_version_prompt_id -> returns "test_prompt.v3" - # 3. get_prompt_by_id("test_prompt.v3") -> returns Spec - - # Setup mocks behavior - def get_prompt_side_effect(prompt_id): - if prompt_id == "test_prompt": - return None - if prompt_id == "test_prompt.v3": - return PromptSpec( - prompt_id="test_prompt.v3", - litellm_params=PromptLiteLLMParams( - prompt_id="test_prompt", prompt_integration="dotprompt" - ), - prompt_info=PromptInfo(prompt_type="db"), - ) - return None - - mock_registry.get_prompt_by_id.side_effect = get_prompt_side_effect - mock_registry.IN_MEMORY_PROMPTS = { - "test_prompt.v1": {}, - "test_prompt.v2": {}, - "test_prompt.v3": {}, - } + mock_registry.resolve_prompt_spec.return_value = PromptSpec( + prompt_id="test_prompt.v3", + litellm_params=PromptLiteLLMParams( + prompt_id="test_prompt", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db"), + ) + mock_registry.has_config_prompt.return_value = False # Patch the prisma client in the endpoint module with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): @@ -150,7 +132,7 @@ async def test_delete_prompt_by_base_id_success(): # 2. Memory deletion should use base ID mock_registry.delete_prompts_by_base_id.assert_called_once_with( - expected_base_id + base_prompt_id=expected_base_id, environment=None ) assert response == { @@ -187,24 +169,8 @@ async def test_get_prompt_info_by_base_id(): prompt_info=PromptInfo(prompt_type="db"), ) - # When get_prompt_by_id is called with "test_prompt", return None (so it searches versions) - # When called with "test_prompt.v3", return the spec - def get_prompt_side_effect(prompt_id): - if prompt_id == "test_prompt": - return None - if prompt_id == "test_prompt.v3": - return prompt_spec_v3 - return None - - mock_registry.get_prompt_by_id.side_effect = get_prompt_side_effect - mock_registry.IN_MEMORY_PROMPTS = { - "test_prompt.v1": {}, - "test_prompt.v2": {}, - "test_prompt.v3": {}, - } - - # We also need to mock get_prompt_callback_by_id to avoid content extraction errors/logic - mock_registry.get_prompt_callback_by_id.return_value = None + mock_registry.resolve_prompt_spec.return_value = prompt_spec_v3 + mock_registry.get_prompt_callback_for_prompt.return_value = None response = await get_prompt_info( prompt_id="test_prompt", user_api_key_dict=mock_user_auth @@ -253,7 +219,7 @@ async def test_patch_prompt_row_deleted_mid_update_returns_404(): "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry, ): - mock_registry.get_prompt_by_id.return_value = existing_prompt + mock_registry.has_config_prompt.return_value = False with pytest.raises(HTTPException) as exc_info: await patch_prompt( @@ -294,7 +260,7 @@ async def test_patch_prompt_merges_unsent_fields_from_db_row_not_stale_memory(): "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry, ): - mock_registry.get_prompt_by_id.return_value = stale_in_memory + mock_registry.has_config_prompt.return_value = False mock_registry.reload_prompt.side_effect = lambda prompt: prompt response = await patch_prompt( @@ -456,7 +422,7 @@ async def test_patch_prompt_info_only_keeps_legacy_keyed_row_patchable(): "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry, ): - mock_registry.get_prompt_by_id.return_value = existing_prompt + mock_registry.has_config_prompt.return_value = False await patch_prompt( prompt_id="agent-prompt", diff --git a/tests/test_litellm/proxy/prompts/test_prompt_environment.py b/tests/test_litellm/proxy/prompts/test_prompt_environment.py index ecd89afefbe..3cb647dea13 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_environment.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_environment.py @@ -191,11 +191,7 @@ async def test_update_prompt_stores_environment_and_created_by(): with patch( "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" ) as mock_registry: - mock_registry.get_prompt_by_id.return_value = PromptSpec( - prompt_id="my_prompt.v1", - litellm_params=request.litellm_params, - prompt_info=PromptInfo(prompt_type="db"), - ) + mock_registry.has_config_prompt.return_value = False mock_registry.initialize_prompt.return_value = PromptSpec( prompt_id="my_prompt.v2", litellm_params=request.litellm_params, @@ -239,7 +235,8 @@ async def test_delete_prompt_scoped_to_environment(): prompt_info=PromptInfo(prompt_type="db"), environment="staging", ) - mock_registry.get_prompt_by_id.return_value = prompt_spec + mock_registry.resolve_prompt_spec.return_value = prompt_spec + mock_registry.has_config_prompt.return_value = False with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): await delete_prompt( @@ -251,3 +248,6 @@ async def test_delete_prompt_scoped_to_environment(): mock_prisma_client.db.litellm_prompttable.delete_many.assert_called_once_with( where={"prompt_id": "test_prompt", "environment": "staging"} ) + mock_registry.delete_prompts_by_base_id.assert_called_once_with( + base_prompt_id="test_prompt", environment="staging" + ) diff --git a/tests/test_litellm/proxy/prompts/test_prompt_registry.py b/tests/test_litellm/proxy/prompts/test_prompt_registry.py index 47f1ba13627..689cbd79873 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_registry.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_registry.py @@ -1,26 +1,35 @@ import pytest import litellm +from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry from litellm.types.prompts.init_prompts import PromptInfo, PromptLiteLLMParams, PromptSpec -def _db_prompt_spec(content: str) -> PromptSpec: +def _db_prompt_spec(content: str, environment: str = "development", version: int = 1) -> PromptSpec: return PromptSpec( - prompt_id="greeting.v1", + prompt_id=f"greeting.v{version}", litellm_params=PromptLiteLLMParams( prompt_id="greeting", prompt_integration="dotprompt", prompt_data={"content": content, "metadata": {}}, ), prompt_info=PromptInfo(prompt_type="db"), + version=version, + environment=environment, ) -def _served_content(registry: InMemoryPromptRegistry) -> str: - callback = registry.get_prompt_callback_by_id("greeting.v1") +def _resolved_callback(registry: InMemoryPromptRegistry, environment: str | None = None) -> CustomPromptManagement: + spec = registry.resolve_prompt_spec("greeting", environment=environment) + assert spec is not None + callback = registry.get_prompt_callback_for_prompt(prompt=spec) assert callback is not None - return callback.prompt_manager.get_prompt("greeting").content + return callback + + +def _served_content(registry: InMemoryPromptRegistry, environment: str | None = None) -> str: + return _resolved_callback(registry, environment=environment).prompt_manager.get_prompt("greeting").content @pytest.fixture @@ -32,32 +41,34 @@ def isolated_callbacks(monkeypatch: pytest.MonkeyPatch) -> list: def test_sync_prompt_from_db_reloads_row_edited_elsewhere(isolated_callbacks: list) -> None: registry = InMemoryPromptRegistry() registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY")) - stale_callback = registry.get_prompt_callback_by_id("greeting.v1") + stale_callback = _resolved_callback(registry) assert _served_content(registry) == "begin every reply with AHOY" registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY")) assert _served_content(registry) == "begin every reply with HOWDY" - assert registry.get_prompt_by_id("greeting.v1").litellm_params.prompt_data["content"] == "begin every reply with HOWDY" + reloaded_spec = registry.resolve_prompt_spec("greeting", environment="development") + assert reloaded_spec is not None + assert reloaded_spec.litellm_params.prompt_data["content"] == "begin every reply with HOWDY" assert stale_callback not in isolated_callbacks - assert isolated_callbacks == [registry.get_prompt_callback_by_id("greeting.v1")] + assert isolated_callbacks == [_resolved_callback(registry)] def test_sync_prompt_from_db_keeps_unchanged_row_in_place(isolated_callbacks: list) -> None: registry = InMemoryPromptRegistry() registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY")) - first_callback = registry.get_prompt_callback_by_id("greeting.v1") + first_callback = _resolved_callback(registry) registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY")) - assert registry.get_prompt_callback_by_id("greeting.v1") is first_callback + assert _resolved_callback(registry) is first_callback assert isolated_callbacks == [first_callback] def test_reload_prompt_replaces_callback_without_leaking_the_old_one(isolated_callbacks: list) -> None: registry = InMemoryPromptRegistry() registry.initialize_prompt(prompt=_db_prompt_spec("begin every reply with AHOY")) - stale_callback = registry.get_prompt_callback_by_id("greeting.v1") + stale_callback = _resolved_callback(registry) reloaded = registry.reload_prompt(prompt=_db_prompt_spec("begin every reply with HOWDY")) @@ -70,7 +81,7 @@ def test_reload_prompt_replaces_callback_without_leaking_the_old_one(isolated_ca def test_reload_prompt_keeps_the_old_template_when_the_replacement_fails(isolated_callbacks: list) -> None: registry = InMemoryPromptRegistry() registry.initialize_prompt(prompt=_db_prompt_spec("begin every reply with AHOY")) - old_callback = registry.get_prompt_callback_by_id("greeting.v1") + old_callback = _resolved_callback(registry) broken = PromptSpec( prompt_id="greeting.v1", @@ -80,11 +91,101 @@ def test_reload_prompt_keeps_the_old_template_when_the_replacement_fails(isolate prompt_data={"content": "begin every reply with HOWDY", "metadata": {}}, ), prompt_info=PromptInfo(prompt_type="db"), + version=1, + environment="development", ) with pytest.raises(ValueError, match="Unsupported prompt"): registry.reload_prompt(prompt=broken) - assert registry.get_prompt_callback_by_id("greeting.v1") is old_callback + assert _resolved_callback(registry) is old_callback assert _served_content(registry) == "begin every reply with AHOY" assert isolated_callbacks == [old_callback] + + +def test_environments_sharing_a_prompt_id_keep_separate_templates(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment="development")) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY", environment="production")) + + assert _served_content(registry, environment="development") == "begin every reply with AHOY" + assert _served_content(registry, environment="production") == "begin every reply with HOWDY" + assert _resolved_callback(registry, environment="development") is not _resolved_callback( + registry, environment="production" + ) + + +def test_default_resolution_prefers_production(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment="development")) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY", environment="production")) + + assert _served_content(registry) == "begin every reply with HOWDY" + + +@pytest.mark.parametrize("environment", ["staging", "qa"]) +def test_default_resolution_serves_the_only_environment_present(isolated_callbacks: list, environment: str) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment=environment)) + + assert _served_content(registry) == "begin every reply with AHOY" + + +def test_resolution_picks_exact_version_and_latest_within_an_environment(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment="development", version=1)) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with YO", environment="development", version=2)) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY", environment="production", version=1)) + + exact = registry.resolve_prompt_spec("greeting", version=1, environment="development") + assert exact is not None + assert exact.litellm_params.prompt_data["content"] == "begin every reply with AHOY" + + latest = registry.resolve_prompt_spec("greeting", environment="development") + assert latest is not None + assert latest.litellm_params.prompt_data["content"] == "begin every reply with YO" + + assert registry.resolve_prompt_spec("greeting", version=3, environment="development") is None + + +def test_resolution_returns_none_for_unknown_environment_or_prompt(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment="development")) + + assert registry.resolve_prompt_spec("greeting", environment="production") is None + assert registry.resolve_prompt_spec("no_such_prompt") is None + + +def test_delete_prompts_by_base_id_scoped_to_one_environment(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with AHOY", environment="development")) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY", environment="production")) + + deleted = registry.delete_prompts_by_base_id(base_prompt_id="greeting", environment="development") + + assert deleted == ["greeting.v1::development"] + assert registry.resolve_prompt_spec("greeting", environment="development") is None + assert _served_content(registry, environment="production") == "begin every reply with HOWDY" + + deleted_rest = registry.delete_prompts_by_base_id(base_prompt_id="greeting") + + assert deleted_rest == ["greeting.v1::production"] + assert registry.resolve_prompt_spec("greeting") is None + + +def test_has_config_prompt_matches_any_version_of_the_base_id(isolated_callbacks: list) -> None: + registry = InMemoryPromptRegistry() + config_spec = PromptSpec( + prompt_id="greeting", + litellm_params=PromptLiteLLMParams( + prompt_id="greeting", + prompt_integration="dotprompt", + prompt_data={"content": "begin every reply with AHOY", "metadata": {}}, + ), + prompt_info=PromptInfo(prompt_type="config"), + ) + registry.initialize_prompt(prompt=config_spec) + registry.sync_prompt_from_db(prompt=_db_prompt_spec("begin every reply with HOWDY", environment="production")) + + assert registry.has_config_prompt(base_prompt_id="greeting") is True + assert registry.has_config_prompt(base_prompt_id="other_prompt") is False diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 4266a13bf11..e2692a99732 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -11379,10 +11379,15 @@ async def test_init_prompts_in_db_reloads_rows_patched_on_another_worker(monkeyp } return row - def served_content() -> str: - callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("greeting_sync.v1") + def served_callback(): + spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec("greeting_sync") + assert spec is not None + callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=spec) assert callback is not None - return callback.prompt_manager.get_prompt("greeting_sync").content + return callback + + def served_content() -> str: + return served_callback().prompt_manager.get_prompt("greeting_sync").content prisma_client = MagicMock() try: @@ -11394,7 +11399,7 @@ async def test_init_prompts_in_db_reloads_rows_patched_on_another_worker(monkeyp await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client) assert served_content() == "Begin every reply with HOWDY" - assert litellm.callbacks == [IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("greeting_sync.v1")] + assert litellm.callbacks == [served_callback()] finally: IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id("greeting_sync") @@ -11433,22 +11438,25 @@ async def test_init_prompts_in_db_syncs_remaining_rows_when_one_row_fails(monkey ) await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client) - assert IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id("broken_sync.v1") is None - assert IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("healthy_sync.v1") is not None - assert litellm.callbacks == [IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("healthy_sync.v1")] + assert IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec("broken_sync") is None + healthy_spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec("healthy_sync") + assert healthy_spec is not None + healthy_callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=healthy_spec) + assert healthy_callback is not None + assert litellm.callbacks == [healthy_callback] finally: IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id("healthy_sync") IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id("broken_sync") @pytest.mark.asyncio -async def test_init_prompts_in_db_serves_the_newest_row_when_environments_collide_on_a_versioned_id(monkeypatch): +async def test_init_prompts_in_db_syncs_every_environment_sharing_a_versioned_id(monkeypatch): from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY from litellm.proxy.proxy_server import ProxyConfig monkeypatch.setattr(litellm, "callbacks", []) - def db_row(environment: str, content: str, updated_at: datetime) -> MagicMock: + def db_row(environment: str, content: str) -> MagicMock: row = MagicMock() row.model_dump.return_value = { "prompt_id": "greeting_env", @@ -11464,30 +11472,41 @@ async def test_init_prompts_in_db_serves_the_newest_row_when_environments_collid ), "prompt_info": json.dumps({"prompt_type": "db"}), "created_at": None, - "updated_at": updated_at, + "updated_at": None, } return row - freshly_patched = db_row( - "production", "Begin every reply with HOWDY", datetime(2026, 8, 26, 12, 0, tzinfo=timezone.utc) - ) - stale_sibling = db_row( - "development", "Begin every reply with AHOY", datetime(2026, 8, 26, 11, 0, tzinfo=timezone.utc) - ) + def served_content(environment: str | None) -> str: + spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec("greeting_env", environment=environment) + assert spec is not None + callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_for_prompt(prompt=spec) + assert callback is not None + return callback.prompt_manager.get_prompt("greeting_env").content prisma_client = MagicMock() try: - prisma_client.db.litellm_prompttable.find_many = AsyncMock(return_value=[freshly_patched, stale_sibling]) + prisma_client.db.litellm_prompttable.find_many = AsyncMock( + return_value=[ + db_row("development", "Begin every reply with AHOY"), + db_row("production", "Begin every reply with HOWDY"), + ] + ) await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client) - first_callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("greeting_env.v1") - assert first_callback is not None - assert first_callback.prompt_manager.get_prompt("greeting_env").content == "Begin every reply with HOWDY" + assert served_content("development") == "Begin every reply with AHOY" + assert served_content("production") == "Begin every reply with HOWDY" + assert served_content(None) == "Begin every reply with HOWDY" + prisma_client.db.litellm_prompttable.find_many = AsyncMock( + return_value=[ + db_row("development", "Begin every reply with YO"), + db_row("production", "Begin every reply with HOWDY"), + ] + ) await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client) - assert IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id("greeting_env.v1") is first_callback - assert litellm.callbacks == [first_callback] + assert served_content("development") == "Begin every reply with YO" + assert served_content("production") == "Begin every reply with HOWDY" finally: IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id("greeting_env") diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 7df39b0ef82..69310124ed3 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -726,10 +726,7 @@ async def test_process_prompt_template_no_op_when_no_prompt_spec(proxy_logging, from litellm.proxy.prompts import prompt_registry monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_callback_by_id", lambda *a, **kw: None - ) - monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_by_id", lambda *a, **kw: None + prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: None ) data: Dict[str, Any] = {"messages": [{"role": "user"}], "model": "m", "temperature": 0.1} await proxy_logging._process_prompt_template( @@ -752,11 +749,11 @@ async def test_process_prompt_template_applies_when_spec_resolves(proxy_logging, monkeypatch.setattr( prompt_registry.IN_MEMORY_PROMPT_REGISTRY, - "get_prompt_callback_by_id", + "get_prompt_callback_for_prompt", lambda *a, **kw: custom_logger, ) monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_by_id", lambda *a, **kw: prompt_spec + prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: prompt_spec ) logging_obj = MagicMock() @@ -802,11 +799,11 @@ async def test_process_prompt_template_async_get_prompt_error_raises(proxy_loggi prompt_spec.litellm_params = MagicMock(prompt_id="x") monkeypatch.setattr( prompt_registry.IN_MEMORY_PROMPT_REGISTRY, - "get_prompt_callback_by_id", + "get_prompt_callback_for_prompt", lambda *a, **kw: custom_logger, ) monkeypatch.setattr( - prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_by_id", lambda *a, **kw: prompt_spec + prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", lambda *a, **kw: prompt_spec ) logging_obj = MagicMock() logging_obj.async_get_chat_completion_prompt = AsyncMock(side_effect=RuntimeError("bad prompt")) @@ -818,3 +815,44 @@ async def test_process_prompt_template_async_get_prompt_error_raises(proxy_loggi prompt_version=None, call_type="completion", ) + + +@pytest.mark.asyncio +async def test_process_prompt_template_resolves_the_requested_environment(proxy_logging, monkeypatch): + from litellm.proxy.prompts import prompt_registry + + prompt_spec = MagicMock() + prompt_spec.litellm_params = MagicMock(prompt_id="greeting") + resolve_calls: list[dict] = [] + + def fake_resolve(prompt_id, version=None, environment=None): + resolve_calls.append({"prompt_id": prompt_id, "version": version, "environment": environment}) + return prompt_spec + + monkeypatch.setattr(prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", fake_resolve) + monkeypatch.setattr( + prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_callback_for_prompt", lambda *a, **kw: MagicMock() + ) + logging_obj = MagicMock() + logging_obj.async_get_chat_completion_prompt = AsyncMock( + return_value=("m", [{"role": "user", "content": "rendered"}], {}) + ) + data: Dict[str, Any] = { + "messages": [{"role": "user", "content": "orig"}], + "model": "m", + "prompt_id": "greeting", + "prompt_version": 1, + "prompt_environment": "development", + } + await proxy_logging._process_prompt_template( + data=data, + litellm_logging_obj=logging_obj, + prompt_id="greeting", + prompt_version=1, + call_type="completion", + ) + + assert resolve_calls == [{"prompt_id": "greeting", "version": 1, "environment": "development"}] + assert "prompt_environment" not in data + assert "prompt_id" not in data + assert data["messages"] == [{"role": "user", "content": "rendered"}] From 6f94554713ea69f108b356ab6691ee2bc72896ac Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:36:16 -0700 Subject: [PATCH 003/132] fix(ui): key prompt table rows by environment and dedupe key prompt options --- .../prompts/_components/PromptTable.tsx | 4 +++- .../components/organisms/create_key_button.tsx | 2 +- .../components/templates/key_edit_view.test.tsx | 16 ++++++++++++++++ .../src/components/templates/key_edit_view.tsx | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx index 47d4f64f254..a64d7dbfe0c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx @@ -74,7 +74,9 @@ const PromptTable: React.FC = ({ prompt.prompt_id || String(index)} + getRowId={(prompt, index) => + prompt.prompt_id ? `${prompt.prompt_id}::${prompt.environment || "development"}` : String(index) + } sortingMode="client" sorting={sorting} onSortingChange={setSorting} diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index e1e6dcfa442..d0da4ddfb2b 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -349,7 +349,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp const fetchPrompts = async () => { try { const response = await getPromptsList(accessToken); - setPromptsList(response.prompts.map((prompt) => prompt.prompt_id)); + setPromptsList(Array.from(new Set(response.prompts.map((prompt) => prompt.prompt_id)))); } catch (error) { console.error("Failed to fetch prompts:", error); } diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index bf8f43b8b5f..08edce6eda1 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -425,6 +425,22 @@ describe("KeyEditView", () => { expect(screen.getByText("Policies")).toBeInTheDocument(); }); + it("lists a prompt existing in several environments once in the dropdown", async () => { + vi.mocked(getPromptsList).mockResolvedValueOnce({ + prompts: [ + { prompt_id: "envgreet", litellm_params: {}, prompt_info: { prompt_type: "db" }, environment: "development" }, + { prompt_id: "envgreet", litellm_params: {}, prompt_info: { prompt_type: "db" }, environment: "production" }, + ], + }); + + renderAs("Admin"); + + const prompts = await screen.findByLabelText(/Prompts/); + await userEvent.type(prompts, "envgreet"); + + expect(await screen.findAllByRole("option", { name: "envgreet" })).toHaveLength(1); + }); + it("should omit both fields and fire neither admin-only request for an internal user", async () => { renderAs("Internal User"); diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index df1af2ca8e9..d4a8a804f29 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -165,7 +165,7 @@ export function KeyEditView({ if (!accessToken) return; try { const response = await getPromptsList(accessToken); - setPromptsList(response.prompts.map((prompt) => prompt.prompt_id)); + setPromptsList(Array.from(new Set(response.prompts.map((prompt) => prompt.prompt_id)))); } catch (error) { console.error("Failed to fetch prompts:", error); } From dcba64ab72a259a14dcd1fd8b631973741338bda Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:16:16 -0700 Subject: [PATCH 004/132] fix(ui): scope prompt row delete and info view to the row's environment --- .../prompts/_components/PromptTable.test.tsx | 8 ++-- .../prompts/_components/PromptTable.tsx | 4 +- .../_components/PromptTableColumns.tsx | 20 ++++++-- .../prompts/_components/index.test.tsx | 48 ++++++++++++++----- .../(dashboard)/prompts/_components/index.tsx | 18 ++++--- .../prompts/_components/prompt_info.test.tsx | 29 +++++++++++ .../prompts/_components/prompt_info.tsx | 17 +++++-- .../src/components/networking.tsx | 7 ++- 8 files changed, 117 insertions(+), 34 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx index 52efd6407f8..edbb897fb2f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.test.tsx @@ -65,11 +65,13 @@ describe("PromptTable", () => { expect(within(rows[1]).getByText("prompt-older")).toBeInTheDocument(); }); - it("should call onPromptClick when the prompt ID is clicked", async () => { + it("should call onPromptClick with the row's environment, defaulting to development", async () => { const user = userEvent.setup(); render(); await user.click(screen.getByRole("button", { name: "prompt-newer" })); - expect(mockOnPromptClick).toHaveBeenCalledWith("prompt-newer"); + expect(mockOnPromptClick).toHaveBeenCalledWith("prompt-newer", "production"); + await user.click(screen.getByRole("button", { name: "prompt-older" })); + expect(mockOnPromptClick).toHaveBeenCalledWith("prompt-older", "development"); }); it("should label the environment and default missing environments to development", () => { @@ -83,7 +85,7 @@ describe("PromptTable", () => { render(); await user.click(screen.getByTestId("prompt-actions-prompt-newer")); await user.click(await screen.findByTestId("prompt-action-delete")); - expect(mockOnDeleteClick).toHaveBeenCalledWith("prompt-newer", "prompt-newer"); + expect(mockOnDeleteClick).toHaveBeenCalledWith("prompt-newer", "prompt-newer", "production"); }); it("should copy the prompt ID through the actions menu", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx index a64d7dbfe0c..c766042ac44 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTable.tsx @@ -13,8 +13,8 @@ import { ModelGroupInfo } from "./prompt_utils"; interface PromptTableProps { promptsList: PromptSpec[]; isLoading: boolean; - onPromptClick?: (id: string) => void; - onDeleteClick?: (id: string, name: string) => void; + onPromptClick?: (id: string, environment: string) => void; + onDeleteClick?: (id: string, name: string, environment: string) => void; accessToken: string | null; isAdmin: boolean; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx index ae584ef6df6..f927a6d1486 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/PromptTableColumns.tsx @@ -64,7 +64,7 @@ function PromptModelCell({ prompt, modelHubData }: { prompt: PromptSpec; modelHu interface PromptRowActionsProps { prompt: PromptSpec; isAdmin: boolean; - onDeleteClick?: (id: string, name: string) => void; + onDeleteClick?: (id: string, name: string, environment: string) => void; } function PromptRowActions({ prompt, isAdmin, onDeleteClick }: PromptRowActionsProps) { @@ -91,7 +91,13 @@ function PromptRowActions({ prompt, isAdmin, onDeleteClick }: PromptRowActionsPr onDeleteClick?.(prompt.prompt_id, prompt.prompt_id || "Unknown Prompt")} + onClick={() => + onDeleteClick?.( + prompt.prompt_id, + prompt.prompt_id || "Unknown Prompt", + prompt.environment || "development", + ) + } > Delete @@ -106,8 +112,8 @@ function PromptRowActions({ prompt, isAdmin, onDeleteClick }: PromptRowActionsPr interface PromptTableColumnsDeps { modelHubData: Map; isAdmin: boolean; - onPromptClick?: (id: string) => void; - onDeleteClick?: (id: string, name: string) => void; + onPromptClick?: (id: string, environment: string) => void; + onDeleteClick?: (id: string, name: string, environment: string) => void; } export const getPromptTableColumns = ({ @@ -128,7 +134,11 @@ export const getPromptTableColumns = ({ title={row.original.prompt_id} titleClassName="font-mono text-xs font-normal" className="max-w-60" - onClick={onPromptClick ? () => onPromptClick(row.original.prompt_id) : undefined} + onClick={ + onPromptClick + ? () => onPromptClick(row.original.prompt_id, row.original.environment || "development") + : undefined + } /> ), }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx index d6a4aaea2e1..334339e6f1d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.test.tsx @@ -15,21 +15,31 @@ vi.mock("./PromptTable", () => ({ __esModule: true, default: ({ isLoading, + onPromptClick, onDeleteClick, }: { isLoading: boolean; - onDeleteClick: (id: string, name: string) => void; + onPromptClick: (id: string, environment: string) => void; + onDeleteClick: (id: string, name: string, environment: string) => void; }) => (
{isLoading ? "table-loading" : "table-loaded"} - +
), })); -vi.mock("./prompt_info", () => ({ __esModule: true, default: () =>
prompt-info-view
})); +vi.mock("./prompt_info", () => ({ + __esModule: true, + default: ({ initialEnvironment }: { initialEnvironment?: string }) => ( +
prompt-info-view:{initialEnvironment ?? "none"}
+ ), +})); vi.mock("./add_prompt_form", () => ({ __esModule: true, default: ({ visible }: { visible: boolean }) => (visible ?
add-prompt-form
: null), @@ -143,6 +153,22 @@ describe("PromptsPanel toolbar", () => { }); }); +describe("PromptsPanel row navigation", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetPromptsList.mockResolvedValue({ prompts: [] } as never); + }); + + it("should open the info view preselected to the clicked row's environment", async () => { + const user = userEvent.setup(); + renderPanel("Admin"); + + await user.click(await screen.findByRole("button", { name: "row-open" })); + + expect(screen.getByText("prompt-info-view:staging")).toBeInTheDocument(); + }); +}); + describe("PromptsPanel delete confirmation", () => { beforeEach(() => { vi.clearAllMocks(); @@ -156,13 +182,13 @@ describe("PromptsPanel delete confirmation", () => { await user.click(await screen.findByRole("button", { name: "row-delete" })); - expect(await screen.findByText(/delete prompt: my-prompt/i)).toBeInTheDocument(); + expect(await screen.findByText(/the staging copy of prompt: my-prompt/i)).toBeInTheDocument(); expect(screen.getByText(/cannot be undone/i)).toBeInTheDocument(); expect(mockDeletePromptCall).not.toHaveBeenCalled(); await user.click(screen.getByRole("button", { name: /^delete$/i })); - await waitFor(() => expect(mockDeletePromptCall).toHaveBeenCalledWith("sk-test", "prompt-1")); + await waitFor(() => expect(mockDeletePromptCall).toHaveBeenCalledWith("sk-test", "prompt-1", "staging")); }); it("should abandon the delete when the confirmation is dismissed", async () => { @@ -170,11 +196,11 @@ describe("PromptsPanel delete confirmation", () => { renderPanel("Admin"); await user.click(await screen.findByRole("button", { name: "row-delete" })); - await screen.findByText(/delete prompt: my-prompt/i); + await screen.findByText(/the staging copy of prompt: my-prompt/i); await user.click(screen.getByRole("button", { name: /cancel/i })); - await waitFor(() => expect(screen.queryByText(/delete prompt: my-prompt/i)).not.toBeInTheDocument()); + await waitFor(() => expect(screen.queryByText(/the staging copy of prompt: my-prompt/i)).not.toBeInTheDocument()); expect(mockDeletePromptCall).not.toHaveBeenCalled(); }); @@ -189,14 +215,14 @@ describe("PromptsPanel delete confirmation", () => { renderPanel("Admin"); await user.click(await screen.findByRole("button", { name: "row-delete" })); - await screen.findByText(/delete prompt: my-prompt/i); + await screen.findByText(/the staging copy of prompt: my-prompt/i); await user.click(screen.getByRole("button", { name: /^delete$/i })); - await waitFor(() => expect(mockDeletePromptCall).toHaveBeenCalledWith("sk-test", "prompt-1")); + await waitFor(() => expect(mockDeletePromptCall).toHaveBeenCalledWith("sk-test", "prompt-1", "staging")); await user.keyboard("{Escape}"); - expect(screen.getByText(/delete prompt: my-prompt/i)).toBeInTheDocument(); + expect(screen.getByText(/the staging copy of prompt: my-prompt/i)).toBeInTheDocument(); finishDelete(); - await waitFor(() => expect(screen.queryByText(/delete prompt: my-prompt/i)).not.toBeInTheDocument()); + await waitFor(() => expect(screen.queryByText(/the staging copy of prompt: my-prompt/i)).not.toBeInTheDocument()); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.tsx index 3594e933704..2d8d905c480 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/index.tsx @@ -41,11 +41,12 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { const [isLoading, setIsLoading] = useState(true); const [selectedEnvironment, setSelectedEnvironment] = useState(undefined); const [selectedPromptId, setSelectedPromptId] = useState(null); + const [selectedPromptEnvironment, setSelectedPromptEnvironment] = useState(undefined); const [isAddModalVisible, setIsAddModalVisible] = useState(false); const [showEditorView, setShowEditorView] = useState(false); const [editPromptData, setEditPromptData] = useState(null); const [isDeleting, setIsDeleting] = useState(false); - const [promptToDelete, setPromptToDelete] = useState<{ id: string; name: string } | null>(null); + const [promptToDelete, setPromptToDelete] = useState<{ id: string; name: string; environment: string } | null>(null); // Admin Viewer follows the read-parity rule: see prompts, no writes. const canModify = userRole ? isProxyAdminRole(userRole) : false; @@ -71,8 +72,9 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { fetchPrompts(); }, [accessToken, selectedEnvironment]); - const handlePromptClick = (promptId: string) => { + const handlePromptClick = (promptId: string, environment: string) => { setSelectedPromptId(promptId); + setSelectedPromptEnvironment(environment); }; const handleAddPrompt = () => { @@ -111,8 +113,8 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { setSelectedPromptId(null); }; - const handleDeleteClick = (promptId: string, promptName: string) => { - setPromptToDelete({ id: promptId, name: promptName }); + const handleDeleteClick = (promptId: string, promptName: string, environment: string) => { + setPromptToDelete({ id: promptId, name: promptName, environment }); }; const handleDeleteConfirm = async () => { @@ -120,8 +122,8 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { setIsDeleting(true); try { - await deletePromptCall(accessToken, promptToDelete.id); - toast.success(`Prompt "${promptToDelete.name}" deleted successfully`); + await deletePromptCall(accessToken, promptToDelete.id, promptToDelete.environment); + toast.success(`Prompt "${promptToDelete.name}" deleted successfully from ${promptToDelete.environment}`); fetchPrompts(); // Refresh the list } catch (error) { console.error("Error deleting prompt:", error); @@ -148,6 +150,7 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { ) : selectedPromptId ? ( setSelectedPromptId(null)} accessToken={accessToken} isAdmin={canModify} @@ -219,7 +222,8 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { Delete Prompt - Are you sure you want to delete prompt: {promptToDelete.name} ? This action cannot be undone. + Are you sure you want to delete the {promptToDelete.environment} copy of prompt: {promptToDelete.name}? + This action cannot be undone. diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx index bb29f12ac42..a5b195e5057 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx @@ -29,6 +29,35 @@ const promptWithoutTemplate = { environments: [], }; +describe("PromptInfoView environment scoping", () => { + beforeEach(() => { + vi.mocked(networking.getPromptInfo).mockReset().mockResolvedValue(promptWithoutTemplate); + vi.mocked(networking.getPromptVersions).mockReset().mockResolvedValue({ prompts: [] }); + }); + + it("fetches the initial environment it was opened with", async () => { + render( + , + ); + + await screen.findByRole("tab", { name: "Raw JSON" }); + expect(networking.getPromptInfo).toHaveBeenCalledWith("sk-test", "support-reply", "staging"); + }); + + it("fetches the serve default when opened without an environment", async () => { + render(); + + await screen.findByRole("tab", { name: "Raw JSON" }); + expect(networking.getPromptInfo).toHaveBeenCalledWith("sk-test", "support-reply", undefined); + }); +}); + describe("PromptInfoView tabs", () => { beforeEach(() => { vi.mocked(networking.getPromptInfo).mockReset().mockResolvedValue(promptWithoutTemplate); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx index c6a6f09fcaa..af4e3bf2121 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx @@ -20,6 +20,7 @@ import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from " export interface PromptInfoProps { promptId: string; + initialEnvironment?: string; onClose: () => void; accessToken: string | null; isAdmin: boolean; @@ -27,7 +28,15 @@ export interface PromptInfoProps { onEdit?: (promptData: any) => void; } -const PromptInfoView: React.FC = ({ promptId, onClose, accessToken, isAdmin, onDelete, onEdit }) => { +const PromptInfoView: React.FC = ({ + promptId, + initialEnvironment, + onClose, + accessToken, + isAdmin, + onDelete, + onEdit, +}) => { const [promptData, setPromptData] = useState(null); const [promptTemplate, setPromptTemplate] = useState(null); const [rawApiResponse, setRawApiResponse] = useState(null); @@ -43,7 +52,7 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo const [selectedVersion, setSelectedVersion] = useState(null); const [loadingVersions, setLoadingVersions] = useState(false); - // Initial fetch — no environment filter, gets default + all environments list + // Fetches the requested environment (or the serve-time default when omitted) plus the environments list const fetchPromptInfo = async (environment?: string) => { try { setLoading(true); @@ -89,7 +98,7 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo setSelectedEnv(null); setEnvironments([]); setVersionHistory([]); - fetchPromptInfo(); + fetchPromptInfo(initialEnvironment); }, [promptId, accessToken]); // When environment changes (user clicks tab), re-fetch — skip initial mount @@ -493,7 +502,7 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo Delete Prompt

- Are you sure you want to delete prompt: {basePromptId}? + Are you sure you want to delete prompt: {basePromptId} from every environment?

This action cannot be undone.

diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 5b6d70b4771..e297bcc5270 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -4625,9 +4625,12 @@ export const updatePromptCall = async (accessToken: string, promptId: string, pr } }; -export const deletePromptCall = async (accessToken: string, promptId: string) => { +export const deletePromptCall = async (accessToken: string, promptId: string, environment?: string) => { try { - const data = await apiClient.delete(`/prompts/${promptId}`, { accessToken }); + const data = await apiClient.delete(`/prompts/${promptId}`, { + accessToken, + query: { environment: environment || undefined }, + }); return data; } catch (error) { console.error("Failed to delete prompt:", error); From adfa42096d99e320e1413c37578158e6ec2d6465 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 23:07:18 -0700 Subject: [PATCH 005/132] fix(prompts): resolve config prompts in /prompts/{id}/info when environment is set --- litellm/proxy/prompts/prompt_endpoints.py | 6 +- .../proxy/prompts/test_prompt_endpoints.py | 92 +++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index 1b7932cdb18..b16611ac157 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -554,8 +554,10 @@ async def get_prompt_info( if env_prompts: prompt_spec = create_versioned_prompt_spec(db_prompt=env_prompts[0]) - if prompt_spec is None and environment is None: - prompt_spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec(prompt_id, version=requested_version) + if prompt_spec is None: + prompt_spec = IN_MEMORY_PROMPT_REGISTRY.resolve_prompt_spec( + prompt_id, version=requested_version, environment=environment + ) if prompt_spec is None: raise HTTPException( diff --git a/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py b/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py index 35402db219f..41c3003af34 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_endpoints.py @@ -368,3 +368,95 @@ class TestAdminViewerReadAccess: assert response.prompt_spec.prompt_id == "jack" assert response.prompt_spec.version == 2 + + +class TestConfigPromptInfoWithEnvironment: + """ + Regression: /prompts/{id}/info with an environment param must still resolve + config-file (in-memory) prompts on a DB-backed proxy instead of 400ing. + """ + + def _registry_with_config_prompt(self): + from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry + + registry = InMemoryPromptRegistry() + registry.IN_MEMORY_PROMPTS["envgreet::development"] = PromptSpec( + prompt_id="envgreet", + litellm_params=PromptLiteLLMParams( + prompt_id="envgreet", + prompt_integration="dotprompt", + dotprompt_content="AHOY {{user_message}}", + ), + prompt_info=PromptInfo(prompt_type="config"), + ) + return registry + + def _prisma_client_with_empty_prompt_table(self): + from unittest.mock import AsyncMock + + mock_prisma = MagicMock() + mock_prisma.db.litellm_prompttable.find_many = AsyncMock(return_value=[]) + return mock_prisma + + @pytest.mark.asyncio + async def test_get_prompt_info_with_environment_falls_back_to_registry(self): + from unittest.mock import patch + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.prompts.prompt_endpoints import get_prompt_info + + admin = UserAPIKeyAuth( + api_key="test_key", user_role=LitellmUserRoles.PROXY_ADMIN + ) + + with ( + patch( # test-quality-ok: endpoint reads prisma_client and the registry from module globals at call time; no injection seam + "litellm.proxy.proxy_server.prisma_client", + self._prisma_client_with_empty_prompt_table(), + ), + patch( # test-quality-ok: endpoint reads prisma_client and the registry from module globals at call time; no injection seam + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY", + self._registry_with_config_prompt(), + ), + ): + response = await get_prompt_info( + prompt_id="envgreet", + environment="development", + user_api_key_dict=admin, + ) + + assert response.prompt_spec.prompt_id == "envgreet" + assert response.prompt_spec.litellm_params.dotprompt_content == "AHOY {{user_message}}" + + @pytest.mark.asyncio + async def test_get_prompt_info_with_wrong_environment_still_400s(self): + from unittest.mock import patch + + from fastapi import HTTPException + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.prompts.prompt_endpoints import get_prompt_info + + admin = UserAPIKeyAuth( + api_key="test_key", user_role=LitellmUserRoles.PROXY_ADMIN + ) + + with ( + patch( # test-quality-ok: endpoint reads prisma_client and the registry from module globals at call time; no injection seam + "litellm.proxy.proxy_server.prisma_client", + self._prisma_client_with_empty_prompt_table(), + ), + patch( # test-quality-ok: endpoint reads prisma_client and the registry from module globals at call time; no injection seam + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY", + self._registry_with_config_prompt(), + ), + ): + with pytest.raises(HTTPException) as exc_info: + await get_prompt_info( + prompt_id="envgreet", + environment="production", + user_api_key_dict=admin, + ) + + assert exc_info.value.status_code == 400 + assert "environment production" in exc_info.value.detail From 3088db3f0ec77fd8256eff2b70dee882755f9a4c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:55:34 -0700 Subject: [PATCH 006/132] fix(prompts): accept a string prompt_version and carry the viewed environment into code snippets --- litellm/proxy/prompts/prompt_registry.py | 10 ++++++ litellm/proxy/utils.py | 5 +-- .../proxy/prompts/test_prompt_registry.py | 10 +++++- .../proxy_logging/test_guardrail_pipeline.py | 35 ++++++++++++++++++ .../PromptCodeSnippets.test.tsx | 24 +++++++++++++ .../prompt_editor_view/PromptCodeSnippets.tsx | 29 ++++++++------- .../PromptEditorHeader.test.tsx | 5 ++- .../prompt_editor_view/PromptEditorHeader.tsx | 1 + .../prompts/_components/prompt_info.test.tsx | 36 ++++++++++++++++++- .../prompts/_components/prompt_info.tsx | 1 + 10 files changed, 139 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/prompts/prompt_registry.py b/litellm/proxy/prompts/prompt_registry.py index ba53809690d..7803352e9e7 100644 --- a/litellm/proxy/prompts/prompt_registry.py +++ b/litellm/proxy/prompts/prompt_registry.py @@ -74,6 +74,16 @@ def registry_key_for_prompt(prompt: PromptSpec) -> str: return f"{prompt.prompt_id}::{prompt_environment_or_default(prompt.environment)}" +def parse_prompt_version(raw_version: object) -> int | None: + if isinstance(raw_version, bool): + return None + if isinstance(raw_version, int): + return raw_version + if isinstance(raw_version, str) and raw_version.isdigit(): + return int(raw_version) + return None + + def _spec_version(prompt: PromptSpec) -> int: return prompt.version if prompt.version is not None else get_version_number(prompt_id=prompt.prompt_id) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 09efa0cc72d..0263ce7598a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1749,7 +1749,6 @@ class ProxyLogging: litellm_logging_obj: Final = cast(Optional["LiteLLMLoggingObj"], data.get("litellm_logging_obj", None)) prompt_id: Final[str | None] = data.get("prompt_id", None) - prompt_version: Final[int | None] = data.get("prompt_version", None) ## PROMPT TEMPLATE CHECK ## @@ -1759,11 +1758,13 @@ class ProxyLogging: and prompt_id is not None and (call_type == "completion" or call_type == "acompletion" or call_type == "aresponses") ): + from litellm.proxy.prompts.prompt_registry import parse_prompt_version + await self._process_prompt_template( data=data, litellm_logging_obj=litellm_logging_obj, prompt_id=prompt_id, - prompt_version=prompt_version, + prompt_version=parse_prompt_version(data.get("prompt_version", None)), call_type=call_type, ) diff --git a/tests/test_litellm/proxy/prompts/test_prompt_registry.py b/tests/test_litellm/proxy/prompts/test_prompt_registry.py index 32930fbbdff..a0743b65dd7 100644 --- a/tests/test_litellm/proxy/prompts/test_prompt_registry.py +++ b/tests/test_litellm/proxy/prompts/test_prompt_registry.py @@ -2,7 +2,7 @@ import pytest import litellm from litellm.integrations.custom_prompt_management import CustomPromptManagement -from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry +from litellm.proxy.prompts.prompt_registry import InMemoryPromptRegistry, parse_prompt_version from litellm.types.prompts.init_prompts import PromptInfo, PromptLiteLLMParams, PromptSpec @@ -214,3 +214,11 @@ def test_remove_prompt_is_a_no_op_for_an_unknown_registry_key(isolated_callbacks assert registry.resolve_prompt_spec("greeting") is not None assert len(isolated_callbacks) == 1 + + +@pytest.mark.parametrize( + ("raw_version", "expected"), + [(2, 2), ("2", 2), (None, None), ("v2", None), (True, None), (2.0, None)], +) +def test_parse_prompt_version_accepts_integers_and_json_strings(raw_version: object, expected: int | None) -> None: + assert parse_prompt_version(raw_version) == expected diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 6589b14e53a..2ba58bd5644 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -858,6 +858,41 @@ async def test_process_prompt_template_resolves_the_requested_environment(proxy_ assert data["messages"] == [{"role": "user", "content": "rendered"}] +@pytest.mark.asyncio +async def test_pre_call_hook_matches_a_prompt_version_sent_as_a_json_string(proxy_logging, monkeypatch): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.prompts import prompt_registry + + prompt_spec = MagicMock() + prompt_spec.litellm_params = MagicMock(prompt_id="greeting") + resolve_calls: list[dict] = [] + + def fake_resolve(prompt_id, version=None, environment=None): + resolve_calls.append({"prompt_id": prompt_id, "version": version, "environment": environment}) + return prompt_spec + + monkeypatch.setattr(prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "resolve_prompt_spec", fake_resolve) + monkeypatch.setattr( + prompt_registry.IN_MEMORY_PROMPT_REGISTRY, "get_prompt_callback_for_prompt", lambda *a, **kw: MagicMock() + ) + logging_obj = MagicMock() + logging_obj.async_get_chat_completion_prompt = AsyncMock( + return_value=("m", [{"role": "user", "content": "rendered"}], {}) + ) + data: Dict[str, Any] = { + "messages": [{"role": "user", "content": "orig"}], + "model": "m", + "prompt_id": "greeting", + "prompt_version": "2", + "litellm_logging_obj": logging_obj, + } + + result = await proxy_logging.pre_call_hook(user_api_key_dict=UserAPIKeyAuth(), data=data, call_type="completion") + + assert resolve_calls == [{"prompt_id": "greeting", "version": 2, "environment": None}] + assert result["messages"] == [{"role": "user", "content": "rendered"}] + + @pytest.mark.asyncio async def test_process_prompt_template_aresponses_swaps_model_and_merges_input(proxy_logging, monkeypatch): from litellm.proxy.prompts import prompt_registry diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.test.tsx index a1b4ad52634..7fa44a4dfe5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.test.tsx @@ -44,4 +44,28 @@ describe("PromptCodeSnippets", () => { expect(screen.getByRole("combobox", { name: "Language" })).toHaveTextContent("Python (OpenAI SDK)"); }); + + it("includes the viewed environment in every generated request", async () => { + const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); + render( + , + ); + await user.click(screen.getByRole("button", { name: /get code/i })); + await screen.findByText("Generated Code"); + + await user.click(screen.getByRole("button", { name: /copy to clipboard/i })); + expect(await navigator.clipboard.readText()).toContain('"prompt_environment": "development"'); + + await user.click(screen.getByRole("tab", { name: "With Version" })); + await user.click(screen.getByRole("button", { name: /copy to clipboard/i })); + const versionSnippet = await navigator.clipboard.readText(); + expect(versionSnippet).toContain('"prompt_environment": "development"'); + expect(versionSnippet).toContain('"prompt_version": 2'); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.tsx index af7d6421265..a6adc160674 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptCodeSnippets.tsx @@ -22,6 +22,7 @@ interface PromptCodeSnippetsProps { promptVariables?: Record; accessToken: string | null; version?: string; + environment?: string; proxySettings?: { PROXY_BASE_URL?: string; LITELLM_UI_API_DOC_BASE_URL?: string | null; @@ -34,6 +35,7 @@ const PromptCodeSnippets: React.FC = ({ promptVariables = {}, accessToken, version = "1", + environment, proxySettings, }) => { const syntaxTheme = useSyntaxTheme(coy); @@ -64,6 +66,9 @@ const PromptCodeSnippets: React.FC = ({ // Generate code based on selected language and tab const generateCode = () => { const hasVariables = Object.keys(promptVariables).length > 0; + const curlEnvironment = environment ? `,\n "prompt_environment": "${environment}"` : ""; + const pythonEnvironment = environment ? `,\n "prompt_environment": "${environment}"` : ""; + const jsEnvironment = environment ? `,\n prompt_environment: "${environment}"` : ""; if (selectedLanguage === "curl") { if (selectedTab === "basic") { @@ -72,7 +77,7 @@ const PromptCodeSnippets: React.FC = ({ -H 'Authorization: Bearer ${effectiveApiKey}' \\ -d '{ "model": "${model}", - "prompt_id": "${promptId}"${ + "prompt_id": "${promptId}"${curlEnvironment}${ hasVariables ? `, "prompt_variables": ${JSON.stringify(promptVariables, null, 6).replace(/\n/g, "\n ")}` @@ -85,7 +90,7 @@ const PromptCodeSnippets: React.FC = ({ -H 'Authorization: Bearer ${effectiveApiKey}' \\ -d '{ "model": "${model}", - "prompt_id": "${promptId}"${ + "prompt_id": "${promptId}"${curlEnvironment}${ hasVariables ? `, "prompt_variables": ${JSON.stringify(promptVariables, null, 6).replace(/\n/g, "\n ")}` @@ -104,7 +109,7 @@ const PromptCodeSnippets: React.FC = ({ -H 'Authorization: Bearer ${effectiveApiKey}' \\ -d '{ "model": "${model}", - "prompt_id": "${promptId}", + "prompt_id": "${promptId}"${curlEnvironment}, "prompt_version": ${version}, "messages": [ { @@ -127,7 +132,7 @@ client = openai.OpenAI( response = client.chat.completions.create( model="${model}", extra_body={ - "prompt_id": "${promptId}"${ + "prompt_id": "${promptId}"${pythonEnvironment}${ hasVariables ? `, "prompt_variables": ${JSON.stringify(promptVariables, null, 8).replace(/\n/g, "\n ")}` @@ -145,7 +150,7 @@ response = client.chat.completions.create( {"role": "user", "content": "hi"} ], extra_body={ - "prompt_id": "${promptId}"${ + "prompt_id": "${promptId}"${pythonEnvironment}${ hasVariables ? `, "prompt_variables": ${JSON.stringify(promptVariables, null, 8).replace(/\n/g, "\n ")}` @@ -163,7 +168,7 @@ response = client.chat.completions.create( {"role": "user", "content": "Who are u"} ], extra_body={ - "prompt_id": "${promptId}", + "prompt_id": "${promptId}"${pythonEnvironment}, "prompt_version": ${version} } ) @@ -186,9 +191,9 @@ async function main() { model: "${model}", ${ hasVariables - ? `prompt_id: "${promptId}", + ? `prompt_id: "${promptId}"${jsEnvironment}, prompt_variables: ${JSON.stringify(promptVariables, null, 8).replace(/\n/g, "\n ")}` - : `prompt_id: "${promptId}"` + : `prompt_id: "${promptId}"${jsEnvironment}` } }); @@ -206,9 +211,9 @@ async function main() { ], ${ hasVariables - ? `prompt_id: "${promptId}", + ? `prompt_id: "${promptId}"${jsEnvironment}, prompt_variables: ${JSON.stringify(promptVariables, null, 8).replace(/\n/g, "\n ")}` - : `prompt_id: "${promptId}"` + : `prompt_id: "${promptId}"${jsEnvironment}` } }); @@ -224,7 +229,7 @@ async function main() { messages: [ { role: "user", content: "Who are u" } ], - prompt_id: "${promptId}", + prompt_id: "${promptId}"${jsEnvironment}, prompt_version: ${version} }); @@ -241,7 +246,7 @@ main();`; if (isModalVisible) { setGeneratedCode(generateCode()); } - }, [isModalVisible, selectedLanguage, selectedTab, promptId, model, promptVariables]); + }, [isModalVisible, selectedLanguage, selectedTab, promptId, model, promptVariables, version, environment]); return ( <> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.test.tsx index 5194cdd4e63..3afc00e37a5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.test.tsx @@ -2,7 +2,9 @@ import { fireEvent, render, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import PromptEditorHeader from "./PromptEditorHeader"; -vi.mock("./PromptCodeSnippets", () => ({ default: () => })); +vi.mock("./PromptCodeSnippets", () => ({ + default: ({ environment }: { environment?: string }) => , +})); describe("PromptEditorHeader", () => { it("preserves navigation, naming, and save actions", () => { @@ -48,5 +50,6 @@ describe("PromptEditorHeader", () => { ); expect(screen.getByRole("combobox", { name: "Environment" })).toHaveTextContent(label); + expect(screen.getByRole("button", { name: "Get Code" })).toHaveAttribute("data-environment", environment); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx index eea9755054f..04cac01365a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/PromptEditorHeader.tsx @@ -89,6 +89,7 @@ const PromptEditorHeader: React.FC = ({ promptVariables={promptVariables} accessToken={accessToken} version={version?.replace("v", "") || "1"} + environment={environment} proxySettings={proxySettings} /> {editMode && onShowHistory && ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx index a5b195e5057..b14d5c3d91f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.test.tsx @@ -12,7 +12,9 @@ vi.mock("@/components/networking", () => ({ })); vi.mock("./prompt_editor_view/PromptCodeSnippets", () => ({ - default: () =>
, + default: ({ environment }: { environment?: string }) => ( +
+ ), })); const promptWithoutTemplate = { @@ -58,6 +60,38 @@ describe("PromptInfoView environment scoping", () => { }); }); +describe("PromptInfoView code snippets", () => { + beforeEach(() => { + vi.mocked(networking.getPromptVersions).mockReset().mockResolvedValue({ prompts: [] }); + }); + + it.each([ + ["a prompt with several environments", "staging", ["development", "staging"]], + ["a config prompt with no environment list", "development", []], + ])("hands the viewed environment of %s to the code snippets", async (_label, environment, environments) => { + vi.mocked(networking.getPromptInfo) + .mockReset() + .mockResolvedValue({ + ...promptWithoutTemplate, + prompt_spec: { ...promptWithoutTemplate.prompt_spec, environment }, + environments, + }); + + render( + , + ); + + await screen.findByRole("tab", { name: "Raw JSON" }); + expect(screen.getByTestId("prompt-code-snippets")).toHaveAttribute("data-environment", environment); + }); +}); + describe("PromptInfoView tabs", () => { beforeEach(() => { vi.mocked(networking.getPromptInfo).mockReset().mockResolvedValue(promptWithoutTemplate); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx index af4e3bf2121..062e1d84a0a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx @@ -221,6 +221,7 @@ const PromptInfoView: React.FC = ({ promptVariables={extractTemplateVariables(promptTemplate?.content)} accessToken={accessToken} version={currentVersion} + environment={selectedEnv ?? promptData.environment} />
{hasActiveSearch - ? "No memories have keys starting with your search." + ? "No memories match your search." : "Memories your agents store under /v1/memory will appear here."}
@@ -81,7 +81,7 @@ export function MemoryTable({ table={table} searchValue={searchValue} onSearchChange={onSearchChange} - searchPlaceholder='Filter by key prefix, e.g. "user:"' + searchPlaceholder="Search by key prefix or memory ID…" onRefresh={onRefresh} isRefreshing={isRefreshing} showViewOptions={false} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.test.tsx index 9ccef5357b9..b703df652c2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.test.tsx @@ -1,8 +1,9 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { act, render, screen } from "@testing-library/react"; +import type { PaginationState } from "@tanstack/react-table"; +import { act, render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import React from "react"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { MemoryRow } from "@/components/networking"; @@ -13,10 +14,13 @@ interface CapturedTableProps { rowCount: number; data: MemoryRow[]; hasActiveSearch: boolean; + onSearchChange: (value: string) => void; + onPaginationChange: (state: PaginationState) => void; onViewClick: (row: MemoryRow) => void; } const captured = vi.hoisted(() => ({ current: null as CapturedTableProps | null })); +const fetchMemoryListMock = vi.hoisted(() => vi.fn()); vi.mock("./MemoryTable", () => ({ MemoryTable: function MemoryTableMock(props: CapturedTableProps) { @@ -25,6 +29,15 @@ vi.mock("./MemoryTable", () => ({ }, })); +vi.mock("@/components/networking", async (importOriginal) => ({ + ...(await importOriginal()), + fetchMemoryList: fetchMemoryListMock, +})); + +vi.mock("@tanstack/react-pacer/debouncer", () => ({ + useDebouncedValue: (value: unknown) => [value, { cancel: vi.fn(), flush: vi.fn() }], +})); + const renderView = (accessToken: string | null) => { const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); return render( @@ -35,6 +48,28 @@ const renderView = (accessToken: string | null) => { }; describe("MemoryView", () => { + beforeEach(() => { + fetchMemoryListMock.mockReset(); + fetchMemoryListMock.mockResolvedValue({ memories: [], total: 0 }); + }); + + it("queries the server with the search box value as `search` and resets to page 1", async () => { + renderView("token"); + await waitFor(() => expect(fetchMemoryListMock).toHaveBeenCalled()); + + act(() => captured.current?.onPaginationChange({ pageIndex: 2, pageSize: 50 })); + await waitFor(() => + expect(fetchMemoryListMock).toHaveBeenLastCalledWith("token", expect.objectContaining({ page: 3 })), + ); + + act(() => captured.current?.onSearchChange("mem-abc123")); + + await waitFor(() => + expect(fetchMemoryListMock).toHaveBeenLastCalledWith("token", { search: "mem-abc123", page: 1, pageSize: 50 }), + ); + expect(captured.current?.hasActiveSearch).toBe(true); + }); + it("keeps the table out of the skeleton state when the token is null (disabled query)", () => { renderView(null); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx index 1d2e5150a62..58d4e42aa96 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx @@ -43,10 +43,8 @@ export const MemoryView: React.FC = ({ accessToken }) => { queryKey: [MEMORY_LIST_KEY, debouncedSearch, pagination.pageIndex, pagination.pageSize], queryFn: () => { if (!accessToken) throw new Error("Access token required"); - // Prefix search matches the Redis-style mental model (namespace scan): - // typing "user:" finds "user:profile", "user:prefs", etc. return fetchMemoryList(accessToken, { - keyPrefix: debouncedSearch || undefined, + search: debouncedSearch || undefined, page: pagination.pageIndex + 1, pageSize: pagination.pageSize, }); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index c69662b69dc..881b0b93ff9 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -542,6 +542,19 @@ describe("server-side filtering – the LIT-4080 regression guard", () => { expect((lastCall[2] ?? {}).userID).toBeUndefined(); }); }); + + it("sends the search box as the combined alias-or-ID search rather than the key-alias filter", async () => { + renderWithProviders(); + + fireEvent.change(screen.getByPlaceholderText(/Search by key alias or ID/), { target: { value: mockKey.token } }); + + await waitFor(() => { + expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ search: mockKey.token })); + }); + const lastOptions = mockUseKeys.mock.calls.at(-1)?.[2]; + expect(lastOptions?.selectedKeyAlias).toBeUndefined(); + expect(lastOptions?.keyHash).toBeUndefined(); + }); }); describe("pagination display – total count comes from useKeys", () => { @@ -663,7 +676,7 @@ describe("table state lives in the URL so it survives leaving and returning to t expect(mockUseKeys).toHaveBeenLastCalledWith( 3, 25, - expect.objectContaining({ selectedKeyAlias: "prod", sortBy: "spend", sortOrder: "asc" }), + expect.objectContaining({ search: "prod", sortBy: "spend", sortOrder: "asc" }), ); }); expect(screen.getByPlaceholderText(/Search by key alias/)).toHaveValue("prod"); @@ -736,7 +749,7 @@ describe("table state lives in the URL so it survives leaving and returning to t fireEvent.change(screen.getByPlaceholderText(/Search by key alias/), { target: { value: "prod" } }); await waitFor(() => { - expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ selectedKeyAlias: "prod" })); + expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ search: "prod" })); }); await waitFor(() => { expect(lastSearchParam(onUrlUpdate, "page")).toBeNull(); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index c424966a0a3..ebedf57af45 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -118,7 +118,7 @@ export function VirtualKeysTable({ headerActions }: VirtualKeysTableProps) { const keyListOptions = { teamID: appliedFilters.team_id || undefined, organizationID: appliedFilters.org_id || undefined, - selectedKeyAlias: searchQuery.trim() || undefined, + search: searchQuery.trim() || undefined, userID: appliedFilters.user_id || undefined, keyHash: appliedFilters.key_hash || undefined, sortBy, @@ -291,7 +291,7 @@ export function VirtualKeysTable({ headerActions }: VirtualKeysTableProps) { table={table} searchValue={searchInput} onSearchChange={handleSearchChange} - searchPlaceholder="Search by key alias…" + searchPlaceholder="Search by key alias or ID…" onRefresh={() => refetch?.()} isRefreshing={isFetching} onOpenFilters={() => setFiltersOpen(true)} diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index 7a220dd4711..9df9e9a9209 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -854,3 +854,48 @@ describe("userListCall search serialization", () => { expect(lastParams(mockFetch).get("user_email")).toBe("ada@example.com"); }); }); + +describe("fetchMemoryList search serialization", () => { + const originalFetch = global.fetch; + + afterEach(() => { + global.fetch = originalFetch; + }); + + const mockOkFetch = () => { + const emptyPage = { memories: [], total: 0 }; + const mockFetch = vi.fn().mockResolvedValue({ ok: true, json: vi.fn().mockResolvedValue(emptyPage) } as any); + global.fetch = mockFetch as any; + return mockFetch; + }; + + const lastParams = (mockFetch: ReturnType) => { + const [url] = mockFetch.mock.calls.at(-1) ?? []; + return new URL(url as string, "http://example.com").searchParams; + }; + + it("sends the search box value as search and omits key_prefix and key", async () => { + const mockFetch = mockOkFetch(); + + await Networking.fetchMemoryList("token", { search: "mem-abc123", page: 1, pageSize: 50 }); + + const params = lastParams(mockFetch); + expect(params.get("search")).toBe("mem-abc123"); + expect(params.has("key_prefix")).toBe(false); + expect(params.has("key")).toBe(false); + expect(params.get("page")).toBe("1"); + expect(params.get("page_size")).toBe("50"); + }); + + it("keeps key_prefix and key working when no search is given", async () => { + const mockFetch = mockOkFetch(); + + await Networking.fetchMemoryList("token", { keyPrefix: "user:" }); + expect(lastParams(mockFetch).get("key_prefix")).toBe("user:"); + expect(lastParams(mockFetch).has("search")).toBe(false); + + await Networking.fetchMemoryList("token", { key: "user:profile" }); + expect(lastParams(mockFetch).get("key")).toBe("user:profile"); + expect(lastParams(mockFetch).has("search")).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index d8762565e08..1384679a88a 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2056,6 +2056,7 @@ interface UiSpendLogsParams { exclude_internal_health_checks?: boolean; group_by_session?: boolean; session_cursor?: string; + search?: string; } interface UiSpendLogsCallOptions { @@ -6563,6 +6564,7 @@ interface UiAuditLogsParams { changed_by_api_key?: string; object_team_id?: string; object_key_hash?: string; + search?: string | null; sort_by?: string; sort_order?: "asc" | "desc"; } @@ -8061,15 +8063,18 @@ export const fetchMemoryList = async ( options: { key?: string; keyPrefix?: string; + search?: string; page?: number; pageSize?: number; } = {}, ): Promise => { const base = proxyBaseUrl ? `${proxyBaseUrl}/v1/memory` : `/v1/memory`; const params = new URLSearchParams(); - // keyPrefix takes precedence — backend also does, but we omit `key` + // Backend precedence is search > key_prefix > key; only the winner is sent // to keep the URL clean and intent obvious. - if (options.keyPrefix) { + if (options.search) { + params.append("search", options.search); + } else if (options.keyPrefix) { params.append("key_prefix", options.keyPrefix); } else if (options.key) { params.append("key", options.key); diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx index 8bf5d639d6c..0d9d0988aa1 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx @@ -30,6 +30,8 @@ vi.mock("@tanstack/react-pacer/debouncer", () => ({ const mockUseKeys = useKeys as MockedFunction; +const KEY_HASH = "88a145505dd6e87e2ea166fcef1e4b53948dbdb32af6431dfd05ec06b571ee52"; + const createMockKey = (overrides: Partial = {}): KeyResponse => ({ token: "sk-test123", @@ -277,7 +279,7 @@ describe("TeamVirtualKeysTable", () => { ); }); - it("maps the search box to a server-side key-alias query", async () => { + it("maps the Key ID drawer filter to a server-side useKeys query and clears it", async () => { const user = userEvent.setup(); mockUseKeys.mockReturnValue({ data: { keys: [createMockKey()], total_count: 1, current_page: 1, total_pages: 1 }, @@ -288,11 +290,42 @@ describe("TeamVirtualKeysTable", () => { renderWithProviders(); - fireEvent.change(await screen.findByTestId("datatable-search"), { target: { value: "check-002" } }); + await user.click(await screen.findByTestId("datatable-filters-trigger")); + const drawerBody = await screen.findByTestId("filter-drawer-body"); + fireEvent.change(within(drawerBody).getByPlaceholderText("Enter Key ID…"), { target: { value: KEY_HASH } }); + await user.click(screen.getByTestId("filter-drawer-apply")); await waitFor(() => - expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ selectedKeyAlias: "check-002" })), + expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ keyHash: KEY_HASH })), ); + expect(screen.getByTestId("filter-chip-key_hash")).toHaveTextContent("Key ID"); + + await user.click(screen.getByTestId("datatable-clear-filters")); + await waitFor(() => + expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ keyHash: undefined })), + ); + }); + + it("maps the search box to the combined alias-or-ID search rather than the key-alias filter", async () => { + mockUseKeys.mockReturnValue({ + data: { keys: [createMockKey()], total_count: 1, current_page: 1, total_pages: 1 }, + isPending: false, + isFetching: false, + refetch: vi.fn(), + } as unknown as ReturnType); + + renderWithProviders(); + + const searchBox = await screen.findByTestId("datatable-search"); + expect(searchBox).toHaveAttribute("placeholder", "Search by key alias or ID…"); + fireEvent.change(searchBox, { target: { value: KEY_HASH } }); + + await waitFor(() => + expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ search: KEY_HASH })), + ); + const lastOptions = mockUseKeys.mock.calls.at(-1)?.[2]; + expect(lastOptions?.selectedKeyAlias).toBeUndefined(); + expect(lastOptions?.keyHash).toBeUndefined(); }); it("should show Loading keys when isPending", async () => { diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx index bd7faa41ee9..aa9df4a0319 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx @@ -68,19 +68,17 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi const pageIndex = tablePagination.pageIndex; const pageSize = tablePagination.pageSize; - const { - data: keys, - isPending: isLoading, - isFetching, - refetch, - } = useKeys(pageIndex + 1, pageSize, { + const keyListOptions = { teamID: teamId, - selectedKeyAlias: searchQuery.trim() || undefined, + search: searchQuery.trim() || undefined, userID: getFilterValue("user_id"), + keyHash: getFilterValue("key_hash"), sortBy: sortBy || undefined, sortOrder: sortOrder || undefined, expand: "user", - }); + }; + + const { data: keys, isPending: isLoading, isFetching, refetch } = useKeys(pageIndex + 1, pageSize, keyListOptions); const displayKeys = useMemo(() => { const kList = keys?.keys || []; @@ -481,11 +479,11 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi table={table} searchValue={searchInput} onSearchChange={handleSearchChange} - searchPlaceholder="Search by key alias…" + searchPlaceholder="Search by key alias or ID…" onRefresh={() => refetch?.()} isRefreshing={isFetching} onOpenFilters={() => setFiltersOpen(true)} - filterLabels={{ user_id: "User ID" }} + filterLabels={{ user_id: "User ID", key_hash: "Key ID" }} /> {({ get, set }) => ( - - set("user_id", event.target.value)} - placeholder="Filter by user ID…" - /> - + <> + + set("user_id", event.target.value)} + placeholder="Filter by user ID…" + /> + + + set("key_hash", event.target.value)} + placeholder="Enter Key ID…" + /> + + )} diff --git a/ui/litellm-dashboard/src/components/view_logs/AuditLogsPanel.test.tsx b/ui/litellm-dashboard/src/components/view_logs/AuditLogsPanel.test.tsx new file mode 100644 index 00000000000..3b27f663b8e --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/AuditLogsPanel.test.tsx @@ -0,0 +1,145 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { chooseSelectOption } from "../../../tests/test-utils"; +import AuditLogsPanel from "./AuditLogsPanel"; + +vi.mock("../networking", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, uiAuditLogsCall: vi.fn() }; +}); + +// Resolve the debounced search synchronously so typed input reaches the query within the test tick. +vi.mock("@tanstack/react-pacer/debouncer", () => ({ + useDebouncedValue: (value: unknown) => [value, { cancel: vi.fn(), flush: vi.fn() }], +})); + +import { uiAuditLogsCall } from "../networking"; + +type AuditLogsParams = NonNullable[0]["params"]>; + +const PAGE_SIZE = 50; + +const ID_PARAM_KEYS = [ + "search", + "object_id", + "changed_by", + "object_team_id", + "object_key_hash", + "action", + "table_name", +] as const satisfies readonly (keyof AuditLogsParams)[]; + +const respondWith = (total: number) => { + const response = { audit_logs: [], total, page: 1, page_size: PAGE_SIZE, total_pages: Math.ceil(total / PAGE_SIZE) }; + return vi.mocked(uiAuditLogsCall).mockResolvedValue(response); +}; + +const lastCall = () => vi.mocked(uiAuditLogsCall).mock.calls.at(-1)?.[0]; +const sentIdParams = () => ID_PARAM_KEYS.filter((key) => lastCall()?.params?.[key] !== undefined); + +const defaultProps = { + accessToken: "sk-test", + token: "jwt-test", + userRole: "Admin", + userID: "user-1", + isActive: true, + premiumUser: true, +}; + +const renderPanel = () => { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + + , + ); +}; + +const TEXT_FILTERS: { filterId: string; placeholder: string; paramKey: keyof AuditLogsParams }[] = [ + { filterId: "object_id", placeholder: "Enter object ID…", paramKey: "object_id" }, + { filterId: "changed_by", placeholder: "Enter user ID…", paramKey: "changed_by" }, + { filterId: "team_id", placeholder: "Enter team ID…", paramKey: "object_team_id" }, + { filterId: "key_hash", placeholder: "Enter key hash…", paramKey: "object_key_hash" }, +]; + +const SELECT_FILTERS: { + label: string; + comboboxIndex: number; + option: string; + paramKey: keyof AuditLogsParams; + value: string; +}[] = [ + { label: "Action", comboboxIndex: 0, option: "Created", paramKey: "action", value: "created" }, + { label: "Table", comboboxIndex: 1, option: "Teams", paramKey: "table_name", value: "LiteLLM_TeamTable" }, +]; + +describe("AuditLogsPanel", () => { + beforeEach(() => { + vi.clearAllMocks(); + respondWith(0); + }); + + it("sends the typed search as params.search and returns to the first page", async () => { + const user = userEvent.setup(); + respondWith(120); + renderPanel(); + await waitFor(() => expect(uiAuditLogsCall).toHaveBeenCalled()); + expect(lastCall()?.params?.search).toBeUndefined(); + + await user.click(screen.getByTestId("pagination-next")); + await waitFor(() => expect(lastCall()?.page).toBe(2)); + + await user.type(screen.getByTestId("datatable-search"), "team-abc"); + + await waitFor(() => expect(lastCall()?.params?.search).toBe("team-abc")); + expect(lastCall()?.page).toBe(1); + expect(sentIdParams()).toEqual(["search"]); + }); + + it("trims the search and drops params.search once the box is cleared", async () => { + const user = userEvent.setup(); + renderPanel(); + const input = await screen.findByTestId("datatable-search"); + + await user.type(input, " abc"); + await waitFor(() => expect(lastCall()?.params?.search).toBe("abc")); + + await user.clear(input); + + await waitFor(() => expect(lastCall()?.params?.search).toBeUndefined()); + expect(sentIdParams()).toEqual([]); + }); + + it.each(TEXT_FILTERS)("maps the $filterId drawer filter to params.$paramKey", async ({ placeholder, paramKey }) => { + const user = userEvent.setup(); + renderPanel(); + await waitFor(() => expect(uiAuditLogsCall).toHaveBeenCalled()); + + await user.click(screen.getByTestId("datatable-filters-trigger")); + fireEvent.change(await screen.findByPlaceholderText(placeholder), { target: { value: "val-1" } }); + await user.click(screen.getByTestId("filter-drawer-apply")); + + await waitFor(() => expect(lastCall()?.params?.[paramKey]).toBe("val-1")); + expect(sentIdParams()).toEqual([paramKey]); + }); + + it.each(SELECT_FILTERS)( + "maps the $label drawer select to params.$paramKey", + async ({ comboboxIndex, option, paramKey, value }) => { + const user = userEvent.setup(); + renderPanel(); + await waitFor(() => expect(uiAuditLogsCall).toHaveBeenCalled()); + + await user.click(screen.getByTestId("datatable-filters-trigger")); + const triggers = await screen.findAllByRole("combobox"); + await chooseSelectOption(user, triggers[comboboxIndex], option); + await user.click(screen.getByTestId("filter-drawer-apply")); + + await waitFor(() => expect(lastCall()?.params?.[paramKey]).toBe(value)); + expect(sentIdParams()).toEqual([paramKey]); + }, + ); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/AuditLogsPanel.tsx b/ui/litellm-dashboard/src/components/view_logs/AuditLogsPanel.tsx index 81bd4a19f76..5ce4ca6053f 100644 --- a/ui/litellm-dashboard/src/components/view_logs/AuditLogsPanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/AuditLogsPanel.tsx @@ -1,7 +1,9 @@ import { useCallback, useState } from "react"; +import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; import { useQuery, keepPreviousData } from "@tanstack/react-query"; import { ColumnFiltersState, OnChangeFn, PaginationState } from "@tanstack/react-table"; import { resolveLogoSrc } from "@/lib/assetPaths"; +import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; import { uiAuditLogsCall } from "../networking"; import { AuditLogEntry } from "./AuditLogsTableColumns"; import { AuditLogsTable } from "./AuditLogsTable"; @@ -39,9 +41,13 @@ export default function AuditLogsPanel({ }: AuditLogsProps) { const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: PAGE_SIZE }); const [columnFilters, setColumnFilters] = useState([]); + const [searchInput, setSearchInput] = useState(""); + const [debouncedSearch] = useDebouncedValue(searchInput, { wait: DEBOUNCE_WAIT_MS }); const [selectedLog, setSelectedLog] = useState(null); const [drawerOpen, setDrawerOpen] = useState(false); + const searchTerm = debouncedSearch.trim(); + const getFilterValue = (columnId: string): string | undefined => { const entry = columnFilters.find((filter) => filter.id === columnId); return typeof entry?.value === "string" && entry.value.trim() ? entry.value.trim() : undefined; @@ -50,7 +56,7 @@ export default function AuditLogsPanel({ const canQueryAuditLogs = !!accessToken && !!token && !!userRole && !!userID && isActive && premiumUser; const query = useQuery({ - queryKey: ["audit_logs", pagination.pageIndex, pagination.pageSize, columnFilters], + queryKey: ["audit_logs", pagination.pageIndex, pagination.pageSize, columnFilters, searchTerm], queryFn: async () => { if (!accessToken) { return { audit_logs: [], total: 0, page: 1, page_size: pagination.pageSize, total_pages: 0 }; @@ -60,6 +66,7 @@ export default function AuditLogsPanel({ page: pagination.pageIndex + 1, page_size: pagination.pageSize, params: { + search: searchTerm || undefined, object_id: getFilterValue("object_id"), changed_by: getFilterValue("changed_by"), object_key_hash: getFilterValue("key_hash"), @@ -80,6 +87,11 @@ export default function AuditLogsPanel({ setPagination((prev) => ({ ...prev, pageIndex: 0 })); }, []); + const handleSearchChange = useCallback((value: string) => { + setSearchInput(value); + setPagination((prev) => ({ ...prev, pageIndex: 0 })); + }, []); + const handleViewLog = useCallback((log: AuditLogEntry) => { setSelectedLog(log); setDrawerOpen(true); @@ -128,6 +140,8 @@ export default function AuditLogsPanel({ onPaginationChange={setPagination} columnFilters={columnFilters} onColumnFiltersChange={handleColumnFiltersChange} + searchValue={searchInput} + onSearchChange={handleSearchChange} onRefresh={() => query.refetch()} onViewLog={handleViewLog} /> diff --git a/ui/litellm-dashboard/src/components/view_logs/AuditLogsTable.test.tsx b/ui/litellm-dashboard/src/components/view_logs/AuditLogsTable.test.tsx index 7349c3019ae..d8e549715af 100644 --- a/ui/litellm-dashboard/src/components/view_logs/AuditLogsTable.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/AuditLogsTable.test.tsx @@ -120,6 +120,24 @@ describe("AuditLogsTable", () => { expect(screen.getByText("No matching audit logs")).toBeInTheDocument(); }); + it("renders the toolbar search box from the search props and forwards typed input", () => { + const onSearchChange = vi.fn(); + renderTable({ searchValue: "team-", onSearchChange }); + + const input = screen.getByPlaceholderText("Search audit logs by ID…"); + expect(input).toHaveValue("team-"); + + fireEvent.change(input, { target: { value: "team-7" } }); + expect(onSearchChange).toHaveBeenCalledWith("team-7"); + }); + + it("treats an active search as a filter for the empty state", () => { + const emptySearchResult = { data: [], rowCount: 0, searchValue: "zzz", onSearchChange: vi.fn() }; + renderTable(emptySearchResult); + + expect(screen.getByText("No matching audit logs")).toBeInTheDocument(); + }); + it("renders active filter chips with human-readable labels", () => { const filters: ColumnFiltersState = [{ id: "action", value: "created" }]; renderTable({ columnFilters: filters }); diff --git a/ui/litellm-dashboard/src/components/view_logs/AuditLogsTable.tsx b/ui/litellm-dashboard/src/components/view_logs/AuditLogsTable.tsx index 799505ed07c..cef828838e2 100644 --- a/ui/litellm-dashboard/src/components/view_logs/AuditLogsTable.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/AuditLogsTable.tsx @@ -24,6 +24,8 @@ interface AuditLogsTableProps { onPaginationChange: OnChangeFn; columnFilters: ColumnFiltersState; onColumnFiltersChange: OnChangeFn; + searchValue?: string; + onSearchChange?: (value: string) => void; onRefresh: () => void; onViewLog: (log: AuditLogEntry) => void; } @@ -102,11 +104,14 @@ export function AuditLogsTable({ onPaginationChange, columnFilters, onColumnFiltersChange, + searchValue, + onSearchChange, onRefresh, onViewLog, }: AuditLogsTableProps) { const [filtersOpen, setFiltersOpen] = useState(false); const columns = useMemo(() => getAuditLogsTableColumns({ onViewLog }), [onViewLog]); + const hasActiveSearch = Boolean(searchValue?.trim()); return ( 0} />} + noDataMessage={ 0 || hasActiveSearch} />} size="compact" toolbar={(table) => ( <> setFiltersOpen(true)} diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx index 93066c106ee..be0d0049c13 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx @@ -54,6 +54,14 @@ vi.mock("./LogDetailsDrawer", () => ({ }, })); +const debounce = vi.hoisted(() => ({ settled: null as string | null })); + +vi.mock("@tanstack/react-pacer/debouncer", () => ({ + useDebouncedValue: vi.fn((value: unknown) => [debounce.settled ?? value, { cancel: vi.fn(), flush: vi.fn() }]), +})); + +import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; +import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; import { uiSpendLogsCall } from "../networking"; const logEntry = (overrides: Partial): LogEntry => ({ @@ -136,6 +144,7 @@ describe("RequestLogsPanel", () => { sessionStorage.clear(); testQueryClient.clear(); respondWith([]); + debounce.settled = null; }); describe("server-grouped session pagination (#38060)", () => { @@ -322,9 +331,8 @@ describe("RequestLogsPanel", () => { }); }); - describe("search by request id (LIT-3981)", () => { - it("sends the typed request id to the server on the first page instead of filtering the loaded rows", async () => { - const user = userEvent.setup(); + describe("search by any id (LIT-3981, LIT-4741)", () => { + it("sends the typed id to the server as search on the first page instead of filtering the loaded rows", async () => { renderPanel(); await waitFor(() => expect(uiSpendLogsCall).toHaveBeenCalled()); @@ -334,9 +342,54 @@ describe("RequestLogsPanel", () => { await waitFor(() => { const call = lastCall(); if (!call) throw new Error("uiSpendLogsCall was not called"); - expect(call.params?.request_id).toBe("req-on-another-page"); + expect(call.params?.search).toBe("req-on-another-page"); expect(call.page).toBe(1); }); + expect(lastCall()?.params?.request_id).toBeUndefined(); + expect(lastCall()?.params?.session_cursor).toBeUndefined(); + }); + + it("sends the debounced value to the server while the box shows what is being typed", async () => { + debounce.settled = "settled-id"; + renderPanel(); + + await waitFor(() => expect(uiSpendLogsCall).toHaveBeenCalled()); + + fireEvent.change(screen.getByTestId("datatable-search"), { target: { value: "still-typing" } }); + + expect(screen.getByTestId("datatable-search")).toHaveValue("still-typing"); + await waitFor(() => + expect(useDebouncedValue).toHaveBeenLastCalledWith("still-typing", { wait: DEBOUNCE_WAIT_MS }), + ); + await waitFor(() => expect(lastCall()?.params?.search).toBe("settled-id")); + const sentLiveValue = vi + .mocked(uiSpendLogsCall) + .mock.calls.some(([options]) => options.params?.search === "still-typing"); + expect(sentLiveValue).toBe(false); + }); + + it("shows a Search chip whose remove button clears the box and restores the unsearched listing", async () => { + const user = userEvent.setup(); + vi.mocked(uiSpendLogsCall).mockImplementation(async ({ params }) => { + const data = + params?.search === "sess-42" + ? [logEntry({ request_id: "req-sess", session_id: "sess-42" })] + : [logEntry({ request_id: "req-initial" })]; + return { data, total: data.length, page: 1, page_size: 50, total_pages: 1 }; + }); + renderPanel(); + + await waitFor(() => expect(row("req-initial")).not.toBeNull()); + fireEvent.change(screen.getByTestId("datatable-search"), { target: { value: "sess-42" } }); + await waitFor(() => expect(row("req-sess")).not.toBeNull()); + expect(row("req-initial")).toBeNull(); + expect(screen.getByTestId("filter-chip-search")).toHaveTextContent("Search:sess-42"); + + await user.click(screen.getByRole("button", { name: "Remove Search filter" })); + + expect(screen.getByTestId("datatable-search")).toHaveValue(""); + await waitFor(() => expect(row("req-initial")).not.toBeNull()); + expect(row("req-sess")).toBeNull(); }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx index 96cf2bd5dde..9b99c6af923 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx @@ -1,11 +1,13 @@ "use client"; +import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; import { useQuery, type UseQueryOptions } from "@tanstack/react-query"; import type { ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table"; import moment from "moment"; import { useCallback, useEffect, useMemo, useState } from "react"; import { AutoRouterModelGroupsProvider } from "@/components/shared/table_cells"; +import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; import type { KeyResponse } from "../key_team_helpers/key_list"; import { keyInfoV1Call, uiSpendLogsCall } from "../networking"; import KeyInfoView from "../templates/key_info_view"; @@ -75,12 +77,22 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, sessionStorage.setItem("excludeInternalHealthChecks", JSON.stringify(excludeInternalHealthChecks)); }, [excludeInternalHealthChecks]); + const searchTerm = useMemo(() => { + const entry = columnFilters.find((filter) => filter.id === LOG_FILTER_IDS.SEARCH); + return typeof entry?.value === "string" ? entry.value : ""; + }, [columnFilters]); + const [debouncedSearch] = useDebouncedValue(searchTerm, { wait: DEBOUNCE_WAIT_MS }); + const queryColumnFilters = useMemo(() => { + const others = columnFilters.filter((filter) => filter.id !== LOG_FILTER_IDS.SEARCH); + return debouncedSearch === "" ? others : [...others, { id: LOG_FILTER_IDS.SEARCH, value: debouncedSearch }]; + }, [columnFilters, debouncedSearch]); + const { logsQuery, filteredLogs, allTeams, usesSessionCursor } = useLogFilterLogic({ accessToken, token, userRole, userID, - columnFilters, + columnFilters: queryColumnFilters, activeTab: isActive ? "request logs" : "inactive", isLiveTail, excludeInternalHealthChecks, @@ -155,15 +167,10 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, const rows: LogEntry[] = filteredLogs.data; - const searchTerm = useMemo(() => { - const entry = columnFilters.find((filter) => filter.id === LOG_FILTER_IDS.REQUEST_ID); - return typeof entry?.value === "string" ? entry.value : ""; - }, [columnFilters]); - const handleSearchChange = useCallback((value: string) => { setColumnFilters((previous) => { - const others = previous.filter((filter) => filter.id !== LOG_FILTER_IDS.REQUEST_ID); - return value === "" ? others : [...others, { id: LOG_FILTER_IDS.REQUEST_ID, value }]; + const others = previous.filter((filter) => filter.id !== LOG_FILTER_IDS.SEARCH); + return value === "" ? others : [...others, { id: LOG_FILTER_IDS.SEARCH, value }]; }); setSessionCursors({}); setPagination((previous) => ({ ...previous, pageIndex: 0 })); diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTable.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTable.tsx index 4159b3b699b..17caa4466fa 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTable.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTable.tsx @@ -108,7 +108,7 @@ export function RequestLogsTable({ table={table} searchValue={searchValue} onSearchChange={onSearchChange} - searchPlaceholder="Search by Request ID" + searchPlaceholder="Search logs by ID…" onRefresh={onRefresh} isRefreshing={isRefreshing} onOpenFilters={() => setFiltersOpen(true)} diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx index 1b738db097d..6af791cc50e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx @@ -91,6 +91,7 @@ describe("useLogFilterLogic", () => { { id: LOG_FILTER_IDS.ERROR_CODE, value: "429", param: "error_code" }, { id: LOG_FILTER_IDS.ERROR_MESSAGE, value: "rate limited", param: "error_message" }, { id: LOG_FILTER_IDS.USER_ID, value: "user-9", param: "user_id" }, + { id: LOG_FILTER_IDS.SEARCH, value: "any-id", param: "search" }, ]; it.each(cases)("sends $id as $param", async ({ id, value, param }) => { diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx index 90f0f0a60f1..3d368527ad9 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx @@ -33,6 +33,7 @@ export const LOG_FILTER_IDS = { PUBLIC_MODEL_OR_SEARCH_TOOL: "model", REQUEST_ID: "request_id", USER_ID: "user_id", + SEARCH: "search", } as const; export const LOG_FILTER_LABELS: Record = { @@ -48,6 +49,7 @@ export const LOG_FILTER_LABELS: Record = { [LOG_FILTER_IDS.SESSION_ID]: "Session ID", [LOG_FILTER_IDS.MODEL_ID]: "Model", [LOG_FILTER_IDS.PUBLIC_MODEL_OR_SEARCH_TOOL]: "Public model / search tool", + [LOG_FILTER_IDS.SEARCH]: "Search", }; export interface LogsWindow { @@ -175,6 +177,7 @@ export function useLogFilterLogic({ api_key: getFilterValue(columnFilters, LOG_FILTER_IDS.KEY_HASH), team_id: getFilterValue(columnFilters, LOG_FILTER_IDS.TEAM_ID), request_id: getFilterValue(columnFilters, LOG_FILTER_IDS.REQUEST_ID), + search: getFilterValue(columnFilters, LOG_FILTER_IDS.SEARCH), session_id: getFilterValue(columnFilters, LOG_FILTER_IDS.SESSION_ID), user_id: userIdFilter, end_user: getFilterValue(columnFilters, LOG_FILTER_IDS.END_USER), From e504477a696ca0b8c82c2083510e4ae0c9a391f6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 3 Sep 2026 15:20:55 -0700 Subject: [PATCH 122/132] chore(ui): regenerate schema.d.ts for the new search params Claude-Session: https://claude.ai/code/session_01Q5sbiogJzPcCRmYSbaHxZf --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6742076fa78..48a8cfd54d6 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -40904,6 +40904,8 @@ export interface operations { object_team_id?: string | null; /** @description Filter by token (key hash) present in before_value or updated_values JSON (PostgreSQL only) */ object_key_hash?: string | null; + /** @description Match a row whose id, object_id, changed_by, or changed_by_api_key equals this value (a raw sk- virtual key is hashed first) */ + search?: string | null; /** @description Column to sort by (e.g. 'updated_at', 'action', 'table_name') */ sort_by?: string | null; /** @description Sort order ('asc' or 'desc') */ @@ -49609,6 +49611,8 @@ export interface operations { key_hash?: string | null; /** @description Filter keys by key alias. Exact match by default; set substring_matching=true (admin only) for case-insensitive substring matching. */ key_alias?: string | null; + /** @description Combined search: matches keys whose token (key hash) equals the value, hashing a raw sk- key first, OR whose key_alias contains it (case-insensitive). */ + search?: string | null; /** @description Return full key object */ return_full_object?: boolean; /** @description Include all keys for teams that user is an admin of. */ @@ -56867,6 +56871,8 @@ export interface operations { group_by_session?: boolean; /** @description Keyset cursor '||' from a previous group_by_session page. UI route only, honored when sorting by startTime */ session_cursor?: string | null; + /** @description Match a log whose request_id, api_key (a raw sk- key is hashed first), team_id, user, end_user, session_id, or model_id equals this value. request_id matches across all time; the other columns match inside start_date/end_date, which stay required */ + search?: string | null; }; header?: never; path?: never; @@ -56983,6 +56989,8 @@ export interface operations { group_by_session?: boolean; /** @description Keyset cursor '||' from a previous group_by_session page. UI route only, honored when sorting by startTime */ session_cursor?: string | null; + /** @description Match a log whose request_id, api_key (a raw sk- key is hashed first), team_id, user, end_user, session_id, or model_id equals this value. request_id matches across all time; the other columns match inside start_date/end_date, which stay required */ + search?: string | null; }; header?: never; path?: never; @@ -63282,6 +63290,8 @@ export interface operations { key?: string | null; /** @description Filter by key prefix (Redis-style namespace scan). Mutually exclusive with `key`; if both are provided, `key_prefix` wins. */ key_prefix?: string | null; + /** @description Match entries whose key starts with this value or whose memory_id equals it. Takes precedence over `key_prefix` and `key` when provided. */ + search?: string | null; page?: number; page_size?: number; }; From 24531ee576b4a5c535f46e840d27e2d89e990878 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:37:56 -0700 Subject: [PATCH 123/132] refactor(cost): drop the docstrings that restate TokenRates and the new tests --- litellm/litellm_core_utils/llm_cost_calc/utils.py | 5 ----- .../llm_cost_calc/test_llm_cost_calc_utils.py | 13 ------------- .../dashscope/test_dashscope_cost_calculator.py | 5 ----- 3 files changed, 23 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index e03c2c93c26..8e24302b440 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -417,11 +417,6 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None = @dataclass(frozen=True, slots=True) class TokenRates: - """The per-token rates one request bills at. reasoning_rate is None when reasoning bills at - output_rate: the model has no dedicated reasoning rate, or the caller resolves reasoning on - its own. - """ - input_rate: float output_rate: float cache_read_rate: float diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 7bc02145841..d5d7b6a47f4 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -815,8 +815,6 @@ def _off_peak_reasoning_usage() -> Usage: def test_generic_cost_per_token_off_peak_reasoning_rate(): - """Regression (LIT-6887): the block's output_cost_per_reasoning_token used to be ignored, so - reasoning tokens billed at the model's standard reasoning rate all through the window.""" from datetime import datetime, timezone model_name = "litellm-test-off-peak-reasoning" @@ -843,8 +841,6 @@ def test_generic_cost_per_token_off_peak_reasoning_rate(): def test_generic_cost_per_token_off_peak_block_without_reasoning_rate(): - """A block that leaves output_cost_per_reasoning_token unset keeps the model's own reasoning - rate, and a model with no reasoning rate at all follows the off-peak output rate.""" from datetime import datetime, timezone inside_window = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) @@ -870,7 +866,6 @@ def test_generic_cost_per_token_off_peak_block_without_reasoning_rate(): def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_tier(): - """Tiered models resolve reasoning on their own path, so the block has to win there too.""" from datetime import datetime, timezone model_name = "litellm-test-off-peak-tiered-reasoning" @@ -914,8 +909,6 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_tier(): def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_service_tier(): - """A priority request bills its service-tier reasoning rate outside the window and the block's - rate inside it.""" from datetime import datetime, timezone model_name = "litellm-test-off-peak-reasoning-service-tier" @@ -946,7 +939,6 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_service_ti def test_apply_off_peak_pricing_treats_bool_as_unset_and_parses_strings(): - """A YAML true never turns into a rate of 1.0, and a quoted number still counts.""" from datetime import datetime, timezone model_name = "litellm-test-off-peak-odd-values" @@ -972,9 +964,6 @@ def test_apply_off_peak_pricing_treats_bool_as_unset_and_parses_strings(): def test_get_token_base_cost_off_peak_cache_creation_rate(): - """Regression (LIT-6887): the block's cache_creation_input_token_cost used to be ignored. It - replaces the five-minute cache-creation rate inside the window; the one-hour rate, and a - block without the key, keep the standard rate.""" from datetime import datetime, timezone from typing import cast @@ -1008,8 +997,6 @@ def test_get_token_base_cost_off_peak_cache_creation_rate(): def test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_creation_rates(): - """The per-token-type breakdown feeds the spend logs, so it has to bill the new keys the same - way the total does.""" from datetime import datetime, timezone model_name = "litellm-test-off-peak-breakdown" diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py index f0949ce041a..a30d35d46f2 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py @@ -650,8 +650,6 @@ class TestDashscopeCostCalculator: assert math.isclose(completion_cost, 200 * 2.4e-06, rel_tol=1e-10) def test_dashscope_off_peak_reasoning_rate_replaces_the_dedicated_reasoning_rate(self): - """Regression (LIT-6887): a block carrying output_cost_per_reasoning_token bills reasoning - tokens at it inside the window, over the model's own reasoning rate, which returns outside.""" self._register_off_peak_flat_model( "dashscope/qwen-reasoning-rate-off-peak-test", { @@ -678,8 +676,6 @@ class TestDashscopeCostCalculator: assert math.isclose(peak_completion_cost, (150 * 4.8e-06) + (50 * 9e-06), rel_tol=1e-10) def test_dashscope_off_peak_cache_creation_rate_replaces_the_standard_rate(self): - """Regression (LIT-6887): a block carrying cache_creation_input_token_cost bills cache-creation - tokens at it inside the window, while the cache-read rate it leaves unset stays standard.""" self._register_off_peak_flat_model( "dashscope/qwen-cache-creation-off-peak-test", {"hours_utc": self.OFF_PEAK_WINDOW, "cache_creation_input_token_cost": 1.5e-06}, @@ -701,7 +697,6 @@ class TestDashscopeCostCalculator: assert math.isclose(peak_prompt_cost, (600 * 2.4e-06) + (300 * 2e-07) + (100 * 3e-06), rel_tol=1e-10) def test_dashscope_off_peak_reasoning_and_cache_creation_rates_override_the_selected_tier(self): - """The new keys override the selected tier the way the input and output rates already do.""" self._register_tiered_model( "dashscope/qwen-tiered-reasoning-off-peak-test", [ From 1d71e306cc13008f551d8964623ca813ca44bccb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:38:15 -0700 Subject: [PATCH 124/132] test(router): assert a non-chat configured mode round-trips With every kept get_configured_mode test using mode "chat", a Router that answered "chat" for any non-blank configured mode passed all four of them (the deleted #39630 pair's audio_speech case was the only test catching it). Read the mode back as audio_speech on an unmapped model so the configured value itself is what the test checks. Six hand-applied mutations of Router.get_configured_mode, including that hardcoded-chat one, are now all killed by the four surviving tests. --- tests/test_litellm/test_router.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 417c58b95f6..228588d974f 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7362,14 +7362,14 @@ def test_get_configured_mode_reads_deployment_model_info(): router = litellm.Router( model_list=[ { - "model_name": "chat-model", - "litellm_params": {"model": "openai/some-unmapped-model"}, - "model_info": {"mode": "chat"}, + "model_name": "tts-model", + "litellm_params": {"model": "openai/some-unmapped-tts-model"}, + "model_info": {"mode": "audio_speech"}, } ] ) - assert router.get_configured_mode("chat-model") == "chat" + assert router.get_configured_mode("tts-model") == "audio_speech" def test_get_configured_mode_returns_none_for_unset_or_unknown(): From 9baa19c7d13c4d5937d7c6f337659b94b514f773 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 3 Sep 2026 15:54:42 -0700 Subject: [PATCH 125/132] fix(proxy): stop hashing raw sk- values in list searches The search= param on /key/list, /audit, and /spend/logs/ui, plus key_hash= on /key/list, now compare the pasted value verbatim. Only a copied key ID (the hash) matches, so a raw virtual key never needs to travel in a GET query string Claude-Session: https://claude.ai/code/session_01Q5sbiogJzPcCRmYSbaHxZf --- .../proxy/audit_logging_endpoints.py | 17 ++--- .../key_management_endpoints.py | 12 ++-- .../spend_management_endpoints.py | 14 ++-- .../key_management_endpoints.py | 2 +- .../proxy/test_audit_logging_endpoints.py | 19 ----- .../test_key_management_endpoints.py | 71 +++---------------- .../test_spend_management_endpoints.py | 37 ++++------ .../(dashboard)/hooks/keys/useKeys.test.ts | 4 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 +-- 9 files changed, 43 insertions(+), 141 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py b/enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py index 72f7a66a420..7df14565c3f 100644 --- a/enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py @@ -18,7 +18,6 @@ from litellm_enterprise.types.proxy.audit_logging_endpoints import ( from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.utils import _hash_token_if_needed from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import AuditLogRepository @@ -50,14 +49,13 @@ def _build_json_field_or_condition(json_key: str, value: str) -> dict[str, objec def _build_search_condition(search: str) -> dict[str, object]: - """Match any id column; a raw sk- key is hashed for the two columns that store key hashes.""" - hashed: Final = _hash_token_if_needed(search) + """Match a row whose id, changed_by, object_id, or changed_by_api_key equals the search value.""" return { "OR": ( {"id": search}, {"changed_by": search}, - {"object_id": hashed}, - {"changed_by_api_key": hashed}, + {"object_id": search}, + {"changed_by_api_key": search}, ) } @@ -99,10 +97,7 @@ async def get_audit_logs( ), search: str | None = Query( None, - description=( - "Match a row whose id, object_id, changed_by, or changed_by_api_key equals this value " - "(a raw sk- virtual key is hashed first)" - ), + description="Match a row whose id, object_id, changed_by, or changed_by_api_key equals this value", ), # Sorting parameters sort_by: str | None = Query( @@ -159,7 +154,7 @@ async def get_audit_logs( {sort_by: sort_order} if sort_by and isinstance(sort_by, str) else {"updated_at": sort_order} ) - audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table + audit_log_table: Final[TableActions[prisma_models.LiteLLM_AuditLog]] = AuditLogRepository(prisma_client).table # Get paginated results audit_logs: Final = await audit_log_table.find_many( @@ -221,7 +216,7 @@ async def get_audit_log_by_id( detail={"message": CommonProxyErrors.db_not_connected_error.value}, ) - audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table + audit_log_table: Final[TableActions[prisma_models.LiteLLM_AuditLog]] = AuditLogRepository(prisma_client).table # Get the audit log by ID audit_log: Final = await audit_log_table.find_unique(where={"id": id}) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 758644ff01b..324d380b85b 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -5803,7 +5803,7 @@ async def list_keys( ), search: str | None = Query( None, - description="Combined search: matches keys whose token (key hash) equals the value, hashing a raw sk- key first, OR whose key_alias contains it (case-insensitive).", + description="Combined search: matches keys whose token (key hash) equals the value OR whose key_alias contains it (case-insensitive).", ), return_full_object: bool = Query(False, description="Return full key object"), include_team_keys: bool = Query(False, description="Include all keys for teams that user is an admin of."), @@ -5867,17 +5867,13 @@ async def list_keys( detail={"error": "Invalid expires value. Supported: 'active', 'expired'."}, ) - hashed_key_hash: Final[str | None] = ( - _hash_token_if_needed(token=key_hash) if isinstance(key_hash, str) else None - ) - complete_user_info: Final = await validate_key_list_check( user_api_key_dict=user_api_key_dict, user_id=user_id, team_id=team_id, organization_id=organization_id, key_alias=key_alias, - key_hash=hashed_key_hash, + key_hash=key_hash, prisma_client=prisma_client, ) @@ -5937,7 +5933,7 @@ async def list_keys( user_id=user_id, team_id=team_id, key_alias=key_alias, - key_hash=hashed_key_hash, + key_hash=key_hash, return_full_object=return_full_object, organization_id=organization_id, admin_team_ids=admin_team_ids, @@ -6175,7 +6171,7 @@ def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, def _build_key_search_where(search: str) -> KeySearchWhere: search_where: Final[KeySearchWhere] = { "OR": ( - {"token": _hash_token_if_needed(token=search)}, + {"token": search}, {"key_alias": {"contains": search, "mode": "insensitive"}}, ) } diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 9ec8dd205a6..b86a877e8f9 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -2240,20 +2240,18 @@ def _build_spend_log_search_condition( end_date: datetime, next_param_index: int, ) -> _SpendLogSearchCondition: - """request_id (indexed) matches across all time; the unindexed id columns only inside the window (sk- keys hashed).""" + """request_id (indexed) matches across all time; the unindexed id columns only inside the window.""" raw: Final = f"${next_param_index}" - hashed: Final = f"${next_param_index + 1}" - window_start: Final = f"${next_param_index + 2}" - window_end: Final = f"${next_param_index + 3}" + window_start: Final = f"${next_param_index + 1}" + window_end: Final = f"${next_param_index + 2}" sql: Final = ( f"(request_id = {raw} OR (" f"\"startTime\" >= ({window_start}::timestamptz AT TIME ZONE 'UTC') " f"AND \"startTime\" <= ({window_end}::timestamptz AT TIME ZONE 'UTC') " - f'AND (api_key = {hashed} OR team_id = {raw} OR "user" = {raw} OR end_user = {raw} ' + f'AND (api_key = {raw} OR team_id = {raw} OR "user" = {raw} OR end_user = {raw} ' f"OR session_id = {raw} OR model_id = {raw})))" ) - hashed_search: Final = hash_token(token=search) if search.startswith("sk-") else search - return _SpendLogSearchCondition(sql=sql, params=(search, hashed_search, start_date, end_date)) + return _SpendLogSearchCondition(sql=sql, params=(search, start_date, end_date)) @router.get( @@ -2359,7 +2357,7 @@ async def ui_view_spend_logs( search: str | None = fastapi.Query( default=None, description=( - "Match a log whose request_id, api_key (a raw sk- key is hashed first), team_id, user, end_user, " + "Match a log whose request_id, api_key (hash), team_id, user, end_user, " "session_id, or model_id equals this value. request_id matches across all time; the other columns " "match inside start_date/end_date, which stay required" ), diff --git a/litellm/types/proxy/management_endpoints/key_management_endpoints.py b/litellm/types/proxy/management_endpoints/key_management_endpoints.py index 5d410d7b55b..9fb5bea81e3 100644 --- a/litellm/types/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/key_management_endpoints.py @@ -16,7 +16,7 @@ class KeyAliasContainsWhere(TypedDict): class KeySearchWhere(TypedDict): - """Prisma filter behind `/key/list?search=`: exact token (sk- keys hashed) or alias substring, case-insensitive.""" + """Prisma filter behind `/key/list?search=`: exact token or case-insensitive alias substring.""" OR: ReadOnly[tuple[KeyTokenWhere, KeyAliasContainsWhere]] diff --git a/tests/enterprise/litellm_enterprise/proxy/test_audit_logging_endpoints.py b/tests/enterprise/litellm_enterprise/proxy/test_audit_logging_endpoints.py index cd2c8b0b904..fd1b05ff060 100644 --- a/tests/enterprise/litellm_enterprise/proxy/test_audit_logging_endpoints.py +++ b/tests/enterprise/litellm_enterprise/proxy/test_audit_logging_endpoints.py @@ -1,4 +1,3 @@ -import hashlib from datetime import datetime, timedelta from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -172,24 +171,6 @@ def test_search_matches_any_id_column_alongside_the_other_filters(mock_prisma_cl } -def test_search_hashes_a_raw_virtual_key_for_the_hashed_columns(mock_prisma_client): - where: Final = _list_audit_logs_where(mock_prisma_client, "search=sk-raw") - - hashed: Final = hashlib.sha256(b"sk-raw").hexdigest() - assert where == { - "AND": ( - { - "OR": ( - {"id": "sk-raw"}, - {"changed_by": "sk-raw"}, - {"object_id": hashed}, - {"changed_by_api_key": hashed}, - ) - }, - ) - } - - def test_an_empty_search_leaves_the_where_clause_unchanged(mock_prisma_client): where: Final = _list_audit_logs_where(mock_prisma_client, "action=create&search=") diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 1e399e4fb58..0e4af9f75a5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -6351,33 +6351,15 @@ def _search_clause(search: str, token: str) -> dict: return {"OR": [{"token": token}, {"key_alias": {"contains": search, "mode": "insensitive"}}]} -def test_build_key_filter_conditions_search_hashes_raw_key_and_ors_alias_contains(): +def test_build_key_filter_conditions_search_ors_token_and_alias_contains(): """ LIT-4741: `search` matches a key by its alias (case-insensitive contains) OR by - its ID. A pasted raw sk- key is hashed to its token first; an already-hashed - value is used verbatim. + its ID (the token column), with the pasted value used verbatim. """ - from litellm.proxy._types import hash_token from litellm.proxy.management_endpoints.key_management_endpoints import ( _build_key_filter_conditions, ) - raw_where = json.loads( - json.dumps( - _build_key_filter_conditions( - user_id=None, - team_id=None, - organization_id=None, - key_alias=None, - key_hash=None, - exclude_team_id=None, - admin_team_ids=None, - search="sk-raw", - ) - ) - ) - assert _search_clause("sk-raw", hash_token("sk-raw")) in raw_where["AND"], f"raw search not ANDed: {raw_where}" - hashed_where = json.loads( json.dumps( _build_key_filter_conditions( @@ -6402,7 +6384,6 @@ def test_build_key_filter_conditions_search_narrows_team_admin_visibility(): LIT-4741, same class as LIT-3243: `search` must be a top-level AND so it narrows a team admin's admin-team branch instead of being bypassed by it. """ - from litellm.proxy._types import hash_token from litellm.proxy.management_endpoints.key_management_endpoints import ( _build_key_filter_conditions, ) @@ -6419,21 +6400,19 @@ def test_build_key_filter_conditions_search_narrows_team_admin_visibility(): admin_team_ids=["team-a"], member_team_ids=["team-a"], include_created_by_keys=False, - search="sk-member", + search="member-key-id", ) ) ) assert where.get("AND"), f"expected top-level AND, got: {where}" - assert _search_clause("sk-member", hash_token("sk-member")) in where["AND"], f"search not ANDed: {where}" + assert _search_clause("member-key-id", "member-key-id") in where["AND"], f"search not ANDed: {where}" assert json.dumps({"team_id": {"in": ["team-a"]}}) in json.dumps(where) @pytest.mark.asyncio async def test_list_key_helper_applies_search_to_prisma_where(): """LIT-4741: `search` given to _list_key_helper must reach the Prisma where clause.""" - from litellm.proxy._types import hash_token - mock_prisma_client = AsyncMock() mock_find_many = AsyncMock(return_value=[]) mock_prisma_client.db.litellm_verificationtoken.find_many = mock_find_many @@ -6448,11 +6427,11 @@ async def test_list_key_helper_applies_search_to_prisma_where(): organization_id=None, key_alias=None, key_hash=None, - search="sk-raw", + search="key-id-123", ) where = json.loads(json.dumps(mock_find_many.call_args.kwargs["where"])) - assert _search_clause("sk-raw", hash_token("sk-raw")) in where["AND"], f"search not in Prisma where: {where}" + assert _search_clause("key-id-123", "key-id-123") in where["AND"], f"search not in Prisma where: {where}" @pytest.mark.asyncio @@ -14978,47 +14957,13 @@ async def test_list_keys_non_admin_cannot_opt_into_substring(): assert kwargs["user_id"] == "alice" -@pytest.mark.asyncio -async def test_list_keys_hashes_raw_key_hash_before_validation(): - """LIT-4741: a raw sk- key pasted as key_hash is hashed before the ownership - check and the query, so a non-admin filtering by their own raw key gets the - row instead of the 'Key Hash not found.' 403.""" - from litellm.proxy._types import hash_token - - user = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice") - validate = AsyncMock( - return_value=LiteLLM_UserTable( - user_id="alice", user_email="alice@example.com", teams=[], organization_memberships=[] - ) - ) - helper = AsyncMock(return_value={"keys": [], "total_count": 0, "current_page": 1, "total_pages": 0}) - with ( - patch("litellm.proxy.proxy_server.prisma_client", AsyncMock()), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints.validate_key_list_check", - validate, - ), - patch("litellm.proxy.management_endpoints.key_management_endpoints._list_key_helper", helper), - ): - await list_keys( - request=MagicMock(), - user_api_key_dict=user, - status=None, - user_id=None, - key_hash="sk-raw", - ) - - assert validate.call_args.kwargs["key_hash"] == hash_token("sk-raw") - assert helper.call_args.kwargs["key_hash"] == hash_token("sk-raw") - - @pytest.mark.asyncio async def test_list_keys_search_is_honored_for_non_admin(): """LIT-4741: unlike substring_matching, `search` is not admin-gated. A non-admin's search reaches the helper while their own-user scoping stays in place.""" user = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice") - kwargs = await _list_keys_capture_helper_kwargs(user, user_id=None, search="sk-raw") - assert kwargs["search"] == "sk-raw" + kwargs = await _list_keys_capture_helper_kwargs(user, user_id=None, search="key-id-123") + assert kwargs["search"] == "key-id-123" assert kwargs["user_id"] == "alice" diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index f869f3ffba2..73a29afd9b9 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -61,7 +61,7 @@ def _filter_logs_by_date_range(logs, where): _SEARCH_CLAUSE_RE = re.compile( r'\(request_id = \$(\d+) OR \("startTime" >= \(\$(\d+)::timestamptz AT TIME ZONE \'UTC\'\) ' r'AND "startTime" <= \(\$(\d+)::timestamptz AT TIME ZONE \'UTC\'\) ' - r'AND \(api_key = \$(\d+) OR team_id = \$\1 OR "user" = \$\1 OR end_user = \$\1 ' + r'AND \(api_key = \$\1 OR team_id = \$\1 OR "user" = \$\1 OR end_user = \$\1 ' r"OR session_id = \$\1 OR model_id = \$\1\)\)\)" ) @@ -72,9 +72,8 @@ def _matches_spend_log_search(log, search): return True if not _filter_logs_by_date_range([log], {"startTime": {"gte": search["gte"], "lte": search["lte"]}}): return False - if log.get("api_key") == search["api_key"]: - return True - return any(log.get(col) == search["value"] for col in ("team_id", "user", "end_user", "session_id", "model_id")) + columns = ("api_key", "team_id", "user", "end_user", "session_id", "model_id") + return any(log.get(col) == search["value"] for col in columns) def _reconstruct_ui_where_from_sql(sql_query, params): @@ -98,10 +97,9 @@ def _reconstruct_ui_where_from_sql(sql_query, params): search_clause = _SEARCH_CLAUSE_RE.search(clause.group(1)) if search_clause: - raw_index, start_index, end_index, hashed_index = (int(g) for g in search_clause.groups()) + raw_index, start_index, end_index = (int(g) for g in search_clause.groups()) where["search"] = { "value": params[raw_index - 1], - "api_key": params[hashed_index - 1], "gte": _iso(params[start_index - 1]), "lte": _iso(params[end_index - 1]), } @@ -2384,31 +2382,20 @@ async def test_ui_view_spend_logs_request_id_owner_scoped_by_id_only( def test_build_spend_log_search_condition_windows_every_branch_except_request_id(): """LIT-4741: request_id matches across all time; the six other id columns only inside the window, - and a raw sk- key is hashed for the api_key branch alone.""" + all comparing the pasted value verbatim.""" start = datetime.datetime(2026, 8, 1, tzinfo=timezone.utc) end = datetime.datetime(2026, 8, 2, tzinfo=timezone.utc) condition = spend_management_endpoints._build_spend_log_search_condition( - search="sk-raw-key", start_date=start, end_date=end, next_param_index=3 + search="key-hash-7", start_date=start, end_date=end, next_param_index=3 ) assert condition.sql == ( - "(request_id = $3 OR (\"startTime\" >= ($5::timestamptz AT TIME ZONE 'UTC') " - "AND \"startTime\" <= ($6::timestamptz AT TIME ZONE 'UTC') " - 'AND (api_key = $4 OR team_id = $3 OR "user" = $3 OR end_user = $3 OR session_id = $3 OR model_id = $3)))' + "(request_id = $3 OR (\"startTime\" >= ($4::timestamptz AT TIME ZONE 'UTC') " + "AND \"startTime\" <= ($5::timestamptz AT TIME ZONE 'UTC') " + 'AND (api_key = $3 OR team_id = $3 OR "user" = $3 OR end_user = $3 OR session_id = $3 OR model_id = $3)))' ) - assert condition.params == ("sk-raw-key", hashlib.sha256(b"sk-raw-key").hexdigest(), start, end) - - -def test_build_spend_log_search_condition_leaves_non_key_values_unhashed(): - start = datetime.datetime(2026, 8, 1, tzinfo=timezone.utc) - end = datetime.datetime(2026, 8, 2, tzinfo=timezone.utc) - - condition = spend_management_endpoints._build_spend_log_search_condition( - search="sess-42", start_date=start, end_date=end, next_param_index=1 - ) - - assert condition.params == ("sess-42", "sess-42", start, end) + assert condition.params == ("key-hash-7", start, end) def _search_fixture_logs(today): @@ -2427,7 +2414,7 @@ def _search_fixture_logs(today): return [ {**base, "request_id": "req-session", "session_id": "sess-42", "startTime": recent}, {**base, "request_id": "req-session-old", "session_id": "sess-42", "startTime": old}, - {**base, "request_id": "req-key", "api_key": hashlib.sha256(b"sk-raw-key").hexdigest(), "startTime": recent}, + {**base, "request_id": "req-key", "api_key": "hashed-7", "startTime": recent}, {**base, "request_id": "req-team", "team_id": "team-7", "startTime": recent}, {**base, "request_id": "req-user", "user": "user-7", "startTime": recent}, {**base, "request_id": "req-end-user", "end_user": "cust-7", "startTime": recent}, @@ -2461,7 +2448,7 @@ def _five_day_window(today): [ ("req-session-old", {"req-session-old"}), ("sess-42", {"req-session"}), - ("sk-raw-key", {"req-key"}), + ("hashed-7", {"req-key"}), ("team-7", {"req-team"}), ("user-7", {"req-user"}), ("cust-7", {"req-end-user"}), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeys.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeys.test.ts index f23fcf811f2..84be7e2ef49 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeys.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeys.test.ts @@ -525,14 +525,14 @@ describe("useKeys", () => { json: async () => mockKeysResponse, }); - const { result } = renderHook(() => useKeys(1, 10, { search: "sk-pasted-key" }), { wrapper }); + const { result } = renderHook(() => useKeys(1, 10, { search: "pasted-key-id" }), { wrapper }); await waitFor(() => { expect(result.current.isLoading).toBe(false); }); const callUrl = new URL(mockFetch.mock.calls[0][0], "http://localhost"); - expect(callUrl.searchParams.get("search")).toBe("sk-pasted-key"); + expect(callUrl.searchParams.get("search")).toBe("pasted-key-id"); expect(callUrl.searchParams.has("key_alias")).toBe(false); expect(callUrl.searchParams.has("key_hash")).toBe(false); }); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 48a8cfd54d6..324085fdae4 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -40904,7 +40904,7 @@ export interface operations { object_team_id?: string | null; /** @description Filter by token (key hash) present in before_value or updated_values JSON (PostgreSQL only) */ object_key_hash?: string | null; - /** @description Match a row whose id, object_id, changed_by, or changed_by_api_key equals this value (a raw sk- virtual key is hashed first) */ + /** @description Match a row whose id, object_id, changed_by, or changed_by_api_key equals this value */ search?: string | null; /** @description Column to sort by (e.g. 'updated_at', 'action', 'table_name') */ sort_by?: string | null; @@ -49611,7 +49611,7 @@ export interface operations { key_hash?: string | null; /** @description Filter keys by key alias. Exact match by default; set substring_matching=true (admin only) for case-insensitive substring matching. */ key_alias?: string | null; - /** @description Combined search: matches keys whose token (key hash) equals the value, hashing a raw sk- key first, OR whose key_alias contains it (case-insensitive). */ + /** @description Combined search: matches keys whose token (key hash) equals the value OR whose key_alias contains it (case-insensitive). */ search?: string | null; /** @description Return full key object */ return_full_object?: boolean; @@ -56871,7 +56871,7 @@ export interface operations { group_by_session?: boolean; /** @description Keyset cursor '||' from a previous group_by_session page. UI route only, honored when sorting by startTime */ session_cursor?: string | null; - /** @description Match a log whose request_id, api_key (a raw sk- key is hashed first), team_id, user, end_user, session_id, or model_id equals this value. request_id matches across all time; the other columns match inside start_date/end_date, which stay required */ + /** @description Match a log whose request_id, api_key (hash), team_id, user, end_user, session_id, or model_id equals this value. request_id matches across all time; the other columns match inside start_date/end_date, which stay required */ search?: string | null; }; header?: never; @@ -56989,7 +56989,7 @@ export interface operations { group_by_session?: boolean; /** @description Keyset cursor '||' from a previous group_by_session page. UI route only, honored when sorting by startTime */ session_cursor?: string | null; - /** @description Match a log whose request_id, api_key (a raw sk- key is hashed first), team_id, user, end_user, session_id, or model_id equals this value. request_id matches across all time; the other columns match inside start_date/end_date, which stay required */ + /** @description Match a log whose request_id, api_key (hash), team_id, user, end_user, session_id, or model_id equals this value. request_id matches across all time; the other columns match inside start_date/end_date, which stay required */ search?: string | null; }; header?: never; From 9464888ee9064df4083eee8843424b16eacd7da0 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 3 Sep 2026 16:04:41 -0700 Subject: [PATCH 126/132] test(proxy): pass search=None in direct ui_view_spend_logs calls Calling the endpoint without going through FastAPI leaves the new search param set to its Query default object, which is not None, so the grouped-session and request_id lookup tests started taking the search branch Claude-Session: https://claude.ai/code/session_01Q5sbiogJzPcCRmYSbaHxZf --- .../proxy/spend_tracking/test_spend_query_optimization.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py index 9ae932ff01f..a7de3f1d8d6 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py @@ -184,6 +184,7 @@ async def test_spend_logs_ui_wraps_params_in_at_time_zone_utc(monkeypatch): api_key=None, user_id=None, request_id=None, + search=None, start_date="2026-02-16 00:00:00", end_date="2026-02-16 23:59:59", page=1, @@ -247,6 +248,7 @@ async def test_spend_logs_ui_uses_bounded_count_not_full_scan(monkeypatch): api_key=None, user_id=None, request_id=None, + search=None, start_date="2026-02-16 00:00:00", end_date="2026-02-16 23:59:59", page=1, @@ -314,6 +316,7 @@ async def test_spend_logs_ui_caps_total_for_large_result_sets(monkeypatch): api_key=None, user_id=None, request_id=None, + search=None, start_date="2026-02-16 00:00:00", end_date="2026-02-16 23:59:59", page=1, @@ -359,6 +362,7 @@ async def test_spend_logs_ui_empty_page_reports_zero_total(monkeypatch): api_key=None, user_id=None, request_id=None, + search=None, start_date="2026-02-16 00:00:00", end_date="2026-02-16 23:59:59", page=1, @@ -406,6 +410,7 @@ async def test_spend_logs_ui_out_of_range_page_keeps_total(monkeypatch): api_key=None, user_id=None, request_id=None, + search=None, start_date="2026-02-16 00:00:00", end_date="2026-02-16 23:59:59", page=99, @@ -552,6 +557,7 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): api_key=None, user_id=None, request_id=None, + search=None, start_date="2026-02-16 00:00:00", end_date="2026-02-16 23:59:59", page=1, @@ -616,6 +622,7 @@ async def test_spend_logs_ui_group_by_session_offset_pages_for_other_sorts(monke api_key=None, user_id=None, request_id=None, + search=None, start_date="2026-02-16 00:00:00", end_date="2026-02-16 23:59:59", page=2, @@ -664,6 +671,7 @@ async def test_spend_logs_ui_request_id_lookup_with_grouping_returns_exact_row(m api_key=None, user_id=None, request_id="req-deep-link", + search=None, start_date=None, end_date=None, page=1, From 7bdd148f38f35e5baed4bced6fd980dd77a83bdd Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 3 Sep 2026 16:10:44 -0700 Subject: [PATCH 127/132] test(proxy-extras): fake run_prisma instead of subprocess.run in the migrate deploy harness --- tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index 9ffb57924b6..3fab20a28ad 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -763,7 +763,7 @@ class _MigrateDeployHarness: "_resolve_specific_migration", staticmethod(self.resolved.append), ) - monkeypatch.setattr(utils_module.subprocess, "run", self._fake_run) + monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", self._fake_run) monkeypatch.setattr(utils_module.time, "sleep", lambda seconds: None) self.baseline_succeeds = True From dc98901dc1645391986e3434a72cd256617837cf Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 3 Sep 2026 19:53:32 +0000 Subject: [PATCH 128/132] fix(scim): apply default_internal_user_params.teams to SCIM-created users Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_endpoints/scim/scim_v2.py | 3 +- .../scim/test_scim_v2_endpoints.py | 80 ++++++++++++++++++- 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index 069f86c852c..0f0124ee9d9 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -1123,7 +1123,6 @@ async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_grou user_id=user_id, user_email=user_id, # We don't have email from group membership user_alias=None, - teams=[], # Teams will be added separately metadata={"created_via": created_via}, auto_create_key=False, user_role=default_role, @@ -1699,7 +1698,7 @@ async def create_user( user_id=user_id, user_email=user_data["user_email"], user_alias=user_data["user_alias"], - teams=user_data["teams"], + teams=user_data["teams"] or None, metadata=metadata, auto_create_key=False, user_role=resolved_role if admin_group is not None else default_role, diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index 1697b77b99a..f4627f82506 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -24,6 +24,7 @@ from litellm.proxy.management_endpoints.scim.scim_v2 import ( SCIMRosterSyncError, UserProvisionerHelpers, _apply_group_patch_updates, + _create_user_if_not_exists, _extract_group_member_ids, _extract_ids_from_path_filter, _handle_group_membership_changes, @@ -37,8 +38,8 @@ from litellm.proxy.management_endpoints.scim.scim_v2 import ( delete_group, delete_user, get_groups, - get_users, get_service_provider_config, + get_users, merge_placeholder, patch_group, patch_team_membership, @@ -304,6 +305,83 @@ async def test_create_user_uses_default_internal_user_params_role(mocker, monkey assert called_args.user_role == LitellmUserRoles.PROXY_ADMIN +def _mock_scim_create_user_deps(mocker: MockerFixture, scim_user: SCIMUser) -> AsyncMock: + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + mock_prisma_client.db.litellm_usertable.find_many = AsyncMock(return_value=()) + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + mocker.patch( # test-quality-ok: endpoint collaborators are module-level, not injectable into create_user + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( # test-quality-ok: endpoint collaborators are module-level, not injectable into create_user + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=scim_user), + ) + return mocker.patch( # test-quality-ok: endpoint collaborators are module-level, not injectable into create_user + "litellm.proxy.management_endpoints.scim.scim_v2.new_user", + AsyncMock(return_value=NewUserRequest(user_id=scim_user.userName)), + ) + + +@pytest.mark.asyncio +async def test_create_user_without_groups_defers_to_default_team(mocker: MockerFixture, monkeypatch): + """IdPs omit groups on POST /Users; teams must stay unset so new_user applies default_internal_user_params.teams""" + scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="new-user", + emails=[SCIMUserEmail(value="new@example.com")], + ) + monkeypatch.setattr( + "litellm.default_internal_user_params", + {"teams": [{"team_id": "default-team", "max_budget_in_team": 25}]}, + raising=False, + ) + new_user_mock = _mock_scim_create_user_deps(mocker, scim_user) + + await create_user(user=scim_user) + + assert new_user_mock.call_args.kwargs["data"].teams is None + + +@pytest.mark.asyncio +async def test_create_user_with_groups_keeps_idp_teams(mocker: MockerFixture, monkeypatch): + scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="new-user", + emails=[SCIMUserEmail(value="new@example.com")], + groups=[SCIMUserGroup(value="idp-team")], + ) + monkeypatch.setattr( + "litellm.default_internal_user_params", + {"teams": [{"team_id": "default-team", "max_budget_in_team": 25}]}, + raising=False, + ) + new_user_mock = _mock_scim_create_user_deps(mocker, scim_user) + + await create_user(user=scim_user) + + assert new_user_mock.call_args.kwargs["data"].teams == ["idp-team"] + + +@pytest.mark.asyncio +async def test_create_user_if_not_exists_defers_to_default_team(mocker: MockerFixture, monkeypatch): + monkeypatch.setattr( + "litellm.default_internal_user_params", + {"teams": [{"team_id": "default-team", "max_budget_in_team": 25}]}, + raising=False, + ) + new_user_mock = mocker.patch( # test-quality-ok: new_user is imported inside the helper, not injectable + "litellm.proxy.management_endpoints.internal_user_endpoints.new_user", + AsyncMock(return_value=NewUserResponse(user_id="group-user", key="k")), + ) + + created = await _create_user_if_not_exists(user_id="group-user") + + assert created is not None + assert new_user_mock.call_args.kwargs["data"].teams is None + + @pytest.mark.asyncio async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeypatch): """ From 0429339204ac41f2c0420d1693f78155244b6205 Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 3 Sep 2026 20:16:09 +0000 Subject: [PATCH 129/132] fix(scim): pass proxy admin auth to new_user so default team add succeeds Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/management_endpoints/scim/scim_v2.py | 6 +++++- .../management_endpoints/scim/test_scim_v2_endpoints.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index 0f0124ee9d9..98770cf9c2b 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -1128,7 +1128,10 @@ async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_grou user_role=default_role, ) - created_user: Final = await new_user(data=new_user_request) + created_user: Final = await new_user( + data=new_user_request, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) verbose_proxy_logger.info("Created user %s via %s", user_id, created_via) return created_user @@ -1716,6 +1719,7 @@ async def create_user( created_user: Final = await new_user( data=new_user_request, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), ) scim_user: Final = await ScimTransformations.transform_litellm_user_to_scim_user(user=created_user) diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index f4627f82506..88f67cfe0e4 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -19,6 +19,7 @@ from litellm.proxy._types import ( NewUserResponse, ProxyErrorTypes, ProxyException, + UserAPIKeyAuth, ) from litellm.proxy.management_endpoints.scim.scim_v2 import ( SCIMRosterSyncError, @@ -342,6 +343,7 @@ async def test_create_user_without_groups_defers_to_default_team(mocker: MockerF await create_user(user=scim_user) assert new_user_mock.call_args.kwargs["data"].teams is None + assert new_user_mock.call_args.kwargs["user_api_key_dict"] == UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) @pytest.mark.asyncio @@ -380,6 +382,7 @@ async def test_create_user_if_not_exists_defers_to_default_team(mocker: MockerFi assert created is not None assert new_user_mock.call_args.kwargs["data"].teams is None + assert new_user_mock.call_args.kwargs["user_api_key_dict"] == UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) @pytest.mark.asyncio From 07dd8a7e47957a020841439da35da1d287998e08 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 3 Sep 2026 15:46:44 -0700 Subject: [PATCH 130/132] fix(scim): keep team memberships when PUT /Users carries no groups Okta sends profile updates as full PUTs with no groups or groups: [], since SCIM User.groups is readOnly and membership is synced through /Groups. The PUT handler diffed that empty list against the stored teams, removed the user from every team (which also deletes their team keys) and recomputed the role from an empty group list. Treat an empty groups list on PUT as unspecified: keep the stored teams and leave the role alone. Explicit non-empty groups still replace memberships as before Claude-Session: https://claude.ai/code/session_01CqwUV4Ywnu5aUjXx1UhJrM --- .../management_endpoints/scim/scim_v2.py | 11 ++-- .../scim/test_scim_v2_endpoints.py | 61 +++++++++++++++++++ type-discipline-budget.json | 2 +- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index 98770cf9c2b..ceb67e3eee8 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -1774,22 +1774,25 @@ async def update_user( roles=user_data["roles"], ) + # SCIM User.groups is readOnly (RFC 7643 4.1.2): IdPs sync membership via /Groups and send + # no groups or `groups: []` on profile PUTs, so empty means unspecified, not "remove from every team" + target_teams: Final = user_data["teams"] or existing_user.teams await _handle_team_membership_changes( user_id=user_id, - existing_teams=existing_user.teams or [], - new_teams=user_data["teams"], + existing_teams=existing_user.teams, + new_teams=target_teams, ) update_data: Final = { "user_email": user_data["user_email"], "user_alias": user_data["user_alias"], "sso_user_id": user_data["sso_user_id"], - "teams": user_data["teams"], + "teams": target_teams, "metadata": safe_dumps(metadata), } admin_group: Final = await _get_scim_admin_group() - if admin_group is not None: + if admin_group is not None and user_data["teams"]: update_data["user_role"] = _resolve_scim_user_role( user.groups or [], admin_group, _default_scim_user_role() ) diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index 88f67cfe0e4..60f9a1a55e2 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -1257,6 +1257,67 @@ async def test_update_user_success(mocker): assert call_args[1]["data"]["teams"] == ["new-team"] +@pytest.mark.asyncio +@pytest.mark.parametrize("groups", [None, []], ids=["groups-omitted", "groups-empty"]) +async def test_update_user_without_groups_preserves_memberships_and_role(mocker, monkeypatch, groups): + """Okta profile PUTs carry no `groups` or `groups: []`; neither may drop teams (and their keys) or recompute role""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": "litellm-admins"}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + existing_user = mocker.MagicMock() + existing_user.teams = ["litellm-admins", "engineering"] + existing_user.metadata = {} + + scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="okta-user", + name=SCIMUserName(familyName="Renamed", givenName="Okta"), + emails=[SCIMUserEmail(value="okta@example.com")], + **({} if groups is None else {"groups": groups}), + ) + response_scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + id="okta-user", + userName="okta-user", + emails=[SCIMUserEmail(value="okta@example.com")], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.update = AsyncMock(return_value={"user_id": "okta-user"}) + + mocker.patch( # test-quality-ok: update_user's collaborators are module-level, not injectable + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( # test-quality-ok: update_user's collaborators are module-level, not injectable + "litellm.proxy.management_endpoints.scim.scim_v2._check_user_exists", + AsyncMock(return_value=existing_user), + ) + mocker.patch( # test-quality-ok: update_user's collaborators are module-level, not injectable + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=response_scim_user), + ) + patch_membership = mocker.patch( # test-quality-ok: roster writes are module-level, not injectable + "litellm.proxy.management_endpoints.scim.scim_v2.patch_team_membership", + AsyncMock(), + ) + + result = await update_user(user_id="okta-user", user=scim_user) + + assert result == response_scim_user + patch_membership.assert_not_awaited() + update_data = mock_prisma_client.db.litellm_usertable.update.call_args.kwargs["data"] + assert update_data["teams"] == ["litellm-admins", "engineering"] + assert "user_role" not in update_data + + @pytest.mark.asyncio async def test_update_user_not_found(mocker): """Should raise 404 when user doesn't exist""" diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 2f85128b4b6..78090779109 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 22328 }, "LIT002": { - "limit": 26760 + "limit": 26758 }, "LIT003": { "limit": 261 From 0b7773dd44aaf5c7da2e591e995ff3a41688ba0b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:38:48 -0700 Subject: [PATCH 131/132] fix(router): count tools and Anthropic system prompt in context-window pre-call check (#39663) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../messages/utils.py | 11 ++ litellm/router.py | 36 +++- tests/test_litellm/test_router.py | 160 +++++++++++++++++- 3 files changed, 198 insertions(+), 9 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py index 9deff950724..242300c7b6d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py @@ -6,6 +6,7 @@ from litellm.types.llms.anthropic import AnthropicMessagesRequestOptionalParams from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) +from litellm.types.llms.openai import ChatCompletionSystemMessage if TYPE_CHECKING: from litellm.exceptions import ContentPolicyViolationError @@ -36,6 +37,16 @@ def safeguard_refusal_error(model: str, stop_details: Mapping[str, object]) -> " ) +def anthropic_system_to_openai_message(system: object) -> ChatCompletionSystemMessage | None: + """ + Return the Anthropic Messages top-level ``system`` (a string or a list of text + blocks) as an OpenAI-style system message, or None when the request has none. + """ + if not isinstance(system, (str, list)) or not system: + return None + return ChatCompletionSystemMessage(role="system", content=system) + + @lru_cache(maxsize=1) def _anthropic_messages_optional_param_keys() -> frozenset[str]: """ diff --git a/litellm/router.py b/litellm/router.py index f33dfbba7bf..dea9aa62729 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -197,6 +197,7 @@ from litellm.router_utils.router_callbacks.track_deployment_metrics import ( from litellm.scheduler import FlowItem, Scheduler from litellm.types.llms.openai import ( AllMessageValues, + ChatCompletionToolParam, FileTypes, OpenAIFileObject, OpenAIFilesPurpose, @@ -11762,7 +11763,7 @@ class Router: self, messages: list[dict[str, str]] | None, input: str | list | None, - instructions: str | None = None, + request_kwargs: Mapping[str, object] | None = None, ) -> int: """ Count input tokens for context-window pre-call checks. @@ -11772,9 +11773,28 @@ class Router: The Responses payload is normalized to chat messages via the shared LiteLLMCompletionResponsesConfig transform so the same token_counter path covers both API surfaces and `instructions` tokens are included in the count. + + Prompt content the message list never carries is read from `request_kwargs`: + `tools` (Chat Completions, Responses and Anthropic Messages shapes) and the + Anthropic Messages top-level `system` block. """ + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + anthropic_system_to_openai_message, + ) + + extras: Final = request_kwargs if request_kwargs is not None else MappingProxyType({}) + raw_instructions: Final = extras.get("instructions") + instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None + raw_tools: Final = extras.get("tools") + tools: Final = ( + cast(list[ChatCompletionToolParam], raw_tools) # cast-ok: token_counter formats any tool dict shape + if isinstance(raw_tools, list) and raw_tools + else None + ) + system_message: Final = anthropic_system_to_openai_message(extras.get("system")) if messages is not None: - return litellm.token_counter(messages=messages) + counted_messages: Final = (system_message, *messages) if system_message is not None else messages + return litellm.token_counter(messages=counted_messages, tools=tools) if input is not None: from openai.types.responses.response_create_params import ResponseInputParam @@ -11787,7 +11807,10 @@ class Router: input=typed_input, responses_api_request={"instructions": instructions} if instructions is not None else {}, ) - return litellm.token_counter(messages=cast(list, input_messages)) # cast-ok: transformed chat messages + return litellm.token_counter( + messages=cast(list, input_messages), # cast-ok: transformed chat messages + tools=tools, + ) raise ValueError("Either messages or input must be provided to count tokens") def _deployment_max_input_tokens(self, model: str, deployment: Mapping[str, object]) -> int | None: @@ -11833,14 +11856,13 @@ class Router: """ if messages is None and input is None: return None - raw_instructions: Final = request_kwargs.get("instructions") if request_kwargs else None try: if not self._pre_call_checks_need_token_count(model, healthy_deployments): return None return await asyncify(self._count_pre_call_check_tokens)( messages=cast(list[dict[str, str]] | None, messages), # cast-ok: forwarded to the sync counter input=cast(str | list | None, input), # cast-ok: forwarded to the sync counter - instructions=raw_instructions if isinstance(raw_instructions, str) else None, + request_kwargs=request_kwargs, ) except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request verbose_router_logger.error( @@ -11887,8 +11909,6 @@ class Router: _rate_limit_error = False parent_otel_span: Final = _get_parent_otel_span_from_kwargs(request_kwargs) - raw_instructions: Final = request_kwargs.get("instructions") if request_kwargs else None - instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None has_countable_input: Final = messages is not None or input is not None ## get model group RPM ## @@ -11919,7 +11939,7 @@ class Router: return _returned_deployments try: input_tokens = self._count_pre_call_check_tokens( - messages=messages, input=input, instructions=instructions + messages=messages, input=input, request_kwargs=request_kwargs ) except Exception as e: verbose_router_logger.error( diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 228588d974f..f7f0d79b4fd 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -3855,7 +3855,7 @@ def test_pre_call_checks_counts_responses_instructions_tokens(monkeypatch): input_only_tokens = router._count_pre_call_check_tokens(messages=None, input=short_input) with_instructions_tokens = router._count_pre_call_check_tokens( - messages=None, input=short_input, instructions=long_instructions + messages=None, input=short_input, request_kwargs={"instructions": long_instructions} ) assert with_instructions_tokens > input_only_tokens @@ -3871,6 +3871,164 @@ def test_pre_call_checks_counts_responses_instructions_tokens(monkeypatch): ) +_OVERSIZED_TOOL_DESCRIPTION = "look up the answer in the knowledge base. " * 40 + + +@pytest.mark.parametrize( + "prompt_kwargs, tool", + [ + pytest.param( + {"messages": [{"role": "user", "content": "hi"}]}, + { + "type": "function", + "function": { + "name": "lookup", + "description": _OVERSIZED_TOOL_DESCRIPTION, + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, + }, + }, + id="chat_completions_tool", + ), + pytest.param( + {"input": "hi"}, + { + "type": "function", + "name": "lookup", + "description": _OVERSIZED_TOOL_DESCRIPTION, + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, + }, + id="responses_tool", + ), + pytest.param( + {"messages": [{"role": "user", "content": "hi"}]}, + { + "name": "lookup", + "description": _OVERSIZED_TOOL_DESCRIPTION, + "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}}, + }, + id="anthropic_messages_tool", + ), + ], +) +def test_pre_call_checks_counts_tool_definition_tokens(monkeypatch, prompt_kwargs, tool): + """ + Tool definitions are sent to the model as prompt tokens but never appear in + `messages` or `input`. A request whose prompt alone fits the context window but + whose prompt plus `tools` exceeds it must be rejected before dispatch, for the + Chat Completions, Responses and Anthropic Messages tool shapes alike. + """ + router = litellm.Router( + model_list=[ + {"model_name": "m", "litellm_params": {"model": "gpt-3.5-turbo"}}, + ], + enable_pre_call_checks=True, + ) + deployments = [ + {"litellm_params": {"model": "gpt-3.5-turbo"}, "model_info": {"id": "d1"}}, + ] + + prompt_only_tokens = router._count_pre_call_check_tokens( + messages=prompt_kwargs.get("messages"), input=prompt_kwargs.get("input") + ) + monkeypatch.setattr( + router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": prompt_only_tokens} + ) + + assert len(router._pre_call_checks(model="m", healthy_deployments=deployments, **prompt_kwargs)) == 1 + with pytest.raises(litellm.ContextWindowExceededError): + router._pre_call_checks( + model="m", + healthy_deployments=deployments, + request_kwargs={"tools": [tool]}, + **prompt_kwargs, + ) + + +@pytest.mark.parametrize( + "system", + [ + pytest.param("You are a meticulous assistant. " * 40, id="system_string"), + pytest.param( + [{"type": "text", "text": "You are a meticulous assistant. " * 40}], + id="system_blocks", + ), + ], +) +def test_pre_call_checks_counts_anthropic_system_tokens(monkeypatch, system): + """ + The Anthropic Messages API carries the system prompt as a top-level `system` field, + not as a message. Its tokens reach the model, so a request whose `messages` fit but + whose `messages` plus `system` exceed the context window must be rejected. + """ + router = litellm.Router( + model_list=[ + {"model_name": "m", "litellm_params": {"model": "gpt-3.5-turbo"}}, + ], + enable_pre_call_checks=True, + ) + deployments = [ + {"litellm_params": {"model": "gpt-3.5-turbo"}, "model_info": {"id": "d1"}}, + ] + messages = [{"role": "user", "content": "hi"}] + + messages_only_tokens = router._count_pre_call_check_tokens(messages=messages, input=None) + monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": messages_only_tokens}) + + assert len(router._pre_call_checks(model="m", healthy_deployments=deployments, messages=messages)) == 1 + with pytest.raises(litellm.ContextWindowExceededError): + router._pre_call_checks( + model="m", + healthy_deployments=deployments, + messages=messages, + request_kwargs={"system": system}, + ) + + +@pytest.mark.asyncio +async def test_aanthropic_messages_enforces_context_window_with_system_and_tools(): + """ + End-to-end router regression for /v1/messages: a request whose only oversized + content lives in the top-level `system` field or in `tools` must trip the pre-call + context-window check instead of being dispatched (the deployment uses mock_response, + so reaching the provider handler would return a response rather than raise). + """ + router = litellm.Router( + model_list=[ + { + "model_name": "small-ctx", + "litellm_params": {"model": "anthropic/claude-3-5-haiku-20241022", "mock_response": "hi"}, + "model_info": {"max_input_tokens": 20}, + } + ], + enable_pre_call_checks=True, + ) + messages = [{"role": "user", "content": "hi"}] + + response = await router.aanthropic_messages(model="small-ctx", messages=messages, max_tokens=5) + assert response is not None + + with pytest.raises(litellm.ContextWindowExceededError): + await router.aanthropic_messages( + model="small-ctx", + messages=messages, + max_tokens=5, + system="You are a meticulous assistant. " * 40, + ) + with pytest.raises(litellm.ContextWindowExceededError): + await router.aanthropic_messages( + model="small-ctx", + messages=messages, + max_tokens=5, + tools=[ + { + "name": "lookup", + "description": _OVERSIZED_TOOL_DESCRIPTION, + "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}}, + } + ], + ) + + def test_count_pre_call_check_tokens_across_api_surfaces(): """ _count_pre_call_check_tokens must count tokens from chat `messages`, a Responses From 39a17898ffdb55e4b54c0a7fe750b4d5afe49ea6 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 3 Sep 2026 19:45:09 -0400 Subject: [PATCH 132/132] test(proxy-extras): repoint the migrate-deploy harness at the run_prisma seam (#39673)