From 17ce2c4e9211a4caf27bfaf58781c9b5f32b633e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:57:15 -0700 Subject: [PATCH 01/11] fix(proxy): resolve named credentials on provider-only batch and files calls --- litellm/proxy/batches_endpoints/endpoints.py | 25 ++ .../openai_files_endpoints/common_utils.py | 24 ++ .../openai_files_endpoints/files_endpoints.py | 36 ++- .../proxy/batches_endpoints/test_endpoints.py | 153 ++++++++++- .../test_files_endpoint.py | 257 ++++++++++++++++++ 5 files changed, 488 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index a2dc5e1caf5..a91b29002e3 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -23,6 +23,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, + apply_team_provider_credentials, decode_model_from_file_id, encode_batch_response_ids, encode_file_id_with_model, @@ -295,6 +296,12 @@ async def create_batch( verbose_proxy_logger.debug(f"Created batch using model: {model_param}") else: # SCENARIO 3: Fallback to custom_llm_provider (uses env variables) + apply_team_provider_credentials( + data=cast(dict, _create_batch_data), # cast-ok: TypedDict is a dict at runtime + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + custom_llm_provider=custom_llm_provider, + ) response = await litellm.acreate_batch( custom_llm_provider=custom_llm_provider, **_create_batch_data, # type: ignore @@ -525,6 +532,12 @@ async def retrieve_batch( or get_custom_llm_provider_from_request_query(request=request) or "openai" ) + apply_team_provider_credentials( + data=data, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + custom_llm_provider=custom_llm_provider, + ) response = await litellm.aretrieve_batch( custom_llm_provider=custom_llm_provider, **data, # type: ignore @@ -718,6 +731,12 @@ async def list_batches( or get_custom_llm_provider_from_request_query(request=request) or "openai" ) + apply_team_provider_credentials( + data=data, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + custom_llm_provider=custom_llm_provider, + ) response = await litellm.alist_batches( custom_llm_provider=custom_llm_provider, # type: ignore after=after, @@ -908,6 +927,12 @@ async def cancel_batch( # Extract batch_id from data to avoid "multiple values for keyword argument" error # data was cast from CancelBatchRequest which already contains batch_id data.pop("batch_id", None) + apply_team_provider_credentials( + data=data, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + custom_llm_provider=custom_llm_provider, + ) _cancel_batch_data = CancelBatchRequest(batch_id=batch_id, **data) response = await litellm.acancel_batch( custom_llm_provider=custom_llm_provider, # type: ignore diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index efd1d6b6cee..d4ac45559ee 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -14,6 +14,7 @@ from litellm.types.utils import SpecialEnums if TYPE_CHECKING: from fastapi import Request + from litellm.proxy._types import UserAPIKeyAuth from litellm.router import Router @@ -373,6 +374,29 @@ def get_team_provider_credentials( return None +def apply_team_provider_credentials( + data: dict, # mutable-ok: credentials are merged into the request payload in place, same contract as prepare_data_with_credentials + llm_router: Optional["Router"], + user_api_key_dict: "UserAPIKeyAuth", + custom_llm_provider: str, +) -> None: + """ + Resolve credentials for a provider-only request (no model pinned) via + ``get_team_provider_credentials`` and merge them into ``data`` in-place. + Leaves ``data`` untouched when no authorized deployment matches, so the + caller falls back to environment-variable credentials exactly as before. + """ + credentials = get_team_provider_credentials( + llm_router=llm_router, + team_models=user_api_key_dict.team_models or [], + custom_llm_provider=custom_llm_provider, + team_id=user_api_key_dict.team_id, + ) + if credentials is None: + return + prepare_data_with_credentials(data=data, credentials=credentials) + + def prepare_data_with_credentials( data: dict, credentials: dict, diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 0c9aa667751..f1bcfbafe58 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -43,10 +43,10 @@ from litellm.litellm_core_utils.cloud_storage_security import ( ) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, + apply_team_provider_credentials, encode_file_id_with_model, extract_file_creation_params, get_credentials_for_model, - get_team_provider_credentials, handle_model_based_routing, prepare_data_with_credentials, validate_managed_files_requirement, @@ -253,6 +253,12 @@ async def route_create_file( _create_file_request=_create_file_request, ) else: + apply_team_provider_credentials( + data=cast(dict, _create_file_request), # cast-ok: TypedDict is a plain dict at runtime; merged in place + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + custom_llm_provider=custom_llm_provider, + ) # get configs for custom_llm_provider llm_provider_config = get_files_provider_config(custom_llm_provider=custom_llm_provider) if llm_provider_config is not None: @@ -735,6 +741,14 @@ async def get_file_content( check_file_id_encoding=True, ) + if not should_route: + apply_team_provider_credentials( + data=data, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + custom_llm_provider=custom_llm_provider, + ) + from litellm.proxy.openai_files_endpoints.file_content_streaming_handler import ( FileContentStreamingHandler, ) @@ -983,6 +997,12 @@ async def get_file( # Remove file_id from data to avoid "multiple values for keyword argument" error # data was initialized with {"file_id": file_id} data.pop("file_id", None) + apply_team_provider_credentials( + data=data, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + custom_llm_provider=custom_llm_provider, + ) response = await litellm.afile_retrieve( custom_llm_provider=custom_llm_provider, file_id=file_id, @@ -1183,6 +1203,12 @@ async def delete_file( ) else: data.pop("file_id", None) + apply_team_provider_credentials( + data=data, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + custom_llm_provider=custom_llm_provider, + ) response = await litellm.afile_delete( custom_llm_provider=custom_llm_provider, file_id=file_id, @@ -1354,14 +1380,12 @@ async def list_files( # No model/target_model_names pinned: resolve upstream credentials from # the team's deployment for this provider so the call is authenticated # against the team's own account (e.g. the team's openai deployment). - team_credentials = get_team_provider_credentials( + apply_team_provider_credentials( + data=data, llm_router=llm_router, - team_models=user_api_key_dict.team_models or [], + user_api_key_dict=user_api_key_dict, custom_llm_provider=custom_llm_provider, - team_id=user_api_key_dict.team_id, ) - if team_credentials is not None: - prepare_data_with_credentials(data=data, credentials=team_credentials) response = await litellm.afile_list( custom_llm_provider=custom_llm_provider, diff --git a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py index 9573fddd435..6a185988c9b 100644 --- a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py @@ -51,7 +51,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.types.llms.openai import BatchJobStatus -from litellm.types.utils import LiteLLMBatch +from litellm.types.utils import CredentialItem, LiteLLMBatch from fastapi import Response @@ -2091,3 +2091,154 @@ async def test_retrieve__unified_no_router_500(retrieve_harness): assert exc.value.code == "500" retrieve_harness.router_aretrieve.assert_not_called() retrieve_harness.litellm_aretrieve.assert_not_called() + + +# =========================================================================== # +# SCENARIO 3 + configured deployments: a provider-only call (custom-llm-provider +# header, no model anywhere) must resolve the gateway/team deployment's named +# credential for that provider and attach it to the provider call kwargs, +# instead of silently falling through to the host environment's default +# credentials (regression: vertex batch jobs landing in the hosting env's GCP +# project because litellm_credential_name never reached the call). +# =========================================================================== # + +VERTEX_NAMED_CREDENTIAL = CredentialItem( + credential_name="vertex-named-cred", + credential_info={}, + credential_values={ + "vertex_project": "customer-project", + "vertex_location": "us-central1", + "vertex_credentials": "/creds/customer-sa.json", + }, +) + + +def vertex_named_credential_router() -> Router: + return Router( + model_list=[ + { + "model_name": "gemini-2.5-pro", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-pro", + "litellm_credential_name": "vertex-named-cred", + }, + } + ] + ) + + +@pytest.mark.asyncio +async def test_create__provider_only_resolves_named_vertex_credentials(harness): + """Provider-only create must attach the configured named credential, and must + NOT turn the call into a model-routed one (no model kwarg injected).""" + set_body( + harness, + { + "input_file_id": "file-plain", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + ) + harness.provider_from_headers.return_value = "vertex_ai" + + with patch.object(litellm, "credential_list", [VERTEX_NAMED_CREDENTIAL]): + with patch.object(proxy_server, "llm_router", vertex_named_credential_router()): + await call_create(harness) + + assert harness.acreate_kwargs() == { + "custom_llm_provider": "vertex_ai", + "input_file_id": "file-plain", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "metadata": None, + "vertex_project": "customer-project", + "vertex_location": "us-central1", + "vertex_credentials": "/creds/customer-sa.json", + } + + +@pytest.mark.asyncio +async def test_create__provider_only_ignores_other_provider_deployments(harness): + """A provider-only vertex call must not pick up credentials from deployments + of a different provider; with no vertex deployment the payload is exactly the + pre-fix env-var fallback.""" + set_body( + harness, + { + "input_file_id": "file-plain", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + ) + harness.provider_from_headers.return_value = "vertex_ai" + openai_only_router = Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-global-openai"}, + } + ] + ) + + with patch.object(proxy_server, "llm_router", openai_only_router): + await call_create(harness) + + assert harness.acreate_kwargs() == { + "custom_llm_provider": "vertex_ai", + "input_file_id": "file-plain", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "metadata": None, + } + + +@pytest.mark.asyncio +async def test_retrieve__provider_only_resolves_named_vertex_credentials(retrieve_harness): + retrieve_harness.provider_from_headers.return_value = "vertex_ai" + + with patch.object(litellm, "credential_list", [VERTEX_NAMED_CREDENTIAL]): + with patch.object(proxy_server, "llm_router", vertex_named_credential_router()): + await call_retrieve(retrieve_harness, "batch-raw-xyz") + + assert retrieve_harness.aretrieve_kwargs() == { + "custom_llm_provider": "vertex_ai", + "batch_id": "batch-raw-xyz", + "vertex_project": "customer-project", + "vertex_location": "us-central1", + "vertex_credentials": "/creds/customer-sa.json", + } + + +@pytest.mark.asyncio +async def test_list__provider_only_resolves_named_vertex_credentials(list_harness): + list_harness.provider_from_headers.return_value = "vertex_ai" + + with patch.object(litellm, "credential_list", [VERTEX_NAMED_CREDENTIAL]): + with patch.object(proxy_server, "llm_router", vertex_named_credential_router()): + await call_list(list_harness) + + assert list_harness.alist_kwargs() == { + "custom_llm_provider": "vertex_ai", + "after": None, + "limit": None, + "vertex_project": "customer-project", + "vertex_location": "us-central1", + "vertex_credentials": "/creds/customer-sa.json", + } + + +@pytest.mark.asyncio +async def test_cancel__provider_only_resolves_named_vertex_credentials(cancel_harness): + cancel_harness.provider_from_headers.return_value = "vertex_ai" + + with patch.object(litellm, "credential_list", [VERTEX_NAMED_CREDENTIAL]): + with patch.object(proxy_server, "llm_router", vertex_named_credential_router()): + await call_cancel(cancel_harness, "batch-raw-xyz") + + assert cancel_harness.acancel_kwargs() == { + "custom_llm_provider": "vertex_ai", + "batch_id": "batch-raw-xyz", + "vertex_project": "customer-project", + "vertex_location": "us-central1", + "vertex_credentials": "/creds/customer-sa.json", + } diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index 46ecb31e1c8..23cd1c71cbe 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -2610,3 +2610,260 @@ def test_list_files_with_all_proxy_models_team_uses_openai_deployment( assert captured_kwargs.get("api_key") == "team-openai-key" assert captured_kwargs.get("custom_llm_provider") == "openai" proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def _setup_vertex_named_credential_router(monkeypatch) -> Router: + from litellm.types.utils import CredentialItem + + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="vertex-named-cred", + credential_info={}, + credential_values={ + "vertex_project": "customer-project", + "vertex_location": "us-central1", + "vertex_credentials": "/creds/customer-sa.json", + }, + ) + ], + ) + return Router( + model_list=[ + { + "model_name": "gemini-2.5-pro", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-pro", + "litellm_credential_name": "vertex-named-cred", + }, + } + ] + ) + + +def _assert_vertex_named_credentials_attached(captured_kwargs: dict) -> None: + assert captured_kwargs.get("custom_llm_provider") == "vertex_ai" + assert captured_kwargs.get("vertex_project") == "customer-project" + assert captured_kwargs.get("vertex_location") == "us-central1" + assert captured_kwargs.get("vertex_credentials") == "/creds/customer-sa.json" + assert captured_kwargs.get("model") is None + + +def test_create_file_provider_only_resolves_named_vertex_credentials( + mocker: MockerFixture, monkeypatch +): + """ + POST /v1/files with only a custom-llm-provider header (no model, no + target_model_names) must attach the configured named vertex credential to + the upstream call instead of falling through to google.auth.default(), + which uploads into the hosting environment's GCP project. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + router = _setup_vertex_named_credential_router(monkeypatch) + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_acreate_file(**kwargs): + captured_kwargs.update(kwargs) + return OpenAIFileObject( + id="file-vertex-123", + object="file", + bytes=2, + created_at=1234567890, + filename="batch.jsonl", + purpose="batch", + status="uploaded", + ) + + monkeypatch.setattr(litellm, "acreate_file", _mock_acreate_file) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="test-user", + ) + + try: + response = client.post( + "/v1/files", + files={"file": ("batch.jsonl", b"{}", "application/jsonl")}, + data={"purpose": "batch"}, + headers={ + "Authorization": "Bearer test-key", + "custom-llm-provider": "vertex_ai", + }, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + _assert_vertex_named_credentials_attached(captured_kwargs) + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_get_file_provider_only_resolves_named_vertex_credentials( + mocker: MockerFixture, monkeypatch +): + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + router = _setup_vertex_named_credential_router(monkeypatch) + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_retrieve(**kwargs): + captured_kwargs.update(kwargs) + return OpenAIFileObject( + id="file-abc123", + object="file", + bytes=2, + created_at=1234567890, + filename="batch.jsonl", + purpose="batch", + status="uploaded", + ) + + monkeypatch.setattr(litellm, "afile_retrieve", _mock_afile_retrieve) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="test-user", + ) + + try: + response = client.get( + "/v1/files/file-abc123", + headers={ + "Authorization": "Bearer test-key", + "custom-llm-provider": "vertex_ai", + }, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("file_id") == "file-abc123" + _assert_vertex_named_credentials_attached(captured_kwargs) + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_get_file_content_provider_only_resolves_named_vertex_credentials( + mocker: MockerFixture, monkeypatch +): + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + router = _setup_vertex_named_credential_router(monkeypatch) + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_content(**kwargs): + captured_kwargs.update(kwargs) + return HttpxBinaryResponseContent( + response=httpx.Response( + status_code=200, + content=b"vertex-bytes", + headers={"content-type": "application/octet-stream"}, + ) + ) + + monkeypatch.setattr(litellm, "afile_content", _mock_afile_content) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="test-user", + ) + + try: + response = client.get( + "/v1/files/file-abc123/content", + headers={ + "Authorization": "Bearer test-key", + "custom-llm-provider": "vertex_ai", + }, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert response.content == b"vertex-bytes" + assert captured_kwargs.get("file_id") == "file-abc123" + _assert_vertex_named_credentials_attached(captured_kwargs) + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_delete_file_provider_only_resolves_named_vertex_credentials( + mocker: MockerFixture, monkeypatch +): + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + router = _setup_vertex_named_credential_router(monkeypatch) + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_delete(**kwargs): + captured_kwargs.update(kwargs) + return OpenAIFileObject( + id="file-abc123", + object="file", + bytes=2, + created_at=1234567890, + filename="batch.jsonl", + purpose="batch", + status="uploaded", + ) + + monkeypatch.setattr(litellm, "afile_delete", _mock_afile_delete) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="test-user", + ) + + try: + response = client.delete( + "/v1/files/file-abc123", + headers={ + "Authorization": "Bearer test-key", + "custom-llm-provider": "vertex_ai", + }, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("file_id") == "file-abc123" + _assert_vertex_named_credentials_attached(captured_kwargs) + proxy_logging_obj.post_call_failure_hook.assert_not_called() From 6d607ca3c228060aa9b706a46e5b00a487798047 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:47:40 -0700 Subject: [PATCH 02/11] fix(router): never resolve another team's deployment credentials for shared model names --- .../openai_files_endpoints/common_utils.py | 2 +- litellm/router.py | 42 ++++++- .../test_files_endpoint.py | 84 +++++++++++++ tests/test_litellm/test_router.py | 119 ++++++++++++++++++ 4 files changed, 243 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index d4ac45559ee..dca4b2a3773 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -319,7 +319,7 @@ def get_team_provider_credentials( return None def _provider_credentials(model_id: str) -> Optional[dict]: - credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id) + credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id, team_id=team_id) if credentials is not None and credentials.get("custom_llm_provider") == custom_llm_provider: return credentials return None diff --git a/litellm/router.py b/litellm/router.py index 487d6a31226..38450fe4ef1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -30,6 +30,7 @@ from typing import ( Generator, List, Literal, + Mapping, Optional, Set, Tuple, @@ -8630,6 +8631,33 @@ class Router: raise Exception("Model Name invalid - {}".format(type(model))) return None + @staticmethod + def _deployment_usable_by_team(model: Union[Mapping, Deployment], team_id: str | None) -> bool: + """ + A team-scoped deployment (``model_info.team_id`` set) is only usable by + callers from that same team; deployments without a team owner are shared. + """ + model_info = model.get("model_info") if isinstance(model, dict) else model.model_info + owner_team_id = model_info.get("team_id") if model_info is not None else None + return owner_team_id is None or owner_team_id == team_id + + def _get_model_group_deployment_usable_by_team( + self, model_group_name: str, team_id: str | None + ) -> Deployment | None: + """ + Like ``get_deployment_by_model_group_name``, but skips deployments owned + by other teams so a shared model name never resolves another team's + credentials. + """ + indices = self.model_name_to_deployment_indices.get(model_group_name) or () + usable = ( + self.model_list[idx] for idx in indices if self._deployment_usable_by_team(self.model_list[idx], team_id) + ) + first_usable = next(usable, None) + if first_usable is None: + return None + return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable + def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ Return (max_input_tokens, max_output_tokens) explicitly configured in a concrete @@ -8664,7 +8692,10 @@ class Router: model_id: Model ID or model name from model_list (e.g., "gpt-4o-litellm") team_id: Optional team id of the caller. When set, team-scoped deployments (indexed by team public model name, including team - wildcard models like "openai/*") are also considered. + wildcard models like "openai/*") are also considered. Name and + wildcard lookups never resolve a deployment owned by a + different team, so shared model names can't leak another + team's credentials. Returns: Dictionary containing api_key, api_base, custom_llm_provider, etc. @@ -8681,7 +8712,7 @@ class Router: # If not found, try by model_group_name if deployment is None: - deployment = self.get_deployment_by_model_group_name(model_group_name=model_id) + deployment = self._get_model_group_deployment_usable_by_team(model_group_name=model_id, team_id=team_id) # If not found, check team-scoped deployments whose team public model # name exactly matches model_id (wildcard team names are matched via @@ -8698,7 +8729,12 @@ class Router: if deployment is None: team_pattern_router = self.team_pattern_routers.get(team_id) if team_id is not None else None team_wildcard_models = (team_pattern_router.route(model_id) or []) if team_pattern_router else [] - potential_wildcard_models = team_wildcard_models or self.pattern_router.route(model_id) or [] + global_wildcard_models = [ + wildcard_model + for wildcard_model in (self.pattern_router.route(model_id) or []) + if self._deployment_usable_by_team(wildcard_model, team_id) + ] + potential_wildcard_models = team_wildcard_models or global_wildcard_models if potential_wildcard_models: # Use the first matching wildcard deployment deployment_dict = potential_wildcard_models[0] diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index 23cd1c71cbe..17cd3f6172a 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -2867,3 +2867,87 @@ def test_delete_file_provider_only_resolves_named_vertex_credentials( assert captured_kwargs.get("file_id") == "file-abc123" _assert_vertex_named_credentials_attached(captured_kwargs) proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_create_file_provider_only_skips_other_team_vertex_deployment( + mocker: MockerFixture, monkeypatch +): + """ + Regression: with a team-scoped vertex deployment indexed before a global + one under the same model name, a provider-only upload from a different + team must use the global deployment's credentials, never the other + team's. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + router = Router( + model_list=[ + { + "model_name": "gemini-2.5-pro", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-pro", + "vertex_project": "team-b-project", + }, + "model_info": { + "id": "team-b-vertex", + "team_id": "team-b", + "team_public_model_name": "gemini-2.5-pro", + }, + }, + { + "model_name": "gemini-2.5-pro", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-pro", + "vertex_project": "shared-project", + }, + }, + ] + ) + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_acreate_file(**kwargs): + captured_kwargs.update(kwargs) + return OpenAIFileObject( + id="file-vertex-456", + object="file", + bytes=2, + created_at=1234567890, + filename="batch.jsonl", + purpose="batch", + status="uploaded", + ) + + monkeypatch.setattr(litellm, "acreate_file", _mock_acreate_file) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="team-a", + team_models=["gemini-2.5-pro"], + ) + + try: + response = client.post( + "/v1/files", + files={"file": ("batch.jsonl", b"{}", "application/jsonl")}, + data={"purpose": "batch"}, + headers={ + "Authorization": "Bearer test-key", + "custom-llm-provider": "vertex_ai", + }, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("vertex_project") == "shared-project" + proxy_logging_obj.post_call_failure_hook.assert_not_called() diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index ad4e430c603..8dcefc0405a 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -3755,6 +3755,125 @@ def test_get_deployment_credentials_with_provider_team_wildcard_priority(): assert global_credentials["api_key"] == "global-key" +def test_get_deployment_credentials_with_provider_skips_other_team_deployment(): + """ + Regression: a team-scoped deployment sharing a model_name with a global + deployment must never resolve for another team's (or an unscoped) caller, + even when it is indexed first; the shared global deployment wins instead. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "gemini-2.5-pro", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-pro", + "vertex_project": "team-b-project", + }, + "model_info": { + "id": "team-b-vertex", + "team_id": "team-b", + "team_public_model_name": "gemini-2.5-pro", + }, + }, + { + "model_name": "gemini-2.5-pro", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-pro", + "vertex_project": "shared-project", + }, + }, + ], + ) + + other_team_credentials = router.get_deployment_credentials_with_provider( + model_id="gemini-2.5-pro", team_id="team-a" + ) + assert other_team_credentials is not None + assert other_team_credentials["vertex_project"] == "shared-project" + + unscoped_credentials = router.get_deployment_credentials_with_provider( + model_id="gemini-2.5-pro" + ) + assert unscoped_credentials is not None + assert unscoped_credentials["vertex_project"] == "shared-project" + + owner_credentials = router.get_deployment_credentials_with_provider( + model_id="gemini-2.5-pro", team_id="team-b" + ) + assert owner_credentials is not None + assert owner_credentials["vertex_project"] == "team-b-project" + + +def test_get_deployment_credentials_with_provider_no_fallback_to_other_team_only_name(): + """ + When the only deployments under a model name belong to another team, other + callers must get None (env fallback) instead of that team's credentials. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "gemini-2.5-pro", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-pro", + "vertex_project": "team-b-project", + }, + "model_info": { + "id": "team-b-vertex", + "team_id": "team-b", + "team_public_model_name": "gemini-2.5-pro", + }, + }, + ], + ) + + assert ( + router.get_deployment_credentials_with_provider( + model_id="gemini-2.5-pro", team_id="team-a" + ) + is None + ) + assert ( + router.get_deployment_credentials_with_provider(model_id="gemini-2.5-pro") + is None + ) + + +def test_get_deployment_credentials_with_provider_skips_other_team_wildcard(): + """ + Global wildcard resolution must skip a team-scoped wildcard deployment for + callers outside that team, falling through to the shared wildcard entry. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "team-b-key"}, + "model_info": { + "id": "team-b-wildcard", + "team_id": "team-b", + "team_public_model_name": "openai/*", + }, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "global-key"}, + }, + ], + ) + + other_team_credentials = router.get_deployment_credentials_with_provider( + model_id="openai/gpt-5.2", team_id="team-a" + ) + assert other_team_credentials is not None + assert other_team_credentials["api_key"] == "global-key" + + owner_credentials = router.get_deployment_credentials_with_provider( + model_id="openai/gpt-5.2", team_id="team-b" + ) + assert owner_credentials is not None + assert owner_credentials["api_key"] == "team-b-key" + + def test_team_wildcard_credentials_not_usable_after_delete_deployment(): """ Regression: team_pattern_routers retained deleted deployments, so a team From 47a9fabb5a3dc3029ed504a959a635f12dea3b1c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:11:50 -0700 Subject: [PATCH 03/11] fix(proxy): honor key-level model allowlist in provider-only credential resolution --- .../openai_files_endpoints/common_utils.py | 80 ++++++++++---- .../test_files_endpoint.py | 100 ++++++++++++++++++ 2 files changed, 159 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index dca4b2a3773..b2e36188681 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -295,9 +295,8 @@ def get_credentials_for_model( def get_team_provider_credentials( llm_router: Optional["Router"], - team_models: List[str], + user_api_key_dict: "UserAPIKeyAuth", custom_llm_provider: str, - team_id: Optional[str] = None, ) -> Optional[dict]: """ Resolve upstream credentials for a provider-scoped file operation @@ -305,19 +304,59 @@ def get_team_provider_credentials( Priority: 1. The team's own (BYOK) deployment for this provider — a deployment whose - ``model_info.team_id`` matches ``team_id``. This keeps team-scoped listings - on the team's own provider account/key instead of a shared global one. - 2. Fallback: any deployment the team is granted access to for this provider, - expanding wildcard routes and the all-proxy-models sentinel. + ``model_info.team_id`` matches the caller's team. This keeps team-scoped + listings on the team's own provider account/key instead of a shared + global one. + 2. Fallback: any deployment the caller is granted access to for this + provider, expanding wildcard routes and the all-proxy-models sentinel. - Credential lookup is always scoped to the team's allowlist, so a team can - never resolve a provider key for a deployment it isn't authorized to use. + Credential lookup is scoped to both the team's allowlist and the key's own + model allowlist (``user_api_key_dict.models``), so neither a team nor a + restricted key within a team can resolve a provider key for a deployment + it isn't authorized to use. A key restricted to an explicit model list + only narrows the team scope; sentinel-bearing keys (all-proxy-models / + all-team-models) defer to the team scope instead of widening past it. Returns None when the router is unavailable or no authorized deployment matches, so the caller can fall back to default credential resolution. """ if llm_router is None: return None + from litellm.proxy._types import SpecialModelNames + from litellm.proxy.auth.model_checks import get_complete_model_list, get_key_models + + team_id = user_api_key_dict.team_id + team_models = user_api_key_dict.team_models or [] + + proxy_model_list = llm_router.get_model_names(team_id=team_id) + model_access_groups = llm_router.get_model_access_groups() + + raw_key_models = user_api_key_dict.models or [] + sentinel_values = { + SpecialModelNames.all_proxy_models.value, + SpecialModelNames.all_team_models.value, + } + key_is_restricted = bool(raw_key_models) and not (set(raw_key_models) & sentinel_values) + key_model_allowlist = ( + tuple( + dict.fromkeys( + get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + ) + ) + ) + if key_is_restricted + else () + ) + key_model_allowlist_set = frozenset(key_model_allowlist) + + def _key_may_use(public_model_name: Optional[str]) -> bool: + if not key_model_allowlist_set: + return True + return public_model_name is not None and public_model_name in key_model_allowlist_set + def _provider_credentials(model_id: str) -> Optional[dict]: credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id, team_id=team_id) if credentials is not None and credentials.get("custom_llm_provider") == custom_llm_provider: @@ -333,27 +372,27 @@ def get_team_provider_credentials( deployment_id = model_info.get("id") if deployment_id is None: continue + if not _key_may_use(model_info.get("team_public_model_name") or deployment.get("model_name")): + continue credentials = _provider_credentials(deployment_id) if credentials is not None: return credentials - # 2. Fall back to deployments the team is allowed to access. The - # all-proxy-models sentinel isn't expanded by get_complete_model_list, so - # normalize it to an empty allowlist, which defers to the team-scoped - # proxy model list. A team with a restricted allowlist (e.g. anthropic - # only) therefore never resolves another provider's key. - from litellm.proxy._types import SpecialModelNames - from litellm.proxy.auth.model_checks import get_complete_model_list - + # 2. Fall back to deployments the caller is allowed to access. The key's + # effective allowlist (sentinels and access groups already expanded by + # get_key_models) wins when set; otherwise the team's allowlist applies. + # The all-proxy-models sentinel isn't expanded by + # get_complete_model_list, so normalize it to an empty allowlist, which + # defers to the team-scoped proxy model list. A team or key with a + # restricted allowlist (e.g. anthropic only) therefore never resolves + # another provider's key. grants_all_models = SpecialModelNames.all_proxy_models.value in team_models effective_team_models = [] if grants_all_models else team_models - proxy_model_list = llm_router.get_model_names(team_id=team_id) - model_access_groups = llm_router.get_model_access_groups() models_to_try = list( dict.fromkeys( get_complete_model_list( - key_models=[], + key_models=list(key_model_allowlist), team_models=effective_team_models, proxy_model_list=proxy_model_list, user_model=None, @@ -388,9 +427,8 @@ def apply_team_provider_credentials( """ credentials = get_team_provider_credentials( llm_router=llm_router, - team_models=user_api_key_dict.team_models or [], + user_api_key_dict=user_api_key_dict, custom_llm_provider=custom_llm_provider, - team_id=user_api_key_dict.team_id, ) if credentials is None: return diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index 17cd3f6172a..ac01c6ae1d1 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -2951,3 +2951,103 @@ def test_create_file_provider_only_skips_other_team_vertex_deployment( assert response.status_code == 200, response.text assert captured_kwargs.get("vertex_project") == "shared-project" proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def _team_openai_plus_global_anthropic_router() -> Router: + return Router( + model_list=[ + { + "model_name": "team-gpt", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "team-openai-key", + }, + "model_info": { + "id": "team-a-openai", + "team_id": "team-a", + "team_public_model_name": "team-gpt", + }, + }, + { + "model_name": "claude-opus-4-6", + "litellm_params": { + "model": "anthropic/claude-opus-4-6", + "api_key": "anthropic-key", + }, + }, + ] + ) + + +def _list_files_captured_kwargs( + mocker: MockerFixture, monkeypatch, router: Router, key_models: list +) -> dict: + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="team-a", + team_models=["team-gpt", "claude-opus-4-6"], + models=key_models, + ) + + try: + response = client.get( + "/v1/files", + headers={ + "Authorization": "Bearer test-key", + "custom-llm-provider": "openai", + }, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + return captured_kwargs + + +def test_list_files_key_restricted_to_other_provider_does_not_leak_team_openai_credentials( + mocker: MockerFixture, monkeypatch +): + """ + Regression: a key restricted to an anthropic model on a team that also has + an openai deployment must not attach the team's openai credentials to a + provider-only openai files call; key-level model restrictions apply to + credential resolution, not just completions. + """ + captured_kwargs = _list_files_captured_kwargs( + mocker, monkeypatch, _team_openai_plus_global_anthropic_router(), ["claude-opus-4-6"] + ) + assert captured_kwargs.get("api_key") != "team-openai-key" + + +def test_list_files_key_allowed_openai_model_still_resolves_team_credentials( + mocker: MockerFixture, monkeypatch +): + """ + A key whose allowlist includes the team's openai model keeps resolving that + deployment's credentials for provider-only openai files calls. + """ + captured_kwargs = _list_files_captured_kwargs( + mocker, monkeypatch, _team_openai_plus_global_anthropic_router(), ["team-gpt"] + ) + assert captured_kwargs.get("api_key") == "team-openai-key" From 6e8655762c3f06e44e1b916f8f230fb47da75bf6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:12:57 -0700 Subject: [PATCH 04/11] test(router): directly cover team-ownership credential filter helpers --- tests/test_litellm/test_router.py | 57 +++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 8dcefc0405a..4df48a6dc56 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -3838,6 +3838,63 @@ def test_get_deployment_credentials_with_provider_no_fallback_to_other_team_only ) +def test_deployment_usable_by_team_helpers(): + """ + Direct coverage of the team-ownership filter: a team-scoped deployment is + usable only by its owning team, shared deployments by anyone, and the + model-group picker returns the first usable deployment or None. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "gemini-2.5-pro", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-pro", + "vertex_project": "team-b-project", + }, + "model_info": { + "id": "team-b-vertex", + "team_id": "team-b", + "team_public_model_name": "gemini-2.5-pro", + }, + }, + { + "model_name": "gemini-2.5-pro", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-pro", + "vertex_project": "shared-project", + }, + }, + ], + ) + + team_owned, shared = router.model_list + assert router._deployment_usable_by_team(team_owned, "team-b") is True + assert router._deployment_usable_by_team(team_owned, "team-a") is False + assert router._deployment_usable_by_team(team_owned, None) is False + assert router._deployment_usable_by_team(shared, "team-a") is True + assert router._deployment_usable_by_team(shared, None) is True + + picked = router._get_model_group_deployment_usable_by_team( + model_group_name="gemini-2.5-pro", team_id="team-a" + ) + assert picked is not None + assert picked.litellm_params.vertex_project == "shared-project" + + owner_picked = router._get_model_group_deployment_usable_by_team( + model_group_name="gemini-2.5-pro", team_id="team-b" + ) + assert owner_picked is not None + assert owner_picked.litellm_params.vertex_project == "team-b-project" + + assert ( + router._get_model_group_deployment_usable_by_team( + model_group_name="unknown-model", team_id="team-a" + ) + is None + ) + + def test_get_deployment_credentials_with_provider_skips_other_team_wildcard(): """ Global wildcard resolution must skip a team-scoped wildcard deployment for From bb769702b13c813eda119d33c8720d321026261b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 29 Jul 2026 11:35:06 -0700 Subject: [PATCH 05/11] feat(ui): deep link team detail page via ?team= query param (#35112) The teams page kept the selected team in React state, so a team detail page had no URL: it could not be shared, bookmarked, or opened from another page, and the browser back button dropped you out of the page instead of closing the detail view Adds useTeamDetailRouting reading ?team= (same pattern as the api-keys, models, and logs deep links) and derives the open team in Teams.tsx from the URL. TeamInfo now also derives team-admin rights from the fetched team data, so team admins arriving via a deep link are not stuck with a read-only view --- .../teams/detailNavigation.test.ts | 53 +++++++++++++++ .../app/(dashboard)/teams/detailNavigation.ts | 32 +++++++++ .../src/components/Teams.test.tsx | 67 +++++++++++++++++++ ui/litellm-dashboard/src/components/Teams.tsx | 11 +-- .../src/components/team/TeamInfo.test.tsx | 23 +++++++ .../src/components/team/TeamInfo.tsx | 10 ++- 6 files changed, 190 insertions(+), 6 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/teams/detailNavigation.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/teams/detailNavigation.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/detailNavigation.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/teams/detailNavigation.test.ts new file mode 100644 index 00000000000..e5d5b1a4073 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/teams/detailNavigation.test.ts @@ -0,0 +1,53 @@ +/* @vitest-environment jsdom */ +import { act, renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useTeamDetailRouting } from "./detailNavigation"; + +vi.mock("next/navigation", () => ({ useSearchParams: () => new URLSearchParams(window.location.search) })); + +describe("useTeamDetailRouting", () => { + beforeEach(() => { + window.history.pushState(null, "", "/teams/"); + }); + + it("openTeam sets ?team= via history.pushState (no full navigation)", () => { + const spy = vi.spyOn(window.history, "pushState"); + const { result } = renderHook(() => useTeamDetailRouting()); + act(() => result.current.openTeam("team-abc123")); + expect(spy).toHaveBeenCalledWith(null, "", expect.stringContaining("team=team-abc123")); + spy.mockRestore(); + }); + + it("openTeam preserves unrelated query params", () => { + window.history.pushState(null, "", "/teams/?foo=bar"); + const spy = vi.spyOn(window.history, "pushState"); + const { result } = renderHook(() => useTeamDetailRouting()); + act(() => result.current.openTeam("team-abc123")); + const url = spy.mock.calls.at(-1)?.[2] as string; + expect(url).toContain("foo=bar"); + expect(url).toContain("team=team-abc123"); + spy.mockRestore(); + }); + + it("close removes only the team param", () => { + window.history.pushState(null, "", "/teams/?foo=bar&team=team-abc123"); + const spy = vi.spyOn(window.history, "pushState"); + const { result } = renderHook(() => useTeamDetailRouting()); + act(() => result.current.close()); + const url = spy.mock.calls.at(-1)?.[2] as string; + expect(url).toContain("foo=bar"); + expect(url).not.toContain("team="); + spy.mockRestore(); + }); + + it("exposes teamId from ?team=", () => { + window.history.pushState(null, "", "/teams/?team=team-abc123"); + const { result } = renderHook(() => useTeamDetailRouting()); + expect(result.current.teamId).toBe("team-abc123"); + }); + + it("teamId is null when no team param is present", () => { + const { result } = renderHook(() => useTeamDetailRouting()); + expect(result.current.teamId).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/detailNavigation.ts b/ui/litellm-dashboard/src/app/(dashboard)/teams/detailNavigation.ts new file mode 100644 index 00000000000..d5208f094cb --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/teams/detailNavigation.ts @@ -0,0 +1,32 @@ +import { useSearchParams } from "next/navigation"; +import { useCallback } from "react"; + +import { navigateWithParams } from "../navigateWithParams"; + +export interface TeamDetailRouting { + teamId: string | null; + openTeam: (id: string) => void; + close: () => void; +} + +export function useTeamDetailRouting(): TeamDetailRouting { + const searchParams = useSearchParams(); + + const openTeam = useCallback((id: string) => { + navigateWithParams((params) => { + params.set("team", id); + }); + }, []); + + const close = useCallback(() => { + navigateWithParams((params) => { + params.delete("team"); + }); + }, []); + + return { + teamId: searchParams?.get("team") ?? null, + openTeam, + close, + }; +} diff --git a/ui/litellm-dashboard/src/components/Teams.test.tsx b/ui/litellm-dashboard/src/components/Teams.test.tsx index 7065b1a5fb6..742a88d864c 100644 --- a/ui/litellm-dashboard/src/components/Teams.test.tsx +++ b/ui/litellm-dashboard/src/components/Teams.test.tsx @@ -72,6 +72,31 @@ vi.mock("@/components/team/TeamInfo", () => ({ }, })); +// The selected team is URL-derived (?team=) via useTeamDetailRouting. Next's real useSearchParams +// re-renders subscribers on history.pushState/replaceState; mirror that so URL changes propagate. +vi.mock("next/navigation", async () => { + const { useSyncExternalStore } = await import("react"); + const LOCATION_CHANGE_EVENT = "test-locationchange"; + for (const method of ["pushState", "replaceState"] as const) { + const original = window.history[method].bind(window.history); + window.history[method] = (...args: Parameters) => { + original(...args); + window.dispatchEvent(new Event(LOCATION_CHANGE_EVENT)); + }; + } + const subscribe = (onChange: () => void) => { + window.addEventListener(LOCATION_CHANGE_EVENT, onChange); + window.addEventListener("popstate", onChange); + return () => { + window.removeEventListener(LOCATION_CHANGE_EVENT, onChange); + window.removeEventListener("popstate", onChange); + }; + }; + return { + useSearchParams: () => new URLSearchParams(useSyncExternalStore(subscribe, () => window.location.search)), + }; +}); + vi.mock("./ModelSelect/ModelSelect", () => { const ModelSelect = React.forwardRef(({ value, onChange, dataTestId, id }: any, ref: any) => { return ( @@ -159,6 +184,7 @@ const renderWithQueryClient = (component: React.ReactElement) => { // Re-establish safe defaults before every test (clearAllMocks keeps return values, so restore them here). beforeEach(() => { mockTeamsTableProps = null; + window.history.replaceState(null, "", "/teams/"); }); describe("Teams - handleCreate organization handling", () => { @@ -436,6 +462,47 @@ describe("Teams - premium props", () => { }); }); +describe("Teams - team detail deep link (?team=)", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockTeamInfoView.mockClear(); + vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue([]); + vi.mocked(fetchMCPAccessGroups).mockResolvedValue([]); + vi.mocked(getGuardrailsList).mockResolvedValue({ guardrails: [] }); + mockUseOrganizations.mockReturnValue({ data: [] }); + }); + + it("selecting a team pushes ?team= to the URL", async () => { + renderWithQueryClient(); + + await waitFor(() => expect(mockTeamsTableProps).not.toBeNull()); + act(() => mockTeamsTableProps.onSelectTeam({ ...baseTableTeam, team_id: "team-deep-link" })); + + expect(window.location.search).toContain("team=team-deep-link"); + await waitFor(() => expect(mockTeamInfoView).toHaveBeenCalled()); + expect(mockTeamInfoView).toHaveBeenLastCalledWith(expect.objectContaining({ teamId: "team-deep-link" })); + }); + + it("opens the team detail view directly from a ?team= deep link", async () => { + window.history.replaceState(null, "", "/teams/?team=team-from-url"); + renderWithQueryClient(); + + await waitFor(() => expect(mockTeamInfoView).toHaveBeenCalled()); + expect(mockTeamInfoView).toHaveBeenLastCalledWith(expect.objectContaining({ teamId: "team-from-url" })); + }); + + it("closing the team detail view removes ?team= from the URL", async () => { + window.history.replaceState(null, "", "/teams/?team=team-from-url"); + renderWithQueryClient(); + + await waitFor(() => expect(mockTeamInfoView).toHaveBeenCalled()); + act(() => mockTeamInfoView.mock.calls.at(-1)?.[0].onClose()); + + expect(window.location.search).not.toContain("team="); + await waitFor(() => expect(screen.queryByTestId("team-info-view")).not.toBeInTheDocument()); + }); +}); + describe("Teams - Create Team CTA is grouped with the tabs on the left", () => { beforeEach(() => { vi.clearAllMocks(); diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index 20e9e78e7e4..0a3c7fc736d 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -12,6 +12,7 @@ import { useQueryClient } from "@tanstack/react-query"; import { PageHeader } from "@/components/shared/PageHeader"; import { Button as UIButton } from "@/components/ui/button"; import { teamsTableKeys } from "@/app/(dashboard)/hooks/teams/useTeams"; +import { useTeamDetailRouting } from "@/app/(dashboard)/teams/detailNavigation"; import { TeamsTable } from "./TeamsPage/TeamsTable"; import AccessGroupSelector from "./common_components/AccessGroupSelector"; import PassThroughRoutesSelector from "./common_components/PassThroughRoutesSelector"; @@ -135,7 +136,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser const [editModalVisible, setEditModalVisible] = useState(false); const [selectedTeam, setSelectedTeam] = useState(null); - const [selectedTeamId, setSelectedTeamId] = useState(null); + const { teamId: selectedTeamId, openTeam, close: closeTeamDetail } = useTeamDetailRouting(); const [editTeam, setEditTeam] = useState(false); const [isTeamModalVisible, setIsTeamModalVisible] = useState(false); @@ -482,12 +483,12 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser userID={userID} onSelectTeam={(team) => { setSelectedTeam(team); - setSelectedTeamId(team.team_id); + openTeam(team.team_id); setEditTeam(false); }} onEditTeam={(team) => { setSelectedTeam(team); - setSelectedTeamId(team.team_id); + openTeam(team.team_id); setEditTeam(true); }} onDeleteTeam={handleDelete} @@ -547,11 +548,11 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser }} onClose={() => { setSelectedTeam(null); - setSelectedTeamId(null); + closeTeamDetail(); setEditTeam(false); }} accessToken={accessToken} - is_team_admin={is_team_admin(selectedTeam)} + is_team_admin={is_team_admin(selectedTeam?.team_id === selectedTeamId ? selectedTeam : null)} is_proxy_admin={userRole == "Admin"} userModels={userModels} editTeam={editTeam} diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index dcc72ccac9c..25365d26a12 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -444,6 +444,29 @@ describe("TeamInfoView", () => { }); }); + it("shows edit tabs when the fetched team data marks the session user as team admin, even without the is_team_admin prop", async () => { + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ + members_with_roles: [ + { + user_id: "user-1", + user_email: "admin@test.com", + role: "admin", + spend: 0, + budget_id: "budget1", + }, + ], + }), + ); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByRole("tab", { name: "Settings" })).toBeInTheDocument(); + }); + expect(screen.getByRole("tab", { name: "Members" })).toBeInTheDocument(); + }); + it("should navigate to settings tab when clicked", async () => { const user = userEvent.setup({ delay: null }); vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData()); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index ff881a68938..acd5a8966a7 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -225,7 +225,15 @@ const TeamInfoView: React.FC = ({ return unfurlWildcardModelsInList(selected, userModels); }, [selectedModelsInForm, teamData, userModels]); - const canEditTeam = is_team_admin || is_proxy_admin || is_org_admin || isOrgAdminForTeam; + const isTeamAdminFromTeamData = useMemo( + () => + teamData?.team_info?.members_with_roles?.some( + (member) => member.user_id != null && member.user_id === userId && member.role === "admin", + ) ?? false, + [teamData, userId], + ); + + const canEditTeam = is_team_admin || is_proxy_admin || is_org_admin || isOrgAdminForTeam || isTeamAdminFromTeamData; const visibleTabs = useMemo(() => getTeamInfoVisibleTabs(canEditTeam), [canEditTeam]); const defaultTabKey = useMemo(() => getTeamInfoDefaultTab(editTeam, canEditTeam), [editTeam, canEditTeam]); From ba7d8ae17fe45dd355f2fc8f629e8822eb6bda57 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 29 Jul 2026 11:56:37 -0700 Subject: [PATCH 06/11] feat(ui): deep link organization detail page via ?org= query param (#35117) * feat(ui): deep link organization detail page via ?org= query param The organizations page kept the selected organization in React state, so an org detail page had no URL: it could not be shared, bookmarked, or opened from another page, and the browser back button dropped you out of the page instead of closing the detail view Adds useOrgDetailRouting reading ?org= (same pattern as the api-keys, models, logs, and teams deep links) and derives the open organization in OrganizationsPanel from the URL * fix(ui): reset org edit mode on plain row selection and type test mocks Greptile P1: with the selected org now URL-derived, browser Back leaves the detail view without running onClose, so a stale editOrg=true made the next plain row click open on the Settings tab. Reset the flag on row selection, matching the teams page Greptile P2: type the panel test's captured table and detail-view props from the real components instead of any --- .../_components/OrganizationsPanel.test.tsx | 113 +++++++++++++++++- .../_components/OrganizationsPanel.tsx | 12 +- .../organizations/detailNavigation.test.ts | 53 ++++++++ .../organizations/detailNavigation.ts | 32 +++++ 4 files changed, 201 insertions(+), 9 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/organizations/detailNavigation.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/organizations/detailNavigation.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.test.tsx index d381e5e65ca..3f9de478069 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.test.tsx @@ -1,7 +1,9 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { render, screen } from "@testing-library/react"; +import { act, render, screen } from "@testing-library/react"; import React from "react"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type OrganizationsTableComponent from "./OrganizationsTable"; +import type OrganizationInfoViewComponent from "@/components/organization/organization_view"; vi.mock("@/components/vector_store_management/VectorStoreSelector", () => ({ __esModule: true, @@ -18,12 +20,50 @@ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ userRole: null, }), })); +type OrganizationsTableProps = React.ComponentProps; +type OrganizationInfoViewProps = React.ComponentProps; + +let capturedTableProps: OrganizationsTableProps | null = null; vi.mock("./OrganizationsTable", () => ({ __esModule: true, - default: (props: { isLoading: boolean }) => ( -
isLoading:{String(props.isLoading)}
- ), + default: (props: OrganizationsTableProps) => { + capturedTableProps = props; + return
isLoading:{String(props.isLoading)}
; + }, })); +const mockOrgInfoView = vi.fn<(props: OrganizationInfoViewProps) => void>(); +vi.mock("@/components/organization/organization_view", () => ({ + __esModule: true, + default: (props: OrganizationInfoViewProps) => { + mockOrgInfoView(props); + return
; + }, +})); + +// The selected org is URL-derived (?org=) via useOrgDetailRouting. Next's real useSearchParams +// re-renders subscribers on history.pushState/replaceState; mirror that so URL changes propagate. +vi.mock("next/navigation", async () => { + const { useSyncExternalStore } = await import("react"); + const LOCATION_CHANGE_EVENT = "test-locationchange"; + for (const method of ["pushState", "replaceState"] as const) { + const original = window.history[method].bind(window.history); + window.history[method] = (...args: Parameters) => { + original(...args); + window.dispatchEvent(new Event(LOCATION_CHANGE_EVENT)); + }; + } + const subscribe = (onChange: () => void) => { + window.addEventListener(LOCATION_CHANGE_EVENT, onChange); + window.addEventListener("popstate", onChange); + return () => { + window.removeEventListener(LOCATION_CHANGE_EVENT, onChange); + window.removeEventListener("popstate", onChange); + }; + }; + return { + useSearchParams: () => new URLSearchParams(useSyncExternalStore(subscribe, () => window.location.search)), + }; +}); import OrganizationsPanel from "./OrganizationsPanel"; @@ -34,6 +74,12 @@ const renderWithQueryClient = (ui: React.ReactElement) => { return render({ui}); }; +beforeEach(() => { + capturedTableProps = null; + mockOrgInfoView.mockClear(); + window.history.replaceState(null, "", "/organizations/"); +}); + describe("OrganizationsPanel", () => { it("gates non-premium users behind the enterprise notice", () => { renderWithQueryClient(); @@ -55,3 +101,60 @@ describe("OrganizationsPanel", () => { expect(screen.getByTestId("organizations-table")).toHaveTextContent("isLoading:false"); }); }); + +describe("OrganizationsPanel - org detail deep link (?org=)", () => { + it("clicking an organization pushes ?org= and opens the detail view", () => { + renderWithQueryClient(); + + act(() => capturedTableProps?.onOrganizationClick("org-deep-link")); + + expect(window.location.search).toContain("org=org-deep-link"); + expect(mockOrgInfoView).toHaveBeenLastCalledWith(expect.objectContaining({ organizationId: "org-deep-link" })); + }); + + it("opens the org detail directly from a ?org= deep link", () => { + window.history.replaceState(null, "", "/organizations/?org=org-from-url"); + renderWithQueryClient(); + + expect(mockOrgInfoView).toHaveBeenLastCalledWith( + expect.objectContaining({ organizationId: "org-from-url", editOrg: false }), + ); + expect(screen.queryByTestId("organizations-table")).not.toBeInTheDocument(); + }); + + it("closing the org detail removes ?org= and returns to the list", () => { + window.history.replaceState(null, "", "/organizations/?org=org-from-url"); + renderWithQueryClient(); + + act(() => mockOrgInfoView.mock.calls.at(-1)?.[0].onClose()); + + expect(window.location.search).not.toContain("org="); + expect(screen.queryByTestId("organization-info-view")).not.toBeInTheDocument(); + expect(screen.getByTestId("organizations-table")).toBeInTheDocument(); + }); + + it("the edit action opens the detail in edit mode with ?org= set", () => { + renderWithQueryClient(); + + act(() => capturedTableProps?.onEditClick("org-edit")); + + expect(window.location.search).toContain("org=org-edit"); + expect(mockOrgInfoView).toHaveBeenLastCalledWith( + expect.objectContaining({ organizationId: "org-edit", editOrg: true }), + ); + }); + + it("a plain row click after leaving an edit view via browser history does not reopen in edit mode", () => { + renderWithQueryClient(); + + act(() => capturedTableProps?.onEditClick("org-edit")); + expect(mockOrgInfoView).toHaveBeenLastCalledWith(expect.objectContaining({ editOrg: true })); + + act(() => window.history.pushState(null, "", "/organizations/")); + act(() => capturedTableProps?.onOrganizationClick("org-plain")); + + expect(mockOrgInfoView).toHaveBeenLastCalledWith( + expect.objectContaining({ organizationId: "org-plain", editOrg: false }), + ); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx index b1c026d3904..a21c0669677 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx @@ -1,5 +1,6 @@ import { organizationKeys, useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; import { useUserModels } from "@/app/(dashboard)/hooks/models/useModels"; +import { useOrgDetailRouting } from "@/app/(dashboard)/organizations/detailNavigation"; import OrganizationFilters, { FilterState } from "@/app/(dashboard)/organizations/OrganizationFilters"; import { useQueryClient } from "@tanstack/react-query"; import React, { useState } from "react"; @@ -19,7 +20,7 @@ interface OrganizationsPanelProps { } const OrganizationsPanel: React.FC = ({ userRole, accessToken, premiumUser }) => { - const [selectedOrgId, setSelectedOrgId] = useState(null); + const { orgId: selectedOrgId, openOrg, close: closeOrgDetail } = useOrgDetailRouting(); const [editOrg, setEditOrg] = useState(false); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const [orgToDelete, setOrgToDelete] = useState(null); @@ -108,7 +109,7 @@ const OrganizationsPanel: React.FC = ({ userRole, acces { - setSelectedOrgId(null); + closeOrgDetail(); setEditOrg(false); }} accessToken={accessToken} @@ -132,9 +133,12 @@ const OrganizationsPanel: React.FC = ({ userRole, acces isLoading={isLoading} userRole={userRole} searchActive={searchActive} - onOrganizationClick={setSelectedOrgId} + onOrganizationClick={(organizationId) => { + setEditOrg(false); + openOrg(organizationId); + }} onEditClick={(organizationId) => { - setSelectedOrgId(organizationId); + openOrg(organizationId); setEditOrg(true); }} onDeleteClick={handleDelete} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/detailNavigation.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/organizations/detailNavigation.test.ts new file mode 100644 index 00000000000..46b7c4313ea --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/detailNavigation.test.ts @@ -0,0 +1,53 @@ +/* @vitest-environment jsdom */ +import { act, renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useOrgDetailRouting } from "./detailNavigation"; + +vi.mock("next/navigation", () => ({ useSearchParams: () => new URLSearchParams(window.location.search) })); + +describe("useOrgDetailRouting", () => { + beforeEach(() => { + window.history.pushState(null, "", "/organizations/"); + }); + + it("openOrg sets ?org= via history.pushState (no full navigation)", () => { + const spy = vi.spyOn(window.history, "pushState"); + const { result } = renderHook(() => useOrgDetailRouting()); + act(() => result.current.openOrg("org-abc123")); + expect(spy).toHaveBeenCalledWith(null, "", expect.stringContaining("org=org-abc123")); + spy.mockRestore(); + }); + + it("openOrg preserves unrelated query params", () => { + window.history.pushState(null, "", "/organizations/?foo=bar"); + const spy = vi.spyOn(window.history, "pushState"); + const { result } = renderHook(() => useOrgDetailRouting()); + act(() => result.current.openOrg("org-abc123")); + const url = spy.mock.calls.at(-1)?.[2] as string; + expect(url).toContain("foo=bar"); + expect(url).toContain("org=org-abc123"); + spy.mockRestore(); + }); + + it("close removes only the org param", () => { + window.history.pushState(null, "", "/organizations/?foo=bar&org=org-abc123"); + const spy = vi.spyOn(window.history, "pushState"); + const { result } = renderHook(() => useOrgDetailRouting()); + act(() => result.current.close()); + const url = spy.mock.calls.at(-1)?.[2] as string; + expect(url).toContain("foo=bar"); + expect(url).not.toContain("org="); + spy.mockRestore(); + }); + + it("exposes orgId from ?org=", () => { + window.history.pushState(null, "", "/organizations/?org=org-abc123"); + const { result } = renderHook(() => useOrgDetailRouting()); + expect(result.current.orgId).toBe("org-abc123"); + }); + + it("orgId is null when no org param is present", () => { + const { result } = renderHook(() => useOrgDetailRouting()); + expect(result.current.orgId).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/detailNavigation.ts b/ui/litellm-dashboard/src/app/(dashboard)/organizations/detailNavigation.ts new file mode 100644 index 00000000000..8c55c7b750c --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/detailNavigation.ts @@ -0,0 +1,32 @@ +import { useSearchParams } from "next/navigation"; +import { useCallback } from "react"; + +import { navigateWithParams } from "../navigateWithParams"; + +export interface OrgDetailRouting { + orgId: string | null; + openOrg: (id: string) => void; + close: () => void; +} + +export function useOrgDetailRouting(): OrgDetailRouting { + const searchParams = useSearchParams(); + + const openOrg = useCallback((id: string) => { + navigateWithParams((params) => { + params.set("org", id); + }); + }, []); + + const close = useCallback(() => { + navigateWithParams((params) => { + params.delete("org"); + }); + }, []); + + return { + orgId: searchParams?.get("org") ?? null, + openOrg, + close, + }; +} From bf5334bc59faf9d7a35bc75fa3d0ed9d8e20f344 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 29 Jul 2026 12:32:55 -0700 Subject: [PATCH 07/11] fix(router): drop duplicate Mapping import that fails ruff F811 (#35122) router.py imports Mapping from collections.abc and again from typing, which ruff flags as a redefinition and fails the lint CI job on every open PR. Keep the collections.abc import --- litellm/router.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 6336b12258b..69535d7c74a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -31,7 +31,6 @@ from typing import ( Generator, List, Literal, - Mapping, Optional, Set, Tuple, From ea783cc35c91d1d6d989421e8cc045f23156feae Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:41:31 -0700 Subject: [PATCH 08/11] refactor(rust): make litellm-core the callable messages() SDK; drop the ai-gateway handler (#35044) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- litellm-rust/ADDING_A_PROVIDER.md | 11 ++-- litellm-rust/AGENTS.md | 26 ++++++-- litellm-rust/CLAUDE.md | 61 +++++++++++++------ litellm-rust/README.md | 41 ++++++++----- .../PROVIDER_CODING_STANDARDS.md | 10 +-- litellm-rust/crates/ai-gateway/AGENTS.md | 9 ++- litellm-rust/crates/ai-gateway/README.md | 10 +-- .../crates/ai-gateway/src/constants.rs | 16 ----- .../crates/ai-gateway/src/io/messages.rs | 1 - litellm-rust/crates/ai-gateway/src/io/mod.rs | 1 - litellm-rust/crates/ai-gateway/src/lib.rs | 5 +- .../crates/ai-gateway/src/messages/mod.rs | 49 --------------- .../crates/ai-gateway/src/messages/types.rs | 24 -------- .../crates/ai-gateway/src/routes/AGENTS.md | 7 ++- .../ai-gateway/src/routes/messages/service.rs | 23 +++---- litellm-rust/crates/core/AGENTS.md | 8 ++- litellm-rust/crates/core/CLAUDE.md | 31 ++++++++-- litellm-rust/crates/core/Cargo.toml | 2 +- litellm-rust/crates/core/src/constants.rs | 16 +++++ .../src/messages/client.rs | 0 .../src/messages/common_utils.rs | 10 +-- .../src/messages/handler.rs | 17 ++---- litellm-rust/crates/core/src/messages/mod.rs | 30 +++++++++ .../src/messages/prepare.rs | 7 +-- .../src/messages/tests.rs | 14 +++-- .../crates/core/src/messages/types.rs | 24 ++++++++ litellm-rust/crates/python-bridge/AGENTS.md | 4 +- litellm-rust/crates/python-bridge/CLAUDE.md | 6 +- litellm-rust/crates/python-bridge/src/lib.rs | 18 ++++-- 29 files changed, 277 insertions(+), 204 deletions(-) delete mode 100644 litellm-rust/crates/ai-gateway/src/io/messages.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/messages/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/messages/types.rs rename litellm-rust/crates/{ai-gateway => core}/src/messages/client.rs (100%) rename litellm-rust/crates/{ai-gateway => core}/src/messages/common_utils.rs (83%) rename litellm-rust/crates/{ai-gateway => core}/src/messages/handler.rs (84%) rename litellm-rust/crates/{ai-gateway => core}/src/messages/prepare.rs (92%) rename litellm-rust/crates/{ai-gateway => core}/src/messages/tests.rs (97%) diff --git a/litellm-rust/ADDING_A_PROVIDER.md b/litellm-rust/ADDING_A_PROVIDER.md index 5f933ec4fa8..857a744e014 100644 --- a/litellm-rust/ADDING_A_PROVIDER.md +++ b/litellm-rust/ADDING_A_PROVIDER.md @@ -1,10 +1,11 @@ # Adding a provider / route to litellm-rust -Three layers, same for every route (see `ocr` and `realtime` as references): +Everything for a route lives in `crates/core/src//`; `crates/core/src/messages` is the reference. A host (the axum gateway, the Python bridge) only calls the route's entrypoint. -1. **Transform contract (pure)** — `crates/core/src//transformation.rs`: a `…ProviderConfig` trait (URL build + request/response transforms) + types in `types.rs`. No network, env, or auth. -2. **Provider config (pure)** — `crates/providers/src///transformation.rs`: implement that trait as a `const __CONFIG`, mirroring the Python provider tree. Add parity unit tests. -3. **HTTP / transport (the host)** — `crates/providers/src/.rs` (e.g. `ocr.rs`, `realtime.rs`): the callable fn (`run_ocr`, `realtime`). It resolves the key, builds the auth header, builds URL + transforms via the config, then does the network call. This is the only layer allowed to do I/O. +1. **Entrypoint** — `mod.rs`: `pub async fn (request) -> CoreResult`, the Rust equivalent of `litellm.()`, plus a `_stream` variant when the route streams. It is the only thing a host touches. +2. **Transform contract** — `transformation.rs`: a `…ProviderConfig` trait (URL build + request/response transforms) with types in `types.rs`. +3. **Provider config** — `crates/core/src/providers///transformation.rs`: implement that trait as a `const __CONFIG`, mirroring the Python provider tree. Add parity unit tests. +4. **Prepare + handler** — `prepare.rs` resolves provider/model, credentials, auth headers, and URL, then transforms the request; `handler.rs` performs the provider call through the shared client in `client.rs` and transforms the response. ## Coding standards @@ -25,4 +26,4 @@ variants of it. The test for a good abstraction is that adding the next provider is a few declarative lines, not a new file of duplicated flow. Only diverge from the base when behavior is genuinely different, and say so explicitly in the PR. -**Calling:** the host invokes the route fn — the Python bridge calls `run_ocr`; the `ai-gateway` server calls `realtime`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`. +**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`. diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index 398eec4685c..36a5ad5a8f4 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -4,14 +4,30 @@ litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes ( ## Crates -| Crate | Role | Pure / I/O | -|-------|------|------------| -| litellm-core | Translation layer — types, route contracts (traits), provider transforms (modules under providers/), and the router. Builds requests/responses; no network. | Pure | -| litellm-ai-gateway | Routes + host — the only crate that touches the network. HTTP/WebSocket I/O (modules under io/) plus the axum server binary (behind the `server` feature). | I/O | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — a thin adapter over litellm-ai-gateway's I/O. | Binding | +| Crate | Role | +|-------|------| +| litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. | +| litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. | +| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +## Where a route lives + +A top-level LiteLLM call is a module under `crates/core/src//`, shaped like `messages`: + +``` +core/src/messages/ + mod.rs # pub async fn messages(..) -> CoreResult<..> (+ messages_stream for SSE) + types.rs # request/response types, MessagesRequest + transformation.rs # the provider template trait + prepare.rs # provider resolution, auth headers, URL + handler.rs # the provider call + client.rs # the shared reqwest client +``` + +Handlers never live in `ai-gateway`. `ocr`, `audio_transcription`, and `realtime` are still hosted there from before this rule; they move to `core` as they are touched. + Adding a crate: default to a MODULE. New crate ONLY on a real trigger — separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these. Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional. diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md index 0659e63df39..fe6ceedbb86 100644 --- a/litellm-rust/CLAUDE.md +++ b/litellm-rust/CLAUDE.md @@ -23,21 +23,34 @@ the base when behavior is genuinely different, and say so explicitly in the PR. ## Crates (exactly three — see AGENTS.md) -`litellm-core` describes work; `litellm-ai-gateway` executes it; `litellm-python-bridge` -exposes it to the Python SDK. A crate is a **layer**, not a route — add modules, not crates. +`litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call. +`litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and +`litellm-python-bridge` exposes it to the Python SDK. A crate is a **layer**, not +a route — add modules, not crates. ## Core Boundary -`litellm-core` is the pure translation layer; the `litellm-ai-gateway` host executes work. +`litellm-core` owns the whole call. The Rust equivalent of `litellm.messages()` +is `litellm_core::messages::messages(request).await`: you call it, it does the +provider call, and you get a typed non-streaming response back. Route-level Rust structure mirrors LiteLLM's Python responsibilities: -- `core/src//` owns the route contract, shared types, and provider - template traits. For OCR, this means `core/src/ocr`. +- `core/src//` owns the route end to end: the public entrypoint fn named + after the route in `mod.rs`, the request/response types (`types.rs`), the + provider template trait (`transformation.rs`), the provider/auth/URL + resolution (`prepare.rs`), the HTTP client (`client.rs`), and the handler that + performs the call (`handler.rs`). `core/src/messages` is the reference. - `core/src/providers///transformation.rs` owns the - provider-specific transform. For Mistral OCR, this means - `core/src/providers/mistral/ocr/transformation.rs`. -- Network execution lives in the host crate `ai-gateway` (`ai-gateway/src/io/`), - never inside `core`. + provider-specific transform. For Anthropic Messages, this means + `core/src/providers/anthropic/messages/transformation.rs`. +- Handlers live in `core`, never in a host. `ai-gateway` must not contain a + route handler that talks to a provider; its axum route reads the HTTP request, + picks a deployment, and calls the `core` entrypoint. `python-bridge` marshals + Python objects and calls the same entrypoint. + +Streaming keeps the same shape: the route entrypoint has a `_stream` +variant in `core` that returns the upstream response so a host can splice it to +its own caller; the host still owns no provider logic. Call-hook and lifecycle instrumentation, including phase timing, usage accumulation, and callback payload construction, always lives in `core`. @@ -45,21 +58,31 @@ Hosts feed observed events into core and dispatch the completed payloads through their I/O logger; hosts must not own callback orchestration. Allowed in `core`: -- Pure request transforms -- Pure response transforms -- Pure stream chunk normalization +- The public entrypoint for a top-level LiteLLM call +- Request/response transforms and stream chunk normalization +- Provider resolution, auth header construction, and URL building +- The provider HTTP call itself, through a shared reused client with connect and + request timeouts - Shared data types and validation errors - Deterministic token/cost helper logic Not allowed in `core`: -- Network calls -- Environment variable or secret reads +- Serving HTTP: axum routes, extractors, and transport concerns stay in the host - Filesystem access -- Database or cache access -- Provider SDK signing or auth flows +- Database access +- Config file reading and rollout state - Logging callbacks, spend writes, or custom callbacks - Global mutable runtime state +Env reads in `core` are limited to credential fallback inside a route's +`prepare.rs` (the `env_lookup` closure), mirroring what the Python SDK does when +no key is passed. Everything else config-shaped is resolved by the host and +passed in. + +Routes still hosted in `ai-gateway` (`ocr`, `audio_transcription`, `realtime`) +predate this rule and are being moved into `core` route modules; do not add new +ones there, and prefer moving one when you touch it. + Python owns rollout state and fallback while Rust is being introduced. Rust paths must be off by default until parity tests prove equivalence with Python. A new provider/route may instead be implemented rust-only with no Python @@ -93,10 +116,10 @@ the first PR: - Preserve Python output shape intentionally. If a field is always serialized as `null` for Python parity, leave a short comment explaining that parity choice. -## Host I/O Rules +## Network I/O Rules -These rules apply when adding future crates or modules that execute network I/O, -such as `ai-gateway`, router hosts, or standalone servers: +These rules apply to every module that executes network I/O, whether it is a +`core` route handler or a host such as `ai-gateway`: - Set connect and full-request timeouts. No unbounded waits. - Reuse HTTP clients; do not construct clients per request. diff --git a/litellm-rust/README.md b/litellm-rust/README.md index 1646c90ad76..bcccf93300b 100644 --- a/litellm-rust/README.md +++ b/litellm-rust/README.md @@ -2,18 +2,31 @@ This workspace contains the staged Rust implementation for LiteLLM. -Rust starts as a pure transform core used by the existing Python host. Python -continues to own auth, configuration, network I/O, retries, routing, logging, +`litellm-core` is the LiteLLM SDK in Rust: one entrypoint per top-level call +that makes the LLM call and hands back a typed response, the same shape as +`litellm.messages()` in Python. + +```rust +let response = litellm_core::messages::messages(MessagesRequest { + model: "claude-sonnet-4-5", + body, + api_key: Some(key), + .. +}) +.await?; +``` + +Python continues to own configuration, retries, routing policy, logging, callbacks, spend tracking, and customer plugins until each Rust path has parity coverage and production evidence. ## Crates -| Crate | Role | Pure / I/O | -|-------|------|------------| -| litellm-core | Translation layer — types, route contracts (traits), provider transforms (modules under providers/), and the router. Builds requests/responses; no network. | Pure | -| litellm-ai-gateway | Routes + host — the only crate that touches the network. HTTP/WebSocket I/O (modules under io/) plus the axum server binary (behind the `server` feature). | I/O | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — a thin adapter over litellm-ai-gateway's I/O. | Binding | +| Crate | Role | +|-------|------| +| litellm-core | The SDK. Per-route entrypoints (`messages::messages()`), types, provider transforms (modules under `providers/`), provider resolution, auth, the provider HTTP call, and the router. | +| litellm-ai-gateway | The axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | +| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. @@ -21,16 +34,16 @@ Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm- ```text crates/ - core/ Route contracts, shared pure types, errors, and templates. - src/ocr/ - providers/ Provider-specific pure transforms. - src/mistral/ocr/transformation.rs + core/ The SDK: route modules + provider transforms. + src/messages/ mod.rs (entrypoint), types, transformation, prepare, handler, client + src/providers/anthropic/messages/transformation.rs + ai-gateway/ Axum server + WebSocket hosts; calls core entrypoints. python-bridge/ PyO3 bridge for Python LiteLLM. ``` -The folder shape should follow the Python provider tree: -`providers/src///transformation.rs`. The bridge should expose -one function per top-level route, starting with `ocr(payload)`. +The folder shape follows the Python provider tree: +`core/src/providers///transformation.rs`. The bridge exposes one +function per top-level route, mirroring the core entrypoints. ## Checks diff --git a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md index ed44dc4c729..a1860d8a9c9 100644 --- a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md +++ b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md @@ -1,6 +1,6 @@ # Provider coding standards (litellm-rust) -Rules for adding or changing an LLM provider/route in `litellm-rust`. OCR (`MISTRAL_OCR_CONFIG`) is the reference; `messages` (`ANTHROPIC_MESSAGES_CONFIG`) is the next port. +Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages` (`core/src/messages`, `ANTHROPIC_MESSAGES_CONFIG`) is the reference: a route is a `core` module with a public entrypoint that makes the call and returns a typed response. ## Provider resolution @@ -16,10 +16,10 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. OCR (`MIST ## Boundaries -7. Layers never cross: `core` = pure transforms/types (no network, env, secrets, auth, logging, global mutable state); `ai-gateway` = all I/O, auth headers, HTTP/SSE, lifecycle hooks; `python-bridge` = thin PyO3 adapter. +7. Layers never cross: `core` = the call itself (entrypoint, types, transforms, provider resolution, auth headers, provider HTTP, lifecycle hooks); `ai-gateway` = serving HTTP/WS (routing, extractors, auth of *our* callers, streaming to the client); `python-bridge` = thin PyO3 adapter. Hosts call the core entrypoint; they never build a provider request. 8. Generic/route files contain zero provider-specific branches. A provider is one module under `core/src/providers///`; a route is a module, never a new crate. -9. Route entry point stays thin: `()` -> `prepare_*` -> `CallLifecycle::run_request`, which owns the pre_call -> during_call -> provider call -> success/failure order and phase timing. Handlers validate and delegate; no business logic in them. -10. Constants (URLs, env-var names, API versions, error messages) live in a crate `constants.rs`, never inline. Env reads happen only at the host/config layer, with the `DEFAULT_*` fallback defined in `constants.rs`. +9. Route entry point stays thin: `core::::()` -> `prepare_*` -> handler (or `CallLifecycle::run_request`, which owns the pre_call -> during_call -> provider call -> success/failure order and phase timing). Axum handlers validate and delegate to a service that calls the entrypoint; no business logic in them. +10. Constants (URLs, env-var names, API versions, error messages) live in a crate `constants.rs`, never inline. Config-shaped env reads happen at the host/config layer with the `DEFAULT_*` fallback defined in `constants.rs`; the only env read in `core` is the credential fallback in a route's `prepare.rs`. ## Types and errors @@ -33,7 +33,7 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. OCR (`MIST 16. Never log request/response bodies, base64 payloads, document contents, or secrets. Truncate and bound any upstream body before it crosses a host boundary. 17. Treat empty/whitespace credentials, URLs, and config values as absent at the host resolution layer. -18. Host I/O sets connect + request timeouts (no unbounded waits), reuses a shared HTTP client, and prefers rustls TLS. +18. Network I/O sets connect + request timeouts (no unbounded waits), reuses a shared HTTP client, and prefers rustls TLS. ## Tests and rollout diff --git a/litellm-rust/crates/ai-gateway/AGENTS.md b/litellm-rust/crates/ai-gateway/AGENTS.md index d9e6e1adde5..92567091cd3 100644 --- a/litellm-rust/crates/ai-gateway/AGENTS.md +++ b/litellm-rust/crates/ai-gateway/AGENTS.md @@ -1,7 +1,9 @@ # ai-gateway — folder architecture The Axum server that fronts the Rust gateway. It owns transport + config + auth -only; deployment selection lives in `core::router`, transforms in `core`/`providers`. +only; deployment selection lives in `core::router`, and the LLM call itself +(transforms, auth headers, provider HTTP) lives behind a `core` route entrypoint +such as `litellm_core::messages::messages`. No provider handler lives here. ``` src/ @@ -32,6 +34,11 @@ src/ args; it runs during extraction. Never re-implement the check per route. - **Handlers are thin.** A handler validates and delegates to its `service`. No business logic, no provider calls, no transforms in handlers. +- **Services call `core`, they don't reimplement it.** A `service` picks the + deployment and calls the `core` route entrypoint. Provider resolution, auth + headers, URL building, and the HTTP call are `core`'s job; a service that + builds a provider request itself is a bug (`routes/messages/service.rs` is + the reference). - **State is shared and cheap to clone.** Long-lived handles live behind `Arc` in `state.rs`; read env/config only in `main.rs` when building state. diff --git a/litellm-rust/crates/ai-gateway/README.md b/litellm-rust/crates/ai-gateway/README.md index f913beff6d5..7a6c620ee84 100644 --- a/litellm-rust/crates/ai-gateway/README.md +++ b/litellm-rust/crates/ai-gateway/README.md @@ -8,11 +8,11 @@ dials OpenAI upstream, and splices the two sockets frame-by-frame. `litellm-rust` is exactly three crates (a crate is a **layer**, not a route): -| Crate | Role | Pure / I/O | -|-------|------|------------| -| litellm-core | Translation layer — types, route contracts (traits), provider transforms (modules under `providers/`), and the router. Builds requests/responses; no network. | Pure | -| litellm-ai-gateway | Routes + host — the only crate that touches the network. HTTP/WebSocket I/O (modules under `io/`) plus the Axum server binary (behind the `server` feature). | I/O | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — a thin adapter over litellm-ai-gateway's I/O. | Binding | +| Crate | Role | +|-------|------| +| litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. | +| litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | +| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. diff --git a/litellm-rust/crates/ai-gateway/src/constants.rs b/litellm-rust/crates/ai-gateway/src/constants.rs index 74808cf1ce6..78af374bf70 100644 --- a/litellm-rust/crates/ai-gateway/src/constants.rs +++ b/litellm-rust/crates/ai-gateway/src/constants.rs @@ -29,18 +29,6 @@ pub(crate) const DEFAULT_FLUSH_INTERVAL_MS: u64 = 500; #[cfg(feature = "server")] pub(crate) const DEFAULT_PROVIDER: &str = "openai"; -/// Full-request timeout ceiling for Anthropic Messages provider calls, in -/// seconds. Mirrors the Python Anthropic Messages default. The per-request -/// timeout from `litellm_params` still overrides this on the request builder. -pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600; - -/// Connect timeout for Anthropic Messages provider calls, in seconds. -pub(crate) const MESSAGES_CONNECT_TIMEOUT_SECS: u64 = 10; - -/// Max characters of an upstream error body echoed across the host boundary -/// before truncation, so provider bodies are bounded and data-minimized. -pub(crate) const MESSAGES_ERROR_BODY_MAX_CHARS: usize = 256; - pub(crate) const DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS: u64 = 10; pub(crate) const DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS: u64 = 300; @@ -48,10 +36,6 @@ pub(crate) const DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS: u64 = 300; #[cfg(feature = "server")] pub(crate) const MESSAGES_ROUTE_PATH: &str = "/v1/messages"; -/// Provider name used by the Anthropic Messages route when a deployment's -/// provider model does not carry an explicit provider prefix. -pub(crate) const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic"; - /// Request headers owned by the gateway and never forwarded upstream. #[cfg(feature = "server")] pub(crate) const MESSAGES_HEADERS_NOT_FORWARDED: &[&str] = diff --git a/litellm-rust/crates/ai-gateway/src/io/messages.rs b/litellm-rust/crates/ai-gateway/src/io/messages.rs deleted file mode 100644 index 86170e45678..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/messages.rs +++ /dev/null @@ -1 +0,0 @@ -pub use crate::messages::{MessagesRequest, messages}; diff --git a/litellm-rust/crates/ai-gateway/src/io/mod.rs b/litellm-rust/crates/ai-gateway/src/io/mod.rs index 6129a808965..cce56dd2121 100644 --- a/litellm-rust/crates/ai-gateway/src/io/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/io/mod.rs @@ -1,5 +1,4 @@ pub mod audio_transcription; -pub mod messages; pub mod ocr; pub mod realtime; pub mod realtime_pool; diff --git a/litellm-rust/crates/ai-gateway/src/lib.rs b/litellm-rust/crates/ai-gateway/src/lib.rs index c44d661c29e..057db6457c4 100644 --- a/litellm-rust/crates/ai-gateway/src/lib.rs +++ b/litellm-rust/crates/ai-gateway/src/lib.rs @@ -4,7 +4,9 @@ //! without pulling in the HTTP server: //! //! - Call-type modules such as [`ocr`]: provider transforms, lifecycle hooks, -//! and provider I/O. Always available — no feature required. +//! and provider I/O. Always available — no feature required. These predate the +//! rule that a route's entrypoint and handler live in `litellm-core` (see +//! `litellm_core::messages`) and move there as they are touched. //! - [`io`]: compatibility exports and realtime WebSocket splice helpers. //! - The server modules ([`auth`], [`routes`], [`state`]) and anything pulling //! `axum` are gated behind the `server` feature, which the `litellm-ai-gateway` @@ -14,7 +16,6 @@ pub mod audio_transcription; mod client; pub mod io; -pub mod messages; pub mod ocr; /// GIL-activity tracking. Pure (atomics only); shared by the `server` routes and diff --git a/litellm-rust/crates/ai-gateway/src/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/messages/mod.rs deleted file mode 100644 index fd2dd546941..00000000000 --- a/litellm-rust/crates/ai-gateway/src/messages/mod.rs +++ /dev/null @@ -1,49 +0,0 @@ -use litellm_core::CoreResult; -use serde_json::Value; - -mod client; -mod common_utils; -mod handler; -mod prepare; -mod types; - -pub use types::MessagesRequest; - -use handler::{execute_messages_provider_call, execute_messages_provider_stream}; -use prepare::prepare_messages_call; - -pub async fn messages(request: MessagesRequest<'_>) -> CoreResult { - match execute_messages(request, false).await? { - MessagesResponse::Json(body) => Ok(body), - MessagesResponse::Stream(response) => { - drop(response); - Err(litellm_core::CoreError::InvalidResponse( - "non-streaming messages execution returned a stream".to_string(), - )) - } - } -} - -pub(crate) enum MessagesResponse { - Json(Value), - Stream(reqwest::Response), -} - -pub(crate) async fn execute_messages( - request: MessagesRequest<'_>, - stream: bool, -) -> CoreResult { - let prepared = prepare_messages_call(request)?; - if stream { - execute_messages_provider_stream(prepared) - .await - .map(MessagesResponse::Stream) - } else { - execute_messages_provider_call(prepared) - .await - .map(MessagesResponse::Json) - } -} - -#[cfg(test)] -mod tests; diff --git a/litellm-rust/crates/ai-gateway/src/messages/types.rs b/litellm-rust/crates/ai-gateway/src/messages/types.rs deleted file mode 100644 index 848fadb4b02..00000000000 --- a/litellm-rust/crates/ai-gateway/src/messages/types.rs +++ /dev/null @@ -1,24 +0,0 @@ -use std::time::Duration; - -use litellm_core::messages::transformation::AnthropicMessagesProviderConfig; -use serde_json::{Map, Value}; - -pub struct MessagesRequest<'a> { - pub model: &'a str, - pub body: Value, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, - pub custom_llm_provider: Option<&'a str>, - pub extra_headers: Option>, - pub timeout: Option, -} - -pub(crate) struct ProviderMessagesRequest { - pub(crate) provider: String, - pub(crate) model: String, - pub(crate) config: &'static dyn AnthropicMessagesProviderConfig, - pub(crate) url: String, - pub(crate) body: Value, - pub(crate) upstream_headers: Vec<(String, String)>, - pub(crate) timeout: Option, -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md b/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md index 02c5f18c4f3..3eee43e7a2f 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md +++ b/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md @@ -19,7 +19,10 @@ async fn handle(...) -> impl IntoResponse { ... } When a route has business logic worth testing without axum, put it in a sibling `service` (a file, or a folder if the route grows). The route file stays the **axum surface** (router + handler + any socket/SSE adapter); `service` is plain -Rust with **no axum types**. `realtime/` is the example: +Rust with **no axum types**, and its job is to pick the deployment and call the +`core` route entrypoint (see `messages/service.rs` calling +`litellm_core::messages::messages`). Never build a provider request, resolve a +key, or perform the provider call here. `realtime/` is the older example: ``` realtime/ mod.rs # axum surface: router() + handler + the WS<->events adapter @@ -33,6 +36,8 @@ genuinely gets hard to read. `crate::auth::RequireMasterKey` to its arguments; it runs during extraction. Never re-implement the check per route. - **Handlers contain no business logic; `service` contains no axum types.** +- **No provider handlers in this crate.** Transforms, auth headers, and the + provider HTTP call live in `core/src//`. - A route owns its paths in its own `router()`; `mod.rs` only merges. - Cross-cutting concerns (logging, CORS, timeouts) → Tower layers in `mod.rs`, not duplicated in handlers. diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs index 75ed26e5be8..5f4c5fe8de4 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs @@ -1,12 +1,12 @@ use std::sync::Arc; +use litellm_core::constants::ANTHROPIC_MESSAGES_PROVIDER; +use litellm_core::messages::types::MessagesRequest; +use litellm_core::messages::{messages, messages_stream}; use litellm_core::router::Router; use litellm_core::{CoreError, CoreResult}; use serde_json::{Map, Value}; -use crate::constants::ANTHROPIC_MESSAGES_PROVIDER; -use crate::messages::{MessagesRequest, execute_messages}; - pub(crate) enum MessagesResponse { Json(Value), Stream(reqwest::Response), @@ -52,13 +52,14 @@ pub async fn run( extra_headers, timeout: None, }; - let stream = request.body.get("stream").and_then(Value::as_bool) == Some(true); - execute_messages(request, stream) - .await - .map(|response| match response { - crate::messages::MessagesResponse::Json(body) => MessagesResponse::Json(body), - crate::messages::MessagesResponse::Stream(upstream) => { - MessagesResponse::Stream(upstream) - } + if request.body.get("stream").and_then(Value::as_bool) == Some(true) { + return messages_stream(request).await.map(MessagesResponse::Stream); + } + + let response = messages(request).await?; + serde_json::to_value(response) + .map(MessagesResponse::Json) + .map_err(|err| { + CoreError::InvalidResponse(format!("failed to serialize messages response: {err}")) }) } diff --git a/litellm-rust/crates/core/AGENTS.md b/litellm-rust/crates/core/AGENTS.md index 8740dccaf01..aee8b4937ef 100644 --- a/litellm-rust/crates/core/AGENTS.md +++ b/litellm-rust/crates/core/AGENTS.md @@ -1,3 +1,7 @@ -litellm-core is the PURE translation layer — types, route contracts (traits), provider transforms (modules under `providers/`), and the router. No network, no I/O, no env reads. +litellm-core is the LiteLLM SDK in Rust — it makes the LLM call. Each top-level call is a module under `src//` exposing a public entrypoint named after the route (`messages::messages()`, the Rust equivalent of `litellm.messages()`): you call it and get a typed non-streaming response back. -Routes (ocr, realtime) and providers (mistral, openai) are modules, not crates. +A route module owns everything the call needs: types, the provider template trait, provider transforms (under `providers/`), provider/auth/URL resolution, and the handler that performs the HTTP call. Handlers belong here, never in a host crate. + +Not here: serving HTTP (axum routes, extractors), config file reading, rollout state, databases, or callback dispatch. Env reads are limited to credential fallback in a route's `prepare.rs`. + +Routes (messages, ocr, realtime) and providers (anthropic, mistral, openai) are modules, not crates. diff --git a/litellm-rust/crates/core/CLAUDE.md b/litellm-rust/crates/core/CLAUDE.md index 20873878967..5d36305ded5 100644 --- a/litellm-rust/crates/core/CLAUDE.md +++ b/litellm-rust/crates/core/CLAUDE.md @@ -4,20 +4,28 @@ Rules for `litellm-rust/crates/core`. ## Responsibility -`core` owns shared data types, typed errors, and deterministic helper contracts. -It must stay pure and host-independent. +`core` is the LiteLLM SDK in Rust: it makes the LLM call. Every top-level +LiteLLM call has a public entrypoint here, named after the route +(`messages::messages()` is the Rust equivalent of `litellm.messages()`), and +calling it returns a typed non-streaming response. Allowed: +- The public entrypoint for a route, plus its `_stream` variant when the + route supports streaming. +- Provider resolution, auth header construction, URL building, and the provider + HTTP call (shared reused client, connect + request timeouts). - Shared request/response structs. - Typed errors with stable, non-sensitive messages. - Deterministic validation helpers. - Serialization helpers that intentionally mirror Python output shape. - Route templates that match Python base config responsibilities, such as - `ocr::transformation::OcrProviderConfig`. + `messages::transformation::AnthropicMessagesProviderConfig`. Not allowed: -- Network, filesystem, database, cache, or environment access. -- Secret reads or auth/header construction. +- Serving HTTP: axum routers, extractors, and other transport concerns. +- Filesystem, database, or cache access. +- Config file reading or rollout state; the host resolves those and passes them + in. Env reads are limited to credential fallback in a route's `prepare.rs`. - Logging callbacks, tracing spans, spend writes, or customer callbacks. - Provider-specific branching that belongs in `providers`. - Panics for user/provider-controlled input. @@ -33,10 +41,21 @@ typed field on a struct, not a raw string threaded through the API. ## Structure -Use route names directly under `src/`: `ocr`, future `messages`, +Use route names directly under `src/`: `messages`, `ocr`, future `chat_completions`, `embeddings`, and similar top-level LiteLLM calls. Do not invent broad names like `engine` for route contracts. +`src/messages` is the reference shape for a route module: + +``` +mod.rs pub async fn messages(..) (+ messages_stream) +types.rs request/response types +transformation.rs the provider template trait +prepare.rs provider resolution, auth headers, URL +handler.rs the provider call +client.rs the shared reqwest client +``` + ## Parity Rules - Every shared type used by a provider transform needs unit tests for diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 65c6db7412c..ab8050734f2 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] rand.workspace = true +reqwest.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true @@ -30,5 +31,4 @@ bedrock-auth = [ ] [dev-dependencies] -reqwest.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 5826a5bc9c1..caada1d98b0 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -1,3 +1,19 @@ pub const OPENAI_DEFAULT_API_BASE: &str = "https://api.openai.com"; pub const OPENAI_RESPONSES_DEFAULT_API_BASE: &str = "https://api.openai.com/v1"; pub const OPENAI_RESPONSES_PATH: &str = "/responses"; + +/// Full-request timeout ceiling for Anthropic Messages provider calls, in +/// seconds. Mirrors the Python Anthropic Messages default. The per-request +/// timeout from the caller still overrides this on the request builder. +pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600; + +/// Connect timeout for Anthropic Messages provider calls, in seconds. +pub(crate) const MESSAGES_CONNECT_TIMEOUT_SECS: u64 = 10; + +/// Max characters of an upstream error body echoed across the call boundary +/// before truncation, so provider bodies are bounded and data-minimized. +pub(crate) const MESSAGES_ERROR_BODY_MAX_CHARS: usize = 256; + +/// Provider name used for Anthropic Messages when a deployment's provider model +/// does not carry an explicit provider prefix. +pub const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic"; diff --git a/litellm-rust/crates/ai-gateway/src/messages/client.rs b/litellm-rust/crates/core/src/messages/client.rs similarity index 100% rename from litellm-rust/crates/ai-gateway/src/messages/client.rs rename to litellm-rust/crates/core/src/messages/client.rs diff --git a/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs b/litellm-rust/crates/core/src/messages/common_utils.rs similarity index 83% rename from litellm-rust/crates/ai-gateway/src/messages/common_utils.rs rename to litellm-rust/crates/core/src/messages/common_utils.rs index 68ecc3f17c1..9dcfcaa71e3 100644 --- a/litellm-rust/crates/ai-gateway/src/messages/common_utils.rs +++ b/litellm-rust/crates/core/src/messages/common_utils.rs @@ -1,11 +1,11 @@ -use litellm_core::CoreResult; -use litellm_core::error::{CoreError, json_type_name}; -use litellm_core::messages::transformation::AnthropicMessagesProviderConfig; -use litellm_core::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG; -use litellm_core::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG; use serde_json::{Map, Value}; use crate::constants::MESSAGES_ERROR_BODY_MAX_CHARS; +use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG; +use crate::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG; + +use super::transformation::AnthropicMessagesProviderConfig; pub(super) fn truncate_error_body(body: &str) -> String { if body.chars().count() <= MESSAGES_ERROR_BODY_MAX_CHARS { diff --git a/litellm-rust/crates/ai-gateway/src/messages/handler.rs b/litellm-rust/crates/core/src/messages/handler.rs similarity index 84% rename from litellm-rust/crates/ai-gateway/src/messages/handler.rs rename to litellm-rust/crates/core/src/messages/handler.rs index 90c12367f50..1c895f66eba 100644 --- a/litellm-rust/crates/ai-gateway/src/messages/handler.rs +++ b/litellm-rust/crates/core/src/messages/handler.rs @@ -1,15 +1,13 @@ -use litellm_core::CoreResult; -use litellm_core::error::CoreError; -use serde_json::Value; +use crate::constants::ANTHROPIC_MESSAGES_PROVIDER; +use crate::error::{CoreError, CoreResult}; use super::client::http_client; use super::common_utils::truncate_error_body; -use super::types::ProviderMessagesRequest; -use crate::constants::ANTHROPIC_MESSAGES_PROVIDER; +use super::types::{AnthropicMessagesResponse, ProviderMessagesRequest}; pub(super) async fn execute_messages_provider_call( request: ProviderMessagesRequest, -) -> CoreResult { +) -> CoreResult { let mut request_builder = http_client().post(&request.url).json(&request.body); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -39,12 +37,7 @@ pub(super) async fn execute_messages_provider_call( let response = serde_json::from_str(&text).map_err(|err| { CoreError::InvalidResponse(format!("invalid messages response JSON: {err}")) })?; - let transformed = request - .config - .transform_response(&request.model, response)?; - serde_json::to_value(transformed).map_err(|err| { - CoreError::InvalidResponse(format!("failed to serialize messages response: {err}")) - }) + request.config.transform_response(&request.model, response) } pub(super) async fn execute_messages_provider_stream( diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index ec2fbb969a6..acb36d89daf 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -1,2 +1,32 @@ +//! The Anthropic Messages call, the Rust equivalent of Python's +//! `litellm.messages()`. +//! +//! [`messages`] is the top-level entrypoint: give it a model, a body, and +//! credentials, and it resolves the provider, transforms the request, calls the +//! provider, and returns a typed non-streaming response. [`messages_stream`] +//! is the streaming variant; it hands the raw upstream response back so a host +//! can splice the event stream to its own caller. + +mod client; +mod common_utils; +mod handler; +mod prepare; pub mod transformation; pub mod types; + +use crate::error::CoreResult; + +use handler::{execute_messages_provider_call, execute_messages_provider_stream}; +use prepare::prepare_messages_call; +use types::{AnthropicMessagesResponse, MessagesRequest}; + +pub async fn messages(request: MessagesRequest<'_>) -> CoreResult { + execute_messages_provider_call(prepare_messages_call(request)?).await +} + +pub async fn messages_stream(request: MessagesRequest<'_>) -> CoreResult { + execute_messages_provider_stream(prepare_messages_call(request)?).await +} + +#[cfg(test)] +mod tests; diff --git a/litellm-rust/crates/ai-gateway/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs similarity index 92% rename from litellm-rust/crates/ai-gateway/src/messages/prepare.rs rename to litellm-rust/crates/core/src/messages/prepare.rs index 9a027490eb6..94b5b1eaed7 100644 --- a/litellm-rust/crates/ai-gateway/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,9 +1,8 @@ -use litellm_core::CoreError; -use litellm_core::CoreResult; -use litellm_core::messages::transformation::MessagesAuthStrategy; -use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; +use crate::error::{CoreError, CoreResult}; +use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}; +use super::transformation::MessagesAuthStrategy; use super::types::{MessagesRequest, ProviderMessagesRequest}; pub(super) fn prepare_messages_call( diff --git a/litellm-rust/crates/ai-gateway/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs similarity index 97% rename from litellm-rust/crates/ai-gateway/src/messages/tests.rs rename to litellm-rust/crates/core/src/messages/tests.rs index 23a53e98045..9fc1763683b 100644 --- a/litellm-rust/crates/ai-gateway/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -1,14 +1,16 @@ use std::time::Duration; -use litellm_core::error::CoreError; use serde_json::{Map, Value, json}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; +use crate::error::CoreError; + use super::common_utils::{ has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body, }; -use super::{MessagesRequest, messages}; +use super::messages; +use super::types::MessagesRequest; async fn read_http_request(socket: &mut TcpStream) -> String { let mut request = Vec::new(); @@ -152,8 +154,8 @@ async fn messages_round_trip_builds_azure_request_and_passes_response_through() .await .expect("messages request succeeds"); - assert_eq!(response["content"][0]["text"], "hi"); - assert_eq!(response["stop_reason"], "end_turn"); + assert_eq!(response.content[0]["text"], "hi"); + assert_eq!(response.stop_reason.as_deref(), Some("end_turn")); let request = server.await.expect("server task completes"); let (head, body) = request.split_once("\r\n\r\n").expect("has body"); @@ -208,8 +210,8 @@ async fn messages_round_trip_builds_native_anthropic_request() { .await .expect("messages request succeeds"); - assert_eq!(response["content"][0]["text"], "hi"); - assert_eq!(response["stop_reason"], "end_turn"); + assert_eq!(response.content[0]["text"], "hi"); + assert_eq!(response.stop_reason.as_deref(), Some("end_turn")); let request = server.await.expect("server task completes"); let (head, _) = request.split_once("\r\n\r\n").expect("has body"); diff --git a/litellm-rust/crates/core/src/messages/types.rs b/litellm-rust/crates/core/src/messages/types.rs index 11fe17ea40f..b9f807c29fd 100644 --- a/litellm-rust/crates/core/src/messages/types.rs +++ b/litellm-rust/crates/core/src/messages/types.rs @@ -1,6 +1,30 @@ +use std::time::Duration; + use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use super::transformation::AnthropicMessagesProviderConfig; + +pub struct MessagesRequest<'a> { + pub model: &'a str, + pub body: Value, + pub api_key: Option<&'a str>, + pub api_base: Option<&'a str>, + pub custom_llm_provider: Option<&'a str>, + pub extra_headers: Option>, + pub timeout: Option, +} + +pub(super) struct ProviderMessagesRequest { + pub(super) provider: String, + pub(super) model: String, + pub(super) config: &'static dyn AnthropicMessagesProviderConfig, + pub(super) url: String, + pub(super) body: Value, + pub(super) upstream_headers: Vec<(String, String)>, + pub(super) timeout: Option, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(untagged)] pub enum SystemPrompt { diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index d6d3d90e6ab..ad3cddfa5fd 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -1,3 +1,3 @@ -litellm-python-bridge is the PyO3 cdylib that exposes Rust to the litellm Python SDK — a thin adapter (Python objects → Rust calls → Python results) over litellm-ai-gateway. +litellm-python-bridge is the PyO3 cdylib that exposes Rust to the litellm Python SDK — a thin adapter (Python objects → Rust calls → Python results) over the litellm-core route entrypoints (e.g. `litellm_core::messages::messages`). -Keep it thin: no business logic, no transforms, no I/O orchestration — just marshal in/out and call into litellm-ai-gateway. +Keep it thin: no business logic, no transforms, no I/O orchestration — just marshal in/out and call the core entrypoint. diff --git a/litellm-rust/crates/python-bridge/CLAUDE.md b/litellm-rust/crates/python-bridge/CLAUDE.md index e5d021ec25b..3ce8b8c639a 100644 --- a/litellm-rust/crates/python-bridge/CLAUDE.md +++ b/litellm-rust/crates/python-bridge/CLAUDE.md @@ -11,11 +11,11 @@ Python-compatible dictionaries. ## Bridge Shape - Prefer one stable method per top-level LiteLLM route, for example - `ocr(payload)`. + `messages(...)`, calling the matching `litellm-core` entrypoint. - Do not add one exported PyO3 function per provider helper unless there is a measured reason. -- Provider dispatch belongs in Rust route modules such as - `litellm_providers::ocr`, not in this PyO3 crate. +- Provider dispatch belongs in the `litellm-core` route module (e.g. + `litellm_core::messages`), not in this PyO3 crate. - Python owns rollout state and fallback. Rust should return errors; Python decides whether to raise or fall back. For a rust-only provider/route (no Python reference), the Python side is a thin dispatch that calls Rust and diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index ee9bdd0b81f..f0cc26a0cca 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -4,10 +4,11 @@ use std::time::Duration; use litellm_ai_gateway::io::audio_transcription::{ AudioTranscriptionRequest, audio_transcription as run_audio_transcription, }; -use litellm_ai_gateway::io::messages::{MessagesRequest, messages as run_messages}; use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr}; use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; use litellm_core::error::CoreError; +use litellm_core::messages::messages as run_messages; +use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyAny, PyDict}; @@ -35,6 +36,15 @@ fn json_to_py(py: Python<'_>, value: Value) -> PyResult> { Ok(json.call_method1("loads", (encoded,))?.unbind()) } +fn messages_response_to_py( + py: Python<'_>, + response: AnthropicMessagesResponse, +) -> PyResult> { + let value = + serde_json::to_value(response).map_err(|err| PyValueError::new_err(err.to_string()))?; + json_to_py(py, value) +} + fn core_error_to_pyerr(err: CoreError) -> PyErr { match err { CoreError::Auth(message) => PyValueError::new_err(message), @@ -382,7 +392,7 @@ fn messages( }); match result { - Ok(value) => json_to_py(py, value), + Ok(response) => messages_response_to_py(py, response), Err(err) => Err(core_error_to_pyerr(err)), } } @@ -404,7 +414,7 @@ fn amessages( marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; pyo3_async_runtimes::tokio::future_into_py(py, async move { - let value = run_messages(MessagesRequest { + let response = run_messages(MessagesRequest { model: &model, body, api_key: api_key.as_deref(), @@ -416,7 +426,7 @@ fn amessages( .await .map_err(core_error_to_pyerr)?; - Python::attach(|py| json_to_py(py, value)) + Python::attach(|py| messages_response_to_py(py, response)) }) } From 440b1bcf654d637967d6036cca269a3db7e4cb9f Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 29 Jul 2026 13:43:33 -0700 Subject: [PATCH 09/11] fix(otel): make OTLP export work against Grafana Cloud (#35060) Three defects kept LiteLLM's OTel metrics from reaching an OTLP backend. OTEL_EXPORTER_OTLP_HEADERS is W3C Baggage encoded per the OTLP spec, so its values are percent-encoded. litellm split the string on "," and "=" and passed the raw value straight to the exporter, so a vendor that documents "Authorization=Basic%20" got a literal "%20" on the wire and the backend rejected the credential. Grafana Cloud documents exactly that shape, which made its OTLP gateway unreachable. Header parsing now delegates to the OTel SDK's own W3C Baggage parser in liberal mode, so percent-encoded values decode and values that were never encoded keep working. It moves from model/utils.py to plumbing/providers.py because model/ is deliberately free of opentelemetry imports; providers.parse_headers was already the entry point every caller used. The OTLP metric exporters then overrode histogram temporality to delta. Prometheus and Mimir, which back Grafana Cloud's OTLP gateway, reject delta histograms outright: the gateway answers 400 "invalid temporality and type combination" and drops the entire batch, so every GenAI metric was silently lost while traces kept flowing. Backends that prefer delta still accept cumulative, so the SDK default is the compatible choice in both directions, and the enterprise billing exporter already relies on it. Three GenAI instruments also carried names no convention or backend defines, so nothing downstream could chart them. Time to first token and time per output token take their semconv names, gen_ai.server.time_to_first_token and gen_ai.server.time_per_output_token; the gen_ai.client.response.* spellings litellm used are not conventions at all. Cost has no semconv instrument, so it takes gen_ai.usage.cost, the name backends already query for spend. All three are listed verbatim in Grafana Cloud's AI Observability integration reference, so its prebuilt panels find them. Both engines now read the names from the shared Metric constants rather than repeating string literals, so v1 and v2 cannot drift. The renames are breaking for anyone charting the former names; the docs and the release changelog carry the migration note. --- litellm/integrations/opentelemetry.py | 21 +++++----- litellm/integrations/otel/model/semconv.py | 22 ++++++++-- litellm/integrations/otel/model/utils.py | 22 +++------- .../integrations/otel/plumbing/providers.py | 35 +++++++++++----- .../otel/test_otel_v2_components.py | 42 +++++++++++++++++++ .../integrations/otel/test_otel_v2_logger.py | 6 +-- .../integrations/otel/test_otel_v2_metrics.py | 6 +-- .../integrations/test_opentelemetry.py | 34 +++++++++++++++ 8 files changed, 142 insertions(+), 46 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 12465377b51..11e7ab062b6 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -27,6 +27,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import ( OTELSemconvCategory, parse_semconv_opt_in, ) +from litellm.integrations.otel.model.semconv import Metric from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.secret_managers.main import get_secret_bool, str_to_bool @@ -597,32 +598,32 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): meter = meter_provider.get_meter(__name__) self._operation_duration_histogram = meter.create_histogram( - name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38 + name=Metric.OPERATION_DURATION, description="GenAI operation duration", unit="s", ) self._token_usage_histogram = meter.create_histogram( - name="gen_ai.client.token.usage", # Replace with semconv constant in otel 1.38 + name=Metric.TOKEN_USAGE, description="GenAI token usage", unit="{token}", ) self._cost_histogram = meter.create_histogram( - name="gen_ai.client.token.cost", + name=Metric.TOKEN_COST, description="GenAI request cost", unit="USD", ) self._time_to_first_token_histogram = meter.create_histogram( - name="gen_ai.client.response.time_to_first_token", + name=Metric.TIME_TO_FIRST_TOKEN, description="Time to first token for streaming requests", unit="s", ) self._time_per_output_token_histogram = meter.create_histogram( - name="gen_ai.client.response.time_per_output_token", + name=Metric.TIME_PER_OUTPUT_TOKEN, description="Average time per output token (generation time / completion tokens)", unit="s", ) self._response_duration_histogram = meter.create_histogram( - name="gen_ai.client.response.duration", + name=Metric.RESPONSE_DURATION, description="Total LLM API generation time (excludes LiteLLM overhead)", unit="s", ) @@ -2980,10 +2981,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): def _get_metric_reader(self): """ Get the appropriate metric reader based on the configuration. + + Histograms keep the SDK's default cumulative temporality: Prometheus-backed + OTLP receivers reject delta histograms and drop the whole batch, while + backends that prefer delta still accept cumulative. """ - from opentelemetry.sdk.metrics import Histogram from opentelemetry.sdk.metrics.export import ( - AggregationTemporality, ConsoleMetricExporter, PeriodicExportingMetricReader, ) @@ -3014,7 +3017,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): exporter = OTLPMetricExporter( endpoint=normalized_endpoint, headers=_split_otel_headers, - preferred_temporality={Histogram: AggregationTemporality.DELTA}, ) return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) @@ -3032,7 +3034,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): exporter = OTLPMetricExporter( endpoint=normalized_endpoint, headers=_split_otel_headers, - preferred_temporality={Histogram: AggregationTemporality.DELTA}, ) return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 44b2f7e0488..1abe8ca33fa 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -257,13 +257,27 @@ class LiteLLM: class Metric: - """GenAI metric instrument names.""" + """GenAI metric instrument names. + + Every name here that a convention or a backend defines uses that name, so a + consumer charting GenAI telemetry finds litellm's series where it looks for + them. ``TOKEN_USAGE``, ``OPERATION_DURATION``, ``TIME_TO_FIRST_TOKEN`` and + ``TIME_PER_OUTPUT_TOKEN`` are semconv instruments, defined in the GenAI + conventions; the ``gen_ai.client.response.*`` spellings litellm used for the + latter two are not conventions at all, so nothing downstream could chart + them. Cost has no semconv instrument, so it takes ``gen_ai.usage.cost``, the + name backends already query for spend. + + ``RESPONSE_DURATION`` keeps its vendor spelling deliberately: the closest + convention, ``gen_ai.server.request.duration``, would collide in meaning with + ``OPERATION_DURATION``, which litellm already emits for the whole operation. + """ TOKEN_USAGE: Final = "gen_ai.client.token.usage" OPERATION_DURATION: Final = "gen_ai.client.operation.duration" - TOKEN_COST: Final = "gen_ai.client.token.cost" - TIME_TO_FIRST_TOKEN: Final = "gen_ai.client.response.time_to_first_token" - TIME_PER_OUTPUT_TOKEN: Final = "gen_ai.client.response.time_per_output_token" + TOKEN_COST: Final = "gen_ai.usage.cost" + TIME_TO_FIRST_TOKEN: Final = "gen_ai.server.time_to_first_token" + TIME_PER_OUTPUT_TOKEN: Final = "gen_ai.server.time_per_output_token" RESPONSE_DURATION: Final = "gen_ai.client.response.duration" diff --git a/litellm/integrations/otel/model/utils.py b/litellm/integrations/otel/model/utils.py index f37afc97879..ab54a558a9a 100644 --- a/litellm/integrations/otel/model/utils.py +++ b/litellm/integrations/otel/model/utils.py @@ -1,9 +1,11 @@ """Shared, OpenTelemetry-free helpers for the otel integration. -Generic value coercion (for reading heterogeneous logging-payload dicts), time -conversion, and header parsing — pulled out of the individual modules so they -live in one place. Deliberately free of any ``opentelemetry`` import so the -OTel-free sources of truth (payloads, semconv, spans, config) can use it too. +Generic value coercion (for reading heterogeneous logging-payload dicts) and +time conversion — pulled out of the individual modules so they live in one +place. Deliberately free of any ``opentelemetry`` import so the OTel-free +sources of truth (payloads, semconv, spans, config) can use it too. OTLP header +parsing lives in :mod:`litellm.integrations.otel.plumbing.providers` instead, +because it delegates to the OTel SDK's own W3C Baggage parser. """ from datetime import datetime @@ -89,15 +91,3 @@ def to_seconds(value: datetime | float | int | str | None) -> float | None: except ValueError: continue return None - - -def parse_headers(raw: str | None) -> dict[str, str]: - """Parse an OTLP ``"k=v,k=v"`` header string into a dict.""" - headers: dict[str, str] = {} - if not raw: - return headers - for pair in raw.split(","): - if "=" in pair: - key, _, value = pair.partition("=") - headers[key.strip()] = value.strip() - return headers diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index ced65aa1ec3..ede9acc6d8f 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -29,15 +29,13 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, ) from opentelemetry.trace import Span, SpanKind, Tracer +from opentelemetry.util.re import parse_env_headers from litellm._version import version as litellm_version from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config from litellm.integrations.otel.model.semconv import LiteLLM from litellm.integrations.otel.model.spans import LiteLLMSpanKind -# Re-exported so ``providers.parse_headers`` remains a stable entry point. -from litellm.integrations.otel.model.utils import parse_headers as parse_headers - if TYPE_CHECKING: from opentelemetry.metrics import Meter from opentelemetry.sdk.metrics.export import MetricReader @@ -119,6 +117,23 @@ def _otlp_traces_endpoint(endpoint: str | None) -> str | None: return endpoint + "/v1/traces" +def parse_headers(raw: str | None) -> dict[str, str]: + """Parse an OTLP ``"k=v,k=v"`` header string into a dict. + + ``OTEL_EXPORTER_OTLP_HEADERS`` is W3C Baggage encoded per the OTLP spec, so + values are percent-decoded: a vendor that documents + ``Authorization=Basic%20`` (Grafana Cloud does, because a bare space + is not representable there) has to reach the exporter as ``Basic ``, + not with a literal ``%20`` that the backend rejects as malformed. The SDK's + own parser is used so litellm decodes exactly what the OTLP exporters do + when they read the env var themselves; ``liberal`` keeps values that are not + percent-encoded (``Authorization=Bearer ``) working unchanged. + """ + if not raw: + return {} + return dict(parse_env_headers(raw, liberal=True)) + + def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter: kind = (spec.kind or "console").lower() factory = _EXPORTER_FACTORIES.get(kind) @@ -191,6 +206,13 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader": ``console`` (and any unrecognized kind) exports to the console; ``otlp_http`` and ``otlp_grpc`` export over OTLP with the configured endpoint/headers. The reader exports on a 5s period, matching v1. + + Histograms keep the SDK's default cumulative temporality. Prometheus-backed + OTLP receivers (Grafana Cloud / Mimir, and the Prometheus OTLP endpoint) + reject delta histograms outright with ``invalid temporality and type + combination``, which drops the whole metric batch, while backends that + prefer delta still accept cumulative. The enterprise billing exporter + already relies on the same default. """ from opentelemetry.sdk.metrics.export import ( ConsoleMetricExporter, @@ -202,18 +224,12 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader": from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( OTLPMetricExporter as HTTPMetricExporter, ) - from opentelemetry.sdk.metrics import Histogram - from opentelemetry.sdk.metrics.export import AggregationTemporality exporter: Any = HTTPMetricExporter( endpoint=_otlp_metrics_endpoint(config.endpoint), headers=parse_headers(config.headers), - preferred_temporality={Histogram: AggregationTemporality.DELTA}, ) elif kind in ("otlp_grpc", "grpc"): - from opentelemetry.sdk.metrics import Histogram - from opentelemetry.sdk.metrics.export import AggregationTemporality - try: from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( OTLPMetricExporter as GRPCMetricExporter, @@ -227,7 +243,6 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader": exporter = GRPCMetricExporter( endpoint=config.endpoint, headers=parse_headers(config.headers), - preferred_temporality={Histogram: AggregationTemporality.DELTA}, ) else: exporter = ConsoleMetricExporter() diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 5191414edeb..d856d6871a3 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -451,6 +451,30 @@ def test_parse_headers(): assert providers.parse_headers("no-equals") == {} +def test_parse_headers_percent_decodes_values(): + """A percent-encoded OTLP header value reaches the exporter decoded. + + ``OTEL_EXPORTER_OTLP_HEADERS`` is W3C Baggage encoded, and Grafana Cloud + documents ``Authorization=Basic%20``. Forwarding the literal ``%20`` + makes the backend reject the export as a malformed credential. + """ + token = "MTMzNzc4MzpnbGNfZXlKdklqb2lNVEl6TkNJPQ==" + assert providers.parse_headers(f"Authorization=Basic%20{token}") == {"authorization": f"Basic {token}"} + assert providers.parse_headers("x-scope-orgid=team%20a") == {"x-scope-orgid": "team a"} + + +def test_parse_headers_keeps_unencoded_values_working(): + """Values that are not percent-encoded keep parsing unchanged. + + Vendors that document a bare space, and litellm's own presets, must survive + the switch to the spec-compliant parser. Base64 padding also means a value + can contain ``=``, so only the first one may split the pair. + """ + assert providers.parse_headers("Authorization=Bearer sk-123") == {"authorization": "Bearer sk-123"} + assert providers.parse_headers("api_key=abc,space_id=xyz") == {"api_key": "abc", "space_id": "xyz"} + assert providers.parse_headers("api_key=YWJjZA==") == {"api_key": "YWJjZA=="} + + def test_otlp_traces_endpoint_normalization(): norm = providers._otlp_traces_endpoint # A base endpoint gets the signal path appended (the common OTLP env shape). @@ -487,6 +511,24 @@ def test_build_span_exporter_variants(): assert "OTLPSpanExporter" in type(http_exporter).__name__ +def test_otlp_metric_exporter_uses_cumulative_histogram_temporality(): + """Histograms must export as cumulative, not delta. + + Prometheus-backed OTLP receivers (Grafana Cloud / Mimir) reject delta + histograms with ``invalid temporality and type combination`` and drop the + entire metric batch, so a delta default silently loses every GenAI metric. + """ + from opentelemetry.sdk.metrics import Histogram + from opentelemetry.sdk.metrics.export import AggregationTemporality + + reader = providers.build_metric_reader( + OpenTelemetryV2Config(exporter="otlp_http", endpoint="http://h:4318") + ) + temporality = reader._exporter._preferred_temporality # noqa: SLF001 # exporter exposes no public accessor + + assert temporality[Histogram] is AggregationTemporality.CUMULATIVE + + def test_otlp_logs_endpoint_normalization(): norm = providers._otlp_logs_endpoint # A base endpoint gets the signal path appended (the common OTLP env shape). diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 02954578644..41c02501acc 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -2120,9 +2120,9 @@ def test_valid_metric_filter_records_six_metrics(monkeypatch): assert _emitted_metric_names(reader) == { "gen_ai.client.operation.duration", "gen_ai.client.token.usage", - "gen_ai.client.token.cost", - "gen_ai.client.response.time_to_first_token", - "gen_ai.client.response.time_per_output_token", + "gen_ai.usage.cost", + "gen_ai.server.time_to_first_token", + "gen_ai.server.time_per_output_token", "gen_ai.client.response.duration", } diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py b/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py index 29067f91b5a..b2d89053ba3 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py @@ -39,9 +39,9 @@ from litellm.integrations.otel.plumbing.providers import ( # noqa: E402 OPERATION_DURATION = "gen_ai.client.operation.duration" TOKEN_USAGE = "gen_ai.client.token.usage" -TOKEN_COST = "gen_ai.client.token.cost" -TIME_TO_FIRST_TOKEN = "gen_ai.client.response.time_to_first_token" -TIME_PER_OUTPUT_TOKEN = "gen_ai.client.response.time_per_output_token" +TOKEN_COST = "gen_ai.usage.cost" +TIME_TO_FIRST_TOKEN = "gen_ai.server.time_to_first_token" +TIME_PER_OUTPUT_TOKEN = "gen_ai.server.time_per_output_token" RESPONSE_DURATION = "gen_ai.client.response.duration" ALL_METRICS = frozenset( diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 7ffd09b931f..05205cb76f2 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -412,6 +412,40 @@ class TestOpenTelemetryProviderInitialization(unittest.TestCase): current_provider is existing_provider ), "Existing TracerProvider should be respected and not overridden" + @patch.dict( + os.environ, {"LITELLM_OTEL_INTEGRATION_ENABLE_METRICS": "true"}, clear=True + ) + def test_init_metrics_creates_instruments_under_their_published_names(self): + """ + The v1 engine's instrument names are a public contract. + + Every name here is what a backend queries: four are GenAI semantic + conventions and gen_ai.usage.cost is the name backends query for spend. + A rename is breaking for anyone charting them, so it has to be a + deliberate edit to the shared Metric constants and to this list, never + a silent drift between the v1 and v2 engines. + """ + from opentelemetry import metrics + + metrics.set_meter_provider(MeterProvider(metric_readers=[InMemoryMetricReader()])) + otel_integration = OpenTelemetry(config=OpenTelemetryConfig.from_env()) + + assert { + otel_integration._operation_duration_histogram.name, + otel_integration._token_usage_histogram.name, + otel_integration._cost_histogram.name, + otel_integration._time_to_first_token_histogram.name, + otel_integration._time_per_output_token_histogram.name, + otel_integration._response_duration_histogram.name, + } == { + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + "gen_ai.usage.cost", + "gen_ai.server.time_to_first_token", + "gen_ai.server.time_per_output_token", + "gen_ai.client.response.duration", + } + @patch.dict( os.environ, {"LITELLM_OTEL_INTEGRATION_ENABLE_METRICS": "true"}, clear=True ) From 5182dfa66bd0c8c8afb3e56f7cf9ac2ec87c6ee5 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 29 Jul 2026 14:20:39 -0700 Subject: [PATCH 10/11] test(e2e): remove the Presidio guardrail suite (#35129) Drops tests/e2e/guardrails/test_presidio_guardrail_e2e.py and the PresidioParamsBody it was the only caller of. Both cases were red on most stage runs between 07-25 and 07-29: pre_call failed 6 of 11 runs, post_call 6 of 11, with post_call reporting the raw address reaching the caller while apply_to_output was set. The cause was propagation, not masking. GuardrailsClient.register() posts /guardrails and returns immediately with no readiness wait, unlike ProxyClient._await_model_servable or GuardrailsClient._await_team, and the data plane only picks a new guardrail up on its next periodic DB sync. Calls issued before that sync pass the raw value through. #34833 has since made both cases poll to the deadline, and on the current build each masks on the first attempt, so the suite is expected to be green now; it is being removed because it spends real provider money on every retry and because a pod replaced mid-poll still reproduces the old failure. The three guardrail.presidio.* rows stay in coverage_registry/guardrail.yaml and go uncovered on purpose, so Presidio reads as a tier-P0 gap in Grafana rather than dropping out of the denominator. --- tests/e2e/guardrails/guardrails_client.py | 11 -- .../guardrails/test_presidio_guardrail_e2e.py | 141 ------------------ 2 files changed, 152 deletions(-) delete mode 100644 tests/e2e/guardrails/test_presidio_guardrail_e2e.py diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 5a54a4f0bbc..93861d19922 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -63,16 +63,6 @@ class OpenAIModerationParamsBody(GuardrailParamsBase): model: str | None = None -class PresidioParamsBody(GuardrailParamsBase): - guardrail: Literal["presidio"] = "presidio" - presidio_analyzer_api_base: str | None = None - presidio_anonymizer_api_base: str | None = None - # apply_to_output masks PII the model itself emitted, which also makes the - # guardrail run post_call. logging_only masks what the proxy logs. - apply_to_output: bool | None = None - logging_only: bool | None = None - - class BlockCodeExecutionParamsBody(GuardrailParamsBase): guardrail: Literal["block_code_execution"] = "block_code_execution" @@ -81,7 +71,6 @@ GuardrailParamsBody = ( ContentFilterParamsBody | BedrockGuardrailParamsBody | OpenAIModerationParamsBody - | PresidioParamsBody | BlockCodeExecutionParamsBody ) diff --git a/tests/e2e/guardrails/test_presidio_guardrail_e2e.py b/tests/e2e/guardrails/test_presidio_guardrail_e2e.py deleted file mode 100644 index 9742dfc6ae7..00000000000 --- a/tests/e2e/guardrails/test_presidio_guardrail_e2e.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Live e2e: the built-in Presidio PII guardrail masks PII on the request and on -the model output. - -Presidio replaces detected PII with `` placeholders (e.g. -``) via a real analyzer + anonymizer. Two modes are checked -independently, each opted into per request (default_on=False) so it never touches -unrelated traffic: - -- pre_call: the prompt is anonymized before it reaches the model, so a - repeat-verbatim request comes back with the placeholder, never the raw email -- post_call (apply_to_output): PII the model itself emits is masked on the way - out, so the caller never receives the raw value the model produced - -A third mode, logging_only, is not covered here: the raw email stayed in the OTEL -span's `gen_ai.input.messages` on every attempt over a full poll deadline while -these two modes masked correctly, so that cell is tracked in LIT-4841 rather than -asserted against known-failing behavior. - -Analyzer/anonymizer bases come from PRESIDIO_ANALYZER_API_BASE / -PRESIDIO_ANONYMIZER_API_BASE (compose provides the in-network hosts; point them at -locally published container ports for a host run). The chat backend is a gemini -deployment created for the test. -""" - -from __future__ import annotations - -import os -import time -from collections.abc import Callable - -import pytest - -from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker -from e2e_http import unwrap -from guardrails_client import GuardrailMode, GuardrailsClient, PresidioParamsBody -from lifecycle import ResourceManager -from models import ChatResponse - -pytestmark = pytest.mark.e2e - -RAW_EMAIL = "alice.example.person@example.com" -PLACEHOLDER = "" - -ECHO_REQUEST = f"Repeat the following text back exactly, verbatim, with no changes: My email is {RAW_EMAIL}" -EMIT_REQUEST = f"Output exactly this one line and nothing else: Please contact {RAW_EMAIL} today" - - -def _content(response: ChatResponse) -> str: - if not response.choices: - return "" - message = response.choices[0].message - return (message.content if message else None) or "" - - -def _presidio_params( - mode: GuardrailMode, *, apply_to_output: bool = False, logging_only: bool = False -) -> PresidioParamsBody: - analyzer = os.environ["PRESIDIO_ANALYZER_API_BASE"] - anonymizer = os.environ["PRESIDIO_ANONYMIZER_API_BASE"] - return PresidioParamsBody( - mode=mode, - default_on=False, - presidio_analyzer_api_base=analyzer, - presidio_anonymizer_api_base=anonymizer, - apply_to_output=apply_to_output, - logging_only=logging_only, - ) - - -def _poll_until_masked(call: Callable[[], str]) -> str: - """Retry a call until the guardrail masks its PII, returning the last content. - - Registering a guardrail is a control-plane write; the data-plane worker that - serves /chat/completions only picks it up on its next periodic DB sync (~30s - in proxy_server.py), so a call issued the instant after the create runs - against a worker that has no guardrail yet and passes the raw value through. - That is in-flight propagation, not a masking failure. Polling to the deadline - waits it out, so the assertions that follow judge the synced state; if the - mask never lands the last unmasked content is returned and they still fail. - """ - deadline = time.monotonic() + POLL_TIMEOUT - last = call() - while time.monotonic() < deadline: - if PLACEHOLDER in last and RAW_EMAIL not in last: - return last - time.sleep(POLL_INTERVAL) - last = call() - return last - - -class TestPresidioGuardrail: - @pytest.mark.covers( - "guardrail.presidio.pre_call.masks", - exercised_on=["chat_completions"], - ) - def test_pre_call_masks_pii_before_the_model_sees_it( - self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str - ) -> None: - model = client.create_backend_model(resources, prefix="e2e-presidio-pre") - name = f"e2e-presidio-pre-{unique_marker()}" - guardrail_id = client.register(name, _presidio_params("pre_call")) - resources.defer(lambda: client.delete_guardrail(guardrail_id)) - - echoed = _poll_until_masked( - lambda: _content( - unwrap(client.chat(scoped_key, model, ECHO_REQUEST, guardrails=[name], max_tokens=128)) - ) - ) - assert RAW_EMAIL not in echoed, ( - "pre_call masking must strip the raw email before the model sees it, but the " - f"model echoed it back: {echoed[:300]!r}" - ) - assert PLACEHOLDER in echoed, ( - "the model should have echoed the masked placeholder the guardrail substituted, " - f"got: {echoed[:300]!r}" - ) - - @pytest.mark.covers( - "guardrail.presidio.post_call.masks", - exercised_on=["chat_completions"], - ) - def test_post_call_masks_pii_in_model_output( - self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str - ) -> None: - model = client.create_backend_model(resources, prefix="e2e-presidio-post") - name = f"e2e-presidio-post-{unique_marker()}" - guardrail_id = client.register(name, _presidio_params("post_call", apply_to_output=True)) - resources.defer(lambda: client.delete_guardrail(guardrail_id)) - - out = _poll_until_masked( - lambda: _content( - unwrap(client.chat(scoped_key, model, EMIT_REQUEST, guardrails=[name], max_tokens=128)) - ) - ) - assert RAW_EMAIL not in out, ( - "post_call masking must strip PII the model emitted, but the raw email reached the " - f"caller: {out[:300]!r}" - ) - assert PLACEHOLDER in out, ( - f"the masked placeholder should replace the model's PII output, got: {out[:300]!r}" - ) From 551e5d097c11f08fd2400a25a651b1844fcf89c2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:22:22 -0700 Subject: [PATCH 11/11] feat(dashscope): add qwen3.7-plus and qwen3.7-max to the model cost map (#35123) * feat(dashscope): add qwen3.7-plus and qwen3.7-max to the model cost map Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore: limit backup cost map diff to the new dashscope entries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost_calculator): adjust tier-only alias assertion for mapped qwen3.7-plus Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(dashscope): drop redundant cost map pinning tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost_calculator): point tier-only alias check at an unmapped model Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: shivam Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 50 +++++++++++++++++++ model_prices_and_context_window.json | 50 +++++++++++++++++++ tests/test_litellm/test_cost_calculator.py | 10 ++-- 3 files changed, 106 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2cef600ea32..87d9b6afc18 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13567,6 +13567,56 @@ } ] }, + "dashscope/qwen3.7-max": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "dashscope", + "max_input_tokens": 991808, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 7.5e-06, + "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "dashscope/qwen3.7-plus": { + "litellm_provider": "dashscope", + "max_input_tokens": 991808, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tiered_pricing": [ + { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "range": [ + 0, + 256000.0 + ] + }, + { + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 4.8e-06, + "range": [ + 256000.0, + 1000000.0 + ] + } + ] + }, "dashscope/qwq-plus": { "input_cost_per_token": 8e-07, "litellm_provider": "dashscope", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index db28118d52b..0edd3bd5f30 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13567,6 +13567,56 @@ } ] }, + "dashscope/qwen3.7-max": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "dashscope", + "max_input_tokens": 991808, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 7.5e-06, + "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "dashscope/qwen3.7-plus": { + "litellm_provider": "dashscope", + "max_input_tokens": 991808, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tiered_pricing": [ + { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "range": [ + 0, + 256000.0 + ] + }, + { + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 4.8e-06, + "range": [ + 256000.0, + 1000000.0 + ] + } + ] + }, "dashscope/qwq-plus": { "input_cost_per_token": 8e-07, "litellm_provider": "dashscope", diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 276ee96ed65..e6ae1f85cfd 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1014,9 +1014,9 @@ def test_tiered_pricing_only_deployment_selects_router_model_id(): router = Router( model_list=[ { - "model_name": "qwen-3.7-plus", + "model_name": "qwen-tier-only", "litellm_params": { - "model": "dashscope/qwen3.7-plus", + "model": "dashscope/qwen-tier-only-test", "api_key": "sk-fake", }, "model_info": { @@ -1037,10 +1037,12 @@ def test_tiered_pricing_only_deployment_selects_router_model_id(): assert entry.get("input_cost_per_token") is None assert entry.get("tiered_pricing") is not None # The stripped shared alias must not carry tiered pricing. - assert litellm.model_cost["dashscope/qwen3.7-plus"].get("tiered_pricing") is None + assert ( + litellm.model_cost["dashscope/qwen-tier-only-test"].get("tiered_pricing") is None + ) selected = _select_model_name_for_cost_calc( - model="dashscope/qwen3.7-plus", + model="dashscope/qwen-tier-only-test", completion_response=None, custom_pricing=True, custom_llm_provider="dashscope",