From 205cfd8044269ac43c2a91c6cafac36146e29171 Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Sat, 24 Oct 2026 16:47:15 -0700 Subject: [PATCH] fix(proxy): restore Claude gateway model discovery (#48204) (#48221) * fix(proxy): restore Claude gateway model discovery (#48204) * test(proxy): cover gateway discovery after OAuth sign-in (#48204) * refactor(proxy): share model listing route policy (#48204) * test(proxy): isolate SSO settings environment (#48204) --- .../proxy/auth/route_checks.py | 4 +- litellm/proxy/_types.py | 11 +-- .../anthropic_endpoints/gateway_endpoints.py | 8 ++ litellm/proxy/auth/auth_checks.py | 7 +- litellm/proxy/auth/route_checks.py | 4 +- .../test_claude_code_gateway_oauth.py | 27 +++++- .../auth/test_managed_authorization.py | 2 +- .../test_gateway_endpoints.py | 92 +++++++++++++++++++ .../test_proxy_setting_endpoints.py | 8 ++ 9 files changed, 148 insertions(+), 15 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/auth/route_checks.py b/enterprise/litellm_enterprise/proxy/auth/route_checks.py index fc57292a8d2..479537c4a90 100644 --- a/enterprise/litellm_enterprise/proxy/auth/route_checks.py +++ b/enterprise/litellm_enterprise/proxy/auth/route_checks.py @@ -2,6 +2,8 @@ import os from fastapi import HTTPException, status +from litellm.proxy._types import MODEL_LIST_ROUTES + class EnterpriseRouteChecks: @staticmethod @@ -43,7 +45,7 @@ class EnterpriseRouteChecks: # Routes that should remain accessible even when LLM API endpoints are disabled. # These are read-only model listing routes needed by the Admin UI. - LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"] + LLM_API_EXEMPT_ROUTES = MODEL_LIST_ROUTES @staticmethod def should_call_route(route: str): diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 3a3306dfdc0..2b01e02bb06 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -219,6 +219,8 @@ from litellm.types.proxy.auth.user_roles import ( # noqa: E402 # re-export aft KeyManagementRoutes as KeyManagementRoutes, # noqa: PLC0414 # public re-export ) +MODEL_LIST_ROUTES: Final = ("/models", "/v1/models", "/claude_code_gateway/v1/models") + class LiteLLMRoutes(enum.Enum): openai_route_names = [ @@ -307,8 +309,7 @@ class LiteLLMRoutes(enum.Enum): "/threads/{thread_id}/runs", "/v1/threads/{thread_id}/runs", # models - "/models", - "/v1/models", + *MODEL_LIST_ROUTES, # token counter "/utils/token_counter", "/utils/model_info", @@ -577,8 +578,7 @@ class LiteLLMRoutes(enum.Enum): "/health/services", "/key/list", "/user/filter/ui", - "/models", - "/v1/models", + *MODEL_LIST_ROUTES, "/sso/get/ui_settings", "/get/user_banner", "/get/latest_release_info", @@ -760,8 +760,7 @@ class LiteLLMRoutes(enum.Enum): "/model/info", "/v2/model/info", "/v2/key/info", - "/models", - "/v1/models", + *MODEL_LIST_ROUTES, "/global/spend", "/global/spend/logs", "/global/spend/keys", diff --git a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py index 4c4155379cd..ba3307bc1ad 100644 --- a/litellm/proxy/anthropic_endpoints/gateway_endpoints.py +++ b/litellm/proxy/anthropic_endpoints/gateway_endpoints.py @@ -58,6 +58,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg ) from litellm.proxy.management_endpoints.sso_helper_utils import CLI_SSO_SESSIONS_TARGET from litellm.proxy.management_endpoints.ui_sso import CliSsoTeamDetail +from litellm.proxy.proxy_server import model_list from litellm.types.llms.base import LiteLLMBaseModel GATEWAY_PREFIX: Final = "/claude_code_gateway" @@ -158,6 +159,13 @@ router: Final = APIRouter( _GATEWAY_ENABLED: Final = (Depends(ensure_gateway_enabled),) _AUTHENTICATED: Final = (Depends(user_api_key_auth),) +router.add_api_route( + "/v1/models", + model_list, + methods=["GET"], + dependencies=_GATEWAY_ENABLED, + include_in_schema=False, +) router.add_api_route( "/v1/messages", anthropic_response, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 2c6768b8423..06c53afcf2c 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -47,6 +47,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.models.project import LiteLLM_ProjectTable from litellm.proxy._types import ( + MODEL_LIST_ROUTES, RBAC_ROLES, UI_TEAM_ID, CallInfo, @@ -910,8 +911,7 @@ def _global_proxy_budget_check(global_proxy_spend: float | None, skip_budget_che and not skip_budget_checks and global_proxy_spend is not None and RouteChecks.is_llm_api_route(route=route) - and route != "/v1/models" - and route != "/models" + and route not in MODEL_LIST_ROUTES ): if math.isfinite(litellm.max_budget) and global_proxy_spend > litellm.max_budget: raise litellm.BudgetExceededError( @@ -1028,8 +1028,7 @@ async def check_tools_allowlist( # budget cannot reach side-effectful routes like /health/services (Slack/email/webhook). See #27923. MODEL_DISCOVERY_ROUTES: Final = frozenset( { - "/v1/models", - "/models", + *MODEL_LIST_ROUTES, "/model/info", "/v1/model/info", "/v2/model/info", diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 3ae671d40ac..c402f111563 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -7,6 +7,7 @@ from fastapi import HTTPException, Request, status from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ( + MODEL_LIST_ROUTES, UI_TEAM_ID, CommonProxyErrors, KeyManagementRoutes, @@ -93,8 +94,7 @@ _SELF_SCOPED_CREDENTIAL_ROUTES: Final = ( _UNBILLED_ROUTES: Final[frozenset[str]] = frozenset( { - "/models", - "/v1/models", + *MODEL_LIST_ROUTES, "/utils/token_counter", "/utils/transform_request", "/responses/input_tokens", diff --git a/tests/integration/authorization/test_claude_code_gateway_oauth.py b/tests/integration/authorization/test_claude_code_gateway_oauth.py index 384bf3a2918..a80eeacad46 100644 --- a/tests/integration/authorization/test_claude_code_gateway_oauth.py +++ b/tests/integration/authorization/test_claude_code_gateway_oauth.py @@ -15,6 +15,7 @@ from urllib.parse import parse_qs, urlencode, urlsplit import httpx import pytest import yaml +from anthropic import Anthropic from anthropic.types import Message from integration._support import claude_code as cc from integration._support.client import Gateway, eventually, gateway_from_environment @@ -397,7 +398,31 @@ def test_device_authorization_polls_complete_sso_and_mints_a_model_restricted_be "anthropic-beta": cc.CLI_BETA, "x-claude-code-session-id": f"oauth-{user_id}", } - request_body: Final = {**request_template, "model": model_a} + standard_listing: Final = enabled_gateway.client.get("/v1/models", headers=bearer_headers) + gateway_listing: Final = enabled_gateway.client.get( + "/claude_code_gateway/v1/models", headers=bearer_headers + ) + assert standard_listing.status_code == 200, standard_listing.text + assert gateway_listing.status_code == 200, gateway_listing.text + assert gateway_listing.json() == standard_listing.json() + with Anthropic( + base_url=f"{str(enabled_gateway.client.base_url).rstrip('/')}/claude_code_gateway", + auth_token=access_token.access_token, + ) as sdk: + discovered_models: Final = tuple(sdk.models.list()) + assert {model.id for model in discovered_models} == {model_a, model_c} + discovered_model: Final = next(model for model in discovered_models if model.id == model_a) + openai_listing: Final = enabled_gateway.client.get( + "/claude_code_gateway/v1/models", + headers={"authorization": f"Bearer {access_token.access_token}"}, + ) + assert openai_listing.status_code == 200, openai_listing.text + assert openai_listing.json()["object"] == "list" + assert {model["id"] for model in openai_listing.json()["data"]} == {model_a, model_c} + unauthenticated: Final = enabled_gateway.client.get("/claude_code_gateway/v1/models") + assert unauthenticated.status_code == 401, unauthenticated.text + assert upstream.drain() == () + request_body: Final = {**request_template, "model": discovered_model.id} expected_message: Final = _MESSAGE.validate_python( { **_JSON_OBJECT.validate_json( diff --git a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py index fb676ae9a61..2b682fda982 100644 --- a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py +++ b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py @@ -588,6 +588,6 @@ def test_registered_inference_routes_have_an_explicit_managed_access_decision(ro "/vector_stores", "/vector_store/", "/search", "/containers", "/skills", "/claude-code/", "/interactions", "/agents", "/responses/{", "/responses/input_tokens", "/realtime/client_secrets", "/realtime/calls", "/realtime/transcription_sessions", - )) or normalized in ("/models", "/cursor/models", "/cursor/v1/models") + )) or normalized in ("/models", "/cursor/models", "/cursor/v1/models", "/claude_code_gateway/v1/models") concrete: Final = route.split("?")[0].replace("{model}", "model").replace("{model_name:path}", "model") assert managed_agent_route_allowed(concrete, None) is not unsupported, route diff --git a/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py index 8ac0e2a6f6d..8c2d8e4f157 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_gateway_endpoints.py @@ -813,3 +813,95 @@ def test_messages_gated_by_enable_flag(): with _gateway_env(enabled=False) as (client, _): resp = client.post("/claude_code_gateway/v1/messages", json={"model": "claude-sonnet-4-5", "messages": []}) assert resp.status_code == 404 + + +@pytest.mark.parametrize("headers", [{}, {"anthropic-version": "2023-06-01"}]) +def test_gateway_model_discovery_preserves_grants_and_format( + monkeypatch: pytest.MonkeyPatch, headers: dict[str, str] +) -> None: + from litellm import Router + from litellm.proxy import proxy_server + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + + model_router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": f"openai/{name}", "api_key": "local-placeholder"}, + } + for name in ("discovery-example", "restricted-example") + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", model_router) + monkeypatch.setattr(proxy_server, "llm_model_list", model_router.model_list) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {"enable_claude_code_gateway": True}) + caller: Final = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + models=["discovery-example"], + ) + app: Final = FastAPI() + app.include_router(proxy_server.router) + app.include_router(gateway_endpoints.router) + app.dependency_overrides[gateway_endpoints.user_api_key_auth] = lambda: caller + with TestClient(app) as client: + standard: Final = client.get("/v1/models", headers=headers) + gateway: Final = client.get("/claude_code_gateway/v1/models", headers=headers) + assert standard.status_code == 200, standard.text + assert gateway.status_code == 200, gateway.text + assert gateway.json() == standard.json() + assert [model["id"] for model in gateway.json()["data"]] == ["discovery-example"] + invalid_scope: Final = client.get("/claude_code_gateway/v1/models?scope=invalid", headers=headers) + assert invalid_scope.status_code == 400 + + +def test_gateway_model_discovery_requires_authentication() -> None: + with _gateway_env(real_auth=True) as (client, _), pytest.raises(ProxyException) as exc_info: + client.get("/claude_code_gateway/v1/models") + assert exc_info.value.code == "401" + + +@pytest.mark.parametrize("settings", [{}, {"enable_claude_code_gateway": False}]) +def test_gateway_model_discovery_disabled(monkeypatch: pytest.MonkeyPatch, settings: dict[str, bool]) -> None: + from litellm.proxy import proxy_server + + with _gateway_env(real_auth=True) as (client, _): + monkeypatch.setattr(proxy_server, "general_settings", settings) + response: Final = client.get( + "/claude_code_gateway/v1/models", headers={"Authorization": f"Bearer {_MASTER_KEY}"} + ) + assert response.status_code == 404, response.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["/models", "/v1/models", "/claude_code_gateway/v1/models"]) +async def test_gateway_model_discovery_preserves_route_exemptions(monkeypatch: pytest.MonkeyPatch, route: str) -> None: + from litellm_enterprise.proxy.auth.route_checks import EnterpriseRouteChecks + + import litellm + from litellm.proxy._types import LiteLLM_TeamTable, UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import common_checks + from litellm.proxy.auth.route_checks import RouteChecks + + monkeypatch.setattr(litellm, "max_budget", 1.0) + result: Final = await common_checks( + request_body={}, + team_object=LiteLLM_TeamTable(team_id="test-team", spend=150.0, max_budget=100.0), + user_object=None, + end_user_object=None, + global_proxy_spend=2.0, + general_settings={}, + route=route, + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=UserAPIKeyAuth(team_id="test-team"), + request=MagicMock(), + ) + assert result is True + assert RouteChecks.is_unbilled_route(route) + assert RouteChecks.is_llm_api_route(route) + monkeypatch.setenv("DISABLE_LLM_API_ENDPOINTS", "true") + monkeypatch.setenv("DISABLE_ADMIN_ENDPOINTS", "true") + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + EnterpriseRouteChecks.should_call_route(route) diff --git a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 0d558f206d8..57b68e6db05 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -1,6 +1,8 @@ import json import os +from collections.abc import Iterator from typing import Final +from unittest.mock import patch import pytest from fastapi.testclient import TestClient @@ -181,6 +183,12 @@ def mock_auth(): app.dependency_overrides.pop(user_api_key_auth, None) +@pytest.fixture(autouse=True) +def isolated_environment() -> Iterator[None]: + with patch.dict(os.environ): + yield + + @pytest.fixture(autouse=True) def fresh_settings_store(monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy import proxy_server