diff --git a/litellm/proxy/common_utils/codex_catalog_source.py b/litellm/proxy/common_utils/codex_catalog_source.py index dcc806c592e..a2389de79ed 100644 --- a/litellm/proxy/common_utils/codex_catalog_source.py +++ b/litellm/proxy/common_utils/codex_catalog_source.py @@ -346,10 +346,11 @@ class CodexCatalogSource: ) -> bytes | _FetchFailed: client: Final = self._client if self._client is not None else _default_client() try: - response: Final = await client.get( - url, headers=headers, timeout=self._timeout, max_response_bytes=CODEX_CATALOG_MAX_BYTES + response: Final = await asyncio.wait_for( + client.get(url, headers=headers, timeout=self._timeout, max_response_bytes=CODEX_CATALOG_MAX_BYTES), + self._timeout, ) - except (httpx.HTTPError, httpx.InvalidURL, HTTPResponseLimitError, ValueError) as error: + except (httpx.HTTPError, httpx.InvalidURL, HTTPResponseLimitError, ValueError, asyncio.TimeoutError) as error: return _FetchFailed(reason=f"{type(error).__name__} fetching {url}") if response.status_code != 200: return _FetchFailed(reason=f"HTTP {response.status_code} from {url}") diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 03b84e690b5..8de5118ff01 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -591,10 +591,21 @@ _DEPLOYMENT_BASE_PARAMS: Final = ("api_base", "base_url") def _is_openai_host(url: object) -> bool: - host: Final = urlparse(url).hostname if isinstance(url, str) else None + if not isinstance(url, str): + return False + try: + host: Final = urlparse(url).hostname + except ValueError: + return False return host == OPENAI_API_HOST +def _every_call_goes_through_a_litellm_proxy() -> bool: + import litellm # noqa: PLC0415 # litellm imports this module while it is initializing + + return litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default() is True + + def _is_openai_origin(bases: Sequence[object]) -> bool: """Whether a deployment's OpenAI requests go to OpenAI itself, so its key may go to OpenAI's catalog endpoint: every `api_base` and `base_url` the deployment and its credential set, else the global @@ -637,17 +648,21 @@ def _deployment_provider(params: Mapping[str, object]) -> str: def _deployment_openai_key(deployment: DeploymentTypedDict) -> str | None: - """The key a deployment sends OpenAI, its named credential's values overriding its own the way the call - path merges them and ``OPENAI_API_KEY`` standing in when neither sets one, when every base either one sets - is OpenAI's.""" + """The key a deployment sends OpenAI, its own values winning over its named credential's the way the call + path fills only what the deployment lacks and ``OPENAI_API_KEY`` standing in when neither sets one, when every + base either one sets is OpenAI's and the call does not go through a LiteLLM proxy.""" params: Final = MappingProxyType[str, object](deployment["litellm_params"]) credential: Final = _credential_values(params.get("litellm_credential_name")) - effective: Final = MappingProxyType({**params, **credential}) + effective: Final = MappingProxyType({**credential, **params}) bases: Final = ( *(params.get(key) for key in _DEPLOYMENT_BASE_PARAMS), *(credential.get(key) for key in _DEPLOYMENT_BASE_PARAMS), ) - if _deployment_provider(effective) != "openai" or not _is_openai_origin(bases): + if ( + effective.get("use_litellm_proxy") is True + or _deployment_provider(effective) != "openai" + or not _is_openai_origin(bases) + ): return None key: Final = _resolved_key(effective.get("api_key")) return None if key is None or key.startswith(FEDRAMP_KEY_PREFIXES) else key @@ -656,6 +671,8 @@ def _deployment_openai_key(deployment: DeploymentTypedDict) -> str | None: def _resolved_key(api_key: object) -> str | None: if not isinstance(api_key, str) or not api_key: return os.getenv("OPENAI_API_KEY") or None + if api_key.startswith("oidc/"): + return None from litellm.secret_managers.main import ( get_secret_str, # noqa: PLC0415 # secret_managers.main imports litellm, which imports this module ) @@ -677,8 +694,8 @@ def codex_catalog_openai_key( OpenAI's own, ``OPENAI_API_KEY`` from the environment for one that sets no key. A key for an OpenAI-compatible server or a FedRAMP key (Codex sends those to ``gov.chatgpt.com``) never leaves for OpenAI's commercial catalog, a listing without an OpenAI deployment sends none, and no key is looked up - while the catalog stays local.""" - if uses_vendored_codex_catalog(): + while the catalog stays local or calls go through a LiteLLM proxy.""" + if uses_vendored_codex_catalog() or _every_call_goes_through_a_litellm_proxy(): return None registry: Final = llm_router.registry() if llm_router is not None else None deployments: Final = ( diff --git a/tests/unit/proxy/common_utils/test_codex_catalog_source.py b/tests/unit/proxy/common_utils/test_codex_catalog_source.py index a127bf7a25c..839179c245a 100644 --- a/tests/unit/proxy/common_utils/test_codex_catalog_source.py +++ b/tests/unit/proxy/common_utils/test_codex_catalog_source.py @@ -370,6 +370,27 @@ def test_a_refused_openai_fetch_falls_back_to_the_release_tag_on_github(status: assert tree.calls[0] == _openai_url(RELEASE) and sorted(tree.calls[1:]) == sorted(_tree(RELEASE)), tree.calls +def test_an_openai_answer_that_never_finishes_falls_back_to_the_release_tag_within_the_fetch_timeout() -> None: + class _NeverFinishingOpenAI(_RawTree): + async def get( + self, + url: str, + *, + headers: dict[str, str] | None = None, + timeout: float | None = None, + max_response_bytes: int | None = None, + ) -> httpx.Response: + if url == _openai_url(RELEASE): + self.calls.append(url) + await asyncio.Event().wait() + return await super().get(url, headers=headers, timeout=timeout, max_response_bytes=max_response_bytes) + + tree: Final = _NeverFinishingOpenAI(_tree(RELEASE)) + source: Final = CodexCatalogSource(client=tree, clock=_Clock(), timeout=0.05, response_deadline=30) + stock: Final = asyncio.run(asyncio.wait_for(source.stock(RELEASE, openai_api_key=OPENAI_KEY), timeout=5)) + assert stock.source == f"github rust-v{RELEASE}" and FRESH_SLUG in stock.models + + def test_an_empty_openai_url_setting_keeps_the_key_home_and_fetches_from_github( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index a12bd76b5fa..21f4ab2d3eb 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -976,7 +976,7 @@ def catalog_fetching(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False) -def _router_with(*litellm_params: Mapping[str, str]) -> litellm.Router: +def _router_with(*litellm_params: Mapping[str, str | bool]) -> litellm.Router: return litellm.Router( model_list=[ {"model_name": f"catalog-model-{index}", "litellm_params": dict(params)} @@ -1126,3 +1126,73 @@ def test_a_base_url_or_a_credential_naming_another_server_keeps_the_key_from_ope entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(4)] assert codex_catalog_openai_key(router, entries[:3]) is None assert codex_catalog_openai_key(router, entries) == "sk-openai-credential" + + +@pytest.mark.usefixtures("catalog_fetching") +def test_a_deployments_own_key_wins_over_its_named_credentials_key_as_on_the_call_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="openai-org", credential_values={"api_key": "sk-openai-credential"}, credential_info={} + ) + ], + ) + router: Final = _router_with( + {"model": "openai/gpt-5.5", "api_key": "sk-own-key", "litellm_credential_name": "openai-org"} + ) + assert codex_catalog_openai_key(router, [("catalog-model-0", "catalog-model-0")]) == "sk-own-key" + + +@pytest.mark.usefixtures("catalog_fetching") +def test_an_api_base_that_does_not_parse_counts_as_another_server(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + router: Final = _router_with( + {"model": "openai/gpt-5.5", "api_key": "sk-unparseable-base", "api_base": "https://[my-vllm]:8000/v1"}, + {"model": "openai/gpt-5.5", "api_key": "sk-plain-openai"}, + ) + entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(2)] + assert codex_catalog_openai_key(router, entries[:1]) is None + assert codex_catalog_openai_key(router, entries) == "sk-plain-openai" + + +@pytest.mark.usefixtures("catalog_fetching") +def test_an_oidc_key_reference_is_never_sent_as_the_catalog_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") + monkeypatch.setenv("CATALOG_OIDC_KEY", "sk-behind-the-reference") + router: Final = _router_with( + {"model": "openai/gpt-5.5", "api_key": "oidc/env/CATALOG_OIDC_KEY"}, + {"model": "openai/gpt-5.5", "api_key": "sk-plain-openai"}, + ) + entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(2)] + assert codex_catalog_openai_key(router, entries[:1]) is None + assert codex_catalog_openai_key(router, entries) == "sk-plain-openai" + + +@pytest.mark.usefixtures("catalog_fetching") +def test_a_deployment_sent_through_a_litellm_proxy_keeps_its_key_from_openai(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("USE_LITELLM_PROXY", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr(litellm, "use_litellm_proxy", False) + router: Final = _router_with( + {"model": "openai/gpt-5.5", "api_key": "sk-downstream-proxy", "use_litellm_proxy": True}, + {"model": "openai/gpt-5.5", "api_key": "sk-plain-openai"}, + ) + entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(2)] + assert codex_catalog_openai_key(router, entries[:1]) is None + assert codex_catalog_openai_key(router, entries) == "sk-plain-openai" + monkeypatch.setenv("USE_LITELLM_PROXY", "True") + assert codex_catalog_openai_key(router, entries) is None