From a8aa5f16fce2c978f2844b349900fb24938eaa55 Mon Sep 17 00:00:00 2001 From: Volodymyr Shkarban Date: Sun, 18 Oct 2026 04:40:44 +0300 Subject: [PATCH 1/3] fix(openai/evals): warn once that OpenAI shuts down the Evals API on 2026-11-30 Calls to api.openai.com/v1/evals fail after OpenAI's shutdown. Log a one-time warning from validate_environment for OpenAI's own host only; Azure and other OpenAI-compatible bases are unaffected. Refs #47212. --- litellm/llms/openai/evals/transformation.py | 26 ++++++++++++++ .../evals/test_openai_evals_transformation.py | 34 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/litellm/llms/openai/evals/transformation.py b/litellm/llms/openai/evals/transformation.py index 24de7f6d479..626843492d2 100644 --- a/litellm/llms/openai/evals/transformation.py +++ b/litellm/llms/openai/evals/transformation.py @@ -4,6 +4,7 @@ OpenAI Evals API configuration and transformations from collections.abc import Mapping from typing import Final +from urllib.parse import urlparse import httpx @@ -33,6 +34,30 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +OPENAI_EVALS_SHUTDOWN_DATE: Final = "2026-11-30" +OPENAI_EVALS_SHUTDOWN_MESSAGE: Final = ( + "OpenAI is shutting down the Evals API on 2026-11-30 (existing evals become read-only on 2026-10-31). " + "Requests to api.openai.com/v1/evals will fail after that date, and OpenAI names no API replacement. " + "See https://developers.openai.com/api/docs/deprecations" +) +_openai_evals_shutdown_warned = False + + +def _warn_openai_evals_shutdown_once(api_base: str | None) -> None: + """Log the shutdown notice once per process, only for OpenAI's own API host. + + Azure and other OpenAI-compatible bases keep their own lifecycles, so they don't get the warning. + """ + global _openai_evals_shutdown_warned + if _openai_evals_shutdown_warned: + return + host: Final = urlparse(api_base).hostname if api_base else "api.openai.com" + if host != "api.openai.com": + return + _openai_evals_shutdown_warned = True + verbose_logger.warning(OPENAI_EVALS_SHUTDOWN_MESSAGE) + + def _parsed_response_json(raw_response: httpx.Response) -> Mapping[str, object]: return raw_response.json() @@ -51,6 +76,7 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): ) if not bearer_token: raise ValueError("OPENAI_API_KEY is required for Evals API") + _warn_openai_evals_shutdown_once(params.api_base) headers["Authorization"] = f"Bearer {bearer_token}" headers["Content-Type"] = "application/json" return headers diff --git a/tests/unit/llms/openai/evals/test_openai_evals_transformation.py b/tests/unit/llms/openai/evals/test_openai_evals_transformation.py index f359f214f97..0486bce90e9 100644 --- a/tests/unit/llms/openai/evals/test_openai_evals_transformation.py +++ b/tests/unit/llms/openai/evals/test_openai_evals_transformation.py @@ -47,6 +47,40 @@ def test_validate_environment_sets_headers(config: OpenAIEvalsConfig): assert result["Content-Type"] == "application/json" +@pytest.fixture() +def reset_shutdown_warning(monkeypatch): + import litellm.llms.openai.evals.transformation as evals_transformation + + monkeypatch.setattr(evals_transformation, "_openai_evals_shutdown_warned", False) + return evals_transformation + + +def test_validate_environment_warns_once_about_openai_evals_shutdown( + config: OpenAIEvalsConfig, reset_shutdown_warning, monkeypatch +): + """OpenAI shuts down the Evals API on 2026-11-30; callers on api.openai.com are told once.""" + warnings: list[str] = [] + monkeypatch.setattr(reset_shutdown_warning.verbose_logger, "warning", lambda msg, *a, **k: warnings.append(msg)) + + for _ in range(3): + config.validate_environment(headers={}, litellm_params=GenericLiteLLMParams(api_key=_API_KEY)) + + assert len(warnings) == 1 + assert "2026-11-30" in warnings[0] + + +@pytest.mark.parametrize("api_base", ["https://my-resource.openai.azure.com", "http://localhost:4000"]) +def test_validate_environment_does_not_warn_for_other_api_bases( + config: OpenAIEvalsConfig, reset_shutdown_warning, monkeypatch, api_base: str +): + warnings: list[str] = [] + monkeypatch.setattr(reset_shutdown_warning.verbose_logger, "warning", lambda msg, *a, **k: warnings.append(msg)) + + config.validate_environment(headers={}, litellm_params=GenericLiteLLMParams(api_key=_API_KEY, api_base=api_base)) + + assert warnings == [] + + def test_validate_environment_requires_api_key(config: OpenAIEvalsConfig, monkeypatch): """Test that validate_environment raises error when no API key is provided""" From 9c5f1536b4fb67fa8c6406d4b980ad5739806348 Mon Sep 17 00:00:00 2001 From: Volodymyr Date: Sun, 18 Oct 2026 05:02:20 +0300 Subject: [PATCH 2/3] style(openai/evals): drop extra blank line after imports (ruff I001) --- litellm/llms/openai/evals/transformation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/llms/openai/evals/transformation.py b/litellm/llms/openai/evals/transformation.py index 626843492d2..d81e13fb54e 100644 --- a/litellm/llms/openai/evals/transformation.py +++ b/litellm/llms/openai/evals/transformation.py @@ -33,7 +33,6 @@ from litellm.types.llms.openai_evals import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders - OPENAI_EVALS_SHUTDOWN_DATE: Final = "2026-11-30" OPENAI_EVALS_SHUTDOWN_MESSAGE: Final = ( "OpenAI is shutting down the Evals API on 2026-11-30 (existing evals become read-only on 2026-10-31). " From 3fdf7d24d64fb2d810c689204adc9baddd492a43 Mon Sep 17 00:00:00 2001 From: Volodymyr Date: Sun, 18 Oct 2026 05:07:43 +0300 Subject: [PATCH 3/3] refactor(openai/evals): warn once via functools.cache instead of a global flag (PLW0603) --- litellm/llms/openai/evals/transformation.py | 17 +++++++++-------- .../evals/test_openai_evals_transformation.py | 7 ++++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/litellm/llms/openai/evals/transformation.py b/litellm/llms/openai/evals/transformation.py index d81e13fb54e..09b9f0f8e3a 100644 --- a/litellm/llms/openai/evals/transformation.py +++ b/litellm/llms/openai/evals/transformation.py @@ -3,6 +3,7 @@ OpenAI Evals API configuration and transformations """ from collections.abc import Mapping +from functools import cache from typing import Final from urllib.parse import urlparse @@ -39,7 +40,12 @@ OPENAI_EVALS_SHUTDOWN_MESSAGE: Final = ( "Requests to api.openai.com/v1/evals will fail after that date, and OpenAI names no API replacement. " "See https://developers.openai.com/api/docs/deprecations" ) -_openai_evals_shutdown_warned = False + + +@cache +def _log_openai_evals_shutdown() -> None: + """Log the shutdown notice; the cache makes it run once per process.""" + verbose_logger.warning(OPENAI_EVALS_SHUTDOWN_MESSAGE) def _warn_openai_evals_shutdown_once(api_base: str | None) -> None: @@ -47,14 +53,9 @@ def _warn_openai_evals_shutdown_once(api_base: str | None) -> None: Azure and other OpenAI-compatible bases keep their own lifecycles, so they don't get the warning. """ - global _openai_evals_shutdown_warned - if _openai_evals_shutdown_warned: - return host: Final = urlparse(api_base).hostname if api_base else "api.openai.com" - if host != "api.openai.com": - return - _openai_evals_shutdown_warned = True - verbose_logger.warning(OPENAI_EVALS_SHUTDOWN_MESSAGE) + if host == "api.openai.com": + _log_openai_evals_shutdown() def _parsed_response_json(raw_response: httpx.Response) -> Mapping[str, object]: diff --git a/tests/unit/llms/openai/evals/test_openai_evals_transformation.py b/tests/unit/llms/openai/evals/test_openai_evals_transformation.py index 0486bce90e9..718bf5f4bec 100644 --- a/tests/unit/llms/openai/evals/test_openai_evals_transformation.py +++ b/tests/unit/llms/openai/evals/test_openai_evals_transformation.py @@ -48,11 +48,12 @@ def test_validate_environment_sets_headers(config: OpenAIEvalsConfig): @pytest.fixture() -def reset_shutdown_warning(monkeypatch): +def reset_shutdown_warning(): import litellm.llms.openai.evals.transformation as evals_transformation - monkeypatch.setattr(evals_transformation, "_openai_evals_shutdown_warned", False) - return evals_transformation + evals_transformation._log_openai_evals_shutdown.cache_clear() + yield evals_transformation + evals_transformation._log_openai_evals_shutdown.cache_clear() def test_validate_environment_warns_once_about_openai_evals_shutdown(