Merge e379703385 into cfe0100e1f

litellm/llms/openai/evals/transformation.py

@@ -2,7 +2,9 @@
 OpenAI Evals API configuration and transformations
 """
 
+from functools import cache
 from typing import Final
+from urllib.parse import urlparse
 
 import httpx
 from pydantic import TypeAdapter
@@ -34,6 +36,29 @@ from litellm.types.utils import LlmProviders
 
 _DECODED_JSON: Final = TypeAdapter(object)
 
+OPENAI_EVALS_SHUTDOWN_DATE: Final = "2026-11-30"
+OPENAI_EVALS_SHUTDOWN_MESSAGE: Final = (
+    "OpenAI is shutting down the Evals API on 2026-11-30 (existing evals become read-only on 2026-10-31). "
+    "Requests to api.openai.com/v1/evals will fail after that date, and OpenAI names no API replacement. "
+    "See https://developers.openai.com/api/docs/deprecations"
+)
+
+
+@cache
+def _log_openai_evals_shutdown() -> None:
+    """Log the shutdown notice; the cache makes it run once per process."""
+    verbose_logger.warning(OPENAI_EVALS_SHUTDOWN_MESSAGE)
+
+
+def _warn_openai_evals_shutdown_once(api_base: str | None) -> None:
+    """Log the shutdown notice once per process, only for OpenAI's own API host.
+
+    Azure and other OpenAI-compatible bases keep their own lifecycles, so they don't get the warning.
+    """
+    host: Final = urlparse(api_base).hostname if api_base else "api.openai.com"
+    if host == "api.openai.com":
+        _log_openai_evals_shutdown()
+
 
 def _parsed_response_json(raw_response: httpx.Response) -> object:
     return _DECODED_JSON.validate_python(raw_response.json())
@@ -53,6 +78,7 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
         )
         if not bearer_token:
             raise ValueError("OPENAI_API_KEY is required for Evals API")
+        _warn_openai_evals_shutdown_once(params.api_base)
         headers["Authorization"] = f"Bearer {bearer_token}"
         headers["Content-Type"] = "application/json"
         return headers

tests/unit/llms/openai/evals/test_openai_evals_transformation.py

@@ -47,6 +47,41 @@ def test_validate_environment_sets_headers(config: OpenAIEvalsConfig):
     assert result["Content-Type"] == "application/json"
 
 
+@pytest.fixture()
+def reset_shutdown_warning():
+    import litellm.llms.openai.evals.transformation as evals_transformation
+
+    evals_transformation._log_openai_evals_shutdown.cache_clear()
+    yield evals_transformation
+    evals_transformation._log_openai_evals_shutdown.cache_clear()
+
+
+def test_validate_environment_warns_once_about_openai_evals_shutdown(
+    config: OpenAIEvalsConfig, reset_shutdown_warning, monkeypatch
+):
+    """OpenAI shuts down the Evals API on 2026-11-30; callers on api.openai.com are told once."""
+    warnings: list[str] = []
+    monkeypatch.setattr(reset_shutdown_warning.verbose_logger, "warning", lambda msg, *a, **k: warnings.append(msg))
+
+    for _ in range(3):
+        config.validate_environment(headers={}, litellm_params=GenericLiteLLMParams(api_key=_API_KEY))
+
+    assert len(warnings) == 1
+    assert "2026-11-30" in warnings[0]
+
+
+@pytest.mark.parametrize("api_base", ["https://my-resource.openai.azure.com", "http://localhost:4000"])
+def test_validate_environment_does_not_warn_for_other_api_bases(
+    config: OpenAIEvalsConfig, reset_shutdown_warning, monkeypatch, api_base: str
+):
+    warnings: list[str] = []
+    monkeypatch.setattr(reset_shutdown_warning.verbose_logger, "warning", lambda msg, *a, **k: warnings.append(msg))
+
+    config.validate_environment(headers={}, litellm_params=GenericLiteLLMParams(api_key=_API_KEY, api_base=api_base))
+
+    assert warnings == []
+
+
 def test_validate_environment_requires_api_key(config: OpenAIEvalsConfig, monkeypatch):
     """Test that validate_environment raises error when no API key is provided"""