diff --git a/litellm/llms/openai/evals/transformation.py b/litellm/llms/openai/evals/transformation.py index 73d211e1613..009191c55a1 100644 --- a/litellm/llms/openai/evals/transformation.py +++ b/litellm/llms/openai/evals/transformation.py @@ -2,7 +2,9 @@ OpenAI Evals API configuration and transformations """ +from functools import cache from typing import Final +from urllib.parse import urlparse import httpx from pydantic import TypeAdapter @@ -34,6 +36,29 @@ from litellm.types.utils import LlmProviders _DECODED_JSON: Final = TypeAdapter(object) +OPENAI_EVALS_SHUTDOWN_DATE: Final = "2026-11-30" +OPENAI_EVALS_SHUTDOWN_MESSAGE: Final = ( + "OpenAI is shutting down the Evals API on 2026-11-30 (existing evals become read-only on 2026-10-31). " + "Requests to api.openai.com/v1/evals will fail after that date, and OpenAI names no API replacement. " + "See https://developers.openai.com/api/docs/deprecations" +) + + +@cache +def _log_openai_evals_shutdown() -> None: + """Log the shutdown notice; the cache makes it run once per process.""" + verbose_logger.warning(OPENAI_EVALS_SHUTDOWN_MESSAGE) + + +def _warn_openai_evals_shutdown_once(api_base: str | None) -> None: + """Log the shutdown notice once per process, only for OpenAI's own API host. + + Azure and other OpenAI-compatible bases keep their own lifecycles, so they don't get the warning. + """ + host: Final = urlparse(api_base).hostname if api_base else "api.openai.com" + if host == "api.openai.com": + _log_openai_evals_shutdown() + def _parsed_response_json(raw_response: httpx.Response) -> object: return _DECODED_JSON.validate_python(raw_response.json()) @@ -53,6 +78,7 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): ) if not bearer_token: raise ValueError("OPENAI_API_KEY is required for Evals API") + _warn_openai_evals_shutdown_once(params.api_base) headers["Authorization"] = f"Bearer {bearer_token}" headers["Content-Type"] = "application/json" return headers diff --git a/tests/unit/llms/openai/evals/test_openai_evals_transformation.py b/tests/unit/llms/openai/evals/test_openai_evals_transformation.py index ed8f86d4727..3e5d123683d 100644 --- a/tests/unit/llms/openai/evals/test_openai_evals_transformation.py +++ b/tests/unit/llms/openai/evals/test_openai_evals_transformation.py @@ -47,6 +47,41 @@ def test_validate_environment_sets_headers(config: OpenAIEvalsConfig): assert result["Content-Type"] == "application/json" +@pytest.fixture() +def reset_shutdown_warning(): + import litellm.llms.openai.evals.transformation as evals_transformation + + evals_transformation._log_openai_evals_shutdown.cache_clear() + yield evals_transformation + evals_transformation._log_openai_evals_shutdown.cache_clear() + + +def test_validate_environment_warns_once_about_openai_evals_shutdown( + config: OpenAIEvalsConfig, reset_shutdown_warning, monkeypatch +): + """OpenAI shuts down the Evals API on 2026-11-30; callers on api.openai.com are told once.""" + warnings: list[str] = [] + monkeypatch.setattr(reset_shutdown_warning.verbose_logger, "warning", lambda msg, *a, **k: warnings.append(msg)) + + for _ in range(3): + config.validate_environment(headers={}, litellm_params=GenericLiteLLMParams(api_key=_API_KEY)) + + assert len(warnings) == 1 + assert "2026-11-30" in warnings[0] + + +@pytest.mark.parametrize("api_base", ["https://my-resource.openai.azure.com", "http://localhost:4000"]) +def test_validate_environment_does_not_warn_for_other_api_bases( + config: OpenAIEvalsConfig, reset_shutdown_warning, monkeypatch, api_base: str +): + warnings: list[str] = [] + monkeypatch.setattr(reset_shutdown_warning.verbose_logger, "warning", lambda msg, *a, **k: warnings.append(msg)) + + config.validate_environment(headers={}, litellm_params=GenericLiteLLMParams(api_key=_API_KEY, api_base=api_base)) + + assert warnings == [] + + def test_validate_environment_requires_api_key(config: OpenAIEvalsConfig, monkeypatch): """Test that validate_environment raises error when no API key is provided"""