diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index b4b7afb9a09..deb7f01004a 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -415,6 +415,7 @@ jobs: tests/unit/containers tests/unit/embeddings tests/unit/endpoints + tests/unit/evals tests/unit/files tests/unit/fine_tuning tests/unit/images diff --git a/litellm/evals/main.py b/litellm/evals/main.py index 2f639d30ca0..34368c4a2d3 100644 --- a/litellm/evals/main.py +++ b/litellm/evals/main.py @@ -5,17 +5,29 @@ Provides create, list, get, update, delete, and cancel operations for evals import asyncio import contextvars -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from functools import partial from typing import Final import httpx import litellm -from litellm.constants import request_timeout +from litellm.constants import ( + BILLED_RETRY_ATTEMPT_METADATA_KEY, + CONSUMED_REQUEST_TAGS_METADATA_KEY, + INTERNAL_CALL_ORIGIN_METADATA_KEY, + MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, + PRE_CALL_EXECUTED_GUARDRAILS_KEY, + ROUTING_REQUEST_TAGS_METADATA_KEY, + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + request_timeout, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.types.integrations.anthropic_cache_control_hook import ( + GATEWAY_INJECTED_CACHE_METADATA_KEY, +) from litellm.types.llms.openai_evals import ( CancelEvalResponse, CancelRunResponse, @@ -40,6 +52,66 @@ from litellm.utils import ProviderConfigManager, client base_llm_http_handler = BaseLLMHTTPHandler() DEFAULT_OPENAI_API_BASE: Final = "https://api.openai.com" +INTERNAL_METADATA_KEY_PREFIXES: Final = ("user_api_key_", "litellm_") + +INTERNAL_METADATA_KEYS: Final = frozenset( + { + "headers", + "requester_metadata", + "user_api_key", + "user_api_end_user_max_budget", + "global_max_parallel_requests", + "endpoint", + "requester_ip_address", + "user_agent", + "spend_logs_metadata", + "proxy_server_request", + "standard_logging_object", + "disable_global_guardrails", + "disable_global_guardrail", + "opted_out_global_guardrails", + "applied_guardrails", + "applied_policies", + "policy_sources", + "guardrail_scan_ids", + "guardrail_scan_metadata", + "routing_decision", + "secret_fields", + "client_disconnected", + "error_information", + "_guardrail_pipelines", + "_pipeline_managed_guardrails", + "pillar_response_headers", + "_pillar_response_headers_trusted", + "pillar_flagged", + "pillar_scanners", + "pillar_evidence", + "pillar_evidence_truncated", + "pillar_session_id_response", + GATEWAY_INJECTED_CACHE_METADATA_KEY, + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + CONSUMED_REQUEST_TAGS_METADATA_KEY, + ROUTING_REQUEST_TAGS_METADATA_KEY, + INTERNAL_CALL_ORIGIN_METADATA_KEY, + BILLED_RETRY_ATTEMPT_METADATA_KEY, + MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, + PRE_CALL_EXECUTED_GUARDRAILS_KEY, + } +) + + +def _is_internal_metadata_key(key: str) -> bool: + return key in INTERNAL_METADATA_KEYS or key.startswith(INTERNAL_METADATA_KEY_PREFIXES) + + +def _user_metadata( + metadata: Mapping[str, object] | None, +) -> dict[str, object] | None: # mutable-ok: request metadata is typed as a dict + if metadata is None: + return None + filtered: Final = {k: v for k, v in metadata.items() if not _is_internal_metadata_key(k)} + return filtered or None + @client async def acreate_eval( @@ -170,6 +242,10 @@ def create_eval( if name is not None: create_request["name"] = name + filtered_metadata: Final = _user_metadata(metadata) + if filtered_metadata is not None: + create_request["metadata"] = filtered_metadata + # Merge extra_body if provided if extra_body: create_request.update(extra_body) @@ -684,44 +760,9 @@ def update_eval( update_request["name"] = name # Filter metadata to exclude internal LiteLLM fields - if metadata is not None: - # List of internal LiteLLM metadata keys that should NOT be sent to OpenAI - internal_keys: Final = { - "headers", - "requester_metadata", - "user_api_key_hash", - "user_api_key_alias", - "user_api_key_spend", - "user_api_key_max_budget", - "user_api_key_team_id", - "user_api_key_user_id", - "user_api_key_org_id", - "user_api_key_team_alias", - "user_api_key_end_user_id", - "user_api_key_user_email", - "user_api_key_request_route", - "user_api_key_budget_reset_at", - "user_api_key_auth_metadata", - "user_api_key", - "user_api_end_user_max_budget", - "user_api_key_auth", - "litellm_api_version", - "global_max_parallel_requests", - "user_api_key_team_max_budget", - "user_api_key_team_spend", - "user_api_key_model_max_budget", - "user_api_key_user_spend", - "user_api_key_user_max_budget", - "user_api_key_metadata", - "endpoint", - "litellm_parent_otel_span", - "requester_ip_address", - "user_agent", - } - # Only include user-provided metadata keys - filtered_metadata: Final = {k: v for k, v in metadata.items() if k not in internal_keys} - if filtered_metadata: # Only add if there's user metadata - update_request["metadata"] = filtered_metadata + filtered_metadata: Final = _user_metadata(metadata) + if filtered_metadata is not None: + update_request["metadata"] = filtered_metadata # Merge extra_body if provided if extra_body: @@ -1218,8 +1259,10 @@ def create_run( } if name is not None: create_request["name"] = name - # if metadata is not None: - # create_request["metadata"] = metadata + + filtered_metadata: Final = _user_metadata(metadata) + if filtered_metadata is not None: + create_request["metadata"] = filtered_metadata # Merge extra_body if provided if extra_body: diff --git a/tests/unit/evals/__init__.py b/tests/unit/evals/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/evals/test_main.py b/tests/unit/evals/test_main.py new file mode 100644 index 00000000000..48a129a2968 --- /dev/null +++ b/tests/unit/evals/test_main.py @@ -0,0 +1,195 @@ +import pytest + +import litellm +from litellm.evals.main import ( + INTERNAL_METADATA_KEY_PREFIXES, + INTERNAL_METADATA_KEYS, + _is_internal_metadata_key, +) +from litellm.types.llms.openai_evals import Eval, Run + +_DATA_SOURCE_CONFIG = {"type": "custom", "item_schema": {"type": "object"}} +_TESTING_CRITERIA = [{"type": "label_model", "name": "grader"}] +_DATA_SOURCE = {"type": "jsonl", "source": {"type": "file_id", "id": "file-123"}} + + +@pytest.fixture +def captured_body(monkeypatch): + seen = {} + + def _capture(handler_name, response): + def _handler(*args, **kwargs): + seen["request_body"] = kwargs["request_body"] + return response + + monkeypatch.setattr(litellm.evals.main.base_llm_http_handler, handler_name, _handler) + + eval_response = Eval( + id="eval-123", + created_at=0, + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + ) + _capture("create_eval_handler", eval_response) + _capture("update_eval_handler", eval_response) + _capture( + "create_run_handler", + Run( + id="run-123", + eval_id="eval-123", + created_at=0, + status="queued", + model="gpt-4o", + data_source=_DATA_SOURCE, + ), + ) + return seen + + +def test_create_eval_forwards_caller_metadata(captured_body): + litellm.create_eval( + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + metadata={"team": "search", "run_by": "nightly"}, + api_key="fake-key", + ) + assert captured_body["request_body"]["metadata"] == { + "team": "search", + "run_by": "nightly", + } + + +def test_create_run_forwards_caller_metadata(captured_body): + litellm.create_run( + eval_id="eval-123", + data_source=_DATA_SOURCE, + metadata={"team": "search"}, + api_key="fake-key", + ) + assert captured_body["request_body"]["metadata"] == {"team": "search"} + + +@pytest.mark.parametrize("internal_key", sorted(INTERNAL_METADATA_KEYS)) +def test_create_eval_strips_internal_metadata_keys(captured_body, internal_key): + litellm.create_eval( + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + metadata={internal_key: {"leaked": True}, "team": "search"}, + api_key="fake-key", + ) + assert captured_body["request_body"]["metadata"] == {"team": "search"} + + +def test_create_eval_omits_metadata_when_only_internal_keys(captured_body): + litellm.create_eval( + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + metadata={"user_api_key_hash": "leaked", "endpoint": "/v1/evals"}, + api_key="fake-key", + ) + assert "metadata" not in captured_body["request_body"] + + +def test_create_eval_omits_metadata_when_not_given(captured_body): + litellm.create_eval( + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + api_key="fake-key", + ) + assert "metadata" not in captured_body["request_body"] + + +@pytest.mark.parametrize( + "internal_key", + [ + "user_api_key_token", + "user_api_key_team_metadata", + "user_api_key_object_permission_id", + "user_api_key_team_object_permission_id", + "user_api_key_budget_reservation", + "user_api_key_org_alias", + "user_api_key_project_id", + "litellm_parent_otel_span", + "litellm_api_version", + ], +) +def test_create_eval_strips_prefixed_internal_keys(captured_body, internal_key): + litellm.create_eval( + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + metadata={internal_key: {"leaked": True}, "team": "search"}, + api_key="fake-key", + ) + assert captured_body["request_body"]["metadata"] == {"team": "search"} + + +def test_every_user_api_key_metadata_key_is_covered(): + assert "user_api_key_" in INTERNAL_METADATA_KEY_PREFIXES + for key in ("user_api_key_token", "user_api_key_hash", "user_api_key_some_field_added_in_2027"): + assert _is_internal_metadata_key(key) + assert not _is_internal_metadata_key("my_user_api_key_note") + assert not _is_internal_metadata_key("team") + + +_PREVIOUS_DENYLIST = { + "headers", + "requester_metadata", + "user_api_key_hash", + "user_api_key_alias", + "user_api_key_spend", + "user_api_key_max_budget", + "user_api_key_team_id", + "user_api_key_user_id", + "user_api_key_org_id", + "user_api_key_team_alias", + "user_api_key_end_user_id", + "user_api_key_user_email", + "user_api_key_request_route", + "user_api_key_budget_reset_at", + "user_api_key_auth_metadata", + "user_api_key", + "user_api_end_user_max_budget", + "user_api_key_auth", + "litellm_api_version", + "global_max_parallel_requests", + "user_api_key_team_max_budget", + "user_api_key_team_spend", + "user_api_key_model_max_budget", + "user_api_key_user_spend", + "user_api_key_user_max_budget", + "user_api_key_metadata", + "endpoint", + "litellm_parent_otel_span", + "requester_ip_address", + "user_agent", +} + + +def test_prefix_filter_covers_everything_the_old_denylist_did(): + missed = sorted(k for k in _PREVIOUS_DENYLIST if not _is_internal_metadata_key(k)) + assert not missed, f"prefix filter no longer strips {missed}" + + +def test_proxy_metadata_control_fields_are_covered(): + pre_call_utils = pytest.importorskip("litellm.proxy.litellm_pre_call_utils") + + missed = sorted(k for k in pre_call_utils._UNTRUSTED_METADATA_CONTROL_FIELDS if not _is_internal_metadata_key(k)) + assert not missed, f"evals filter does not strip proxy-injected {missed}" + + +def test_update_eval_filtering_is_unchanged(captured_body): + litellm.update_eval( + eval_id="eval-123", + metadata={"team": "search", "user_api_key_hash": "leaked", "user_api_key_token": "leaked"}, + api_key="fake-key", + ) + assert captured_body["request_body"]["metadata"] == {"team": "search"} + + +def test_update_eval_omits_metadata_when_only_internal_keys(captured_body): + litellm.update_eval( + eval_id="eval-123", + metadata={"user_api_key_hash": "leaked"}, + api_key="fake-key", + ) + assert "metadata" not in captured_body["request_body"]