From 79798c7e37fc1f3b0f42b78638c69166f0b68de4 Mon Sep 17 00:00:00 2001 From: Vineeth Sai Date: Mon, 26 Oct 2026 15:09:59 -0400 Subject: [PATCH] fix(evals): forward the caller's metadata on create_eval and create_run create_eval dropped metadata and create_run had the assignment commented out, so a caller's metadata never reached the provider. update_eval forwarded it through an exact-name denylist that misses several user_api_key_* keys, user_api_key_token among them. Match internal keys by prefix, as the proxy does when it strips spoofable auth metadata, and use one helper at all three call sites. Register the new tests directory in the endpoints shard. --- .github/workflows/test-unit.yml | 1 + litellm/evals/main.py | 127 ++++++++++++++------- tests/unit/evals/__init__.py | 0 tests/unit/evals/test_main.py | 195 ++++++++++++++++++++++++++++++++ 4 files changed, 281 insertions(+), 42 deletions(-) create mode 100644 tests/unit/evals/__init__.py create mode 100644 tests/unit/evals/test_main.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 76af64c8ccd..df2afa78c7a 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -414,6 +414,7 @@ jobs: tests/unit/containers tests/unit/embeddings tests/unit/endpoints + tests/unit/evals tests/unit/files tests/unit/fine_tuning tests/unit/images diff --git a/litellm/evals/main.py b/litellm/evals/main.py index 2f639d30ca0..34368c4a2d3 100644 --- a/litellm/evals/main.py +++ b/litellm/evals/main.py @@ -5,17 +5,29 @@ Provides create, list, get, update, delete, and cancel operations for evals import asyncio import contextvars -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from functools import partial from typing import Final import httpx import litellm -from litellm.constants import request_timeout +from litellm.constants import ( + BILLED_RETRY_ATTEMPT_METADATA_KEY, + CONSUMED_REQUEST_TAGS_METADATA_KEY, + INTERNAL_CALL_ORIGIN_METADATA_KEY, + MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, + PRE_CALL_EXECUTED_GUARDRAILS_KEY, + ROUTING_REQUEST_TAGS_METADATA_KEY, + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + request_timeout, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.types.integrations.anthropic_cache_control_hook import ( + GATEWAY_INJECTED_CACHE_METADATA_KEY, +) from litellm.types.llms.openai_evals import ( CancelEvalResponse, CancelRunResponse, @@ -40,6 +52,66 @@ from litellm.utils import ProviderConfigManager, client base_llm_http_handler = BaseLLMHTTPHandler() DEFAULT_OPENAI_API_BASE: Final = "https://api.openai.com" +INTERNAL_METADATA_KEY_PREFIXES: Final = ("user_api_key_", "litellm_") + +INTERNAL_METADATA_KEYS: Final = frozenset( + { + "headers", + "requester_metadata", + "user_api_key", + "user_api_end_user_max_budget", + "global_max_parallel_requests", + "endpoint", + "requester_ip_address", + "user_agent", + "spend_logs_metadata", + "proxy_server_request", + "standard_logging_object", + "disable_global_guardrails", + "disable_global_guardrail", + "opted_out_global_guardrails", + "applied_guardrails", + "applied_policies", + "policy_sources", + "guardrail_scan_ids", + "guardrail_scan_metadata", + "routing_decision", + "secret_fields", + "client_disconnected", + "error_information", + "_guardrail_pipelines", + "_pipeline_managed_guardrails", + "pillar_response_headers", + "_pillar_response_headers_trusted", + "pillar_flagged", + "pillar_scanners", + "pillar_evidence", + "pillar_evidence_truncated", + "pillar_session_id_response", + GATEWAY_INJECTED_CACHE_METADATA_KEY, + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + CONSUMED_REQUEST_TAGS_METADATA_KEY, + ROUTING_REQUEST_TAGS_METADATA_KEY, + INTERNAL_CALL_ORIGIN_METADATA_KEY, + BILLED_RETRY_ATTEMPT_METADATA_KEY, + MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, + PRE_CALL_EXECUTED_GUARDRAILS_KEY, + } +) + + +def _is_internal_metadata_key(key: str) -> bool: + return key in INTERNAL_METADATA_KEYS or key.startswith(INTERNAL_METADATA_KEY_PREFIXES) + + +def _user_metadata( + metadata: Mapping[str, object] | None, +) -> dict[str, object] | None: # mutable-ok: request metadata is typed as a dict + if metadata is None: + return None + filtered: Final = {k: v for k, v in metadata.items() if not _is_internal_metadata_key(k)} + return filtered or None + @client async def acreate_eval( @@ -170,6 +242,10 @@ def create_eval( if name is not None: create_request["name"] = name + filtered_metadata: Final = _user_metadata(metadata) + if filtered_metadata is not None: + create_request["metadata"] = filtered_metadata + # Merge extra_body if provided if extra_body: create_request.update(extra_body) @@ -684,44 +760,9 @@ def update_eval( update_request["name"] = name # Filter metadata to exclude internal LiteLLM fields - if metadata is not None: - # List of internal LiteLLM metadata keys that should NOT be sent to OpenAI - internal_keys: Final = { - "headers", - "requester_metadata", - "user_api_key_hash", - "user_api_key_alias", - "user_api_key_spend", - "user_api_key_max_budget", - "user_api_key_team_id", - "user_api_key_user_id", - "user_api_key_org_id", - "user_api_key_team_alias", - "user_api_key_end_user_id", - "user_api_key_user_email", - "user_api_key_request_route", - "user_api_key_budget_reset_at", - "user_api_key_auth_metadata", - "user_api_key", - "user_api_end_user_max_budget", - "user_api_key_auth", - "litellm_api_version", - "global_max_parallel_requests", - "user_api_key_team_max_budget", - "user_api_key_team_spend", - "user_api_key_model_max_budget", - "user_api_key_user_spend", - "user_api_key_user_max_budget", - "user_api_key_metadata", - "endpoint", - "litellm_parent_otel_span", - "requester_ip_address", - "user_agent", - } - # Only include user-provided metadata keys - filtered_metadata: Final = {k: v for k, v in metadata.items() if k not in internal_keys} - if filtered_metadata: # Only add if there's user metadata - update_request["metadata"] = filtered_metadata + filtered_metadata: Final = _user_metadata(metadata) + if filtered_metadata is not None: + update_request["metadata"] = filtered_metadata # Merge extra_body if provided if extra_body: @@ -1218,8 +1259,10 @@ def create_run( } if name is not None: create_request["name"] = name - # if metadata is not None: - # create_request["metadata"] = metadata + + filtered_metadata: Final = _user_metadata(metadata) + if filtered_metadata is not None: + create_request["metadata"] = filtered_metadata # Merge extra_body if provided if extra_body: diff --git a/tests/unit/evals/__init__.py b/tests/unit/evals/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/evals/test_main.py b/tests/unit/evals/test_main.py new file mode 100644 index 00000000000..48a129a2968 --- /dev/null +++ b/tests/unit/evals/test_main.py @@ -0,0 +1,195 @@ +import pytest + +import litellm +from litellm.evals.main import ( + INTERNAL_METADATA_KEY_PREFIXES, + INTERNAL_METADATA_KEYS, + _is_internal_metadata_key, +) +from litellm.types.llms.openai_evals import Eval, Run + +_DATA_SOURCE_CONFIG = {"type": "custom", "item_schema": {"type": "object"}} +_TESTING_CRITERIA = [{"type": "label_model", "name": "grader"}] +_DATA_SOURCE = {"type": "jsonl", "source": {"type": "file_id", "id": "file-123"}} + + +@pytest.fixture +def captured_body(monkeypatch): + seen = {} + + def _capture(handler_name, response): + def _handler(*args, **kwargs): + seen["request_body"] = kwargs["request_body"] + return response + + monkeypatch.setattr(litellm.evals.main.base_llm_http_handler, handler_name, _handler) + + eval_response = Eval( + id="eval-123", + created_at=0, + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + ) + _capture("create_eval_handler", eval_response) + _capture("update_eval_handler", eval_response) + _capture( + "create_run_handler", + Run( + id="run-123", + eval_id="eval-123", + created_at=0, + status="queued", + model="gpt-4o", + data_source=_DATA_SOURCE, + ), + ) + return seen + + +def test_create_eval_forwards_caller_metadata(captured_body): + litellm.create_eval( + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + metadata={"team": "search", "run_by": "nightly"}, + api_key="fake-key", + ) + assert captured_body["request_body"]["metadata"] == { + "team": "search", + "run_by": "nightly", + } + + +def test_create_run_forwards_caller_metadata(captured_body): + litellm.create_run( + eval_id="eval-123", + data_source=_DATA_SOURCE, + metadata={"team": "search"}, + api_key="fake-key", + ) + assert captured_body["request_body"]["metadata"] == {"team": "search"} + + +@pytest.mark.parametrize("internal_key", sorted(INTERNAL_METADATA_KEYS)) +def test_create_eval_strips_internal_metadata_keys(captured_body, internal_key): + litellm.create_eval( + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + metadata={internal_key: {"leaked": True}, "team": "search"}, + api_key="fake-key", + ) + assert captured_body["request_body"]["metadata"] == {"team": "search"} + + +def test_create_eval_omits_metadata_when_only_internal_keys(captured_body): + litellm.create_eval( + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + metadata={"user_api_key_hash": "leaked", "endpoint": "/v1/evals"}, + api_key="fake-key", + ) + assert "metadata" not in captured_body["request_body"] + + +def test_create_eval_omits_metadata_when_not_given(captured_body): + litellm.create_eval( + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + api_key="fake-key", + ) + assert "metadata" not in captured_body["request_body"] + + +@pytest.mark.parametrize( + "internal_key", + [ + "user_api_key_token", + "user_api_key_team_metadata", + "user_api_key_object_permission_id", + "user_api_key_team_object_permission_id", + "user_api_key_budget_reservation", + "user_api_key_org_alias", + "user_api_key_project_id", + "litellm_parent_otel_span", + "litellm_api_version", + ], +) +def test_create_eval_strips_prefixed_internal_keys(captured_body, internal_key): + litellm.create_eval( + data_source_config=_DATA_SOURCE_CONFIG, + testing_criteria=_TESTING_CRITERIA, + metadata={internal_key: {"leaked": True}, "team": "search"}, + api_key="fake-key", + ) + assert captured_body["request_body"]["metadata"] == {"team": "search"} + + +def test_every_user_api_key_metadata_key_is_covered(): + assert "user_api_key_" in INTERNAL_METADATA_KEY_PREFIXES + for key in ("user_api_key_token", "user_api_key_hash", "user_api_key_some_field_added_in_2027"): + assert _is_internal_metadata_key(key) + assert not _is_internal_metadata_key("my_user_api_key_note") + assert not _is_internal_metadata_key("team") + + +_PREVIOUS_DENYLIST = { + "headers", + "requester_metadata", + "user_api_key_hash", + "user_api_key_alias", + "user_api_key_spend", + "user_api_key_max_budget", + "user_api_key_team_id", + "user_api_key_user_id", + "user_api_key_org_id", + "user_api_key_team_alias", + "user_api_key_end_user_id", + "user_api_key_user_email", + "user_api_key_request_route", + "user_api_key_budget_reset_at", + "user_api_key_auth_metadata", + "user_api_key", + "user_api_end_user_max_budget", + "user_api_key_auth", + "litellm_api_version", + "global_max_parallel_requests", + "user_api_key_team_max_budget", + "user_api_key_team_spend", + "user_api_key_model_max_budget", + "user_api_key_user_spend", + "user_api_key_user_max_budget", + "user_api_key_metadata", + "endpoint", + "litellm_parent_otel_span", + "requester_ip_address", + "user_agent", +} + + +def test_prefix_filter_covers_everything_the_old_denylist_did(): + missed = sorted(k for k in _PREVIOUS_DENYLIST if not _is_internal_metadata_key(k)) + assert not missed, f"prefix filter no longer strips {missed}" + + +def test_proxy_metadata_control_fields_are_covered(): + pre_call_utils = pytest.importorskip("litellm.proxy.litellm_pre_call_utils") + + missed = sorted(k for k in pre_call_utils._UNTRUSTED_METADATA_CONTROL_FIELDS if not _is_internal_metadata_key(k)) + assert not missed, f"evals filter does not strip proxy-injected {missed}" + + +def test_update_eval_filtering_is_unchanged(captured_body): + litellm.update_eval( + eval_id="eval-123", + metadata={"team": "search", "user_api_key_hash": "leaked", "user_api_key_token": "leaked"}, + api_key="fake-key", + ) + assert captured_body["request_body"]["metadata"] == {"team": "search"} + + +def test_update_eval_omits_metadata_when_only_internal_keys(captured_body): + litellm.update_eval( + eval_id="eval-123", + metadata={"user_api_key_hash": "leaked"}, + api_key="fake-key", + ) + assert "metadata" not in captured_body["request_body"]