scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

fix(evals): forward the caller's metadata on create_eval and create_run

Browse source 
create_eval dropped metadata and create_run had the assignment commented out, so a caller's metadata never reached the provider. update_eval forwarded it through an exact-name denylist that misses several user_api_key_* keys, user_api_key_token among them.

Match internal keys by prefix, as the proxy does when it strips spoofable auth metadata, and use one helper at all three call sites. Register the new tests directory in the endpoints shard.
This commit is contained in:
Vineeth Sai 2026-10-26 15:09:59 -04:00
parent dab39d7402
commit 79798c7e37
4 changed files with 281 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/test-unit.yml vendored
View file

@ -414,6 +414,7 @@ jobs:
tests/unit/containers
tests/unit/embeddings
tests/unit/endpoints
tests/unit/evals
tests/unit/files
tests/unit/fine_tuning
tests/unit/images

127
litellm/evals/main.py
View file

@ -5,17 +5,29 @@ Provides create, list, get, update, delete, and cancel operations for evals
import asyncio
import contextvars
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from functools import partial
from typing import Final
import httpx
import litellm
from litellm.constants import request_timeout
from litellm.constants import (
BILLED_RETRY_ATTEMPT_METADATA_KEY,
CONSUMED_REQUEST_TAGS_METADATA_KEY,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
ROUTING_REQUEST_TAGS_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
request_timeout,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.integrations.anthropic_cache_control_hook import (
GATEWAY_INJECTED_CACHE_METADATA_KEY,
)
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
CancelRunResponse,
@ -40,6 +52,66 @@ from litellm.utils import ProviderConfigManager, client
base_llm_http_handler = BaseLLMHTTPHandler()
DEFAULT_OPENAI_API_BASE: Final = "https://api.openai.com"
INTERNAL_METADATA_KEY_PREFIXES: Final = ("user_api_key_", "litellm_")
INTERNAL_METADATA_KEYS: Final = frozenset(
{
"headers",
"requester_metadata",
"user_api_key",
"user_api_end_user_max_budget",
"global_max_parallel_requests",
"endpoint",
"requester_ip_address",
"user_agent",
"spend_logs_metadata",
"proxy_server_request",
"standard_logging_object",
"disable_global_guardrails",
"disable_global_guardrail",
"opted_out_global_guardrails",
"applied_guardrails",
"applied_policies",
"policy_sources",
"guardrail_scan_ids",
"guardrail_scan_metadata",
"routing_decision",
"secret_fields",
"client_disconnected",
"error_information",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
"pillar_response_headers",
"_pillar_response_headers_trusted",
"pillar_flagged",
"pillar_scanners",
"pillar_evidence",
"pillar_evidence_truncated",
"pillar_session_id_response",
GATEWAY_INJECTED_CACHE_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
CONSUMED_REQUEST_TAGS_METADATA_KEY,
ROUTING_REQUEST_TAGS_METADATA_KEY,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
BILLED_RETRY_ATTEMPT_METADATA_KEY,
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
}
)
def _is_internal_metadata_key(key: str) -> bool:
return key in INTERNAL_METADATA_KEYS or key.startswith(INTERNAL_METADATA_KEY_PREFIXES)
def _user_metadata(
metadata: Mapping[str, object] | None,
) -> dict[str, object] | None: # mutable-ok: request metadata is typed as a dict
if metadata is None:
return None
filtered: Final = {k: v for k, v in metadata.items() if not _is_internal_metadata_key(k)}
return filtered or None
@client
async def acreate_eval(
@ -170,6 +242,10 @@ def create_eval(
if name is not None:
create_request["name"] = name
filtered_metadata: Final = _user_metadata(metadata)
if filtered_metadata is not None:
create_request["metadata"] = filtered_metadata
# Merge extra_body if provided
if extra_body:
create_request.update(extra_body)
@ -684,44 +760,9 @@ def update_eval(
update_request["name"] = name
# Filter metadata to exclude internal LiteLLM fields
if metadata is not None:
# List of internal LiteLLM metadata keys that should NOT be sent to OpenAI
internal_keys: Final = {
"headers",
"requester_metadata",
"user_api_key_hash",
"user_api_key_alias",
"user_api_key_spend",
"user_api_key_max_budget",
"user_api_key_team_id",
"user_api_key_user_id",
"user_api_key_org_id",
"user_api_key_team_alias",
"user_api_key_end_user_id",
"user_api_key_user_email",
"user_api_key_request_route",
"user_api_key_budget_reset_at",
"user_api_key_auth_metadata",
"user_api_key",
"user_api_end_user_max_budget",
"user_api_key_auth",
"litellm_api_version",
"global_max_parallel_requests",
"user_api_key_team_max_budget",
"user_api_key_team_spend",
"user_api_key_model_max_budget",
"user_api_key_user_spend",
"user_api_key_user_max_budget",
"user_api_key_metadata",
"endpoint",
"litellm_parent_otel_span",
"requester_ip_address",
"user_agent",
}
# Only include user-provided metadata keys
filtered_metadata: Final = {k: v for k, v in metadata.items() if k not in internal_keys}
if filtered_metadata: # Only add if there's user metadata
update_request["metadata"] = filtered_metadata
filtered_metadata: Final = _user_metadata(metadata)
if filtered_metadata is not None:
update_request["metadata"] = filtered_metadata
# Merge extra_body if provided
if extra_body:
@ -1218,8 +1259,10 @@ def create_run(
}
if name is not None:
create_request["name"] = name
# if metadata is not None:
# create_request["metadata"] = metadata
filtered_metadata: Final = _user_metadata(metadata)
if filtered_metadata is not None:
create_request["metadata"] = filtered_metadata
# Merge extra_body if provided
if extra_body:

0
tests/unit/evals/__init__.py Normal file
View file

195
tests/unit/evals/test_main.py Normal file
View file

@ -0,0 +1,195 @@
import pytest
import litellm
from litellm.evals.main import (
INTERNAL_METADATA_KEY_PREFIXES,
INTERNAL_METADATA_KEYS,
_is_internal_metadata_key,
)
from litellm.types.llms.openai_evals import Eval, Run
_DATA_SOURCE_CONFIG = {"type": "custom", "item_schema": {"type": "object"}}
_TESTING_CRITERIA = [{"type": "label_model", "name": "grader"}]
_DATA_SOURCE = {"type": "jsonl", "source": {"type": "file_id", "id": "file-123"}}
@pytest.fixture
def captured_body(monkeypatch):
seen = {}
def _capture(handler_name, response):
def _handler(*args, **kwargs):
seen["request_body"] = kwargs["request_body"]
return response
monkeypatch.setattr(litellm.evals.main.base_llm_http_handler, handler_name, _handler)
eval_response = Eval(
id="eval-123",
created_at=0,
data_source_config=_DATA_SOURCE_CONFIG,
testing_criteria=_TESTING_CRITERIA,
)
_capture("create_eval_handler", eval_response)
_capture("update_eval_handler", eval_response)
_capture(
"create_run_handler",
Run(
id="run-123",
eval_id="eval-123",
created_at=0,
status="queued",
model="gpt-4o",
data_source=_DATA_SOURCE,
),
)
return seen
def test_create_eval_forwards_caller_metadata(captured_body):
litellm.create_eval(
data_source_config=_DATA_SOURCE_CONFIG,
testing_criteria=_TESTING_CRITERIA,
metadata={"team": "search", "run_by": "nightly"},
api_key="fake-key",
)
assert captured_body["request_body"]["metadata"] == {
"team": "search",
"run_by": "nightly",
}
def test_create_run_forwards_caller_metadata(captured_body):
litellm.create_run(
eval_id="eval-123",
data_source=_DATA_SOURCE,
metadata={"team": "search"},
api_key="fake-key",
)
assert captured_body["request_body"]["metadata"] == {"team": "search"}
@pytest.mark.parametrize("internal_key", sorted(INTERNAL_METADATA_KEYS))
def test_create_eval_strips_internal_metadata_keys(captured_body, internal_key):
litellm.create_eval(
data_source_config=_DATA_SOURCE_CONFIG,
testing_criteria=_TESTING_CRITERIA,
metadata={internal_key: {"leaked": True}, "team": "search"},
api_key="fake-key",
)
assert captured_body["request_body"]["metadata"] == {"team": "search"}
def test_create_eval_omits_metadata_when_only_internal_keys(captured_body):
litellm.create_eval(
data_source_config=_DATA_SOURCE_CONFIG,
testing_criteria=_TESTING_CRITERIA,
metadata={"user_api_key_hash": "leaked", "endpoint": "/v1/evals"},
api_key="fake-key",
)
assert "metadata" not in captured_body["request_body"]
def test_create_eval_omits_metadata_when_not_given(captured_body):
litellm.create_eval(
data_source_config=_DATA_SOURCE_CONFIG,
testing_criteria=_TESTING_CRITERIA,
api_key="fake-key",
)
assert "metadata" not in captured_body["request_body"]
@pytest.mark.parametrize(
"internal_key",
[
"user_api_key_token",
"user_api_key_team_metadata",
"user_api_key_object_permission_id",
"user_api_key_team_object_permission_id",
"user_api_key_budget_reservation",
"user_api_key_org_alias",
"user_api_key_project_id",
"litellm_parent_otel_span",
"litellm_api_version",
],
)
def test_create_eval_strips_prefixed_internal_keys(captured_body, internal_key):
litellm.create_eval(
data_source_config=_DATA_SOURCE_CONFIG,
testing_criteria=_TESTING_CRITERIA,
metadata={internal_key: {"leaked": True}, "team": "search"},
api_key="fake-key",
)
assert captured_body["request_body"]["metadata"] == {"team": "search"}
def test_every_user_api_key_metadata_key_is_covered():
assert "user_api_key_" in INTERNAL_METADATA_KEY_PREFIXES
for key in ("user_api_key_token", "user_api_key_hash", "user_api_key_some_field_added_in_2027"):
assert _is_internal_metadata_key(key)
assert not _is_internal_metadata_key("my_user_api_key_note")
assert not _is_internal_metadata_key("team")
_PREVIOUS_DENYLIST = {
"headers",
"requester_metadata",
"user_api_key_hash",
"user_api_key_alias",
"user_api_key_spend",
"user_api_key_max_budget",
"user_api_key_team_id",
"user_api_key_user_id",
"user_api_key_org_id",
"user_api_key_team_alias",
"user_api_key_end_user_id",
"user_api_key_user_email",
"user_api_key_request_route",
"user_api_key_budget_reset_at",
"user_api_key_auth_metadata",
"user_api_key",
"user_api_end_user_max_budget",
"user_api_key_auth",
"litellm_api_version",
"global_max_parallel_requests",
"user_api_key_team_max_budget",
"user_api_key_team_spend",
"user_api_key_model_max_budget",
"user_api_key_user_spend",
"user_api_key_user_max_budget",
"user_api_key_metadata",
"endpoint",
"litellm_parent_otel_span",
"requester_ip_address",
"user_agent",
}
def test_prefix_filter_covers_everything_the_old_denylist_did():
missed = sorted(k for k in _PREVIOUS_DENYLIST if not _is_internal_metadata_key(k))
assert not missed, f"prefix filter no longer strips {missed}"
def test_proxy_metadata_control_fields_are_covered():
pre_call_utils = pytest.importorskip("litellm.proxy.litellm_pre_call_utils")
missed = sorted(k for k in pre_call_utils._UNTRUSTED_METADATA_CONTROL_FIELDS if not _is_internal_metadata_key(k))
assert not missed, f"evals filter does not strip proxy-injected {missed}"
def test_update_eval_filtering_is_unchanged(captured_body):
litellm.update_eval(
eval_id="eval-123",
metadata={"team": "search", "user_api_key_hash": "leaked", "user_api_key_token": "leaked"},
api_key="fake-key",
)
assert captured_body["request_body"]["metadata"] == {"team": "search"}
def test_update_eval_omits_metadata_when_only_internal_keys(captured_body):
litellm.update_eval(
eval_id="eval-123",
metadata={"user_api_key_hash": "leaked"},
api_key="fake-key",
)
assert "metadata" not in captured_body["request_body"]