diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 71abdfa5e9e..36711f44d17 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -1,6 +1,6 @@ import json import re -from typing import Any, Collection, Dict, List, Optional +from typing import Any, Collection, Dict, FrozenSet, List, Optional, Tuple import orjson from fastapi import Request, UploadFile, status @@ -12,6 +12,115 @@ from litellm.proxy.common_utils.callback_utils import ( ) from litellm.types.router import Deployment +# Fields the proxy populates itself during request processing. These are +# removed from the incoming request body so downstream code works with the +# expected shape — caller-supplied values here would collide with or shadow +# the proxy's own. disable_global_guardrails and opted_out_global_guardrails +# are intentionally excluded; they're handled by the auth layer's +# _guardrail_modification_check. +_RESTRICTED_TOP_LEVEL_FIELDS: FrozenSet[str] = frozenset( + { + "proxy_server_request", + "standard_logging_object", + "secret_fields", + "litellm_logging_obj", + } +) + +# mock_response / mock_tool_calls short-circuit the real LLM call and are +# meant for test/dev use. Removed from client payloads by default; set +# ``general_settings.allow_client_side_mock_response: true`` in config.yaml +# to accept client-supplied values. Internal code (guardrails, health +# checks) sets these server-side after ingress and is unaffected. +_MOCK_RESPONSE_FIELDS: FrozenSet[str] = frozenset({"mock_response", "mock_tool_calls"}) + +_RESTRICTED_METADATA_FIELDS: FrozenSet[str] = frozenset( + { + "applied_guardrails", + "applied_policies", + "policy_sources", + "pillar_response_headers", + "pillar_flagged", + "pillar_scanners", + "pillar_evidence", + "pillar_evidence_truncated", + "pillar_session_id_response", + "semantic-similarity", # hyphenated, matches redis_semantic_cache.py + "_guardrail_pipelines", + "_pipeline_managed_guardrails", + } +) + +# The proxy writes user_api_key_* fields into metadata during +# add_litellm_data_to_request; any incoming values of the same shape are +# removed to avoid collisions. Safe here because this strip only runs on +# raw ingress bodies, before the proxy populates its own values. +_RESTRICTED_METADATA_PREFIXES: Tuple[str, ...] = ("user_api_key_",) + +_METADATA_CONTAINER_KEYS: Tuple[str, ...] = ("metadata", "litellm_metadata") + + +def _client_mock_response_allowed() -> bool: + """Check the proxy's ``general_settings.allow_client_side_mock_response``. + + Lazy-imported because ``general_settings`` lives on ``proxy_server`` and + we don't want an import cycle (``proxy_server`` imports this module). + In non-proxy contexts (SDK, unit tests that don't start the proxy) the + import succeeds but the dict is empty, so the function returns False + and the fields are removed by default. + """ + try: + from litellm.proxy.proxy_server import general_settings + except ImportError: + return False + return bool(general_settings.get("allow_client_side_mock_response")) + + +def _strip_internal_metadata_keys(metadata: dict) -> bool: + """Remove restricted keys from `metadata` in place; return True if anything was removed.""" + removed = False + for key in list(metadata.keys()): + if key in _RESTRICTED_METADATA_FIELDS or key.startswith( + _RESTRICTED_METADATA_PREFIXES + ): + del metadata[key] + removed = True + return removed + + +def strip_internal_control_fields(data: dict) -> None: + """Remove proxy-internal fields from a user-supplied request body in place. + + Metadata containers can arrive as either a dict or a JSON string (the + latter happens with multipart/form-data and some extra_body paths); + both shapes are handled. Idempotent — safe to call on an + already-cleaned dict. + + Only runs at ingress (before the proxy enriches `data` with its own + user_api_key_* / internal fields). Do not call on an enriched body. + """ + if not isinstance(data, dict): + return + + for key in _RESTRICTED_TOP_LEVEL_FIELDS: + data.pop(key, None) + + if not _client_mock_response_allowed(): + for key in _MOCK_RESPONSE_FIELDS: + data.pop(key, None) + + for container_key in _METADATA_CONTAINER_KEYS: + container = data.get(container_key) + if isinstance(container, dict): + _strip_internal_metadata_keys(container) + elif isinstance(container, str): + try: + parsed = json.loads(container) + except (json.JSONDecodeError, ValueError): + continue + if isinstance(parsed, dict) and _strip_internal_metadata_keys(parsed): + data[container_key] = json.dumps(parsed) + async def _read_request_body(request: Optional[Request]) -> Dict: """ @@ -32,6 +141,11 @@ async def _read_request_body(request: Optional[Request]) -> Dict: request=request ) if _cached_request_body is not None: + # The cache was populated by this function after the strip + # ran, so it's already in the expected shape. Do NOT re-strip + # here — by this point the proxy may have enriched the body + # with its own user_api_key_* / internal fields, which would + # be incorrectly removed. return _cached_request_body _request_headers: dict = _safe_get_request_headers(request=request) @@ -80,6 +194,10 @@ async def _read_request_body(request: Optional[Request]) -> Dict: code=status.HTTP_400_BAD_REQUEST, ) + # Strip proxy-internal fields before anything downstream (including + # the cache) sees the body. + strip_internal_control_fields(parsed_body) + # Cache the parsed result _safe_set_request_parsed_body(request=request, parsed_body=parsed_body) return parsed_body diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index b4343f6b2e1..21c22a8c055 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -853,3 +853,258 @@ class TestGetTagsFromRequestBodyStringCoerce: tags = get_tags_from_request_body({"metadata": {"tags": ["x"]}}) assert tags == ["x"] + + +class TestStripInternalControlFields: + """Coverage for ``strip_internal_control_fields``: proxy-internal + fields supplied in the request body must not reach downstream + readers. Tests exercise the helper directly and the end-to-end path + through ``_read_request_body``. + """ + + @staticmethod + def _clear_allow_mock(monkeypatch): + """Ensure general_settings.allow_client_side_mock_response is unset.""" + from litellm.proxy import proxy_server + + monkeypatch.setitem( + proxy_server.general_settings, "allow_client_side_mock_response", False + ) + + @staticmethod + def _allow_mock(monkeypatch): + from litellm.proxy import proxy_server + + monkeypatch.setitem( + proxy_server.general_settings, "allow_client_side_mock_response", True + ) + + def test_mock_response_stripped_by_default(self, monkeypatch): + from litellm.proxy.common_utils.http_parsing_utils import ( + strip_internal_control_fields, + ) + + self._clear_allow_mock(monkeypatch) + data = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + "mock_response": "canned", + "mock_tool_calls": [{"id": "x"}], + } + strip_internal_control_fields(data) + assert "mock_response" not in data + assert "mock_tool_calls" not in data + # Other fields are preserved. + assert data["model"] == "gpt-4o" + assert data["messages"] == [{"role": "user", "content": "hi"}] + + def test_mock_response_preserved_with_opt_in_setting(self, monkeypatch): + from litellm.proxy.common_utils.http_parsing_utils import ( + strip_internal_control_fields, + ) + + self._allow_mock(monkeypatch) + data = { + "model": "x", + "mock_response": "legit testing", + "mock_tool_calls": [{"id": "x"}], + } + strip_internal_control_fields(data) + assert data["mock_response"] == "legit testing" + assert data["mock_tool_calls"] == [{"id": "x"}] + + def test_internal_objects_always_stripped(self): + from litellm.proxy.common_utils.http_parsing_utils import ( + strip_internal_control_fields, + ) + + data = { + "model": "x", + "proxy_server_request": {"url": "caller-supplied"}, + "standard_logging_object": {"caller_supplied": True}, + "secret_fields": {"raw_headers": "caller-supplied"}, + "litellm_logging_obj": object(), + } + strip_internal_control_fields(data) + assert data == {"model": "x"} + + def test_metadata_internal_fields_stripped(self): + from litellm.proxy.common_utils.http_parsing_utils import ( + strip_internal_control_fields, + ) + + data = { + "model": "x", + "metadata": { + "applied_guardrails": ["presidio-pii"], + "applied_policies": ["dlp"], + "policy_sources": {"x": "y"}, + "pillar_response_headers": {"Set-Cookie": "v"}, + "pillar_flagged": True, + "semantic-similarity": 0.92, + "_guardrail_pipelines": [{"name": "p"}], + "tags": ["ok"], + }, + } + strip_internal_control_fields(data) + # Every restricted key is removed; user-controlled tags survive. + assert data["metadata"] == {"tags": ["ok"]} + + def test_litellm_metadata_alias_stripped(self): + from litellm.proxy.common_utils.http_parsing_utils import ( + strip_internal_control_fields, + ) + + data = { + "model": "x", + "litellm_metadata": { + "applied_guardrails": ["caller"], + "user_tag": "keep", + }, + } + strip_internal_control_fields(data) + assert data["litellm_metadata"] == {"user_tag": "keep"} + + def test_user_api_key_prefix_stripped_from_metadata(self): + """User-supplied user_api_key_* fields in metadata are removed so + they don't collide with the values the proxy writes itself.""" + from litellm.proxy.common_utils.http_parsing_utils import ( + strip_internal_control_fields, + ) + + data = { + "metadata": { + "user_api_key_user_id": "caller-supplied", + "user_api_key_team_id": "caller-supplied", + "user_api_key_alias": "caller-supplied", + "tags": ["ok"], + } + } + strip_internal_control_fields(data) + assert data["metadata"] == {"tags": ["ok"]} + + def test_json_string_metadata_is_also_sanitized(self): + """Multipart form-data / extra_body can deliver metadata as a JSON + string; the strip parses, removes, and re-serializes so the + restricted fields don't survive downstream dict coercion.""" + from litellm.proxy.common_utils.http_parsing_utils import ( + strip_internal_control_fields, + ) + + data = { + "model": "x", + "metadata": json.dumps( + { + "applied_guardrails": ["caller"], + "pillar_response_headers": {"X-Caller": "yes"}, + "tag": "ok", + } + ), + } + strip_internal_control_fields(data) + assert isinstance(data["metadata"], str) + parsed = json.loads(data["metadata"]) + assert parsed == {"tag": "ok"} + + def test_clean_json_string_metadata_not_reserialized(self): + """When nothing is removed from a JSON-string metadata, the + original byte representation is preserved (no ordering / whitespace / + unicode-escape rewrite).""" + from litellm.proxy.common_utils.http_parsing_utils import ( + strip_internal_control_fields, + ) + + # Choose a string whose Python-round-tripped form would differ + # (non-ASCII char that json.dumps would default to \u-escape). + original = '{"tag":"café","count":1}' + data = {"metadata": original} + strip_internal_control_fields(data) + assert data["metadata"] == original + + def test_non_dict_input_noop(self): + from litellm.proxy.common_utils.http_parsing_utils import ( + strip_internal_control_fields, + ) + + # Must not raise for any of these. + strip_internal_control_fields(None) # type: ignore[arg-type] + strip_internal_control_fields("string") # type: ignore[arg-type] + strip_internal_control_fields([1, 2]) # type: ignore[arg-type] + + def test_idempotent(self, monkeypatch): + from litellm.proxy.common_utils.http_parsing_utils import ( + strip_internal_control_fields, + ) + + self._clear_allow_mock(monkeypatch) + data = { + "mock_response": "x", + "metadata": {"applied_guardrails": ["a"], "keep": 1}, + } + strip_internal_control_fields(data) + snapshot = json.dumps(data, sort_keys=True) + strip_internal_control_fields(data) + assert json.dumps(data, sort_keys=True) == snapshot + + @pytest.mark.asyncio + async def test_read_request_body_strips_on_fresh_parse(self, monkeypatch): + """End-to-end: a body containing restricted fields is sanitized + before any downstream reader sees it.""" + self._clear_allow_mock(monkeypatch) + mock_request = MagicMock() + mock_request.body = AsyncMock( + return_value=orjson.dumps( + { + "model": "x", + "messages": [{"role": "user", "content": "hi"}], + "mock_response": "canned", + "metadata": { + "applied_guardrails": ["caller"], + "pillar_response_headers": {"Set-Cookie": "v"}, + "tags": ["ok"], + }, + } + ) + ) + mock_request.headers = {"content-type": "application/json"} + mock_request.scope = {} + + result = await _read_request_body(mock_request) + + assert "mock_response" not in result + assert result["metadata"] == {"tags": ["ok"]} + + @pytest.mark.asyncio + async def test_read_request_body_caches_stripped_form(self, monkeypatch): + """The cache stores the sanitized body, and a handler that + enriches metadata with user_api_key_* afterwards does NOT see + those values removed on a subsequent read (regression against an + earlier defensive re-strip that would have wiped proxy-populated + fields).""" + self._clear_allow_mock(monkeypatch) + mock_request = MagicMock() + mock_request.body = AsyncMock( + return_value=orjson.dumps( + { + "model": "x", + "mock_response": "canned", + "metadata": {"applied_guardrails": ["caller"]}, + } + ) + ) + mock_request.headers = {"content-type": "application/json"} + mock_request.scope = {} + + first = await _read_request_body(mock_request) + assert "mock_response" not in first + assert first["metadata"] == {} + + # Simulate the proxy enriching metadata after ingress. + first["metadata"]["user_api_key_user_id"] = "real-user" + + # Second read returns the cached (and enriched) body without + # removing the proxy-populated user_api_key_user_id. + mock_request.body.reset_mock() + second = await _read_request_body(mock_request) + mock_request.body.assert_not_called() + assert second["metadata"]["user_api_key_user_id"] == "real-user"