diff --git a/litellm/litellm_core_utils/internal_key_emission_guard.py b/litellm/litellm_core_utils/internal_key_emission_guard.py index 09d32973b52..bb2375c813d 100644 --- a/litellm/litellm_core_utils/internal_key_emission_guard.py +++ b/litellm/litellm_core_utils/internal_key_emission_guard.py @@ -1,5 +1,5 @@ import threading -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from typing import Final, cast from litellm._logging import verbose_logger @@ -10,6 +10,7 @@ OWNED_KEYS: Final = frozenset(key for key in ("litellm_params", *all_litellm_par OWNED_KEY_SCOPES: Final = frozenset( ("", "metadata", "additionalModelRequestFields", "additionalModelRequestFields.extra_body") ) +_CONTAINER_TYPES: Final[tuple[type[object], ...]] = (dict, list, tuple) class LeakCounter: @@ -34,37 +35,32 @@ def is_litellm_owned_key(key: str, scope: str) -> bool: return key.startswith(OWNED_KEY_PREFIX) or (scope in OWNED_KEY_SCOPES and key in OWNED_KEYS) -def _as_mapping(value: object) -> Mapping[object, object] | None: - if not isinstance(value, Mapping): - return None - return cast("Mapping[object, object]", value) # cast-ok: isinstance narrows only to Mapping[Unknown, Unknown] +def _walk_mapping(value: Mapping[object, object], prefix: str, found: list[str]) -> None: + scoped: Final = prefix in OWNED_KEY_SCOPES + for key, child in value.items(): + if not isinstance(key, str): + continue + if key.startswith(OWNED_KEY_PREFIX) or (scoped and key in OWNED_KEYS): + found.append(f"{prefix}.{key}" if prefix else key) + if isinstance(child, _CONTAINER_TYPES): + _walk(cast(object, child), f"{prefix}.{key}" if prefix else key, found) # cast-ok: undo Unknown narrowing -def _as_sequence(value: object) -> Sequence[object] | None: - if not isinstance(value, Sequence) or isinstance(value, (str, bytes, bytearray)): - return None - return cast("Sequence[object]", value) # cast-ok: isinstance narrows only to Sequence[Unknown] - - -def _child_paths(value: object, prefix: str) -> tuple[str, ...]: - mapping: Final = _as_mapping(value) - if mapping is not None: - return tuple(path for key, child in mapping.items() for path in _entry_paths(key, child, prefix)) - sequence: Final = _as_sequence(value) - if sequence is not None: - return tuple(path for index, child in enumerate(sequence) for path in _child_paths(child, f"{prefix}[{index}]")) - return () - - -def _entry_paths(key: object, child: object, prefix: str) -> tuple[str, ...]: - name: Final = str(key) - path: Final = f"{prefix}.{name}" if prefix else name - own: Final = (path,) if is_litellm_owned_key(name, prefix) else () - return (*own, *_child_paths(child, path)) +def _walk(value: object, prefix: str, found: list[str]) -> None: + if isinstance(value, dict): + _walk_mapping(cast("dict[object, object]", value), prefix, found) # cast-ok: isinstance leaves Unknown params + return + if not isinstance(value, (list, tuple)): + return + for index, child in enumerate(cast("list[object] | tuple[object, ...]", value)): # cast-ok: same as above + if isinstance(child, _CONTAINER_TYPES): + _walk(cast(object, child), f"{prefix}[{index}]", found) # cast-ok: undo Unknown narrowing def litellm_owned_key_paths(body: Mapping[str, object]) -> tuple[str, ...]: - return _child_paths(body, "") + found: Final[list[str]] = [] # mutable-ok: single accumulator for the hot-path walk, sealed into a tuple below + _walk_mapping(cast("Mapping[object, object]", body), "", found) # cast-ok: widen invariant key type + return tuple(found) def observe_internal_keys(body: Mapping[str, object], provider: str) -> tuple[str, ...]: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index cdf4041de1a..21e21f573fe 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1076,7 +1076,7 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, ) - data: Final = provider_config.transform_embedding_request( + data: Final[dict[str, object]] = provider_config.transform_embedding_request( model=model, input=input, optional_params=optional_params, @@ -1084,6 +1084,7 @@ class BaseLLMHTTPHandler: ) if embedding_extra_body: data.update(embedding_extra_body) + observe_internal_keys(data, custom_llm_provider) # Some providers (e.g. OCI) require request signing after the body is built. # The default BaseConfig.sign_request returns (headers, None) — a no-op for @@ -2702,6 +2703,7 @@ class BaseLLMHTTPHandler: if extra_body: data.update(extra_body) + observe_internal_keys(data, custom_llm_provider) stream = bool(stream or data.get("stream")) # Preserve the OpenAI-style request context (not sent to the provider) for streaming @@ -2890,6 +2892,7 @@ class BaseLLMHTTPHandler: if extra_body: data.update(extra_body) + observe_internal_keys(data, custom_llm_provider) stream = bool(stream or data.get("stream")) # Preserve the OpenAI-style request context (not sent to the provider) for streaming diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 3b7e947d5ca..4f4d8d599c8 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -51,6 +51,15 @@ "tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_converse_body_is_observed_at_emission": [ "other.provider_wire.internal_key_emission_observed_bedrock_converse" ], + "tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_embedding_body_is_observed_at_emission": [ + "other.provider_wire.internal_key_emission_observed_embedding" + ], + "tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_responses_body_is_observed_at_emission": [ + "other.provider_wire.internal_key_emission_observed_responses" + ], + "tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_async_responses_body_is_observed_at_emission": [ + "other.provider_wire.internal_key_emission_observed_responses_async" + ], "tests/integration/pricing/test_configured_prices.py::test_loaded_router_preserves_cached_defaults_during_real_requests": [ "quota_management.spend_tracking.default_prices.loaded_router_preserves_cached_defaults" ], diff --git a/tests/integration/providers/test_request_boundary.py b/tests/integration/providers/test_request_boundary.py index 24e550d68cd..27b33b86b04 100644 --- a/tests/integration/providers/test_request_boundary.py +++ b/tests/integration/providers/test_request_boundary.py @@ -1,3 +1,4 @@ +import asyncio import json import logging from typing import Final @@ -25,6 +26,43 @@ def converse_peer(request: Request) -> Reply: return Reply(body=CONVERSE_REPLY) +EMBEDDING_REPLY: Final = json.dumps( + { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.25, 0.5]}], + "model": "synthetic-embedding", + "usage": {"prompt_tokens": 3, "total_tokens": 3}, + } +).encode() +RESPONSES_REPLY: Final = json.dumps( + { + "id": "resp_synthetic", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "synthetic-responses", + "output": [ + { + "type": "message", + "id": "msg_synthetic", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "responses guard control", "annotations": []}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 3, "total_tokens": 6}, + } +).encode() + + +def embedding_peer(request: Request) -> Reply: + return Reply(body=EMBEDDING_REPLY) + + +def responses_peer(request: Request) -> Reply: + return Reply(body=RESPONSES_REPLY) + + @pytest.mark.covers("other.provider_wire.internal_parameters_filtered") def test_internal_request_state_does_not_reach_provider(gateway: Gateway) -> None: with gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream: @@ -117,6 +155,81 @@ def test_internal_key_forced_into_converse_body_is_observed_at_emission( ] +@pytest.mark.covers("other.provider_wire.internal_key_emission_observed_embedding") +def test_internal_key_forced_into_embedding_body_is_observed_at_emission(caplog: pytest.LogCaptureFixture) -> None: + before: Final = internal_key_leak_counter.value + with wire_server(embedding_peer) as wire, caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = litellm.embedding( + model="hosted_vllm/synthetic-embedding", + api_base=f"{wire.url}/v1", + api_key="sk-synthetic", + input=["embedding emission guard"], + extra_body={"litellm_call_id": "forced-through-extra-body"}, + timeout=5, + num_retries=0, + ) + received: Final = wire.drain() + assert isinstance(result, litellm.EmbeddingResponse) + assert len(received) == 1 + assert received[0].target == "/v1/embeddings" + assert JSON_OBJECT.validate_json(received[0].body)["litellm_call_id"] == "forced-through-extra-body" + assert internal_key_leak_counter.value == before + 1 + assert [record.getMessage() for record in caplog.records if record.levelno == logging.WARNING] == [ + "LiteLLM internal keys reached the hosted_vllm provider request body: litellm_call_id" + ] + + +def assert_responses_body_observed( + response: object, received: tuple[Request, ...], before: int, caplog: pytest.LogCaptureFixture +) -> None: + assert isinstance(response, litellm.ResponsesAPIResponse) + assert len(received) == 1 + assert received[0].target == "/v1/responses" + assert JSON_OBJECT.validate_json(received[0].body)["litellm_call_id"] == "forced-through-extra-body" + assert internal_key_leak_counter.value == before + 1 + assert [record.getMessage() for record in caplog.records if record.levelno == logging.WARNING] == [ + "LiteLLM internal keys reached the openai provider request body: litellm_call_id" + ] + + +@pytest.mark.covers("other.provider_wire.internal_key_emission_observed_responses") +def test_internal_key_forced_into_responses_body_is_observed_at_emission(caplog: pytest.LogCaptureFixture) -> None: + before: Final = internal_key_leak_counter.value + with wire_server(responses_peer) as wire, caplog.at_level(logging.WARNING, logger="LiteLLM"): + response: Final = litellm.responses( + model="openai/synthetic-responses", + api_base=f"{wire.url}/v1", + api_key="sk-synthetic", + input="responses emission guard", + extra_body={"litellm_call_id": "forced-through-extra-body"}, + timeout=5, + num_retries=0, + ) + received: Final = wire.drain() + assert_responses_body_observed(response, received, before, caplog) + + +@pytest.mark.covers("other.provider_wire.internal_key_emission_observed_responses_async") +def test_internal_key_forced_into_async_responses_body_is_observed_at_emission( + caplog: pytest.LogCaptureFixture, +) -> None: + before: Final = internal_key_leak_counter.value + with wire_server(responses_peer) as wire, caplog.at_level(logging.WARNING, logger="LiteLLM"): + response: Final = asyncio.run( + litellm.aresponses( + model="openai/synthetic-responses", + api_base=f"{wire.url}/v1", + api_key="sk-synthetic", + input="responses emission guard", + extra_body={"litellm_call_id": "forced-through-extra-body"}, + timeout=5, + num_retries=0, + ) + ) + received: Final = wire.drain() + assert_responses_body_observed(response, received, before, caplog) + + @pytest.mark.covers("other.provider_wire.validator_rejects_corruption") def test_upstream_rejects_corruption_and_accepts_supported_metadata(gateway: Gateway) -> None: with httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream: