perf(core): walk provider bodies with a single accumulator and observe embedding and responses merges

This commit is contained in:
shrey kharbanda 2026-09-22 19:58:19 +00:00
parent 22fc88298d
commit 029eeaccf7
4 changed files with 149 additions and 28 deletions

View file

@ -1,5 +1,5 @@
import threading
from collections.abc import Mapping, Sequence
from collections.abc import Mapping
from typing import Final, cast
from litellm._logging import verbose_logger
@ -10,6 +10,7 @@ OWNED_KEYS: Final = frozenset(key for key in ("litellm_params", *all_litellm_par
OWNED_KEY_SCOPES: Final = frozenset(
("", "metadata", "additionalModelRequestFields", "additionalModelRequestFields.extra_body")
)
_CONTAINER_TYPES: Final[tuple[type[object], ...]] = (dict, list, tuple)
class LeakCounter:
@ -34,37 +35,32 @@ def is_litellm_owned_key(key: str, scope: str) -> bool:
return key.startswith(OWNED_KEY_PREFIX) or (scope in OWNED_KEY_SCOPES and key in OWNED_KEYS)
def _as_mapping(value: object) -> Mapping[object, object] | None:
if not isinstance(value, Mapping):
return None
return cast("Mapping[object, object]", value) # cast-ok: isinstance narrows only to Mapping[Unknown, Unknown]
def _walk_mapping(value: Mapping[object, object], prefix: str, found: list[str]) -> None:
scoped: Final = prefix in OWNED_KEY_SCOPES
for key, child in value.items():
if not isinstance(key, str):
continue
if key.startswith(OWNED_KEY_PREFIX) or (scoped and key in OWNED_KEYS):
found.append(f"{prefix}.{key}" if prefix else key)
if isinstance(child, _CONTAINER_TYPES):
_walk(cast(object, child), f"{prefix}.{key}" if prefix else key, found) # cast-ok: undo Unknown narrowing
def _as_sequence(value: object) -> Sequence[object] | None:
if not isinstance(value, Sequence) or isinstance(value, (str, bytes, bytearray)):
return None
return cast("Sequence[object]", value) # cast-ok: isinstance narrows only to Sequence[Unknown]
def _child_paths(value: object, prefix: str) -> tuple[str, ...]:
mapping: Final = _as_mapping(value)
if mapping is not None:
return tuple(path for key, child in mapping.items() for path in _entry_paths(key, child, prefix))
sequence: Final = _as_sequence(value)
if sequence is not None:
return tuple(path for index, child in enumerate(sequence) for path in _child_paths(child, f"{prefix}[{index}]"))
return ()
def _entry_paths(key: object, child: object, prefix: str) -> tuple[str, ...]:
name: Final = str(key)
path: Final = f"{prefix}.{name}" if prefix else name
own: Final = (path,) if is_litellm_owned_key(name, prefix) else ()
return (*own, *_child_paths(child, path))
def _walk(value: object, prefix: str, found: list[str]) -> None:
if isinstance(value, dict):
_walk_mapping(cast("dict[object, object]", value), prefix, found) # cast-ok: isinstance leaves Unknown params
return
if not isinstance(value, (list, tuple)):
return
for index, child in enumerate(cast("list[object] | tuple[object, ...]", value)): # cast-ok: same as above
if isinstance(child, _CONTAINER_TYPES):
_walk(cast(object, child), f"{prefix}[{index}]", found) # cast-ok: undo Unknown narrowing
def litellm_owned_key_paths(body: Mapping[str, object]) -> tuple[str, ...]:
return _child_paths(body, "")
found: Final[list[str]] = [] # mutable-ok: single accumulator for the hot-path walk, sealed into a tuple below
_walk_mapping(cast("Mapping[object, object]", body), "", found) # cast-ok: widen invariant key type
return tuple(found)
def observe_internal_keys(body: Mapping[str, object], provider: str) -> tuple[str, ...]:

View file

@ -1076,7 +1076,7 @@ class BaseLLMHTTPHandler:
litellm_params=litellm_params,
)
data: Final = provider_config.transform_embedding_request(
data: Final[dict[str, object]] = provider_config.transform_embedding_request(
model=model,
input=input,
optional_params=optional_params,
@ -1084,6 +1084,7 @@ class BaseLLMHTTPHandler:
)
if embedding_extra_body:
data.update(embedding_extra_body)
observe_internal_keys(data, custom_llm_provider)
# Some providers (e.g. OCI) require request signing after the body is built.
# The default BaseConfig.sign_request returns (headers, None) — a no-op for
@ -2702,6 +2703,7 @@ class BaseLLMHTTPHandler:
if extra_body:
data.update(extra_body)
observe_internal_keys(data, custom_llm_provider)
stream = bool(stream or data.get("stream"))
# Preserve the OpenAI-style request context (not sent to the provider) for streaming
@ -2890,6 +2892,7 @@ class BaseLLMHTTPHandler:
if extra_body:
data.update(extra_body)
observe_internal_keys(data, custom_llm_provider)
stream = bool(stream or data.get("stream"))
# Preserve the OpenAI-style request context (not sent to the provider) for streaming

View file

@ -51,6 +51,15 @@
"tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_converse_body_is_observed_at_emission": [
"other.provider_wire.internal_key_emission_observed_bedrock_converse"
],
"tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_embedding_body_is_observed_at_emission": [
"other.provider_wire.internal_key_emission_observed_embedding"
],
"tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_responses_body_is_observed_at_emission": [
"other.provider_wire.internal_key_emission_observed_responses"
],
"tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_async_responses_body_is_observed_at_emission": [
"other.provider_wire.internal_key_emission_observed_responses_async"
],
"tests/integration/pricing/test_configured_prices.py::test_loaded_router_preserves_cached_defaults_during_real_requests": [
"quota_management.spend_tracking.default_prices.loaded_router_preserves_cached_defaults"
],

View file

@ -1,3 +1,4 @@
import asyncio
import json
import logging
from typing import Final
@ -25,6 +26,43 @@ def converse_peer(request: Request) -> Reply:
return Reply(body=CONVERSE_REPLY)
EMBEDDING_REPLY: Final = json.dumps(
{
"object": "list",
"data": [{"object": "embedding", "index": 0, "embedding": [0.25, 0.5]}],
"model": "synthetic-embedding",
"usage": {"prompt_tokens": 3, "total_tokens": 3},
}
).encode()
RESPONSES_REPLY: Final = json.dumps(
{
"id": "resp_synthetic",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "synthetic-responses",
"output": [
{
"type": "message",
"id": "msg_synthetic",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "responses guard control", "annotations": []}],
}
],
"usage": {"input_tokens": 3, "output_tokens": 3, "total_tokens": 6},
}
).encode()
def embedding_peer(request: Request) -> Reply:
return Reply(body=EMBEDDING_REPLY)
def responses_peer(request: Request) -> Reply:
return Reply(body=RESPONSES_REPLY)
@pytest.mark.covers("other.provider_wire.internal_parameters_filtered")
def test_internal_request_state_does_not_reach_provider(gateway: Gateway) -> None:
with gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream:
@ -117,6 +155,81 @@ def test_internal_key_forced_into_converse_body_is_observed_at_emission(
]
@pytest.mark.covers("other.provider_wire.internal_key_emission_observed_embedding")
def test_internal_key_forced_into_embedding_body_is_observed_at_emission(caplog: pytest.LogCaptureFixture) -> None:
before: Final = internal_key_leak_counter.value
with wire_server(embedding_peer) as wire, caplog.at_level(logging.WARNING, logger="LiteLLM"):
result: Final = litellm.embedding(
model="hosted_vllm/synthetic-embedding",
api_base=f"{wire.url}/v1",
api_key="sk-synthetic",
input=["embedding emission guard"],
extra_body={"litellm_call_id": "forced-through-extra-body"},
timeout=5,
num_retries=0,
)
received: Final = wire.drain()
assert isinstance(result, litellm.EmbeddingResponse)
assert len(received) == 1
assert received[0].target == "/v1/embeddings"
assert JSON_OBJECT.validate_json(received[0].body)["litellm_call_id"] == "forced-through-extra-body"
assert internal_key_leak_counter.value == before + 1
assert [record.getMessage() for record in caplog.records if record.levelno == logging.WARNING] == [
"LiteLLM internal keys reached the hosted_vllm provider request body: litellm_call_id"
]
def assert_responses_body_observed(
response: object, received: tuple[Request, ...], before: int, caplog: pytest.LogCaptureFixture
) -> None:
assert isinstance(response, litellm.ResponsesAPIResponse)
assert len(received) == 1
assert received[0].target == "/v1/responses"
assert JSON_OBJECT.validate_json(received[0].body)["litellm_call_id"] == "forced-through-extra-body"
assert internal_key_leak_counter.value == before + 1
assert [record.getMessage() for record in caplog.records if record.levelno == logging.WARNING] == [
"LiteLLM internal keys reached the openai provider request body: litellm_call_id"
]
@pytest.mark.covers("other.provider_wire.internal_key_emission_observed_responses")
def test_internal_key_forced_into_responses_body_is_observed_at_emission(caplog: pytest.LogCaptureFixture) -> None:
before: Final = internal_key_leak_counter.value
with wire_server(responses_peer) as wire, caplog.at_level(logging.WARNING, logger="LiteLLM"):
response: Final = litellm.responses(
model="openai/synthetic-responses",
api_base=f"{wire.url}/v1",
api_key="sk-synthetic",
input="responses emission guard",
extra_body={"litellm_call_id": "forced-through-extra-body"},
timeout=5,
num_retries=0,
)
received: Final = wire.drain()
assert_responses_body_observed(response, received, before, caplog)
@pytest.mark.covers("other.provider_wire.internal_key_emission_observed_responses_async")
def test_internal_key_forced_into_async_responses_body_is_observed_at_emission(
caplog: pytest.LogCaptureFixture,
) -> None:
before: Final = internal_key_leak_counter.value
with wire_server(responses_peer) as wire, caplog.at_level(logging.WARNING, logger="LiteLLM"):
response: Final = asyncio.run(
litellm.aresponses(
model="openai/synthetic-responses",
api_base=f"{wire.url}/v1",
api_key="sk-synthetic",
input="responses emission guard",
extra_body={"litellm_call_id": "forced-through-extra-body"},
timeout=5,
num_retries=0,
)
)
received: Final = wire.drain()
assert_responses_body_observed(response, received, before, caplog)
@pytest.mark.covers("other.provider_wire.validator_rejects_corruption")
def test_upstream_rejects_corruption_and_accepts_supported_metadata(gateway: Gateway) -> None:
with httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream: