mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
perf(core): walk provider bodies with a single accumulator and observe embedding and responses merges
This commit is contained in:
parent
22fc88298d
commit
029eeaccf7
4 changed files with 149 additions and 28 deletions
|
|
@ -1,5 +1,5 @@
|
|||
import threading
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Mapping
|
||||
from typing import Final, cast
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -10,6 +10,7 @@ OWNED_KEYS: Final = frozenset(key for key in ("litellm_params", *all_litellm_par
|
|||
OWNED_KEY_SCOPES: Final = frozenset(
|
||||
("", "metadata", "additionalModelRequestFields", "additionalModelRequestFields.extra_body")
|
||||
)
|
||||
_CONTAINER_TYPES: Final[tuple[type[object], ...]] = (dict, list, tuple)
|
||||
|
||||
|
||||
class LeakCounter:
|
||||
|
|
@ -34,37 +35,32 @@ def is_litellm_owned_key(key: str, scope: str) -> bool:
|
|||
return key.startswith(OWNED_KEY_PREFIX) or (scope in OWNED_KEY_SCOPES and key in OWNED_KEYS)
|
||||
|
||||
|
||||
def _as_mapping(value: object) -> Mapping[object, object] | None:
|
||||
if not isinstance(value, Mapping):
|
||||
return None
|
||||
return cast("Mapping[object, object]", value) # cast-ok: isinstance narrows only to Mapping[Unknown, Unknown]
|
||||
def _walk_mapping(value: Mapping[object, object], prefix: str, found: list[str]) -> None:
|
||||
scoped: Final = prefix in OWNED_KEY_SCOPES
|
||||
for key, child in value.items():
|
||||
if not isinstance(key, str):
|
||||
continue
|
||||
if key.startswith(OWNED_KEY_PREFIX) or (scoped and key in OWNED_KEYS):
|
||||
found.append(f"{prefix}.{key}" if prefix else key)
|
||||
if isinstance(child, _CONTAINER_TYPES):
|
||||
_walk(cast(object, child), f"{prefix}.{key}" if prefix else key, found) # cast-ok: undo Unknown narrowing
|
||||
|
||||
|
||||
def _as_sequence(value: object) -> Sequence[object] | None:
|
||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes, bytearray)):
|
||||
return None
|
||||
return cast("Sequence[object]", value) # cast-ok: isinstance narrows only to Sequence[Unknown]
|
||||
|
||||
|
||||
def _child_paths(value: object, prefix: str) -> tuple[str, ...]:
|
||||
mapping: Final = _as_mapping(value)
|
||||
if mapping is not None:
|
||||
return tuple(path for key, child in mapping.items() for path in _entry_paths(key, child, prefix))
|
||||
sequence: Final = _as_sequence(value)
|
||||
if sequence is not None:
|
||||
return tuple(path for index, child in enumerate(sequence) for path in _child_paths(child, f"{prefix}[{index}]"))
|
||||
return ()
|
||||
|
||||
|
||||
def _entry_paths(key: object, child: object, prefix: str) -> tuple[str, ...]:
|
||||
name: Final = str(key)
|
||||
path: Final = f"{prefix}.{name}" if prefix else name
|
||||
own: Final = (path,) if is_litellm_owned_key(name, prefix) else ()
|
||||
return (*own, *_child_paths(child, path))
|
||||
def _walk(value: object, prefix: str, found: list[str]) -> None:
|
||||
if isinstance(value, dict):
|
||||
_walk_mapping(cast("dict[object, object]", value), prefix, found) # cast-ok: isinstance leaves Unknown params
|
||||
return
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return
|
||||
for index, child in enumerate(cast("list[object] | tuple[object, ...]", value)): # cast-ok: same as above
|
||||
if isinstance(child, _CONTAINER_TYPES):
|
||||
_walk(cast(object, child), f"{prefix}[{index}]", found) # cast-ok: undo Unknown narrowing
|
||||
|
||||
|
||||
def litellm_owned_key_paths(body: Mapping[str, object]) -> tuple[str, ...]:
|
||||
return _child_paths(body, "")
|
||||
found: Final[list[str]] = [] # mutable-ok: single accumulator for the hot-path walk, sealed into a tuple below
|
||||
_walk_mapping(cast("Mapping[object, object]", body), "", found) # cast-ok: widen invariant key type
|
||||
return tuple(found)
|
||||
|
||||
|
||||
def observe_internal_keys(body: Mapping[str, object], provider: str) -> tuple[str, ...]:
|
||||
|
|
|
|||
|
|
@ -1076,7 +1076,7 @@ class BaseLLMHTTPHandler:
|
|||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
data: Final = provider_config.transform_embedding_request(
|
||||
data: Final[dict[str, object]] = provider_config.transform_embedding_request(
|
||||
model=model,
|
||||
input=input,
|
||||
optional_params=optional_params,
|
||||
|
|
@ -1084,6 +1084,7 @@ class BaseLLMHTTPHandler:
|
|||
)
|
||||
if embedding_extra_body:
|
||||
data.update(embedding_extra_body)
|
||||
observe_internal_keys(data, custom_llm_provider)
|
||||
|
||||
# Some providers (e.g. OCI) require request signing after the body is built.
|
||||
# The default BaseConfig.sign_request returns (headers, None) — a no-op for
|
||||
|
|
@ -2702,6 +2703,7 @@ class BaseLLMHTTPHandler:
|
|||
|
||||
if extra_body:
|
||||
data.update(extra_body)
|
||||
observe_internal_keys(data, custom_llm_provider)
|
||||
stream = bool(stream or data.get("stream"))
|
||||
|
||||
# Preserve the OpenAI-style request context (not sent to the provider) for streaming
|
||||
|
|
@ -2890,6 +2892,7 @@ class BaseLLMHTTPHandler:
|
|||
|
||||
if extra_body:
|
||||
data.update(extra_body)
|
||||
observe_internal_keys(data, custom_llm_provider)
|
||||
stream = bool(stream or data.get("stream"))
|
||||
|
||||
# Preserve the OpenAI-style request context (not sent to the provider) for streaming
|
||||
|
|
|
|||
|
|
@ -51,6 +51,15 @@
|
|||
"tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_converse_body_is_observed_at_emission": [
|
||||
"other.provider_wire.internal_key_emission_observed_bedrock_converse"
|
||||
],
|
||||
"tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_embedding_body_is_observed_at_emission": [
|
||||
"other.provider_wire.internal_key_emission_observed_embedding"
|
||||
],
|
||||
"tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_responses_body_is_observed_at_emission": [
|
||||
"other.provider_wire.internal_key_emission_observed_responses"
|
||||
],
|
||||
"tests/integration/providers/test_request_boundary.py::test_internal_key_forced_into_async_responses_body_is_observed_at_emission": [
|
||||
"other.provider_wire.internal_key_emission_observed_responses_async"
|
||||
],
|
||||
"tests/integration/pricing/test_configured_prices.py::test_loaded_router_preserves_cached_defaults_during_real_requests": [
|
||||
"quota_management.spend_tracking.default_prices.loaded_router_preserves_cached_defaults"
|
||||
],
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Final
|
||||
|
|
@ -25,6 +26,43 @@ def converse_peer(request: Request) -> Reply:
|
|||
return Reply(body=CONVERSE_REPLY)
|
||||
|
||||
|
||||
EMBEDDING_REPLY: Final = json.dumps(
|
||||
{
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.25, 0.5]}],
|
||||
"model": "synthetic-embedding",
|
||||
"usage": {"prompt_tokens": 3, "total_tokens": 3},
|
||||
}
|
||||
).encode()
|
||||
RESPONSES_REPLY: Final = json.dumps(
|
||||
{
|
||||
"id": "resp_synthetic",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "synthetic-responses",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_synthetic",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "responses guard control", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 3, "total_tokens": 6},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def embedding_peer(request: Request) -> Reply:
|
||||
return Reply(body=EMBEDDING_REPLY)
|
||||
|
||||
|
||||
def responses_peer(request: Request) -> Reply:
|
||||
return Reply(body=RESPONSES_REPLY)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.internal_parameters_filtered")
|
||||
def test_internal_request_state_does_not_reach_provider(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream:
|
||||
|
|
@ -117,6 +155,81 @@ def test_internal_key_forced_into_converse_body_is_observed_at_emission(
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.internal_key_emission_observed_embedding")
|
||||
def test_internal_key_forced_into_embedding_body_is_observed_at_emission(caplog: pytest.LogCaptureFixture) -> None:
|
||||
before: Final = internal_key_leak_counter.value
|
||||
with wire_server(embedding_peer) as wire, caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
result: Final = litellm.embedding(
|
||||
model="hosted_vllm/synthetic-embedding",
|
||||
api_base=f"{wire.url}/v1",
|
||||
api_key="sk-synthetic",
|
||||
input=["embedding emission guard"],
|
||||
extra_body={"litellm_call_id": "forced-through-extra-body"},
|
||||
timeout=5,
|
||||
num_retries=0,
|
||||
)
|
||||
received: Final = wire.drain()
|
||||
assert isinstance(result, litellm.EmbeddingResponse)
|
||||
assert len(received) == 1
|
||||
assert received[0].target == "/v1/embeddings"
|
||||
assert JSON_OBJECT.validate_json(received[0].body)["litellm_call_id"] == "forced-through-extra-body"
|
||||
assert internal_key_leak_counter.value == before + 1
|
||||
assert [record.getMessage() for record in caplog.records if record.levelno == logging.WARNING] == [
|
||||
"LiteLLM internal keys reached the hosted_vllm provider request body: litellm_call_id"
|
||||
]
|
||||
|
||||
|
||||
def assert_responses_body_observed(
|
||||
response: object, received: tuple[Request, ...], before: int, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
assert isinstance(response, litellm.ResponsesAPIResponse)
|
||||
assert len(received) == 1
|
||||
assert received[0].target == "/v1/responses"
|
||||
assert JSON_OBJECT.validate_json(received[0].body)["litellm_call_id"] == "forced-through-extra-body"
|
||||
assert internal_key_leak_counter.value == before + 1
|
||||
assert [record.getMessage() for record in caplog.records if record.levelno == logging.WARNING] == [
|
||||
"LiteLLM internal keys reached the openai provider request body: litellm_call_id"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.internal_key_emission_observed_responses")
|
||||
def test_internal_key_forced_into_responses_body_is_observed_at_emission(caplog: pytest.LogCaptureFixture) -> None:
|
||||
before: Final = internal_key_leak_counter.value
|
||||
with wire_server(responses_peer) as wire, caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
response: Final = litellm.responses(
|
||||
model="openai/synthetic-responses",
|
||||
api_base=f"{wire.url}/v1",
|
||||
api_key="sk-synthetic",
|
||||
input="responses emission guard",
|
||||
extra_body={"litellm_call_id": "forced-through-extra-body"},
|
||||
timeout=5,
|
||||
num_retries=0,
|
||||
)
|
||||
received: Final = wire.drain()
|
||||
assert_responses_body_observed(response, received, before, caplog)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.internal_key_emission_observed_responses_async")
|
||||
def test_internal_key_forced_into_async_responses_body_is_observed_at_emission(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
before: Final = internal_key_leak_counter.value
|
||||
with wire_server(responses_peer) as wire, caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
response: Final = asyncio.run(
|
||||
litellm.aresponses(
|
||||
model="openai/synthetic-responses",
|
||||
api_base=f"{wire.url}/v1",
|
||||
api_key="sk-synthetic",
|
||||
input="responses emission guard",
|
||||
extra_body={"litellm_call_id": "forced-through-extra-body"},
|
||||
timeout=5,
|
||||
num_retries=0,
|
||||
)
|
||||
)
|
||||
received: Final = wire.drain()
|
||||
assert_responses_body_observed(response, received, before, caplog)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.validator_rejects_corruption")
|
||||
def test_upstream_rejects_corruption_and_accepts_supported_metadata(gateway: Gateway) -> None:
|
||||
with httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue