From 0bc103911e8b1b4e0e77f8b9cbe81cfbef728875 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 11 Nov 2026 17:12:51 -0800 Subject: [PATCH] fix(proxy): bill anthropic pass-through calls at the deployment's custom price (#51267) * fix(proxy): bill anthropic pass-through calls at the deployment's custom price The /anthropic route never attached a router deployment's model_info, so spend fell back to the shared cost-map key, which carries no per-deployment prices. Attach the model_info of the one deployment in the requested group that serves that exact Anthropic model and carries custom pricing, so streaming and non-streaming calls, and dated snapshot responses, bill at it. Fixes #49596 * fix(proxy): validate deployment rows before reading anthropic pass-through pricing Read the untyped router rows and request body through a pydantic adapter so the pricing lookup adds no basedpyright unknown-type errors. * test(proxy): drive the real anthropic pass-through relay when asserting the deployment price Stub only the upstream http client and assert the x-litellm-response-cost the route returns, instead of doubling create_pass_through_route. * fix(proxy): match an anthropic pass-through deployment by its configured or resolved model name get_llm_provider rewrites -latest aliases, so a deployment configured with the alias the caller sends never matched and lost its custom price. * fix(proxy): price an anthropic pass-through call by the model its custom_body forwards The relay forwards custom_body when the caller sends one, so resolving the deployment from the outer model let a caller bill an expensive call at a cheaper deployment's price. * fix(proxy): only price an anthropic pass-through call with a deployment the caller's team may use A teamless key could pick up a team-owned deployment's price because the group lookup does not filter by team when the key has none. * fix(proxy): resolve anthropic pass-through pricing from the body forwarded after pre-call hooks The route stamped the deployment price before the pass-through ran its pre-call hooks, so a hook that swapped the model billed the original model's deployment. The route now hands the relay a resolver, and the relay applies it to the final forwarded body, which already reflects the custom_body envelope and every hook. * refactor(proxy): clear pass-through request state keys in one helper Keeps create_pass_through_route within the C901 ceiling now that the deployment pricing resolver adds a fourth request.state key. --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../llm_passthrough_endpoints.py | 71 ++++- .../pass_through_endpoints.py | 31 ++- .../pass_through_endpoints.py | 11 +- ...t_anthropic_passthrough_logging_handler.py | 68 +++++ .../test_llm_pass_through_endpoints.py | 250 ++++++++++++++++++ 5 files changed, 423 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 321c921b5bd..782b5698f3c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -177,7 +177,9 @@ from litellm.types.passthrough_endpoints.eu_jev import EuJevDecisionRequest, EuJ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, + LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, + DeploymentPricingResolver, WebSocketSessionMeter, ) from litellm.types.passthrough_endpoints.tinyfish import ( @@ -188,7 +190,12 @@ from litellm.types.passthrough_endpoints.tinyfish import ( ) from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials from litellm.types.router import LiteLLMParamsTypedDict -from litellm.types.utils import CallTypes, LlmProviders +from litellm.types.utils import ( + DEPLOYMENT_SCOPED_PRICING_FIELDS, + SERVER_DERIVED_PRICING_FIELDS, + CallTypes, + LlmProviders, +) from litellm.types.vector_stores import LiteLLM_ManagedVectorStore, LiteLLM_ManagedVectorStoreIndex from litellm.utils import ProviderConfigManager, with_named_credential @@ -248,6 +255,58 @@ def _models_served_by_group(llm_router: litellm.Router, model_group: str) -> fro ) +_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_DEPLOYMENT_PRICING_FIELDS: Final[frozenset[str]] = SERVER_DERIVED_PRICING_FIELDS | DEPLOYMENT_SCOPED_PRICING_FIELDS + + +def _object_mapping(value: object) -> Mapping[str, object]: + return _OBJECT_MAPPING.validate_python(value if isinstance(value, Mapping) else {}) + + +def _anthropic_deployment_model_info(deployment: object, model: str) -> Mapping[str, object] | None: + fields: Final = _object_mapping(deployment) + params: Final = _object_mapping(fields.get("litellm_params")) + backend: Final = params.get("model") + provider_override: Final = params.get("custom_llm_provider") + if not isinstance(backend, str): + return None + try: + backend_model, provider, _, _ = get_llm_provider( + model=backend, custom_llm_provider=provider_override if isinstance(provider_override, str) else None + ) + except litellm.BadRequestError: + return None + if provider != LlmProviders.ANTHROPIC.value or model not in (backend_model, backend.removeprefix(f"{provider}/")): + return None + return { + **_object_mapping(fields.get("model_info")), + **{field: value for field in _DEPLOYMENT_PRICING_FIELDS if (value := params.get(field)) is not None}, + } + + +def anthropic_deployment_pricing_model_info( + llm_router: litellm.Router | None, request_body: Mapping[str, object], team_id: str | None +) -> Mapping[str, object] | None: + """ + The pass-through selects no deployment, so ``request_body`` is the body it forwards. Only a single deployment in + that model's group serving that exact Anthropic model can say what it costs: with several, any pick could bill + another deployment's negotiated price. + """ + model: Final = request_body.get("model") + if llm_router is None or not isinstance(model, str) or not model: + return None + candidates: Final = tuple( + model_info + for deployment in llm_router.registry().deployments(model, team_id=team_id) or () + if llm_router.registry().includes(deployment, team_id=team_id) + and (model_info := _anthropic_deployment_model_info(deployment, model)) is not None + ) + if len(candidates) != 1: + return None + (only_match,) = candidates + return only_match if any(only_match.get(field) is not None for field in _DEPLOYMENT_PRICING_FIELDS) else None + + def is_passthrough_request_streaming(request_body: object) -> bool: """ Returns True if the request is streaming. @@ -1236,6 +1295,16 @@ async def anthropic_proxy_route( ## check for streaming is_streaming_request: Final = await is_streaming_request_fn(request) + from litellm.proxy.proxy_server import llm_router + + setattr( + request.state, + LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY, + DeploymentPricingResolver( + resolve=partial(anthropic_deployment_pricing_model_info, llm_router, team_id=user_api_key_dict.team_id) + ), + ) + ## CREATE PASS-THROUGH auth_header: Final = await AnthropicModelInfo.aget_auth_header( anthropic_api_key or None, allow_workload_identity=True diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 00e2aeb9e6e..8a728a7335a 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -167,8 +167,10 @@ from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, + LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, + DeploymentPricingResolver, EndpointType, PassthroughStandardLoggingPayload, WebSocketSessionMeter, @@ -448,6 +450,20 @@ async def chat_completion_pass_through_endpoint( ) +_PASS_THROUGH_REQUEST_STATE_KEYS: Final = ( + LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, + LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, + LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, + LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY, +) + + +def _clear_pass_through_request_state(state: object) -> None: + for key in _PASS_THROUGH_REQUEST_STATE_KEYS: + if hasattr(state, key): + delattr(state, key) + + def _is_provider_payload(request: Request) -> bool: return is_opaque_pass_through_request(get_request_route(request), request.headers.get("content-type", "")) @@ -798,6 +814,14 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): ) if isinstance(deployment_model_info, Mapping): _metadata["model_info"] = dict(deployment_model_info) + deployment_pricing: Final = getattr(_request_state, LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY, None) + priced_model_info: Final = ( + deployment_pricing.resolve(server_marker_free_body) + if isinstance(deployment_pricing, DeploymentPricingResolver) + else None + ) + if priced_model_info is not None: + _metadata["model_info"] = dict(priced_model_info) kwargs: Final = { "litellm_params": { @@ -2536,12 +2560,7 @@ def create_pass_through_route( keep_target_query=keep_target_query, ) finally: - if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY): - delattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY) - if hasattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY): - delattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY) - if hasattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY): - delattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY) + _clear_pass_through_request_state(request.state) # The upstream withholds its response headers until its first token, so # the whole time-to-first-token is spent inside _relay with nothing on diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py index b6bab77ab33..8a2b2a8f99c 100644 --- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py +++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime from enum import Enum @@ -20,6 +20,15 @@ LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY: Final = "litellm_pass_through_raw_body" # `model_info` of the router deployment a provider route (e.g. Vertex) resolved for this request. LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY: Final = "litellm_pass_through_deployment_model_info" +# Resolves that `model_info` from the body actually forwarded, after the `custom_body` envelope and pre-call hooks. +LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY: Final = "litellm_pass_through_deployment_pricing" + + +@dataclass(frozen=True, slots=True) +class DeploymentPricingResolver: + resolve: Callable[[Mapping[str, object]], Mapping[str, object] | None] + + # Attribute set on the FastAPI endpoint function of every user-defined pass-through # route. Auth reads it off the dispatched endpoint (``request.scope["endpoint"]``) to # decide whether a request body ``model`` names an upstream model rather than a diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index ad12716ef2a..df011a8296f 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -3102,3 +3102,71 @@ def test_anthropic_passthrough_stream_bills_a_bedrock_guardrail_trace_whose_json ) (entry,) = _recorded_guardrail_entries(mock_logging_obj) assert entry["guardrail_usage"] == {"topicPolicyUnits": 3, "contentPolicyUnits": 3} + + +_DEPLOYMENT_PRICING: Final = {"input_cost_per_token": 3e-03, "output_cost_per_token": 7e-03} + + +def _deployment_priced_logging_obj(monkeypatch: pytest.MonkeyPatch, stream: bool) -> LiteLLMLoggingObj: + monkeypatch.setattr(litellm, "model_cost", {"sonnet-dep": {"litellm_provider": "anthropic", **_DEPLOYMENT_PRICING}}) + logging_obj: Final = LiteLLMLoggingObj( + model="claude-sonnet-4-5", + messages=[], + stream=stream, + call_type="pass_through_endpoint", + start_time=datetime(2025, 1, 1), + litellm_call_id="deployment-priced", + function_id="1", + ) + litellm_params: Final = {"metadata": {"model_info": {"id": "sonnet-dep", **_DEPLOYMENT_PRICING}}} + logging_obj.litellm_params = litellm_params + logging_obj.model_call_details["litellm_params"] = litellm_params + logging_obj.model_call_details["custom_llm_provider"] = "anthropic" + logging_obj.model_call_details["stream"] = stream + return logging_obj + + +def _deployment_price_of(usage: litellm.Usage) -> float: + return ( + usage.prompt_tokens * _DEPLOYMENT_PRICING["input_cost_per_token"] + + usage.completion_tokens * _DEPLOYMENT_PRICING["output_cost_per_token"] + ) + + +def test_a_response_naming_a_dated_snapshot_is_billed_at_the_resolved_deployment_price( + monkeypatch, mock_httpx_response +): + logging_obj: Final = _deployment_priced_logging_obj(monkeypatch, stream=False) + + result: Final = AnthropicPassthroughLoggingHandler.anthropic_passthrough_handler( + httpx_response=mock_httpx_response, + response_body=mock_httpx_response.json(), + logging_obj=logging_obj, + url_route="/v1/messages", + result="success", + start_time=datetime(2025, 1, 1), + end_time=datetime(2025, 1, 1), + cache_hit=False, + request_body={"model": "claude-sonnet-4-5", "messages": []}, + ) + + assert result["result"].usage.prompt_tokens > 0 + assert result["kwargs"]["response_cost"] == pytest.approx(_deployment_price_of(result["result"].usage)) + + +def test_a_stream_naming_a_dated_snapshot_is_billed_at_the_resolved_deployment_price(monkeypatch, all_chunks): + logging_obj: Final = _deployment_priced_logging_obj(monkeypatch, stream=True) + + result: Final = AnthropicPassthroughLoggingHandler.handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + request_body={"model": "claude-sonnet-4-5", "stream": True, "messages": []}, + endpoint_type=MagicMock(), + start_time=datetime(2025, 1, 1), + all_chunks=all_chunks, + end_time=datetime(2025, 1, 1), + ) + + assert result["result"].usage.prompt_tokens > 0 + assert result["kwargs"]["response_cost"] == pytest.approx(_deployment_price_of(result["result"].usage)) diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 7f118fb3186..ca04787e592 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -7256,6 +7256,256 @@ class TestAnthropicProxyRoute: assert thread_ids and thread_ids[0] != threading.get_ident() +_HAIKU_PRICING: Final = MappingProxyType({"input_cost_per_token": 1e-07, "output_cost_per_token": 5e-07}) + + +def _anthropic_deployment( + deployment_id: str, + model_name: str = "claude-haiku-5-5", + backend: str = "anthropic/claude-haiku-5-5", + model_info_pricing: Mapping[str, float] = _HAIKU_PRICING, + params_pricing: Mapping[str, float] = MappingProxyType({}), + team_id: str | None = None, +) -> dict[str, object]: + return { + "model_name": model_name, + "litellm_params": {"model": backend, "api_key": "sk-ant-test", **params_pricing}, + "model_info": {"id": deployment_id, **model_info_pricing, **({"team_id": team_id} if team_id else {})}, + } + + +def _passthrough_pricing( + monkeypatch: pytest.MonkeyPatch, + *deployments: dict[str, object], + request_body: Mapping[str, object] = MappingProxyType({"model": "claude-haiku-5-5"}), + team_id: str | None = None, +) -> Mapping[str, object] | None: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + anthropic_deployment_pricing_model_info, + ) + + monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) + router: Final = litellm.Router(model_list=[*deployments]) + return anthropic_deployment_pricing_model_info(router, request_body, team_id) + + +def test_anthropic_passthrough_prices_with_the_one_deployment_serving_the_requested_model(monkeypatch): + model_info: Final = _passthrough_pricing(monkeypatch, _anthropic_deployment("haiku-dep")) + + assert model_info is not None + assert model_info["id"] == "haiku-dep" + assert {key: model_info[key] for key in _HAIKU_PRICING} == _HAIKU_PRICING + + +def test_anthropic_passthrough_reads_custom_pricing_set_in_litellm_params(monkeypatch): + model_info: Final = _passthrough_pricing( + monkeypatch, + _anthropic_deployment("haiku-dep", model_info_pricing=MappingProxyType({}), params_pricing=_HAIKU_PRICING), + ) + + assert model_info is not None + assert model_info["id"] == "haiku-dep" + assert {key: model_info[key] for key in _HAIKU_PRICING} == _HAIKU_PRICING + + +def test_anthropic_passthrough_prices_a_deployment_configured_with_the_alias_the_caller_sends(monkeypatch): + model_info: Final = _passthrough_pricing( + monkeypatch, + _anthropic_deployment( + "haiku-latest-dep", model_name="claude-haiku-latest", backend="anthropic/claude-haiku-latest" + ), + request_body={"model": "claude-haiku-latest"}, + ) + + assert model_info is not None + assert model_info["id"] == "haiku-latest-dep" + + +@pytest.mark.parametrize( + ("caller_team_id", "priced"), + [ + pytest.param("team-a", True, id="owning-team"), + pytest.param("team-b", False, id="other-team"), + pytest.param(None, False, id="teamless-key"), + ], +) +def test_anthropic_passthrough_prices_a_team_deployment_only_for_its_team(monkeypatch, caller_team_id, priced): + model_info: Final = _passthrough_pricing( + monkeypatch, _anthropic_deployment("team-a-haiku-dep", team_id="team-a"), team_id=caller_team_id + ) + + assert (model_info is not None) is priced + + +@pytest.mark.parametrize( + "deployments", + [ + pytest.param( + ( + _anthropic_deployment("haiku-dep-a"), + _anthropic_deployment("haiku-dep-b", model_info_pricing={"input_cost_per_token": 9e-07}), + ), + id="two-deployments-could-bill-either-price", + ), + pytest.param( + (_anthropic_deployment("opus-dep", backend="anthropic/claude-opus-5-5"),), + id="group-serves-a-different-upstream-model", + ), + pytest.param( + (_anthropic_deployment("bedrock-dep", backend="bedrock/claude-haiku-5-5"),), + id="deployment-is-not-anthropic", + ), + pytest.param( + (_anthropic_deployment("other-group-dep", model_name="haiku-alias"),), + id="model-group-not-requested", + ), + pytest.param( + (_anthropic_deployment("unpriced-dep", model_info_pricing=MappingProxyType({})),), + id="deployment-has-no-custom-pricing", + ), + ], +) +def test_anthropic_passthrough_attaches_no_deployment_pricing_without_one_priced_match(monkeypatch, deployments): + assert _passthrough_pricing(monkeypatch, *deployments) is None + + +_RELAY_HAIKU_PRICING: Final = MappingProxyType({"input_cost_per_token": 1e-03, "output_cost_per_token": 2e-03}) +_RELAY_OPUS_PRICING: Final = MappingProxyType({"input_cost_per_token": 5e-03, "output_cost_per_token": 6e-03}) +_RELAY_MESSAGES: Final = ({"role": "user", "content": "hi"},) + + +async def _hook_passes_body_through(*, data: dict[str, object], **_: object) -> dict[str, object]: + return data + + +async def _hook_upgrades_to_opus(*, data: dict[str, object], **_: object) -> dict[str, object]: + return {**data, "model": "claude-opus-5-5"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_body", "pre_call_hook", "forwarded_model", "pricing"), + [ + pytest.param( + {"model": "claude-haiku-5-5", "messages": [*_RELAY_MESSAGES]}, + _hook_passes_body_through, + "claude-haiku-5-5", + _RELAY_HAIKU_PRICING, + id="requested-model", + ), + pytest.param( + {"model": "claude-haiku-5-5", "custom_body": {"model": "claude-opus-5-5", "messages": [*_RELAY_MESSAGES]}}, + _hook_passes_body_through, + "claude-opus-5-5", + _RELAY_OPUS_PRICING, + id="custom-body-envelope", + ), + pytest.param( + {"model": "claude-haiku-5-5", "messages": [*_RELAY_MESSAGES]}, + _hook_upgrades_to_opus, + "claude-opus-5-5", + _RELAY_OPUS_PRICING, + id="pre-call-hook-rewrite", + ), + ], +) +async def test_anthropic_proxy_route_bills_the_custom_price_of_the_forwarded_model( + monkeypatch, request_body, pre_call_hook, forwarded_model, pricing +): + for name in ( + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_API_BASE", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_FEDERATION_RULE_ID", + "ANTHROPIC_ORGANIZATION_ID", + "ANTHROPIC_IDENTITY_TOKEN_FILE", + "ANTHROPIC_IDENTITY_TOKEN", + "SERVER_ROOT_PATH", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test") + monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) + monkeypatch.setattr( + "litellm.proxy.proxy_server.llm_router", + litellm.Router( + model_list=[ + _anthropic_deployment("haiku-dep", model_info_pricing=_RELAY_HAIKU_PRICING), + _anthropic_deployment( + "opus-dep", + model_name="claude-opus-5-5", + backend="anthropic/claude-opus-5-5", + model_info_pricing=_RELAY_OPUS_PRICING, + ), + ] + ), + ) + upstream_response: Final = httpx.Response( + 200, + json={ + "id": "msg_relay", + "type": "message", + "role": "assistant", + "model": forwarded_model, + "content": [{"type": "text", "text": "pong"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 18, "output_tokens": 4}, + }, + request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), + ) + httpx_client: Final = MagicMock() + httpx_client.build_request = MagicMock(return_value=MagicMock()) + httpx_client.send = AsyncMock(return_value=upstream_response) + client_wrapper: Final = MagicMock() + client_wrapper.client = httpx_client + body: Final = json.dumps(request_body).encode() + + async def receive() -> dict[str, object]: + return {"type": "http.request", "body": body, "more_body": False} + + request: Final = Request( + { + "type": "http", + "http_version": "1.1", + "method": "POST", + "scheme": "https", + "path": "/anthropic/v1/messages", + "raw_path": b"/anthropic/v1/messages", + "root_path": "", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + "client": ("127.0.0.1", 51234), + "server": ("proxy.local", 4000), + "state": {}, + }, + receive, + ) + + with ( + patch( # test-quality-ok: the upstream http client is the boundary, it answers with a real Anthropic body + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client", + return_value=client_wrapper, + ), + patch( # test-quality-ok: the relay calls these hooks, and they need a db this test has no use for + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_logging_obj, + ): + mock_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) + mock_logging_obj.post_call_success_hook = AsyncMock() + mock_logging_obj.post_call_failure_hook = AsyncMock() + mock_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + response: Final = await anthropic_proxy_route( + endpoint="v1/messages", + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-caller-virtual-key"), + ) + + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx( + 18 * pricing["input_cost_per_token"] + 4 * pricing["output_cost_per_token"] + ) + + class TestAnthropicProxyRouteCallerAuthHeaders: """Regression for a caller credential riding upstream next to a server-owned one.