mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
fix(proxy): bill anthropic pass-through calls at the deployment's custom price (#51267)
* fix(proxy): bill anthropic pass-through calls at the deployment's custom price The /anthropic route never attached a router deployment's model_info, so spend fell back to the shared cost-map key, which carries no per-deployment prices. Attach the model_info of the one deployment in the requested group that serves that exact Anthropic model and carries custom pricing, so streaming and non-streaming calls, and dated snapshot responses, bill at it. Fixes #49596 * fix(proxy): validate deployment rows before reading anthropic pass-through pricing Read the untyped router rows and request body through a pydantic adapter so the pricing lookup adds no basedpyright unknown-type errors. * test(proxy): drive the real anthropic pass-through relay when asserting the deployment price Stub only the upstream http client and assert the x-litellm-response-cost the route returns, instead of doubling create_pass_through_route. * fix(proxy): match an anthropic pass-through deployment by its configured or resolved model name get_llm_provider rewrites -latest aliases, so a deployment configured with the alias the caller sends never matched and lost its custom price. * fix(proxy): price an anthropic pass-through call by the model its custom_body forwards The relay forwards custom_body when the caller sends one, so resolving the deployment from the outer model let a caller bill an expensive call at a cheaper deployment's price. * fix(proxy): only price an anthropic pass-through call with a deployment the caller's team may use A teamless key could pick up a team-owned deployment's price because the group lookup does not filter by team when the key has none. * fix(proxy): resolve anthropic pass-through pricing from the body forwarded after pre-call hooks The route stamped the deployment price before the pass-through ran its pre-call hooks, so a hook that swapped the model billed the original model's deployment. The route now hands the relay a resolver, and the relay applies it to the final forwarded body, which already reflects the custom_body envelope and every hook. * refactor(proxy): clear pass-through request state keys in one helper Keeps create_pass_through_route within the C901 ceiling now that the deployment pricing resolver adds a fourth request.state key. --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
This commit is contained in:
joshua-berri
GitHub
parent
a5d04ec9ac
commit
0bc103911e
5 changed files with 423 additions and 8 deletions
|
|
@ -177,7 +177,9 @@ from litellm.types.passthrough_endpoints.eu_jev import EuJevDecisionRequest, EuJ
|
|||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
|
||||
DeploymentPricingResolver,
|
||||
WebSocketSessionMeter,
|
||||
)
|
||||
from litellm.types.passthrough_endpoints.tinyfish import (
|
||||
|
|
@ -188,7 +190,12 @@ from litellm.types.passthrough_endpoints.tinyfish import (
|
|||
)
|
||||
from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials
|
||||
from litellm.types.router import LiteLLMParamsTypedDict
|
||||
from litellm.types.utils import CallTypes, LlmProviders
|
||||
from litellm.types.utils import (
|
||||
DEPLOYMENT_SCOPED_PRICING_FIELDS,
|
||||
SERVER_DERIVED_PRICING_FIELDS,
|
||||
CallTypes,
|
||||
LlmProviders,
|
||||
)
|
||||
from litellm.types.vector_stores import LiteLLM_ManagedVectorStore, LiteLLM_ManagedVectorStoreIndex
|
||||
from litellm.utils import ProviderConfigManager, with_named_credential
|
||||
|
||||
|
|
@ -248,6 +255,58 @@ def _models_served_by_group(llm_router: litellm.Router, model_group: str) -> fro
|
|||
)
|
||||
|
||||
|
||||
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_DEPLOYMENT_PRICING_FIELDS: Final[frozenset[str]] = SERVER_DERIVED_PRICING_FIELDS | DEPLOYMENT_SCOPED_PRICING_FIELDS
|
||||
|
||||
|
||||
def _object_mapping(value: object) -> Mapping[str, object]:
|
||||
return _OBJECT_MAPPING.validate_python(value if isinstance(value, Mapping) else {})
|
||||
|
||||
|
||||
def _anthropic_deployment_model_info(deployment: object, model: str) -> Mapping[str, object] | None:
|
||||
fields: Final = _object_mapping(deployment)
|
||||
params: Final = _object_mapping(fields.get("litellm_params"))
|
||||
backend: Final = params.get("model")
|
||||
provider_override: Final = params.get("custom_llm_provider")
|
||||
if not isinstance(backend, str):
|
||||
return None
|
||||
try:
|
||||
backend_model, provider, _, _ = get_llm_provider(
|
||||
model=backend, custom_llm_provider=provider_override if isinstance(provider_override, str) else None
|
||||
)
|
||||
except litellm.BadRequestError:
|
||||
return None
|
||||
if provider != LlmProviders.ANTHROPIC.value or model not in (backend_model, backend.removeprefix(f"{provider}/")):
|
||||
return None
|
||||
return {
|
||||
**_object_mapping(fields.get("model_info")),
|
||||
**{field: value for field in _DEPLOYMENT_PRICING_FIELDS if (value := params.get(field)) is not None},
|
||||
}
|
||||
|
||||
|
||||
def anthropic_deployment_pricing_model_info(
|
||||
llm_router: litellm.Router | None, request_body: Mapping[str, object], team_id: str | None
|
||||
) -> Mapping[str, object] | None:
|
||||
"""
|
||||
The pass-through selects no deployment, so ``request_body`` is the body it forwards. Only a single deployment in
|
||||
that model's group serving that exact Anthropic model can say what it costs: with several, any pick could bill
|
||||
another deployment's negotiated price.
|
||||
"""
|
||||
model: Final = request_body.get("model")
|
||||
if llm_router is None or not isinstance(model, str) or not model:
|
||||
return None
|
||||
candidates: Final = tuple(
|
||||
model_info
|
||||
for deployment in llm_router.registry().deployments(model, team_id=team_id) or ()
|
||||
if llm_router.registry().includes(deployment, team_id=team_id)
|
||||
and (model_info := _anthropic_deployment_model_info(deployment, model)) is not None
|
||||
)
|
||||
if len(candidates) != 1:
|
||||
return None
|
||||
(only_match,) = candidates
|
||||
return only_match if any(only_match.get(field) is not None for field in _DEPLOYMENT_PRICING_FIELDS) else None
|
||||
|
||||
|
||||
def is_passthrough_request_streaming(request_body: object) -> bool:
|
||||
"""
|
||||
Returns True if the request is streaming.
|
||||
|
|
@ -1236,6 +1295,16 @@ async def anthropic_proxy_route(
|
|||
## check for streaming
|
||||
is_streaming_request: Final = await is_streaming_request_fn(request)
|
||||
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
setattr(
|
||||
request.state,
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY,
|
||||
DeploymentPricingResolver(
|
||||
resolve=partial(anthropic_deployment_pricing_model_info, llm_router, team_id=user_api_key_dict.team_id)
|
||||
),
|
||||
)
|
||||
|
||||
## CREATE PASS-THROUGH
|
||||
auth_header: Final = await AnthropicModelInfo.aget_auth_header(
|
||||
anthropic_api_key or None, allow_workload_identity=True
|
||||
|
|
|
|||
|
|
@ -167,8 +167,10 @@ from litellm.types.llms.custom_http import httpxSpecialProvider
|
|||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
|
||||
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
|
||||
DeploymentPricingResolver,
|
||||
EndpointType,
|
||||
PassthroughStandardLoggingPayload,
|
||||
WebSocketSessionMeter,
|
||||
|
|
@ -448,6 +450,20 @@ async def chat_completion_pass_through_endpoint(
|
|||
)
|
||||
|
||||
|
||||
_PASS_THROUGH_REQUEST_STATE_KEYS: Final = (
|
||||
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY,
|
||||
)
|
||||
|
||||
|
||||
def _clear_pass_through_request_state(state: object) -> None:
|
||||
for key in _PASS_THROUGH_REQUEST_STATE_KEYS:
|
||||
if hasattr(state, key):
|
||||
delattr(state, key)
|
||||
|
||||
|
||||
def _is_provider_payload(request: Request) -> bool:
|
||||
return is_opaque_pass_through_request(get_request_route(request), request.headers.get("content-type", ""))
|
||||
|
||||
|
|
@ -798,6 +814,14 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
)
|
||||
if isinstance(deployment_model_info, Mapping):
|
||||
_metadata["model_info"] = dict(deployment_model_info)
|
||||
deployment_pricing: Final = getattr(_request_state, LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY, None)
|
||||
priced_model_info: Final = (
|
||||
deployment_pricing.resolve(server_marker_free_body)
|
||||
if isinstance(deployment_pricing, DeploymentPricingResolver)
|
||||
else None
|
||||
)
|
||||
if priced_model_info is not None:
|
||||
_metadata["model_info"] = dict(priced_model_info)
|
||||
|
||||
kwargs: Final = {
|
||||
"litellm_params": {
|
||||
|
|
@ -2536,12 +2560,7 @@ def create_pass_through_route(
|
|||
keep_target_query=keep_target_query,
|
||||
)
|
||||
finally:
|
||||
if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY):
|
||||
delattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY)
|
||||
if hasattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY):
|
||||
delattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY)
|
||||
if hasattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY):
|
||||
delattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY)
|
||||
_clear_pass_through_request_state(request.state)
|
||||
|
||||
# The upstream withholds its response headers until its first token, so
|
||||
# the whole time-to-first-token is spent inside _relay with nothing on
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
|
|
@ -20,6 +20,15 @@ LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY: Final = "litellm_pass_through_raw_body"
|
|||
# `model_info` of the router deployment a provider route (e.g. Vertex) resolved for this request.
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY: Final = "litellm_pass_through_deployment_model_info"
|
||||
|
||||
# Resolves that `model_info` from the body actually forwarded, after the `custom_body` envelope and pre-call hooks.
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_PRICING_STATE_KEY: Final = "litellm_pass_through_deployment_pricing"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeploymentPricingResolver:
|
||||
resolve: Callable[[Mapping[str, object]], Mapping[str, object] | None]
|
||||
|
||||
|
||||
# Attribute set on the FastAPI endpoint function of every user-defined pass-through
|
||||
# route. Auth reads it off the dispatched endpoint (``request.scope["endpoint"]``) to
|
||||
# decide whether a request body ``model`` names an upstream model rather than a
|
||||
|
|
|
|||
|
|
@ -3102,3 +3102,71 @@ def test_anthropic_passthrough_stream_bills_a_bedrock_guardrail_trace_whose_json
|
|||
)
|
||||
(entry,) = _recorded_guardrail_entries(mock_logging_obj)
|
||||
assert entry["guardrail_usage"] == {"topicPolicyUnits": 3, "contentPolicyUnits": 3}
|
||||
|
||||
|
||||
_DEPLOYMENT_PRICING: Final = {"input_cost_per_token": 3e-03, "output_cost_per_token": 7e-03}
|
||||
|
||||
|
||||
def _deployment_priced_logging_obj(monkeypatch: pytest.MonkeyPatch, stream: bool) -> LiteLLMLoggingObj:
|
||||
monkeypatch.setattr(litellm, "model_cost", {"sonnet-dep": {"litellm_provider": "anthropic", **_DEPLOYMENT_PRICING}})
|
||||
logging_obj: Final = LiteLLMLoggingObj(
|
||||
model="claude-sonnet-4-5",
|
||||
messages=[],
|
||||
stream=stream,
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=datetime(2025, 1, 1),
|
||||
litellm_call_id="deployment-priced",
|
||||
function_id="1",
|
||||
)
|
||||
litellm_params: Final = {"metadata": {"model_info": {"id": "sonnet-dep", **_DEPLOYMENT_PRICING}}}
|
||||
logging_obj.litellm_params = litellm_params
|
||||
logging_obj.model_call_details["litellm_params"] = litellm_params
|
||||
logging_obj.model_call_details["custom_llm_provider"] = "anthropic"
|
||||
logging_obj.model_call_details["stream"] = stream
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _deployment_price_of(usage: litellm.Usage) -> float:
|
||||
return (
|
||||
usage.prompt_tokens * _DEPLOYMENT_PRICING["input_cost_per_token"]
|
||||
+ usage.completion_tokens * _DEPLOYMENT_PRICING["output_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
def test_a_response_naming_a_dated_snapshot_is_billed_at_the_resolved_deployment_price(
|
||||
monkeypatch, mock_httpx_response
|
||||
):
|
||||
logging_obj: Final = _deployment_priced_logging_obj(monkeypatch, stream=False)
|
||||
|
||||
result: Final = AnthropicPassthroughLoggingHandler.anthropic_passthrough_handler(
|
||||
httpx_response=mock_httpx_response,
|
||||
response_body=mock_httpx_response.json(),
|
||||
logging_obj=logging_obj,
|
||||
url_route="/v1/messages",
|
||||
result="success",
|
||||
start_time=datetime(2025, 1, 1),
|
||||
end_time=datetime(2025, 1, 1),
|
||||
cache_hit=False,
|
||||
request_body={"model": "claude-sonnet-4-5", "messages": []},
|
||||
)
|
||||
|
||||
assert result["result"].usage.prompt_tokens > 0
|
||||
assert result["kwargs"]["response_cost"] == pytest.approx(_deployment_price_of(result["result"].usage))
|
||||
|
||||
|
||||
def test_a_stream_naming_a_dated_snapshot_is_billed_at_the_resolved_deployment_price(monkeypatch, all_chunks):
|
||||
logging_obj: Final = _deployment_priced_logging_obj(monkeypatch, stream=True)
|
||||
|
||||
result: Final = AnthropicPassthroughLoggingHandler.handle_logging_anthropic_collected_chunks(
|
||||
litellm_logging_obj=logging_obj,
|
||||
passthrough_success_handler_obj=MagicMock(),
|
||||
url_route="/v1/messages",
|
||||
request_body={"model": "claude-sonnet-4-5", "stream": True, "messages": []},
|
||||
endpoint_type=MagicMock(),
|
||||
start_time=datetime(2025, 1, 1),
|
||||
all_chunks=all_chunks,
|
||||
end_time=datetime(2025, 1, 1),
|
||||
)
|
||||
|
||||
assert result["result"].usage.prompt_tokens > 0
|
||||
assert result["kwargs"]["response_cost"] == pytest.approx(_deployment_price_of(result["result"].usage))
|
||||
|
|
|
|||
|
|
@ -7256,6 +7256,256 @@ class TestAnthropicProxyRoute:
|
|||
assert thread_ids and thread_ids[0] != threading.get_ident()
|
||||
|
||||
|
||||
_HAIKU_PRICING: Final = MappingProxyType({"input_cost_per_token": 1e-07, "output_cost_per_token": 5e-07})
|
||||
|
||||
|
||||
def _anthropic_deployment(
|
||||
deployment_id: str,
|
||||
model_name: str = "claude-haiku-5-5",
|
||||
backend: str = "anthropic/claude-haiku-5-5",
|
||||
model_info_pricing: Mapping[str, float] = _HAIKU_PRICING,
|
||||
params_pricing: Mapping[str, float] = MappingProxyType({}),
|
||||
team_id: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"litellm_params": {"model": backend, "api_key": "sk-ant-test", **params_pricing},
|
||||
"model_info": {"id": deployment_id, **model_info_pricing, **({"team_id": team_id} if team_id else {})},
|
||||
}
|
||||
|
||||
|
||||
def _passthrough_pricing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*deployments: dict[str, object],
|
||||
request_body: Mapping[str, object] = MappingProxyType({"model": "claude-haiku-5-5"}),
|
||||
team_id: str | None = None,
|
||||
) -> Mapping[str, object] | None:
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
anthropic_deployment_pricing_model_info,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
|
||||
router: Final = litellm.Router(model_list=[*deployments])
|
||||
return anthropic_deployment_pricing_model_info(router, request_body, team_id)
|
||||
|
||||
|
||||
def test_anthropic_passthrough_prices_with_the_one_deployment_serving_the_requested_model(monkeypatch):
|
||||
model_info: Final = _passthrough_pricing(monkeypatch, _anthropic_deployment("haiku-dep"))
|
||||
|
||||
assert model_info is not None
|
||||
assert model_info["id"] == "haiku-dep"
|
||||
assert {key: model_info[key] for key in _HAIKU_PRICING} == _HAIKU_PRICING
|
||||
|
||||
|
||||
def test_anthropic_passthrough_reads_custom_pricing_set_in_litellm_params(monkeypatch):
|
||||
model_info: Final = _passthrough_pricing(
|
||||
monkeypatch,
|
||||
_anthropic_deployment("haiku-dep", model_info_pricing=MappingProxyType({}), params_pricing=_HAIKU_PRICING),
|
||||
)
|
||||
|
||||
assert model_info is not None
|
||||
assert model_info["id"] == "haiku-dep"
|
||||
assert {key: model_info[key] for key in _HAIKU_PRICING} == _HAIKU_PRICING
|
||||
|
||||
|
||||
def test_anthropic_passthrough_prices_a_deployment_configured_with_the_alias_the_caller_sends(monkeypatch):
|
||||
model_info: Final = _passthrough_pricing(
|
||||
monkeypatch,
|
||||
_anthropic_deployment(
|
||||
"haiku-latest-dep", model_name="claude-haiku-latest", backend="anthropic/claude-haiku-latest"
|
||||
),
|
||||
request_body={"model": "claude-haiku-latest"},
|
||||
)
|
||||
|
||||
assert model_info is not None
|
||||
assert model_info["id"] == "haiku-latest-dep"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("caller_team_id", "priced"),
|
||||
[
|
||||
pytest.param("team-a", True, id="owning-team"),
|
||||
pytest.param("team-b", False, id="other-team"),
|
||||
pytest.param(None, False, id="teamless-key"),
|
||||
],
|
||||
)
|
||||
def test_anthropic_passthrough_prices_a_team_deployment_only_for_its_team(monkeypatch, caller_team_id, priced):
|
||||
model_info: Final = _passthrough_pricing(
|
||||
monkeypatch, _anthropic_deployment("team-a-haiku-dep", team_id="team-a"), team_id=caller_team_id
|
||||
)
|
||||
|
||||
assert (model_info is not None) is priced
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployments",
|
||||
[
|
||||
pytest.param(
|
||||
(
|
||||
_anthropic_deployment("haiku-dep-a"),
|
||||
_anthropic_deployment("haiku-dep-b", model_info_pricing={"input_cost_per_token": 9e-07}),
|
||||
),
|
||||
id="two-deployments-could-bill-either-price",
|
||||
),
|
||||
pytest.param(
|
||||
(_anthropic_deployment("opus-dep", backend="anthropic/claude-opus-5-5"),),
|
||||
id="group-serves-a-different-upstream-model",
|
||||
),
|
||||
pytest.param(
|
||||
(_anthropic_deployment("bedrock-dep", backend="bedrock/claude-haiku-5-5"),),
|
||||
id="deployment-is-not-anthropic",
|
||||
),
|
||||
pytest.param(
|
||||
(_anthropic_deployment("other-group-dep", model_name="haiku-alias"),),
|
||||
id="model-group-not-requested",
|
||||
),
|
||||
pytest.param(
|
||||
(_anthropic_deployment("unpriced-dep", model_info_pricing=MappingProxyType({})),),
|
||||
id="deployment-has-no-custom-pricing",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_anthropic_passthrough_attaches_no_deployment_pricing_without_one_priced_match(monkeypatch, deployments):
|
||||
assert _passthrough_pricing(monkeypatch, *deployments) is None
|
||||
|
||||
|
||||
_RELAY_HAIKU_PRICING: Final = MappingProxyType({"input_cost_per_token": 1e-03, "output_cost_per_token": 2e-03})
|
||||
_RELAY_OPUS_PRICING: Final = MappingProxyType({"input_cost_per_token": 5e-03, "output_cost_per_token": 6e-03})
|
||||
_RELAY_MESSAGES: Final = ({"role": "user", "content": "hi"},)
|
||||
|
||||
|
||||
async def _hook_passes_body_through(*, data: dict[str, object], **_: object) -> dict[str, object]:
|
||||
return data
|
||||
|
||||
|
||||
async def _hook_upgrades_to_opus(*, data: dict[str, object], **_: object) -> dict[str, object]:
|
||||
return {**data, "model": "claude-opus-5-5"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_body", "pre_call_hook", "forwarded_model", "pricing"),
|
||||
[
|
||||
pytest.param(
|
||||
{"model": "claude-haiku-5-5", "messages": [*_RELAY_MESSAGES]},
|
||||
_hook_passes_body_through,
|
||||
"claude-haiku-5-5",
|
||||
_RELAY_HAIKU_PRICING,
|
||||
id="requested-model",
|
||||
),
|
||||
pytest.param(
|
||||
{"model": "claude-haiku-5-5", "custom_body": {"model": "claude-opus-5-5", "messages": [*_RELAY_MESSAGES]}},
|
||||
_hook_passes_body_through,
|
||||
"claude-opus-5-5",
|
||||
_RELAY_OPUS_PRICING,
|
||||
id="custom-body-envelope",
|
||||
),
|
||||
pytest.param(
|
||||
{"model": "claude-haiku-5-5", "messages": [*_RELAY_MESSAGES]},
|
||||
_hook_upgrades_to_opus,
|
||||
"claude-opus-5-5",
|
||||
_RELAY_OPUS_PRICING,
|
||||
id="pre-call-hook-rewrite",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_anthropic_proxy_route_bills_the_custom_price_of_the_forwarded_model(
|
||||
monkeypatch, request_body, pre_call_hook, forwarded_model, pricing
|
||||
):
|
||||
for name in (
|
||||
"ANTHROPIC_AUTH_TOKEN",
|
||||
"ANTHROPIC_API_BASE",
|
||||
"ANTHROPIC_BASE_URL",
|
||||
"ANTHROPIC_FEDERATION_RULE_ID",
|
||||
"ANTHROPIC_ORGANIZATION_ID",
|
||||
"ANTHROPIC_IDENTITY_TOKEN_FILE",
|
||||
"ANTHROPIC_IDENTITY_TOKEN",
|
||||
"SERVER_ROOT_PATH",
|
||||
):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test")
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
_anthropic_deployment("haiku-dep", model_info_pricing=_RELAY_HAIKU_PRICING),
|
||||
_anthropic_deployment(
|
||||
"opus-dep",
|
||||
model_name="claude-opus-5-5",
|
||||
backend="anthropic/claude-opus-5-5",
|
||||
model_info_pricing=_RELAY_OPUS_PRICING,
|
||||
),
|
||||
]
|
||||
),
|
||||
)
|
||||
upstream_response: Final = httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_relay",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": forwarded_model,
|
||||
"content": [{"type": "text", "text": "pong"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 18, "output_tokens": 4},
|
||||
},
|
||||
request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"),
|
||||
)
|
||||
httpx_client: Final = MagicMock()
|
||||
httpx_client.build_request = MagicMock(return_value=MagicMock())
|
||||
httpx_client.send = AsyncMock(return_value=upstream_response)
|
||||
client_wrapper: Final = MagicMock()
|
||||
client_wrapper.client = httpx_client
|
||||
body: Final = json.dumps(request_body).encode()
|
||||
|
||||
async def receive() -> dict[str, object]:
|
||||
return {"type": "http.request", "body": body, "more_body": False}
|
||||
|
||||
request: Final = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"http_version": "1.1",
|
||||
"method": "POST",
|
||||
"scheme": "https",
|
||||
"path": "/anthropic/v1/messages",
|
||||
"raw_path": b"/anthropic/v1/messages",
|
||||
"root_path": "",
|
||||
"query_string": b"",
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
"client": ("127.0.0.1", 51234),
|
||||
"server": ("proxy.local", 4000),
|
||||
"state": {},
|
||||
},
|
||||
receive,
|
||||
)
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: the upstream http client is the boundary, it answers with a real Anthropic body
|
||||
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client",
|
||||
return_value=client_wrapper,
|
||||
),
|
||||
patch( # test-quality-ok: the relay calls these hooks, and they need a db this test has no use for
|
||||
"litellm.proxy.proxy_server.proxy_logging_obj"
|
||||
) as mock_logging_obj,
|
||||
):
|
||||
mock_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook)
|
||||
mock_logging_obj.post_call_success_hook = AsyncMock()
|
||||
mock_logging_obj.post_call_failure_hook = AsyncMock()
|
||||
mock_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={})
|
||||
|
||||
response: Final = await anthropic_proxy_route(
|
||||
endpoint="v1/messages",
|
||||
request=request,
|
||||
fastapi_response=MagicMock(spec=Response),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-caller-virtual-key"),
|
||||
)
|
||||
|
||||
assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(
|
||||
18 * pricing["input_cost_per_token"] + 4 * pricing["output_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
class TestAnthropicProxyRouteCallerAuthHeaders:
|
||||
"""Regression for a caller credential riding upstream next to a server-owned one.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue