mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 619818e04e into 1c022d1d06
This commit is contained in:
commit
34a476ce01
5 changed files with 440 additions and 7 deletions
|
|
@ -148,6 +148,7 @@ from .streaming_handler import PassThroughStreamingHandler
|
|||
from .success_handler import PassThroughEndpointLogging
|
||||
from .upstream_usage_headers import (
|
||||
UpstreamReportedUsage,
|
||||
apply_upstream_reported_model,
|
||||
apply_upstream_reported_usage,
|
||||
)
|
||||
|
||||
|
|
@ -852,6 +853,7 @@ def _build_passthrough_failure_request_payload(
|
|||
logging_obj: LiteLLMLoggingObj | None,
|
||||
custom_llm_provider: str | None,
|
||||
upstream_usage: UpstreamReportedUsage | None = None,
|
||||
upstream_model: str | None = None,
|
||||
) -> dict:
|
||||
"""Build the ``request_data`` dict passed to ``post_call_failure_hook``.
|
||||
|
||||
|
|
@ -862,6 +864,8 @@ def _build_passthrough_failure_request_payload(
|
|||
``upstream_usage`` carries the cost and tokens an upstream reported on an
|
||||
error response. Spend tracking only attributes a recovered cost when it
|
||||
comes paired with a usage object, so both keys are written together.
|
||||
``upstream_model`` is the model name the upstream reported, which replaces
|
||||
the body's ``model`` on the failure row just as it does on a success row.
|
||||
"""
|
||||
request_payload: Final[dict] = dict(cast(Mapping[str, object], parsed_body or {})) # cast-ok: json body
|
||||
if kwargs:
|
||||
|
|
@ -875,6 +879,8 @@ def _build_passthrough_failure_request_payload(
|
|||
if upstream_usage is not None:
|
||||
request_payload["response_cost"] = upstream_usage.response_cost or 0.0
|
||||
request_payload["combined_usage_object"] = Usage(total_tokens=upstream_usage.total_tokens or 0)
|
||||
if upstream_model is not None:
|
||||
request_payload["model"] = upstream_model
|
||||
return request_payload
|
||||
|
||||
|
||||
|
|
@ -1538,6 +1544,10 @@ async def pass_through_request(
|
|||
logging_obj=logging_obj,
|
||||
headers=response.headers,
|
||||
)
|
||||
upstream_model = apply_upstream_reported_model( # rebind-ok: bound once per branch, like upstream_usage
|
||||
logging_obj=logging_obj,
|
||||
headers=response.headers,
|
||||
)
|
||||
|
||||
relay_response: Final = await _log_passthrough_upstream_failure(
|
||||
response=response,
|
||||
|
|
@ -1548,6 +1558,7 @@ async def pass_through_request(
|
|||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
upstream_usage=upstream_usage,
|
||||
upstream_model=upstream_model,
|
||||
),
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
|
@ -1626,6 +1637,10 @@ async def pass_through_request(
|
|||
logging_obj=logging_obj,
|
||||
headers=response.headers,
|
||||
)
|
||||
upstream_model = apply_upstream_reported_model( # rebind-ok: bound once per branch, like upstream_usage
|
||||
logging_obj=logging_obj,
|
||||
headers=response.headers,
|
||||
)
|
||||
|
||||
if _is_streaming_response(response) is True:
|
||||
logging_obj.stream = True
|
||||
|
|
@ -1640,6 +1655,7 @@ async def pass_through_request(
|
|||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
upstream_usage=upstream_usage,
|
||||
upstream_model=upstream_model,
|
||||
),
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
|
@ -1740,6 +1756,7 @@ async def pass_through_request(
|
|||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
upstream_usage=upstream_usage,
|
||||
upstream_model=upstream_model,
|
||||
)
|
||||
failure_request_payload["response_body"] = response_body
|
||||
await _log_passthrough_upstream_failure(
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@
|
|||
Some pass-through targets invoke several models internally, so LiteLLM cannot
|
||||
price the request from the response body. Those targets report the totals for
|
||||
the whole HTTP request in ``x-litellm-*`` response headers instead; LiteLLM
|
||||
records the reported values without recomputing them.
|
||||
records the reported values without recomputing them. A target may also name
|
||||
the request in ``x-litellm-upstream-model`` so its spend can be told apart from
|
||||
other requests that carry no ``model`` in the body.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
|
@ -14,11 +16,15 @@ from typing import Final
|
|||
import httpx
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import MAX_SPEND_LOG_MODEL_NAME_LENGTH
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
UPSTREAM_RESPONSE_COST_HEADER: Final = "x-litellm-response-cost"
|
||||
UPSTREAM_TOTAL_TOKENS_HEADER: Final = "x-litellm-total-tokens"
|
||||
# Deliberately not x-litellm-model-name/-id/-group: the gateway already sends
|
||||
# those to clients, and pass-through relays upstream headers back to the client.
|
||||
UPSTREAM_MODEL_HEADER: Final = "x-litellm-upstream-model"
|
||||
|
||||
# model_call_details key holding what the upstream reported, so later stages of
|
||||
# the success path can tell an upstream-reported cost apart from one LiteLLM
|
||||
|
|
@ -128,6 +134,39 @@ def apply_upstream_reported_usage(
|
|||
return reported
|
||||
|
||||
|
||||
def parse_upstream_reported_model(headers: httpx.Headers) -> str | None:
|
||||
"""Read the model name an upstream reported, or ``None`` when it sent none.
|
||||
|
||||
An empty or whitespace-only value counts as absent. A repeated header reads
|
||||
as one comma-separated list (RFC 9110 field combining, which is how httpx
|
||||
joins it), so only its first element is used. The name is cut to the spend
|
||||
log's model-name limit, since it becomes a spend row, rollup key and metric
|
||||
label.
|
||||
"""
|
||||
raw_model: Final[str | None] = headers.get(UPSTREAM_MODEL_HEADER)
|
||||
model: Final = raw_model.split(",", 1)[0].strip() if raw_model is not None else ""
|
||||
return model[:MAX_SPEND_LOG_MODEL_NAME_LENGTH].rstrip() or None
|
||||
|
||||
|
||||
def apply_upstream_reported_model(
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
headers: httpx.Headers,
|
||||
) -> str | None:
|
||||
"""Record the upstream's model name as the request's model.
|
||||
|
||||
The name wins over a ``model`` in the request body. It is a label only:
|
||||
it never marks the request as upstream-priced (see
|
||||
``has_upstream_reported_usage``) and nothing prices the request from it,
|
||||
so a name outside the price map leaves the recorded cost alone.
|
||||
"""
|
||||
reported_model: Final = parse_upstream_reported_model(headers)
|
||||
if reported_model is None:
|
||||
return None
|
||||
logging_obj.model = reported_model # rebind-ok: the request's logging object is where the recorded model lives
|
||||
logging_obj.model_call_details["model"] = reported_model # rebind-ok: read by the standard logging payload
|
||||
return reported_model
|
||||
|
||||
|
||||
def has_upstream_reported_usage(logging_obj: LiteLLMLoggingObj) -> bool:
|
||||
"""Whether the upstream spoke this contract on the request's response.
|
||||
|
||||
|
|
|
|||
228
tests/integration/spend/test_passthrough_upstream_model.py
Normal file
228
tests/integration/spend/test_passthrough_upstream_model.py
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from hashlib import sha256
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, JsonValue, Scenario, eventually, object_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.shares_worker_state
|
||||
|
||||
_REPORTED_MODEL: Final = "support-summarizer"
|
||||
_REPORTED_COST: Final = 0.0123
|
||||
_REPORTED_TOKENS: Final = 4210
|
||||
_USAGE_HEADERS: Final = {
|
||||
"x-litellm-response-cost": str(_REPORTED_COST),
|
||||
"x-litellm-total-tokens": str(_REPORTED_TOKENS),
|
||||
}
|
||||
_MODEL_HEADER: Final = {"x-litellm-upstream-model": _REPORTED_MODEL}
|
||||
|
||||
|
||||
def _upstream(
|
||||
marker: str, headers: Mapping[str, str], *, status: int = 200, stream: bool = False
|
||||
) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST", request
|
||||
assert json.loads(request.body)["ticket_id"] == marker, request
|
||||
if stream:
|
||||
return Reply(
|
||||
status=status,
|
||||
content_type="text/event-stream",
|
||||
chunks=(f'data: {{"delta": "{marker}"}}\n\n'.encode(), b"data: [DONE]\n\n"),
|
||||
headers=headers,
|
||||
)
|
||||
return Reply(status=status, body=json.dumps({"summary": marker}).encode(), headers=headers)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def _endpoint(gateway: Gateway, scenario: Scenario, marker: str, target: str, **fields: JsonValue) -> str:
|
||||
path: Final = f"/support-ai-{marker}"
|
||||
created: Final = gateway.post(
|
||||
"/config/pass_through_endpoint", {"path": path, "target": target, "auth": True, **fields}
|
||||
)
|
||||
endpoint_id: Final = object_value(TypeAdapter(list[JsonValue]).validate_python(created["endpoints"])[0])["id"]
|
||||
scenario.cleanups.callback(
|
||||
lambda: gateway.request("DELETE", "/config/pass_through_endpoint", params={"endpoint_id": str(endpoint_id)})
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def _spend_row(key: str) -> dict[str, JsonValue]:
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT model, spend, total_tokens, status FROM "LiteLLM_SpendLogs" WHERE api_key=%s AND call_type=%s',
|
||||
(sha256(key.encode()).hexdigest(), "pass_through_endpoint"),
|
||||
),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
return rows[0]
|
||||
|
||||
|
||||
def _row_values(row: Mapping[str, JsonValue]) -> tuple[JsonValue, float, JsonValue, JsonValue]:
|
||||
spend: Final = row["spend"]
|
||||
assert isinstance(spend, (int, float)), row
|
||||
return row["model"], float(spend), row["total_tokens"], row["status"]
|
||||
|
||||
|
||||
def test_upstream_reported_model_names_the_spend_row_daily_rollup_and_usage_breakdown(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_upstream(marker, {**_USAGE_HEADERS, **_MODEL_HEADER})) as wire, gateway.scenario() as scenario:
|
||||
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize")
|
||||
team: Final = scenario.team()
|
||||
key: Final = scenario.key(team_id=team, allowed_passthrough_routes=[path])
|
||||
|
||||
response: Final = gateway.request("POST", path, {"ticket_id": marker}, key=key)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"summary": marker}, response.text
|
||||
assert response.headers["x-litellm-upstream-model"] == _REPORTED_MODEL, response.headers
|
||||
|
||||
assert _row_values(_spend_row(key)) == (
|
||||
_REPORTED_MODEL,
|
||||
pytest.approx(_REPORTED_COST),
|
||||
_REPORTED_TOKENS,
|
||||
"success",
|
||||
)
|
||||
daily: Final = eventually(
|
||||
lambda: read_rows('SELECT model, spend FROM "LiteLLM_DailyTeamSpend" WHERE team_id=%s', (team,)),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert (daily[0]["model"], daily[0]["spend"]) == (_REPORTED_MODEL, pytest.approx(_REPORTED_COST)), daily
|
||||
|
||||
today: Final = datetime.now(timezone.utc)
|
||||
activity: Final = gateway.request(
|
||||
"GET",
|
||||
"/team/daily/activity",
|
||||
params={
|
||||
"team_ids": team,
|
||||
"start_date": (today - timedelta(days=1)).strftime("%Y-%m-%d"),
|
||||
"end_date": (today + timedelta(days=1)).strftime("%Y-%m-%d"),
|
||||
},
|
||||
)
|
||||
assert activity.status_code == 200, activity.text
|
||||
results: Final = object_value(activity.json())["results"]
|
||||
assert isinstance(results, list) and len(results) == 1, activity.text
|
||||
models: Final = object_value(object_value(object_value(results[0])["breakdown"])["models"])
|
||||
assert set(models) == {_REPORTED_MODEL}, activity.text
|
||||
assert object_value(object_value(models[_REPORTED_MODEL])["metrics"])["spend"] == pytest.approx(
|
||||
_REPORTED_COST
|
||||
), activity.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streamed"])
|
||||
def test_upstream_reported_model_wins_over_the_request_body_model(gateway: Gateway, stream: bool) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with (
|
||||
wire_server(_upstream(marker, {**_USAGE_HEADERS, **_MODEL_HEADER}, stream=stream)) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize")
|
||||
key: Final = scenario.key(allowed_passthrough_routes=[path])
|
||||
|
||||
response: Final = gateway.request("POST", path, {"ticket_id": marker, "model": "gpt-4o"}, key=key)
|
||||
assert response.status_code == 200, response.text
|
||||
assert marker in response.text, response.text
|
||||
|
||||
assert _row_values(_spend_row(key)) == (
|
||||
_REPORTED_MODEL,
|
||||
pytest.approx(_REPORTED_COST),
|
||||
_REPORTED_TOKENS,
|
||||
"success",
|
||||
)
|
||||
|
||||
|
||||
def test_upstream_error_row_carries_the_reported_model_and_cost(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with (
|
||||
wire_server(_upstream(marker, {**_USAGE_HEADERS, **_MODEL_HEADER}, status=500)) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize")
|
||||
key: Final = scenario.key(allowed_passthrough_routes=[path])
|
||||
|
||||
response: Final = gateway.request("POST", path, {"ticket_id": marker, "model": "gpt-4o"}, key=key)
|
||||
assert response.status_code == 500, response.text
|
||||
|
||||
assert _row_values(_spend_row(key)) == (
|
||||
_REPORTED_MODEL,
|
||||
pytest.approx(_REPORTED_COST),
|
||||
_REPORTED_TOKENS,
|
||||
"failure",
|
||||
)
|
||||
|
||||
|
||||
def test_model_header_alone_keeps_the_configured_cost_per_request(gateway: Gateway) -> None:
|
||||
"""The name is a label: an upstream that reports only a model does not own
|
||||
the cost, so the endpoint's flat cost_per_request still applies."""
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_upstream(marker, _MODEL_HEADER)) as wire, gateway.scenario() as scenario:
|
||||
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize", cost_per_request=0.5)
|
||||
key: Final = scenario.key(allowed_passthrough_routes=[path])
|
||||
|
||||
response: Final = gateway.request("POST", path, {"ticket_id": marker}, key=key)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
model, spend, _, status = _row_values(_spend_row(key))
|
||||
assert (model, spend, status) == (_REPORTED_MODEL, pytest.approx(0.5), "success")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body_model", "header_value", "expected_model"),
|
||||
[(None, None, "unknown"), ("gpt-4o", None, "gpt-4o"), ("gpt-4o", " ", "gpt-4o")],
|
||||
ids=["no_header_no_body_model", "no_header_body_model", "blank_header_body_model"],
|
||||
)
|
||||
def test_without_a_usable_model_header_the_row_keeps_the_request_model(
|
||||
gateway: Gateway, body_model: str | None, header_value: str | None, expected_model: str
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
headers: Final = {**_USAGE_HEADERS, **({} if header_value is None else {"x-litellm-upstream-model": header_value})}
|
||||
body: Final[dict[str, JsonValue]] = {"ticket_id": marker, **({} if body_model is None else {"model": body_model})}
|
||||
with wire_server(_upstream(marker, headers)) as wire, gateway.scenario() as scenario:
|
||||
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize")
|
||||
key: Final = scenario.key(allowed_passthrough_routes=[path])
|
||||
|
||||
response: Final = gateway.request("POST", path, body, key=key)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
assert _row_values(_spend_row(key)) == (
|
||||
expected_model,
|
||||
pytest.approx(_REPORTED_COST),
|
||||
_REPORTED_TOKENS,
|
||||
"success",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("header_value", "expected_model"),
|
||||
[
|
||||
(f"{_REPORTED_MODEL}, other-name", _REPORTED_MODEL),
|
||||
("m" * 5000, "m" * 256),
|
||||
],
|
||||
ids=["repeated_header_keeps_first", "long_name_cut_to_256"],
|
||||
)
|
||||
@pytest.mark.parametrize("status", [200, 500], ids=["success", "failure"])
|
||||
def test_spend_row_keeps_a_bounded_first_model_name(
|
||||
gateway: Gateway, header_value: str, expected_model: str, status: int
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
headers: Final = {**_USAGE_HEADERS, "x-litellm-upstream-model": header_value}
|
||||
with wire_server(_upstream(marker, headers, status=status)) as wire, gateway.scenario() as scenario:
|
||||
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize")
|
||||
key: Final = scenario.key(allowed_passthrough_routes=[path])
|
||||
|
||||
response: Final = gateway.request("POST", path, {"ticket_id": marker}, key=key)
|
||||
assert response.status_code == status, response.text
|
||||
|
||||
assert _row_values(_spend_row(key)) == (
|
||||
expected_model,
|
||||
pytest.approx(_REPORTED_COST),
|
||||
_REPORTED_TOKENS,
|
||||
"success" if status == 200 else "failure",
|
||||
)
|
||||
|
|
@ -6726,6 +6726,7 @@ async def _run_upstream_reporting_passthrough(
|
|||
status_code: int = 200,
|
||||
cost_per_request: float | None = None,
|
||||
request_headers: Mapping[str, str] | None = None,
|
||||
request_body: Mapping[str, object] | None = None,
|
||||
) -> tuple[list[object], MagicMock]:
|
||||
"""Drive a generic pass-through against an upstream that reports its own
|
||||
cost/usage. Returns (recorded standard logging payloads, proxy logging mock)."""
|
||||
|
|
@ -6741,13 +6742,14 @@ async def _run_upstream_reporting_passthrough(
|
|||
)
|
||||
try:
|
||||
with ExitStack() as stack:
|
||||
mock_proxy_logging, enqueued = _enter_upstream_usage_mocks(stack, {})
|
||||
mock_proxy_logging, enqueued = _enter_upstream_usage_mocks(stack, dict(request_body or {}))
|
||||
with _recording_success_callback() as recorder:
|
||||
await pass_through_request(
|
||||
request=_relay_client_request(method="POST", headers=request_headers),
|
||||
target="http://internal-api.test/v1/summarize",
|
||||
custom_headers={},
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-upstream-usage", team_id="team-fil"),
|
||||
custom_body=dict(request_body) if request_body else None,
|
||||
cost_per_request=cost_per_request,
|
||||
)
|
||||
for coroutine in enqueued:
|
||||
|
|
@ -6846,12 +6848,72 @@ async def test_passthrough_error_response_without_usage_headers_records_no_spend
|
|||
assert "combined_usage_object" not in request_data
|
||||
|
||||
|
||||
_UPSTREAM_SUMMARIZER_HEADERS: Final = MappingProxyType(
|
||||
{
|
||||
"x-litellm-response-cost": "0.0123",
|
||||
"x-litellm-total-tokens": "4210",
|
||||
"x-litellm-upstream-model": "support-summarizer",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_passthrough_records_cost_and_tokens_reported_by_upstream():
|
||||
@pytest.mark.parametrize("request_body", [None, {"model": "gpt-4o"}], ids=["no_body_model", "body_model"])
|
||||
async def test_passthrough_records_model_reported_by_upstream(request_body: Mapping[str, object] | None):
|
||||
"""A product endpoint takes no `model`, so without the header its spend lands
|
||||
under `unknown`. The upstream's name wins over a body `model` too, and the
|
||||
reported cost is kept even though the name is not in the price map."""
|
||||
payloads, _ = await _run_upstream_reporting_passthrough(_UPSTREAM_SUMMARIZER_HEADERS, request_body=request_body)
|
||||
|
||||
assert len(payloads) == 1
|
||||
logged: Final = cast(types_utils.StandardLoggingPayload, payloads[0])
|
||||
assert (logged["model"], logged["response_cost"], logged["total_tokens"]) == (
|
||||
"support-summarizer",
|
||||
0.0123,
|
||||
4210,
|
||||
), logged
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_body", "expected_model"),
|
||||
[(None, "unknown"), ({"model": "gpt-4o"}, "gpt-4o")],
|
||||
ids=["no_body_model", "body_model"],
|
||||
)
|
||||
async def test_passthrough_without_model_header_keeps_request_model(
|
||||
request_body: Mapping[str, object] | None, expected_model: str
|
||||
):
|
||||
payloads, _ = await _run_upstream_reporting_passthrough(
|
||||
{"x-litellm-response-cost": "0.0123", "x-litellm-total-tokens": "4210"}, request_body=request_body
|
||||
)
|
||||
|
||||
assert len(payloads) == 1
|
||||
assert cast(types_utils.StandardLoggingPayload, payloads[0])["model"] == expected_model, payloads[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_passthrough_records_upstream_reported_model_on_error_response():
|
||||
import litellm
|
||||
|
||||
_, mock_proxy_logging = await _run_upstream_reporting_passthrough(
|
||||
_UPSTREAM_SUMMARIZER_HEADERS, status_code=500, request_body={"model": "gpt-4o"}
|
||||
)
|
||||
|
||||
mock_proxy_logging.post_call_failure_hook.assert_awaited_once()
|
||||
request_data: Final = mock_proxy_logging.post_call_failure_hook.await_args.kwargs["request_data"]
|
||||
assert (request_data["model"], request_data["response_cost"], request_data["combined_usage_object"]) == (
|
||||
"support-summarizer",
|
||||
0.0123,
|
||||
litellm.Usage(total_tokens=4210),
|
||||
), request_data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_passthrough_records_cost_tokens_and_model_reported_by_upstream():
|
||||
"""
|
||||
The totals are reported in the response headers, which are known before the
|
||||
first byte of the stream, so a streamed pass-through must record the same
|
||||
spend a buffered one does.
|
||||
The totals and model name are reported in the response headers, which are
|
||||
known before the first byte of the stream, so a streamed pass-through must
|
||||
record the same spend and model a buffered one does.
|
||||
"""
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
|
|
@ -6864,6 +6926,7 @@ async def test_streaming_passthrough_records_cost_and_tokens_reported_by_upstrea
|
|||
"content-type": "text/event-stream",
|
||||
"x-litellm-response-cost": "0.00312",
|
||||
"x-litellm-total-tokens": "4021",
|
||||
"x-litellm-upstream-model": "support-summarizer",
|
||||
},
|
||||
stream=_RecordingUpstreamByteStream((b'data: {"delta": "hi"}\n\n',)),
|
||||
),
|
||||
|
|
@ -6890,6 +6953,7 @@ async def test_streaming_passthrough_records_cost_and_tokens_reported_by_upstrea
|
|||
assert len(recorder.payloads) == 1
|
||||
assert recorder.payloads[0]["response_cost"] == 0.00312
|
||||
assert recorder.payloads[0]["total_tokens"] == 4021
|
||||
assert recorder.payloads[0]["model"] == "support-summarizer"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -1,12 +1,16 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
|
||||
from litellm.constants import MAX_SPEND_LOG_MODEL_NAME_LENGTH
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy.pass_through_endpoints.upstream_usage_headers import (
|
||||
UpstreamReportedUsage,
|
||||
apply_upstream_reported_model,
|
||||
apply_upstream_reported_usage,
|
||||
has_upstream_reported_usage,
|
||||
parse_upstream_reported_model,
|
||||
parse_upstream_reported_usage,
|
||||
)
|
||||
from litellm.types.utils import Usage
|
||||
|
|
@ -126,3 +130,84 @@ def test_apply_only_overwrites_what_upstream_reported():
|
|||
|
||||
assert logging_obj.model_call_details["response_cost"] == 0.5
|
||||
assert logging_obj.model_call_details["combined_usage_object"] == Usage(total_tokens=42)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw_model", "expected"),
|
||||
[
|
||||
("support-summarizer", "support-summarizer"),
|
||||
(" support-summarizer \t", "support-summarizer"),
|
||||
("", None),
|
||||
(" ", None),
|
||||
],
|
||||
)
|
||||
def test_parse_reads_reported_model_and_treats_blank_as_absent(raw_model: str, expected: str | None):
|
||||
assert parse_upstream_reported_model(_headers(**{"x-litellm-upstream-model": raw_model})) == expected
|
||||
|
||||
|
||||
def test_parse_returns_no_model_when_upstream_sends_none():
|
||||
assert parse_upstream_reported_model(_headers(**{"x-litellm-response-cost": "0.5"})) is None
|
||||
|
||||
|
||||
def test_parse_uses_the_first_value_of_a_repeated_model_header():
|
||||
headers: Final = httpx.Headers(
|
||||
[("x-litellm-upstream-model", "support-summarizer"), ("x-litellm-upstream-model", "other")]
|
||||
)
|
||||
|
||||
assert parse_upstream_reported_model(headers) == "support-summarizer"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw_model", "expected"),
|
||||
[
|
||||
("m" * 5000, "m" * MAX_SPEND_LOG_MODEL_NAME_LENGTH),
|
||||
("m" * MAX_SPEND_LOG_MODEL_NAME_LENGTH, "m" * MAX_SPEND_LOG_MODEL_NAME_LENGTH),
|
||||
("m" * (MAX_SPEND_LOG_MODEL_NAME_LENGTH - 1) + " tail", "m" * (MAX_SPEND_LOG_MODEL_NAME_LENGTH - 1)),
|
||||
],
|
||||
ids=["over_limit", "at_limit", "cut_lands_on_a_space"],
|
||||
)
|
||||
def test_parse_cuts_a_long_model_name_to_the_spend_log_limit(raw_model: str, expected: str):
|
||||
assert parse_upstream_reported_model(_headers(**{"x-litellm-upstream-model": raw_model})) == expected
|
||||
|
||||
|
||||
def _logging_obj_for_model(model: str) -> LiteLLMLoggingObj:
|
||||
logging_obj: Final = _logging_obj()
|
||||
logging_obj.update_environment_variables(
|
||||
model=model, user="unknown", optional_params={}, litellm_params={}, call_type="pass_through_endpoint"
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
|
||||
def test_apply_model_replaces_the_request_model():
|
||||
logging_obj: Final = _logging_obj_for_model("gpt-4o")
|
||||
|
||||
reported: Final = apply_upstream_reported_model(
|
||||
logging_obj=logging_obj,
|
||||
headers=_headers(**{"x-litellm-upstream-model": "support-summarizer"}),
|
||||
)
|
||||
|
||||
assert reported == "support-summarizer"
|
||||
assert (logging_obj.model, logging_obj.model_call_details["model"]) == ("support-summarizer", "support-summarizer")
|
||||
|
||||
|
||||
def test_apply_model_leaves_request_model_alone_when_header_is_blank():
|
||||
logging_obj: Final = _logging_obj_for_model("gpt-4o")
|
||||
|
||||
assert (
|
||||
apply_upstream_reported_model(logging_obj=logging_obj, headers=_headers(**{"x-litellm-upstream-model": " "}))
|
||||
is None
|
||||
)
|
||||
assert (logging_obj.model, logging_obj.model_call_details["model"]) == ("gpt-4o", "gpt-4o")
|
||||
|
||||
|
||||
def test_model_header_alone_does_not_make_the_upstream_own_the_cost():
|
||||
"""The model name is a label: without a cost or token header the request
|
||||
must still fall back to cost_per_request, not be recorded at 0."""
|
||||
logging_obj: Final = _logging_obj()
|
||||
headers: Final = _headers(**{"x-litellm-upstream-model": "support-summarizer"})
|
||||
|
||||
apply_upstream_reported_usage(logging_obj=logging_obj, headers=headers)
|
||||
apply_upstream_reported_model(logging_obj=logging_obj, headers=headers)
|
||||
|
||||
assert has_upstream_reported_usage(logging_obj) is False
|
||||
assert "response_cost" not in logging_obj.model_call_details
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue