mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(count_tokens): let the caller's drop_params replace the deployment's like inference, and pin the Bedrock count betas in tests
The count now passes the request's drop_params through as sent, the same way router inference reads it, so " true " drops, False keeps the deployment's options, and a value inference cannot parse counts as not dropping even on a deployment that drops. Tests cover each case against a recorded inference request, the output-format-only runtime count's betas, and the Mantle retry's expected betas
This commit is contained in:
parent
502ce4a887
commit
0fa2f90ab6
3 changed files with 76 additions and 15 deletions
|
|
@ -14775,7 +14775,7 @@ _DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
||||||
|
|
||||||
|
|
||||||
def _deployment_with_request_drop_params(
|
def _deployment_with_request_drop_params(
|
||||||
deployment: Mapping[str, object] | None, drop_params: bool | None
|
deployment: Mapping[str, object] | None, drop_params: object
|
||||||
) -> Mapping[str, object] | None:
|
) -> Mapping[str, object] | None:
|
||||||
if drop_params is None:
|
if drop_params is None:
|
||||||
return deployment
|
return deployment
|
||||||
|
|
@ -15475,9 +15475,7 @@ async def _count_request_tokens(
|
||||||
model_to_use = _model
|
model_to_use = _model
|
||||||
|
|
||||||
sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params}
|
sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params}
|
||||||
count_deployment: Final = _deployment_with_request_drop_params(
|
count_deployment: Final = _deployment_with_request_drop_params(deployment, sent_params.get("drop_params"))
|
||||||
deployment, normalize_drop_params(sent_params.get("drop_params"))
|
|
||||||
)
|
|
||||||
conversation_empty: Final = not messages and not contents
|
conversation_empty: Final = not messages and not contents
|
||||||
if prompt and not conversation_empty and not deployment_drops_params(count_deployment):
|
if prompt and not conversation_empty and not deployment_drops_params(count_deployment):
|
||||||
raise ProxyException(
|
raise ProxyException(
|
||||||
|
|
|
||||||
|
|
@ -485,9 +485,13 @@ _CLEAR_THINKING_EDITS: Final[dict[str, JsonValue]] = {
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("request_options", (_DISPLAY_THINKING, _EFFORT), ids=("thinking_display", "effort"))
|
@pytest.mark.parametrize(
|
||||||
|
("request_options", "expected_betas"),
|
||||||
|
((_DISPLAY_THINKING, (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,)), (_EFFORT, ())),
|
||||||
|
ids=("thinking_display", "effort"),
|
||||||
|
)
|
||||||
async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends(
|
async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_sends(
|
||||||
request_options: dict[str, JsonValue],
|
request_options: dict[str, JsonValue], expected_betas: tuple[str, ...]
|
||||||
) -> None:
|
) -> None:
|
||||||
inference, bedrock = await _messages_inference_and_count(
|
inference, bedrock = await _messages_inference_and_count(
|
||||||
"bedrock/invoke/us.anthropic.claude-opus-5-5",
|
"bedrock/invoke/us.anthropic.claude-opus-5-5",
|
||||||
|
|
@ -499,7 +503,7 @@ async def test_invoke_count_retried_on_mantle_sends_the_betas_invoke_inference_s
|
||||||
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
|
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
|
||||||
sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta", [])
|
sent_betas: Final = _JSON_BODY.validate_json(inference.requests[0].content).get("anthropic_beta", [])
|
||||||
assert isinstance(sent_betas, list)
|
assert isinstance(sent_betas, list)
|
||||||
assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas))
|
assert _sent_betas(bedrock.requests[1]) == tuple(sorted(sent_betas)) == expected_betas
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
@ -531,6 +535,21 @@ async def test_runtime_count_of_an_effort_sends_the_effort_beta_once_beside_the_
|
||||||
assert set(counted_betas) - {ANTHROPIC_EFFORT_BETA_HEADER} == set(sent_betas) - {ANTHROPIC_EFFORT_BETA_HEADER}
|
assert set(counted_betas) - {ANTHROPIC_EFFORT_BETA_HEADER} == set(sent_betas) - {ANTHROPIC_EFFORT_BETA_HEADER}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runtime_count_of_an_output_format_sends_only_the_betas_invoke_inference_sends() -> None:
|
||||||
|
inference, bedrock = await _messages_inference_and_count(
|
||||||
|
"bedrock/invoke/us.anthropic.claude-sonnet-4-6",
|
||||||
|
"us.anthropic.claude-sonnet-4-6",
|
||||||
|
{**_CITY_FORMAT, "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||||
|
counted: Final = _invoke_body(bedrock.requests[0])
|
||||||
|
sent: Final = _JSON_BODY.validate_json(inference.requests[0].content)
|
||||||
|
assert counted.get("output_config") == sent.get("output_config")
|
||||||
|
assert counted.get("anthropic_beta") == sent.get("anthropic_beta")
|
||||||
|
|
||||||
|
|
||||||
_CONVERSE_REPLY: Final = {
|
_CONVERSE_REPLY: Final = {
|
||||||
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
|
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
|
||||||
"stopReason": "end_turn",
|
"stopReason": "end_turn",
|
||||||
|
|
@ -547,10 +566,9 @@ _NO_PARALLEL_TOOL_USE: Final[dict[str, JsonValue]] = {
|
||||||
("deployment_model", "inference_reply"),
|
("deployment_model", "inference_reply"),
|
||||||
(
|
(
|
||||||
("bedrock/converse/us.anthropic.claude-sonnet-4-6", _CONVERSE_REPLY),
|
("bedrock/converse/us.anthropic.claude-sonnet-4-6", _CONVERSE_REPLY),
|
||||||
("bedrock/converse/us.anthropic.claude-opus-4-6-v1", _CONVERSE_REPLY),
|
|
||||||
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLAUDE_REPLY),
|
("bedrock/invoke/us.anthropic.claude-sonnet-4-6", _CLAUDE_REPLY),
|
||||||
),
|
),
|
||||||
ids=("converse", "converse_parallel_tool_use_config_model", "invoke"),
|
ids=("converse", "invoke"),
|
||||||
)
|
)
|
||||||
async def test_runtime_count_sends_disable_parallel_tool_use_only_when_messages_inference_sends_it(
|
async def test_runtime_count_sends_disable_parallel_tool_use_only_when_messages_inference_sends_it(
|
||||||
deployment_model: str, inference_reply: Mapping[str, object]
|
deployment_model: str, inference_reply: Mapping[str, object]
|
||||||
|
|
|
||||||
|
|
@ -1744,20 +1744,28 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_options_i
|
||||||
_CALLER_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value
|
_CALLER_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value
|
||||||
|
|
||||||
|
|
||||||
|
def _claude_router(deployment_model: str, deployment_params: Mapping[str, JsonValue]) -> Router:
|
||||||
|
return Router(
|
||||||
|
model_list=[
|
||||||
|
{
|
||||||
|
"model_name": "claude-agent",
|
||||||
|
"litellm_params": {"model": deployment_model, "api_key": "sk-ant-test", **deployment_params},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _count_on_a_claude_deployment(
|
async def _count_on_a_claude_deployment(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
deployment_model: str,
|
deployment_model: str,
|
||||||
body: Mapping[str, JsonValue],
|
body: Mapping[str, JsonValue],
|
||||||
reply: Callable[[httpx.Request], httpx.Response],
|
reply: Callable[[httpx.Request], httpx.Response],
|
||||||
user_agent: str = "python-httpx/0.28.1",
|
user_agent: str = "python-httpx/0.28.1",
|
||||||
|
deployment_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||||
) -> tuple[object, tuple[httpx.Request, ...]]:
|
) -> tuple[object, tuple[httpx.Request, ...]]:
|
||||||
import litellm.proxy.proxy_server as proxy_server
|
import litellm.proxy.proxy_server as proxy_server
|
||||||
|
|
||||||
router: Final = Router(
|
router: Final = _claude_router(deployment_model, deployment_params)
|
||||||
model_list=[
|
|
||||||
{"model_name": "claude-agent", "litellm_params": {"model": deployment_model, "api_key": "sk-ant-test"}}
|
|
||||||
]
|
|
||||||
)
|
|
||||||
request: Final = _json_request(
|
request: Final = _json_request(
|
||||||
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
|
{"model": "claude-agent", "messages": [{"role": "user", "content": "Weather in SF?"}], **body},
|
||||||
user_agent,
|
user_agent,
|
||||||
|
|
@ -2078,7 +2086,6 @@ async def _claude_inference_body(model: str, options: Mapping[str, JsonValue]) -
|
||||||
("request_drop_params", "user_agent"),
|
("request_drop_params", "user_agent"),
|
||||||
(
|
(
|
||||||
({"drop_params": True}, "python-httpx/0.28.1"),
|
({"drop_params": True}, "python-httpx/0.28.1"),
|
||||||
({"drop_params": " true "}, "python-httpx/0.28.1"),
|
|
||||||
({}, "claude-cli/2.1.0 (external, cli)"),
|
({}, "claude-cli/2.1.0 (external, cli)"),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
@ -2106,6 +2113,44 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("deployment_drop_params", "request_drop_params"),
|
||||||
|
((True, "maybe"), (True, False), (None, " true ")),
|
||||||
|
)
|
||||||
|
async def test_anthropic_count_tokens_reads_the_request_drop_params_over_the_deployments_like_inference(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, deployment_drop_params: bool | None, request_drop_params: JsonValue
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||||
|
thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"}
|
||||||
|
deployment_params: Final[Mapping[str, JsonValue]] = (
|
||||||
|
{} if deployment_drop_params is None else {"drop_params": deployment_drop_params}
|
||||||
|
)
|
||||||
|
with respx.mock:
|
||||||
|
inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
|
||||||
|
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
|
||||||
|
)
|
||||||
|
await _claude_router("anthropic/claude-opus-4-7", deployment_params).aanthropic_messages(
|
||||||
|
model="claude-agent",
|
||||||
|
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||||
|
max_tokens=4096,
|
||||||
|
thinking=thinking,
|
||||||
|
drop_params=request_drop_params,
|
||||||
|
)
|
||||||
|
sent: Final = json.loads(inference_route.calls.last.request.content)
|
||||||
|
|
||||||
|
response, (count,) = await _count_on_a_claude_deployment(
|
||||||
|
monkeypatch,
|
||||||
|
"anthropic/claude-opus-4-7",
|
||||||
|
{"thinking": thinking, "drop_params": request_drop_params},
|
||||||
|
lambda _: httpx.Response(200, json={"input_tokens": 7}),
|
||||||
|
deployment_params=deployment_params,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not isinstance(response, JSONResponse)
|
||||||
|
assert json.loads(count.content).get("thinking") == sent.get("thinking")
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_anthropic_count_tokens_on_a_claude_deployment_counts_the_output_format_as_its_output_config_format(
|
async def test_anthropic_count_tokens_on_a_claude_deployment_counts_the_output_format_as_its_output_config_format(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue