mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(count_tokens): let an explicit null drop_params replace the deployment's on the count like inference
The count now checks whether the request carries drop_params instead of whether its value is None, so a caller's null keeps summary and other fields on a drop_params deployment the way Router inference does. The test covers the null and the absent key and moves to claude-opus-5-5
This commit is contained in:
parent
a7b0bdc13c
commit
54ec2c8330
2 changed files with 17 additions and 11 deletions
|
|
@ -15303,13 +15303,13 @@ _DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
|||
|
||||
|
||||
def _deployment_with_request_drop_params(
|
||||
deployment: Mapping[str, object] | None, drop_params: object
|
||||
deployment: Mapping[str, object] | None, sent_params: Mapping[str, object]
|
||||
) -> Mapping[str, object] | None:
|
||||
if drop_params is None:
|
||||
if "drop_params" not in sent_params:
|
||||
return deployment
|
||||
resolved_deployment: Final = deployment or {}
|
||||
litellm_params: Final = _DEPLOYMENT_LITELLM_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {})
|
||||
return {**resolved_deployment, "litellm_params": {**litellm_params, "drop_params": drop_params}}
|
||||
return {**resolved_deployment, "litellm_params": {**litellm_params, "drop_params": sent_params["drop_params"]}}
|
||||
|
||||
|
||||
def _counted_request_messages(
|
||||
|
|
@ -16151,7 +16151,7 @@ async def _count_request_tokens(
|
|||
model_to_use = _model
|
||||
|
||||
sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params}
|
||||
count_deployment: Final = _deployment_with_request_drop_params(deployment, sent_params.get("drop_params"))
|
||||
count_deployment: Final = _deployment_with_request_drop_params(deployment, sent_params)
|
||||
conversation_empty: Final = not messages and not contents
|
||||
if prompt and not conversation_empty and not deployment_drops_params(count_deployment):
|
||||
raise ProxyException(
|
||||
|
|
|
|||
|
|
@ -2191,11 +2191,17 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_sends_the_thinking_
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_drop_params", "request_drop_params"),
|
||||
((True, "maybe"), (True, False), (None, " true ")),
|
||||
("deployment_drop_params", "request_params"),
|
||||
(
|
||||
(True, {"drop_params": "maybe"}),
|
||||
(True, {"drop_params": False}),
|
||||
(True, {"drop_params": None}),
|
||||
(True, {}),
|
||||
(None, {"drop_params": " true "}),
|
||||
),
|
||||
)
|
||||
async def test_anthropic_count_tokens_reads_the_request_drop_params_over_the_deployments_like_inference(
|
||||
monkeypatch: pytest.MonkeyPatch, deployment_drop_params: bool | None, request_drop_params: JsonValue
|
||||
monkeypatch: pytest.MonkeyPatch, deployment_drop_params: bool | None, request_params: Mapping[str, JsonValue]
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
thinking: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2000, "summary": "auto"}
|
||||
|
|
@ -2206,19 +2212,19 @@ async def test_anthropic_count_tokens_reads_the_request_drop_params_over_the_dep
|
|||
inference_route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=httpx.Response(200, json=_CLAUDE_MESSAGE_REPLY)
|
||||
)
|
||||
await _claude_router("anthropic/claude-opus-4-7", deployment_params).aanthropic_messages(
|
||||
await _claude_router("anthropic/claude-opus-5-5", deployment_params).aanthropic_messages(
|
||||
model="claude-agent",
|
||||
messages=[{"role": "user", "content": "Weather in SF?"}],
|
||||
max_tokens=4096,
|
||||
thinking=thinking,
|
||||
drop_params=request_drop_params,
|
||||
**request_params,
|
||||
)
|
||||
sent: Final = json.loads(inference_route.calls.last.request.content)
|
||||
|
||||
response, (count,) = await _count_on_a_claude_deployment(
|
||||
monkeypatch,
|
||||
"anthropic/claude-opus-4-7",
|
||||
{"thinking": thinking, "drop_params": request_drop_params},
|
||||
"anthropic/claude-opus-5-5",
|
||||
{"thinking": thinking, **request_params},
|
||||
lambda _: httpx.Response(200, json={"input_tokens": 7}),
|
||||
deployment_params=deployment_params,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue