From 44d99475a559490f987be9091eedd8cc67428e3e Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sun, 18 Oct 2026 02:58:05 -0700 Subject: [PATCH 01/50] test(integration): stop GET /__observations from draining the upstream log (#47256) * test(integration): stop GET /__observations from draining the upstream log GET /__observations used to empty the queue, so a second read in a test saw nothing and a "never reached upstream" check could pass whether or not the request was forwarded. GET now returns every request since the last reset and DELETE /__observations is the reset. An autouse fixture resets the worker upstream before each test, which also covers tests that read without the gateway fixture. Discarded reads used as resets become DELETEs, helpers that accumulated drained reads now read one snapshot, and tests with several phases reset between them. The three vacuous negative checks (rag allowlist, guardrail opt-out chaos, xAI batch output) now assert the absence on the same read that shows a request which did reach the upstream. * style(integration): space the observations unit test like its neighbours --- tests/integration/README.md | 2 +- tests/integration/_support/upstream.py | 24 +++++- .../authorization/_guardrail_opt_out.py | 4 +- .../authorization/_hidden_alias_budget.py | 8 +- .../test_access_group_model_listing.py | 17 +++- .../authorization/test_end_user_block.py | 8 +- .../test_hidden_alias_budget_bypass_chaos.py | 5 +- .../test_key_guardrail_opt_out_chaos.py | 12 ++- .../test_rag_query_vector_store_allowlist.py | 15 +++- .../test_team_member_permissions.py | 4 +- .../authorization/test_team_scoped_models.py | 4 +- .../test_wildcard_model_access.py | 3 +- .../caching/response_cache_case.py | 11 +-- .../compatibility/_status_code_audit.py | 20 ++--- .../test_missing_body_param_status.py | 2 +- .../test_openai_sdk_client_paths_chaos.py | 2 +- .../test_status_code_followups.py | 2 +- .../test_status_code_followups_chaos.py | 4 +- .../test_status_code_followups_defaults.py | 6 -- .../test_status_code_followups_requests.py | 6 -- .../test_status_code_followups_usage_chat.py | 2 +- .../configuration/test_effective_settings.py | 4 +- tests/integration/conftest.py | 8 +- .../test_batch_output_file_listing.py | 4 +- .../test_fallback_management_public_names.py | 4 +- .../management/test_model_health_check.py | 2 +- .../management/test_organization_lifecycle.py | 2 + .../integration/management/test_team_block.py | 9 ++- .../management/test_team_member_lifecycle.py | 13 ++- .../test_user_delete_revokes_keys.py | 1 + ..._user_updates_wedged_coordination_redis.py | 8 +- .../test_vector_store_config_ownership.py | 4 +- .../_logging_only_scope_support.py | 23 ++++-- .../observability/test_guardrail_effects.py | 14 ++-- .../test_logging_only_scope_chaos.py | 25 +++--- .../test_logging_only_scope_config.py | 60 ++++++++------ .../test_logging_only_scope_runtime.py | 79 +++++++++++-------- .../test_presidio_entity_masking.py | 2 +- .../pricing/test_configured_prices.py | 4 +- .../pricing/test_cost_margin_discount.py | 6 +- .../pricing/test_per_second_pricing.py | 4 +- .../pricing/test_service_tier_pricing.py | 4 +- .../test_databricks_decisions_wire.py | 8 ++ .../test_decision_model_guardrail.py | 16 ++-- .../providers/test_deepseek_vision_wire.py | 2 +- .../providers/test_provider_lookup_status.py | 2 +- ..._transcription_guardrail_session_update.py | 9 +++ .../providers/test_request_boundary.py | 2 +- .../test_xai_batch_output_file_wire.py | 9 +-- .../routing/test_end_user_region_routing.py | 4 +- .../routing/test_key_max_parallel_requests.py | 2 +- .../routing/test_observed_routing.py | 4 +- .../test_pre_call_response_format_filter.py | 12 +-- .../routing/test_redis_recovery.py | 4 +- .../routing/test_stale_cost_map_boot.py | 2 +- .../routing/test_team_tag_routing.py | 2 +- .../spend/_background_responses.py | 3 +- .../test_background_interaction_settlement.py | 20 +++-- ...est_background_response_poll_retirement.py | 3 +- .../integration/spend/test_cache_and_quota.py | 17 ++-- .../spend/test_chaos_burst_spend_once.py | 2 +- .../test_disable_end_user_cost_tracking.py | 8 +- .../spend/test_end_user_budget_enforcement.py | 13 +-- .../test_end_user_spend_without_proxy_user.py | 18 +++-- .../spend/test_global_spend_reset.py | 4 +- .../spend/test_key_budget_lockout.py | 16 ++-- .../test_org_budget_cli_session_token.py | 3 +- .../spend/test_team_budget_enforcement.py | 2 +- .../spend/test_team_member_spend.py | 19 ++--- .../spend/test_user_budget_on_team_keys.py | 2 +- .../unit/integration_support/test_upstream.py | 26 +++++- 71 files changed, 413 insertions(+), 262 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index 496c88050ee..c32fda2f944 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -34,7 +34,7 @@ The CircleCI workflow starts its own database and Redis, restricts test-phase eg There is no hand-maintained list of test cases. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration//test_*.py` file in a scheduled group as owned by CircleCI -Provider sentinels currently use the controlled server, not live recordings. The provider shard also runs the existing strict replay controls for changed requests, exhausted interactions, leftover interactions and no provider connection. Future recorded scenarios must use that replay-only implementation; missing recordings cannot fall back to a real provider. The observation endpoint is destructive and the current selection runs serially against one owned upstream +Provider sentinels currently use the controlled server, not live recordings. The provider shard also runs the existing strict replay controls for changed requests, exhausted interactions, leftover interactions and no provider connection. Future recorded scenarios must use that replay-only implementation; missing recordings cannot fall back to a real provider. `GET /__observations` returns every request the upstream recorded since the last `DELETE /__observations`, and reading it never clears it. An autouse fixture resets it before each test, so a test that needs a clean slate between phases sends the DELETE itself. A check that a request never reached the upstream asserts on the same read that shows a request sent after it did reach the upstream, so the absence cannot pass on an empty or reset log Translation tests in `translation/` compare exact provider requests and LiteLLM responses against a shared fake provider; `translation/README.md` has their rules diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index 186bc345fab..85da60a37ad 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -11,7 +11,6 @@ import zlib from collections import deque from collections.abc import AsyncIterator, Callable, Iterator, Mapping from dataclasses import dataclass, field -from queue import SimpleQueue from types import MappingProxyType from typing import Final, cast @@ -184,9 +183,23 @@ class ScenarioStore: return self._scenarios.get(scenario_id) +class ObservationLog: + def __init__(self) -> None: + self._entries: list[Observation] = [] + + def put(self, observation: Observation) -> None: + self._entries.append(observation) + + def snapshot(self) -> tuple[Observation, ...]: + return tuple(self._entries) + + def clear(self) -> None: + self._entries.clear() + + @dataclass(frozen=True, slots=True) class Provider: - observations: SimpleQueue[Observation] = field(default_factory=SimpleQueue) + observations: ObservationLog = field(default_factory=ObservationLog) scripts: dict[str, deque[int]] = field(default_factory=dict) scenario_store: ScenarioStore = field(default_factory=ScenarioStore) interactions: dict[str, InteractionState] = field(default_factory=dict) @@ -271,7 +284,10 @@ class Provider: return JSONResponse({"configured": len(statuses)}) async def observed(self, request: Request) -> Response: - values: Final = tuple(self.observations.get() for _ in range(self.observations.qsize())) + if request.method == "DELETE": + self.observations.clear() + return Response(status_code=204) + values: Final = self.observations.snapshot() return JSONResponse( { "requests": [ @@ -622,7 +638,7 @@ class Provider: return Starlette( routes=[ Route("/health", health), - Route("/__observations", self.observed), + Route("/__observations", self.observed, methods=["GET", "DELETE"]), Route("/__scripts/{model}", self.script, methods=["POST", "DELETE", "GET"]), Route("/__scenarios", self.register_scenario, methods=["POST"]), Route("/__scenarios/{scenario_id}", self.delete_scenario, methods=["DELETE"]), diff --git a/tests/integration/authorization/_guardrail_opt_out.py b/tests/integration/authorization/_guardrail_opt_out.py index e813993edf4..38854a29dc2 100644 --- a/tests/integration/authorization/_guardrail_opt_out.py +++ b/tests/integration/authorization/_guardrail_opt_out.py @@ -61,8 +61,8 @@ def chat(candidate: Gateway, model: str, key: str, marker: str, *, stream: bool def upstream_observations(gateway: Gateway) -> tuple[dict[str, JsonValue], ...]: with httpx.Client(timeout=5, trust_env=False) as client: - drained: Final = object_value(client.get(f"{gateway.upstream_url}/__observations").json()) - requests: Final = drained["requests"] + observed: Final = object_value(client.get(f"{gateway.upstream_url}/__observations").json()) + requests: Final = observed["requests"] assert isinstance(requests, list) return tuple(object_value(entry) for entry in requests) diff --git a/tests/integration/authorization/_hidden_alias_budget.py b/tests/integration/authorization/_hidden_alias_budget.py index eb099e56026..bee2a585cff 100644 --- a/tests/integration/authorization/_hidden_alias_budget.py +++ b/tests/integration/authorization/_hidden_alias_budget.py @@ -201,10 +201,14 @@ def exhausted_key(rig: AliasRig, scenario: Scenario) -> str: def upstream_requests(upstream_url: str) -> tuple[str, ...]: - drained: Final = JSON_OBJECT.validate_json( + observed: Final = JSON_OBJECT.validate_json( httpx.get(f"{upstream_url}/__observations", timeout=15, trust_env=False).content ) - return tuple(json.dumps(entry) for entry in JSON_LIST.validate_python(drained["requests"])) + return tuple(json.dumps(entry) for entry in JSON_LIST.validate_python(observed["requests"])) + + +def reset_upstream_requests(upstream_url: str) -> None: + httpx.delete(f"{upstream_url}/__observations", timeout=15, trust_env=False).raise_for_status() def upstream_hits(observed: tuple[str, ...], marker: str) -> int: diff --git a/tests/integration/authorization/test_access_group_model_listing.py b/tests/integration/authorization/test_access_group_model_listing.py index 13439d19769..c8757bdab9d 100644 --- a/tests/integration/authorization/test_access_group_model_listing.py +++ b/tests/integration/authorization/test_access_group_model_listing.py @@ -79,6 +79,10 @@ def _observed(upstream: httpx.Client) -> list[dict[str, JsonValue]]: return requests +def _reset_observations(upstream: httpx.Client) -> None: + upstream.delete("/__observations").raise_for_status() + + def _chat(proxy: Gateway, model: str, key: str, text: str) -> httpx.Response: return proxy.request( "POST", @@ -135,7 +139,7 @@ def test_access_group_assignment_grants_and_revokes_on_gateway_and_peer( team_key: Final = scenario.key(team_id=team_id) assigned_key: Final = scenario.key(models=["no-default-models"]) hashed_key: Final = sha256(assigned_key.encode()).hexdigest() - _observed(upstream) + _reset_observations(upstream) _assert_denied(_chat(gateway, model, team_key, "control team"), model, "team_model_access_denied") _assert_denied(_chat(gateway, model, assigned_key, "control key"), model, "key_model_access_denied") assert _observed(upstream) == [] @@ -173,6 +177,7 @@ def test_access_group_assignment_grants_and_revokes_on_gateway_and_peer( ) expected.append(_observation(text)) assert _observed(upstream) == expected + _reset_observations(upstream) removed_team: Final = gateway.request("PUT", f"{route_prefix}/{access_group_id}", {"assigned_team_ids": []}) assert removed_team.status_code == 200, removed_team.text @@ -183,6 +188,7 @@ def test_access_group_assignment_grants_and_revokes_on_gateway_and_peer( still_granted: Final = f"still granted {uuid.uuid4().hex}" _assert_served(_chat(gateway, model, assigned_key, still_granted)) assert _observed(upstream) == [_observation(still_granted)] + _reset_observations(upstream) removed_key: Final = gateway.request("PUT", f"{route_prefix}/{access_group_id}", {"assigned_key_ids": []}) assert removed_key.status_code == 200, removed_key.text @@ -203,6 +209,7 @@ def test_access_group_assignment_grants_and_revokes_on_gateway_and_peer( _assert_served(_chat(gateway, model, key, text)) regranted.append(_observation(text)) assert _observed(upstream) == regranted + _reset_observations(upstream) deleted: Final = gateway.request("DELETE", f"{route_prefix}/{access_group_id}") assert deleted.status_code == 204, deleted.text @@ -266,6 +273,7 @@ def test_deleting_an_access_group_revokes_a_still_assigned_team_and_key(gateway: ) _assert_served(key_response) assert _observed(upstream) == [_observation(team_text), _observation(key_text)] + _reset_observations(upstream) deleted: Final = gateway.request("DELETE", f"{route_prefix}/{access_group_id}") assert deleted.status_code == 204, deleted.text @@ -310,12 +318,13 @@ def test_access_group_revocation_reaches_a_warmed_peer_before_the_cache_ttl( assert created.status_code == 201, created.text access_group_id: Final = string_value(created.json()["access_group_id"]) scenario.cleanups.callback(gateway.request, "DELETE", f"{route_prefix}/{access_group_id}") - _observed(upstream) + _reset_observations(upstream) if revocation == "team": warmed: Final = _chat(peer, model, team_key, "warm team") assert warmed.status_code == 200, warmed.text assert _observed(upstream) == [_observation("warm team")] + _reset_observations(upstream) team_update: Final = gateway.request("PUT", f"{route_prefix}/{access_group_id}", {"assigned_team_ids": []}) assert team_update.status_code == 200, team_update.text _assert_denied( @@ -331,6 +340,7 @@ def test_access_group_revocation_reaches_a_warmed_peer_before_the_cache_ttl( warmed: Final = _chat(peer, model, assigned_key, "warm key") assert warmed.status_code == 200, warmed.text assert _observed(upstream) == [_observation("warm key")] + _reset_observations(upstream) key_update: Final = gateway.request("PUT", f"{route_prefix}/{access_group_id}", {"assigned_key_ids": []}) assert key_update.status_code == 200, key_update.text _assert_denied( @@ -351,6 +361,7 @@ def test_access_group_revocation_reaches_a_warmed_peer_before_the_cache_ttl( _observation("warm delete team"), _observation("warm delete key"), ] + _reset_observations(upstream) deleted: Final = gateway.request("DELETE", f"{route_prefix}/{access_group_id}") assert deleted.status_code == 204, deleted.text _assert_denied( @@ -383,7 +394,7 @@ def test_raw_key_in_assigned_key_ids_grants_the_same_access_as_the_hashed_key(ga model: Final = scenario.model() key: Final = scenario.key(models=["no-default-models"]) hashed_key: Final = sha256(key.encode()).hexdigest() - _observed(upstream) + _reset_observations(upstream) _assert_denied(_chat(gateway, model, key, "control"), model, "key_model_access_denied") assert _observed(upstream) == [] diff --git a/tests/integration/authorization/test_end_user_block.py b/tests/integration/authorization/test_end_user_block.py index fd283380d8f..7b1eb8da74a 100644 --- a/tests/integration/authorization/test_end_user_block.py +++ b/tests/integration/authorization/test_end_user_block.py @@ -36,6 +36,10 @@ def _observed(upstream: httpx.Client) -> list[dict[str, JsonValue]]: return requests +def _reset_observations(upstream: httpx.Client) -> None: + upstream.delete("/__observations").raise_for_status() + + def _chat(gateway: Gateway, model: str, key: str, end_user: str) -> httpx.Response: return gateway.request( "POST", @@ -215,7 +219,7 @@ def test_blocked_end_user_is_refused_on_every_proxy_and_unblock_restores( ): rig: Final = _two_callback_proxies(gateway, tmp_path, stack) with rig.first.scenario() as scenario: - _observed(upstream) + _reset_observations(upstream) end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" created: Final = rig.first.request("POST", "/customer/new", {"user_id": end_user, "max_budget": 100.0}) assert created.status_code == 200, created.text @@ -224,7 +228,7 @@ def test_blocked_end_user_is_refused_on_every_proxy_and_unblock_restores( key: Final = scenario.key(models=[model]) served: Final = _chat(rig.first, model, key, end_user) assert served.status_code == 200, served.text - _observed(upstream) + _reset_observations(upstream) rig.first.request("POST", block_route, {"user_ids": [end_user]}) assert eventually( diff --git a/tests/integration/authorization/test_hidden_alias_budget_bypass_chaos.py b/tests/integration/authorization/test_hidden_alias_budget_bypass_chaos.py index 3e6acfc72de..980e7251a66 100644 --- a/tests/integration/authorization/test_hidden_alias_budget_bypass_chaos.py +++ b/tests/integration/authorization/test_hidden_alias_budget_bypass_chaos.py @@ -26,6 +26,7 @@ from integration.authorization._hidden_alias_budget import ( install_aliases, landed_all_once, remove_aliases, + reset_upstream_requests, settle_candidate, settle_chat, upstream_hits, @@ -73,7 +74,7 @@ def test_mixed_concurrent_burst_through_hidden_free_aliases_lands_each_call_once markers: Final = tuple(f"burst-{_KINDS[index % len(_KINDS)]}-{index}-{uuid.uuid4().hex}" for index in range(30)) with rig.gateway.scenario() as scenario: key: Final = exhausted_key(rig, scenario) - upstream_requests(rig.gateway.upstream_url) + reset_upstream_requests(rig.gateway.upstream_url) responses: Final = _send_all(lambda marker: _mixed_call(rig, key, marker), markers) assert [response.status_code for response in responses] == [200] * len(markers), [ response.text for response in responses if response.status_code != 200 @@ -96,7 +97,7 @@ def test_alias_flipped_to_a_free_group_during_a_burst_only_ever_serves_or_refuse settle_chat(rig, alias, rig.gateway.key, 200) key: Final = exhausted_key(rig, scenario) settle_chat(rig, alias, key, BUDGET_EXCEEDED) - upstream_requests(rig.gateway.upstream_url) + reset_upstream_requests(rig.gateway.upstream_url) def wave(candidate: Gateway) -> frozenset[int]: markers: Final = _burst_markers("flip", _WAVE) diff --git a/tests/integration/authorization/test_key_guardrail_opt_out_chaos.py b/tests/integration/authorization/test_key_guardrail_opt_out_chaos.py index 3ceae540ba9..4815eebe2f1 100644 --- a/tests/integration/authorization/test_key_guardrail_opt_out_chaos.py +++ b/tests/integration/authorization/test_key_guardrail_opt_out_chaos.py @@ -21,7 +21,6 @@ from integration.authorization._guardrail_opt_out import ( guardrail_config, non_admin_caller, stored_metadata, - upstream_hits, upstream_observations, ) @@ -226,9 +225,14 @@ def test_exempt_burst_survives_guardrail_sink_outage(gateway: Gateway, tmp_path: "synthetic policy denial" in response.text or response.status_code == 500 for response in non_exempt_outage ), [response.text for response in non_exempt_outage if response.status_code not in {400, 500}] - assert all(upstream_hits(gateway, f"outage-{index}-") == 0 for index in range(1, 20, 2)), ( - upstream_observations(gateway) - ) + control_marker: Final = "control-" + uuid.uuid4().hex + control: Final = chat(candidate, model, exempt, control_marker) + assert control.status_code == 200, control.text + observed: Final = upstream_observations(gateway) + bodies: Final = tuple(json.dumps(entry.get("body")) for entry in observed) + assert any(control_marker in body for body in bodies), observed + assert all(any(f"outage-{index}-" in body for body in bodies) for index in range(0, 20, 2)), observed + assert not any(any(f"outage-{index}-" in body for body in bodies) for index in range(1, 20, 2)), observed sink.start() recovered: Final = chat(candidate, model, plain, "recovered-" + uuid.uuid4().hex) diff --git a/tests/integration/authorization/test_rag_query_vector_store_allowlist.py b/tests/integration/authorization/test_rag_query_vector_store_allowlist.py index c1777c66c39..c90b689599b 100644 --- a/tests/integration/authorization/test_rag_query_vector_store_allowlist.py +++ b/tests/integration/authorization/test_rag_query_vector_store_allowlist.py @@ -734,14 +734,21 @@ def test_explicit_false_flag_keeps_legacy_vector_store_outcomes(tmp_path: Path) models=_json_array(model), object_permission=_permission_for_stores("vs_some_other_store") ) - allowed: Final = _rag_query(gateway, model, allowed_marker, no_permission_key) denied: Final = _rag_query(gateway, model, denied_marker, excluding_key) + allowed: Final = _rag_query(gateway, model, allowed_marker, no_permission_key) - assert allowed.status_code == 200, allowed.text - assert len(_searches_for_marker(upstream_gateway, allowed_marker)) == 1 assert denied.status_code == 401, denied.text assert denied.json()["error"]["type"] == "key_vector_store_access_denied" - assert _searches_for_marker(upstream_gateway, denied_marker) == () + assert allowed.status_code == 200, allowed.text + searches: Final = tuple( + str(observation["body"]) + for observation in upstream_observations(upstream_gateway) + if observation["path"] == f"/vector_stores/{CONFIG_STORE_ID}/search" + ) + assert ( + sum(allowed_marker in body for body in searches), + sum(denied_marker in body for body in searches), + ) == (1, 0), searches def _strict_no_registry_config(directory: Path) -> Path: diff --git a/tests/integration/authorization/test_team_member_permissions.py b/tests/integration/authorization/test_team_member_permissions.py index 5d29aceb10c..22752becfcb 100644 --- a/tests/integration/authorization/test_team_member_permissions.py +++ b/tests/integration/authorization/test_team_member_permissions.py @@ -71,8 +71,8 @@ class _HTTPErrorResponse(BaseModel): @pytest.fixture def upstream(gateway: Gateway) -> Iterator[httpx.Client]: with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client: - response: Final = client.get("/__observations") - assert response.status_code == 200, response.text + response: Final = client.delete("/__observations") + assert response.status_code == 204, response.text yield client diff --git a/tests/integration/authorization/test_team_scoped_models.py b/tests/integration/authorization/test_team_scoped_models.py index 2982fbdb4eb..8f8e12ae498 100644 --- a/tests/integration/authorization/test_team_scoped_models.py +++ b/tests/integration/authorization/test_team_scoped_models.py @@ -41,7 +41,7 @@ class _ErrorResponse(BaseModel): @pytest.fixture def upstream(gateway: Gateway) -> Iterator[httpx.Client]: with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as client: - client.get("/__observations").raise_for_status() + client.delete("/__observations").raise_for_status() yield client @@ -166,6 +166,7 @@ def test_team_model_alias_routes_a_team_key_to_its_target( assert string_value(response.json()["model"]) == alias observed: Final = _observed_requests(upstream) assert _calls_to(observed, provider_model) == 1, observed + upstream.delete("/__observations").raise_for_status() unaliased: Final = _chat(gateway, f"alias-{uuid.uuid4().hex}", key) assert unaliased.status_code == 403, unaliased.text assert unaliased.json()["error"]["type"] == "key_model_access_denied" @@ -189,6 +190,7 @@ def test_team_model_add_and_delete_change_what_a_warmed_team_key_can_call( _model_denied(initially_denied) observed_before_add: Final = _observed_provider_models(upstream) assert observed_before_add == (provider_one,), repr(observed_before_add) + upstream.delete("/__observations").raise_for_status() added: Final = gateway.request("POST", "/team/model/add", {"team_id": team, "models": [model_two, model_three]}) assert added.status_code == 200, added.text diff --git a/tests/integration/authorization/test_wildcard_model_access.py b/tests/integration/authorization/test_wildcard_model_access.py index f63c2aac84a..9aa55be0da5 100644 --- a/tests/integration/authorization/test_wildcard_model_access.py +++ b/tests/integration/authorization/test_wildcard_model_access.py @@ -40,7 +40,7 @@ def candidate(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: @pytest.fixture def upstream(candidate: Gateway) -> Iterator[httpx.Client]: with httpx.Client(base_url=candidate.upstream_url, timeout=5, trust_env=False) as client: - client.get("/__observations").raise_for_status() + client.delete("/__observations").raise_for_status() yield client @@ -78,6 +78,7 @@ def test_a_key_without_models_inherits_the_users_exact_and_wildcard_grants( assert _chat(candidate, f"anthropic/{wildcard_model}", key).status_code == 200 assert _chat(candidate, "good-model", key).status_code == 200 assert _observed_models(upstream) == [wildcard_model, "good-model-upstream"] + upstream.delete("/__observations").raise_for_status() denied: Final = tuple( _chat(candidate, outside, key) for outside in (f"groq/{wildcard_model}", f"bedrock/anthropic.{wildcard_model}") diff --git a/tests/integration/caching/response_cache_case.py b/tests/integration/caching/response_cache_case.py index 8dfb6a0a479..a1850ebdb9d 100644 --- a/tests/integration/caching/response_cache_case.py +++ b/tests/integration/caching/response_cache_case.py @@ -216,22 +216,19 @@ MESSAGE_STREAM_WITHOUT_CONTENT_BLOCKS: Final = SseResponse( class Upstream: - """The owned upstream's observations, drained once per read and kept, filtered per deployment.""" + """The owned upstream's observations, filtered per deployment.""" def __init__(self, url: str = CONTROL_URL) -> None: self._url: Final = url.rstrip("/") - self._seen: tuple[dict[str, JsonValue], ...] = () def received(self, identity: str) -> tuple[dict[str, JsonValue], ...]: with httpx.Client(timeout=10, trust_env=False) as client: payload: Final = JSON_OBJECT.validate_json(client.get(f"{self._url}/__observations").content) requests: Final = payload["requests"] assert isinstance(requests, list) - self._seen = ( - *self._seen, - *(object_value(item) for item in requests), - ) # rebind-ok: drained observations accumulate - return tuple(item for item in self._seen if f"/{identity}/" in string_value(item.get("path", ""))) + return tuple( + item for item in map(object_value, requests) if f"/{identity}/" in string_value(item.get("path", "")) + ) def calls(self, identity: str) -> int: """The LLM calls the deployment took; the proxy's boot-time ``GET /v1/models`` discovery of a config diff --git a/tests/integration/compatibility/_status_code_audit.py b/tests/integration/compatibility/_status_code_audit.py index 4ba1ee6b102..e0e457fb11d 100644 --- a/tests/integration/compatibility/_status_code_audit.py +++ b/tests/integration/compatibility/_status_code_audit.py @@ -6,7 +6,6 @@ upstream's observation feed, so a cell asserts both what the caller got and what from __future__ import annotations import json -import os import uuid from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager @@ -140,35 +139,30 @@ def openai_error(message: str, param: str) -> JsonResponse: class Upstream: - """Reads the scripted upstream's destructive observation feed and keeps every record it drained.""" - def __init__(self, url: str) -> None: self.url = url.rstrip("/") - self.items: tuple[dict[str, JsonValue], ...] = () - def drain(self) -> None: + def reset(self) -> None: + with httpx.Client(timeout=10, trust_env=False) as client: + client.delete(f"{self.url}/__observations").raise_for_status() + + def observed(self) -> tuple[dict[str, JsonValue], ...]: with httpx.Client(timeout=10, trust_env=False) as client: payload: Final = JSON_OBJECT.validate_python( client.get(f"{self.url}/__observations?include_method=true").json() ) requests: Final = payload.get("requests") assert isinstance(requests, list) - self.items = (*self.items, *(object_value(item) for item in requests if isinstance(item, dict))) + return tuple(object_value(item) for item in requests if isinstance(item, dict)) def calls(self, identity: str) -> tuple[dict[str, JsonValue], ...]: - self.drain() return tuple( item - for item in self.items + for item in self.observed() if f"/{identity}/" in str(item.get("path")) and not str(item.get("path", "")).endswith("/models") ) -def drain_rig_upstream() -> None: - """Drops every observation earlier cells left on the shared upstream, so a cell reads only its own calls.""" - Upstream(os.environ["INTEGRATION_UPSTREAM_URL"]).drain() - - def assert_no_provider_call(gateway: Gateway, *identities: str) -> None: upstream: Final = Upstream(gateway.upstream_url) assert {identity: upstream.calls(identity) for identity in identities} == dict.fromkeys(identities, ()) diff --git a/tests/integration/compatibility/test_missing_body_param_status.py b/tests/integration/compatibility/test_missing_body_param_status.py index 673fb0acc51..b2c5b1c74b6 100644 --- a/tests/integration/compatibility/test_missing_body_param_status.py +++ b/tests/integration/compatibility/test_missing_body_param_status.py @@ -641,7 +641,7 @@ class _Observations: ) requests: Final = payload.get("requests") assert isinstance(requests, list) - self.items = (*self.items, *(object_value(item) for item in requests if isinstance(item, dict))) + self.items = tuple(object_value(item) for item in requests if isinstance(item, dict)) return self.items def for_scenario(self, identity: str) -> tuple[dict[str, JsonValue], ...]: diff --git a/tests/integration/compatibility/test_openai_sdk_client_paths_chaos.py b/tests/integration/compatibility/test_openai_sdk_client_paths_chaos.py index 9710d8a1b5c..9a6b9fcd60b 100644 --- a/tests/integration/compatibility/test_openai_sdk_client_paths_chaos.py +++ b/tests/integration/compatibility/test_openai_sdk_client_paths_chaos.py @@ -128,7 +128,7 @@ class _Observations: payload: Final = JSON_OBJECT.validate_python(client.get(f"{self.url}/__observations").json()) requests: Final = payload.get("requests") assert isinstance(requests, list) - self.items = (*self.items, *(object_value(item) for item in requests if isinstance(item, dict))) + self.items = tuple(object_value(item) for item in requests if isinstance(item, dict)) return self.items diff --git a/tests/integration/compatibility/test_status_code_followups.py b/tests/integration/compatibility/test_status_code_followups.py index 662f37aed1a..b6f1ebb07ca 100644 --- a/tests/integration/compatibility/test_status_code_followups.py +++ b/tests/integration/compatibility/test_status_code_followups.py @@ -89,7 +89,7 @@ class _Observations: ) requests: Final = payload.get("requests") assert isinstance(requests, list) - self.items = (*self.items, *(object_value(item) for item in requests if isinstance(item, dict))) + self.items = tuple(object_value(item) for item in requests if isinstance(item, dict)) return self.items def provider_calls(self, identity: str) -> tuple[dict[str, JsonValue], ...]: diff --git a/tests/integration/compatibility/test_status_code_followups_chaos.py b/tests/integration/compatibility/test_status_code_followups_chaos.py index f1ea9e60330..dfff737d17a 100644 --- a/tests/integration/compatibility/test_status_code_followups_chaos.py +++ b/tests/integration/compatibility/test_status_code_followups_chaos.py @@ -183,9 +183,7 @@ def _burst(chaos: _Chaos, phase: str) -> tuple[_Outcome, ...]: def _upstream_hits(slot: UpstreamSlot, markers: frozenset[str]) -> Mapping[str, int]: - upstream: Final = Upstream(slot.url) - upstream.drain() - bodies: Final = tuple(json.dumps(item.get("body")) for item in upstream.items) + bodies: Final = tuple(json.dumps(item.get("body")) for item in Upstream(slot.url).observed()) return {marker: sum(marker in body for body in bodies) for marker in markers} diff --git a/tests/integration/compatibility/test_status_code_followups_defaults.py b/tests/integration/compatibility/test_status_code_followups_defaults.py index a0c73d12dec..13a6b1b73eb 100644 --- a/tests/integration/compatibility/test_status_code_followups_defaults.py +++ b/tests/integration/compatibility/test_status_code_followups_defaults.py @@ -28,7 +28,6 @@ from integration.compatibility._status_code_audit import ( USER_MESSAGES, assembled_text, assert_no_provider_call, - drain_rig_upstream, deployment, error_body, invalid_request, @@ -47,11 +46,6 @@ from tests.integration.cost_calculation.cost_tracking_case import BinaryResponse pytestmark: Final = pytest.mark.timeout(180) -@pytest.fixture(autouse=True) -def _drained_upstream() -> None: - drain_rig_upstream() - - @dataclass(frozen=True, slots=True) class _Rig: gateway: Gateway diff --git a/tests/integration/compatibility/test_status_code_followups_requests.py b/tests/integration/compatibility/test_status_code_followups_requests.py index cffaf09ec55..29822a21d34 100644 --- a/tests/integration/compatibility/test_status_code_followups_requests.py +++ b/tests/integration/compatibility/test_status_code_followups_requests.py @@ -20,7 +20,6 @@ from integration.compatibility._status_code_audit import ( RESPONSES_FRAMES, assembled_text, assert_no_provider_call, - drain_rig_upstream, invalid_request, json_response, one_outbound, @@ -38,11 +37,6 @@ from tests.integration.cost_calculation.cost_tracking_case import StoredResponse pytestmark: Final = pytest.mark.timeout(180) -@pytest.fixture(autouse=True) -def _drained_upstream() -> None: - drain_rig_upstream() - - _OCR: Final[dict[str, JsonValue]] = { "pages": [{"index": 0, "markdown": "scripted page", "images": [], "dimensions": None}], "model": "mistral-ocr-latest", diff --git a/tests/integration/compatibility/test_status_code_followups_usage_chat.py b/tests/integration/compatibility/test_status_code_followups_usage_chat.py index 6d9d9507158..de9ec16cf30 100644 --- a/tests/integration/compatibility/test_status_code_followups_usage_chat.py +++ b/tests/integration/compatibility/test_status_code_followups_usage_chat.py @@ -71,7 +71,7 @@ def _tool_results(usage_chat: _UsageChat, start_date: str, end_date: str) -> tup """Runs one chat turn whose model calls every tool with these dates; returns the tool messages the follow-up model call carried, keyed by tool, and the tool statuses the SSE stream reported.""" register_scenario(usage_chat.identity, json_response(_tool_calls(start_date, end_date))) - Upstream(usage_chat.gateway.upstream_url).drain() + Upstream(usage_chat.gateway.upstream_url).reset() with usage_chat.gateway.client.stream( "POST", "/usage/ai/chat", diff --git a/tests/integration/configuration/test_effective_settings.py b/tests/integration/configuration/test_effective_settings.py index 029a5c7220e..acdaacfcfb5 100644 --- a/tests/integration/configuration/test_effective_settings.py +++ b/tests/integration/configuration/test_effective_settings.py @@ -868,7 +868,7 @@ def test_saved_retry_setting_controls_real_attempts_and_restores(gateway: Gatewa assert object_value(gateway.get("/router/settings")["current_values"])["num_retries"] == retries configured: Final = upstream.post(f"/__scripts/{provider_model}", json={"statuses": [500, 200]}) assert configured.status_code == 200, configured.text - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() response: Final = gateway.request( "POST", "/v1/chat/completions", @@ -1110,7 +1110,7 @@ def test_credential_value_update_and_model_reload_reach_provider(gateway: Gatewa "PATCH", f"/model/{identity}/update", {"model_info": {"description": value}} ) assert response.status_code == 200, response.text - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() assert ( object_value(gateway.chat(model, text=f"{name} {value} reload={reload}")["usage"])["total_tokens"] == 40 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 672ab7b6c6a..d605bd3d7d7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -323,10 +323,16 @@ def pytest_runtest_teardown(item: pytest.Item, nextitem: pytest.Item | None) -> return (yield) +@pytest.fixture(autouse=True) +def _reset_upstream_observations() -> None: + upstream: Final = os.environ.get("INTEGRATION_UPSTREAM_URL") + if upstream is not None: + httpx.delete(f"{upstream}/__observations", timeout=5, trust_env=False).raise_for_status() + + @pytest.fixture def gateway() -> Iterator[Gateway]: with gateway_from_environment() as value: - httpx.get(f"{value.upstream_url}/__observations", timeout=5, trust_env=False).raise_for_status() yield value diff --git a/tests/integration/management/test_batch_output_file_listing.py b/tests/integration/management/test_batch_output_file_listing.py index 26d1ae8de46..8aa01b0564e 100644 --- a/tests/integration/management/test_batch_output_file_listing.py +++ b/tests/integration/management/test_batch_output_file_listing.py @@ -389,7 +389,7 @@ def test_fallback_output_file_details_refresh_once_provider_recovers(gateway: Ga assert details["bytes"] == OUTPUT_BYTES, details assert (details["filename"], details["purpose"]) == ("output.jsonl", "batch_output"), details assert "litellm_details_fallback" not in details, details - assert _metadata_hit_count(gateway, batch.scenario) == 1 + assert _metadata_hit_count(gateway, batch.scenario) - metadata_hits_before_recovery == 1 refreshed_files: Final = _list_files(gateway, batch.owner_key, purpose="batch_output") refreshed_output: Final = next((file for file in refreshed_files if file.get("id") == output_id), None) @@ -647,5 +647,5 @@ def test_repeated_batch_retrieve_does_not_refetch_saved_file_details(gateway: Ga None, ) assert output_after == output_before, output_after - additional_hits: Final = _metadata_hit_count(gateway, batch.scenario) + additional_hits: Final = _metadata_hit_count(gateway, batch.scenario) - hits_before assert additional_hits == 0, f"Repeated batch retrieval fetched metadata {additional_hits} more times" diff --git a/tests/integration/management/test_fallback_management_public_names.py b/tests/integration/management/test_fallback_management_public_names.py index a5b311e38eb..fcbeee9dc60 100644 --- a/tests/integration/management/test_fallback_management_public_names.py +++ b/tests/integration/management/test_fallback_management_public_names.py @@ -53,7 +53,7 @@ def migrated_database(tmp_path_factory: pytest.TempPathFactory) -> None: @pytest.fixture def upstream(gateway: Gateway) -> Iterator[httpx.Client]: with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as client: - client.get("/__observations").raise_for_status() + client.delete("/__observations").raise_for_status() yield client @@ -206,7 +206,7 @@ def test_rule_by_team_public_name_fires_on_chat_completions(gateway: Gateway, up created: Final = _create_fallback(gateway, scenario, primary, [fallback]) assert created.status_code == 200, created.text upstream.post(f"/__scripts/{primary_provider}", json={"statuses": [500]}).raise_for_status() - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() answered: Final = eventually( lambda: gateway.request( "POST", diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 9945d16af07..57a05daca94 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -110,7 +110,7 @@ def test_health_check_of_a_model_added_through_the_api_calls_its_upstream_and_re assert ( object_value(gateway.chat(model, key=key, text=f"health {uuid.uuid4().hex}")["usage"])["total_tokens"] == 40 ) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() health: Final = gateway.request("GET", "/health", params={"model": model}) assert health.status_code == 200, health.text report: Final = health.json() diff --git a/tests/integration/management/test_organization_lifecycle.py b/tests/integration/management/test_organization_lifecycle.py index cdbf3c15b18..dbd8cce542e 100644 --- a/tests/integration/management/test_organization_lifecycle.py +++ b/tests/integration/management/test_organization_lifecycle.py @@ -179,6 +179,7 @@ def test_organization_delete_revokes_org_and_team_keys_on_gateway_and_peer(gatew response: Final = _chat(proxy, model, key, text) assert response.status_code == 200, response.text assert _observed(upstream) == [_observation(text)] + upstream.delete("/__observations").raise_for_status() deleted: Final = gateway.request("DELETE", "/organization/delete", {"organization_ids": [organization_id]}) assert deleted.status_code == 200, deleted.text @@ -207,6 +208,7 @@ def test_organization_delete_revokes_org_and_team_keys_on_gateway_and_peer(gatew response: Final = _chat(proxy, model, other_key, text) assert response.status_code == 200, response.text assert _observed(upstream) == [_observation(text)] + upstream.delete("/__observations").raise_for_status() hashed: Final = tuple(sha256(key.encode()).hexdigest() for key in (organization_key, team_key)) assert _key_rows(*hashed) == [] diff --git a/tests/integration/management/test_team_block.py b/tests/integration/management/test_team_block.py index 37c9da33c06..3d1f9e430ea 100644 --- a/tests/integration/management/test_team_block.py +++ b/tests/integration/management/test_team_block.py @@ -38,8 +38,7 @@ class _ErrorResponse(BaseModel): @pytest.fixture def upstream(gateway: Gateway) -> Iterator[httpx.Client]: with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client: - response: Final = client.get("/__observations") - assert response.status_code == 200, response.text + client.delete("/__observations").raise_for_status() yield client @@ -74,6 +73,7 @@ def test_team_block_and_unblock_update_endpoint_info_and_db_state( assert peer_before.status_code == 200, peer_before.text observed_before_block: Final = _models(upstream) assert observed_before_block == (provider_model, provider_model), repr(observed_before_block) + upstream.delete("/__observations").raise_for_status() blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team}) assert blocked_response.status_code == 200, blocked_response.text @@ -96,6 +96,7 @@ def test_team_block_and_unblock_update_endpoint_info_and_db_state( assert _ErrorResponse.model_validate_json(cold_denied.text).error.type == "auth_error", cold_denied.text after_cold_denied: Final = _models(upstream) assert after_cold_denied == (), repr(after_cold_denied) + upstream.delete("/__observations").raise_for_status() unblocked_response: Final = gateway.request("POST", "/team/unblock", {"team_id": team}) assert unblocked_response.status_code == 200, unblocked_response.text @@ -139,6 +140,7 @@ def test_unblocking_a_team_restores_warmed_keys_on_both_proxies( assert peer_before.status_code == 200, peer_before.text observed_before_block: Final = _models(upstream) assert observed_before_block == (provider_model, provider_model), repr(observed_before_block) + upstream.delete("/__observations").raise_for_status() blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team}) assert blocked_response.status_code == 200, blocked_response.text @@ -148,6 +150,7 @@ def test_unblocking_a_team_restores_warmed_keys_on_both_proxies( assert _ErrorResponse.model_validate_json(cold_denied.text).error.type == "auth_error", cold_denied.text after_cold_denied: Final = _models(upstream) assert after_cold_denied == (), repr(after_cold_denied) + upstream.delete("/__observations").raise_for_status() unblocked_response: Final = gateway.request("POST", "/team/unblock", {"team_id": team}) assert unblocked_response.status_code == 200, unblocked_response.text @@ -177,6 +180,7 @@ def test_warmed_team_keys_are_refused_on_both_proxies_after_block( assert peer_before.status_code == 200, peer_before.text observed_before_block: Final = _models(upstream) assert observed_before_block == (provider_model, provider_model), repr(observed_before_block) + upstream.delete("/__observations").raise_for_status() blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team}) assert blocked_response.status_code == 200, blocked_response.text @@ -210,6 +214,7 @@ def test_a_new_team_key_is_refused_on_a_peer_that_cached_the_team_before_block( assert peer_before.status_code == 200, peer_before.text observed_before_block: Final = _models(upstream) assert observed_before_block == (provider_model,), repr(observed_before_block) + upstream.delete("/__observations").raise_for_status() blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team}) assert blocked_response.status_code == 200, blocked_response.text diff --git a/tests/integration/management/test_team_member_lifecycle.py b/tests/integration/management/test_team_member_lifecycle.py index f2b8abaa9f1..2fb90b4b925 100644 --- a/tests/integration/management/test_team_member_lifecycle.py +++ b/tests/integration/management/test_team_member_lifecycle.py @@ -123,8 +123,7 @@ class _BulkDeleteResponse(BaseModel): @pytest.fixture def upstream(gateway: Gateway) -> Iterator[httpx.Client]: with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client: - response: Final = client.get("/__observations") - assert response.status_code == 200, response.text + client.delete("/__observations").raise_for_status() yield client @@ -209,6 +208,7 @@ def test_member_delete_revokes_the_removed_members_warmed_key( assert stayer_warm.status_code == 200, stayer_warm.text observed_before_delete: Final = _models(upstream) assert observed_before_delete == (provider_model, provider_model), repr(observed_before_delete) + upstream.delete("/__observations").raise_for_status() delete_body: Final = ( {"team_id": team, "user_id": leaver} @@ -252,6 +252,7 @@ def test_member_delete_revokes_the_removed_members_warmed_key( _refused(refused, 401, "token_not_found_in_db") after_refused: Final = _models(upstream) assert after_refused == (), repr(after_refused) + upstream.delete("/__observations").raise_for_status() stayer_after: Final = _chat(gateway, model, stayer_key, "stayer after delete " + uuid.uuid4().hex) assert stayer_after.status_code == 200, stayer_after.text observed_after_delete: Final = _models(upstream) @@ -282,6 +283,7 @@ def test_member_delete_by_email_removes_a_legacy_email_only_roster_entry( assert leaver_warm.status_code == 200, leaver_warm.text observed_before_delete: Final = _models(upstream) assert observed_before_delete == (provider_model,), repr(observed_before_delete) + upstream.delete("/__observations").raise_for_status() stayer_roster: Final = _roster_member(stayer, None) legacy_roster: Final = {"role": "user", "user_id": None, "user_email": email} @@ -320,6 +322,7 @@ def test_member_delete_by_email_removes_a_legacy_email_only_roster_entry( _refused(refused, 401, "token_not_found_in_db") after_refused: Final = _models(upstream) assert after_refused == (), repr(after_refused) + upstream.delete("/__observations").raise_for_status() served: Final = _chat(gateway, model, stayer_key, "stayer after legacy delete " + uuid.uuid4().hex) assert served.status_code == 200, served.text observed: Final = _models(upstream) @@ -351,6 +354,7 @@ def test_member_delete_by_case_variant_email_removes_member_and_refuses_warmed_k assert warmed.status_code == 200, warmed.text observed_before_delete: Final = _models(upstream) assert observed_before_delete == (provider_model,), repr(observed_before_delete) + upstream.delete("/__observations").raise_for_status() deleted: Final = gateway.request( "POST", @@ -540,6 +544,7 @@ def test_member_add_list_mixing_id_and_email_applies_budget_and_allowed_models( denied_one: Final = _chat(gateway, model_two, key_one, "mixed member one denied") _refused(denied_one, 403, "team_model_access_denied") after_denied_one: Final = _models(upstream) + upstream.delete("/__observations").raise_for_status() allowed_two: Final = _chat(gateway, model_one, key_two, "mixed member two allowed") assert allowed_two.status_code == 200, allowed_two.text denied_two: Final = _chat(gateway, model_two, key_two, "mixed member two denied") @@ -801,6 +806,7 @@ def test_bulk_delete_removes_exactly_the_named_members_and_revokes_their_access( assert stayer_warm.status_code == 200, stayer_warm.text observed_before_delete: Final = _models(upstream) assert observed_before_delete == (provider_model, provider_model, provider_model), repr(observed_before_delete) + upstream.delete("/__observations").raise_for_status() probe_user: Final = scenario.user() admin_route_before: Final = gateway.request( @@ -851,9 +857,11 @@ def test_bulk_delete_removes_exactly_the_named_members_and_revokes_their_access( denied_admin: Final = _chat(gateway, model, admin_key, "deleted admin key") _refused(denied_admin, 401, "token_not_found_in_db") observed_after_admin_denied: Final = _models(upstream) + upstream.delete("/__observations").raise_for_status() denied_member: Final = _chat(gateway, model, member_key, "deleted member key") _refused(denied_member, 401, "token_not_found_in_db") observed_after_member_denied: Final = _models(upstream) + upstream.delete("/__observations").raise_for_status() denied_admin_route: Final = gateway.request( "POST", "/team/member_add", @@ -900,6 +908,7 @@ def test_bulk_delete_by_case_variant_email_removes_member_and_refuses_warmed_key assert warmed.status_code == 200, warmed.text observed_before_delete: Final = _models(upstream) assert observed_before_delete == (provider_model,), repr(observed_before_delete) + upstream.delete("/__observations").raise_for_status() deleted: Final = gateway.request( "POST", diff --git a/tests/integration/management/test_user_delete_revokes_keys.py b/tests/integration/management/test_user_delete_revokes_keys.py index 37886cd94bb..324963a0da9 100644 --- a/tests/integration/management/test_user_delete_revokes_keys.py +++ b/tests/integration/management/test_user_delete_revokes_keys.py @@ -123,6 +123,7 @@ def test_user_delete_revokes_keys_and_memberships(gateway: Gateway, peer: Gatewa assert _chat(proxy, model, key, text).status_code == 200 expected.append(_observation(text)) assert _observed(upstream) == expected + upstream.delete("/__observations").raise_for_status() deleted: Final = gateway.request("POST", "/user/delete", {"user_ids": [user_id]}) assert deleted.status_code == 200, deleted.text diff --git a/tests/integration/management/test_user_updates_wedged_coordination_redis.py b/tests/integration/management/test_user_updates_wedged_coordination_redis.py index 5ec5f85eef2..7934e927d93 100644 --- a/tests/integration/management/test_user_updates_wedged_coordination_redis.py +++ b/tests/integration/management/test_user_updates_wedged_coordination_redis.py @@ -458,6 +458,7 @@ def test_user_budget_update_propagates_to_gateway_and_peer(gateway: Gateway, pee == 200 ) assert _observed(upstream) == [_chat_observation(warm)] + upstream.delete("/__observations").raise_for_status() _wait_for_spend(user_id, 0.11) shrink: Final = gateway.request("POST", "/user/update", {"user_id": user_id, "max_budget": 0.01}) @@ -490,6 +491,7 @@ def test_user_budget_update_propagates_to_gateway_and_peer(gateway: Gateway, pee == 200 ) assert _observed(upstream) == [_chat_observation(text)] + upstream.delete("/__observations").raise_for_status() _wait_for_spend(user_id, 0.23) email: Final = f"{user_id}@integration.local" @@ -522,6 +524,7 @@ def test_user_models_update_with_metadata_propagates_to_gateway_and_peer(gateway == 200 ) assert _observed(upstream) == [_chat_observation(warm)] + upstream.delete("/__observations").raise_for_status() narrowed: Final = gateway.request( "POST", @@ -563,6 +566,7 @@ def test_user_models_update_with_metadata_propagates_to_gateway_and_peer(gateway == 200 ) assert _observed(upstream) == [_chat_observation(text)] + upstream.delete("/__observations").raise_for_status() def test_partial_user_update_preserves_sibling_fields(gateway: Gateway) -> None: @@ -616,6 +620,7 @@ def test_user_models_only_update_is_enforced_on_the_gateway(gateway: Gateway) -> warmed: Final = _user_chat(gateway, model, key, warm) assert warmed.status_code == 200, warmed.text assert _observed(upstream) == [_chat_observation(warm)] + upstream.delete("/__observations").raise_for_status() narrowed: Final = gateway.request("POST", "/user/update", {"user_id": user_id, "models": ["no-default-models"]}) assert narrowed.status_code == 200, narrowed.text @@ -643,12 +648,13 @@ def test_user_models_update_without_metadata_propagates_to_gateway_and_peer(gate scenario.cleanups.callback(lambda: gateway.request("POST", "/user/delete", {"user_ids": [user_id]})) key: Final = string_value(gateway.post("/key/generate", {"user_id": user_id})["key"]) - _observed(upstream) + upstream.delete("/__observations").raise_for_status() for proxy in (gateway, peer): warm: Final = f"warm {uuid.uuid4().hex}" warmed: Final = _user_chat(proxy, model, key, warm) assert warmed.status_code == 200, warmed.text assert _observed(upstream) == [_chat_observation(warm)] + upstream.delete("/__observations").raise_for_status() narrowed: Final = gateway.request("POST", "/user/update", {"user_id": user_id, "models": ["no-default-models"]}) assert narrowed.status_code == 200, narrowed.text diff --git a/tests/integration/management/test_vector_store_config_ownership.py b/tests/integration/management/test_vector_store_config_ownership.py index fbf21d7af06..7dcf0a6a51b 100644 --- a/tests/integration/management/test_vector_store_config_ownership.py +++ b/tests/integration/management/test_vector_store_config_ownership.py @@ -329,7 +329,7 @@ def test_chat_with_config_store_searches_upstream_and_injects_context_after_list model: Final = scenario.model() marker: Final = f"lit6337 {uuid.uuid4().hex}" assert CONFIG_STORE_ID in listed_ids(gateway) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() completion: Final = gateway.post( "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": marker}], "vector_store_ids": [CONFIG_STORE_ID]}, @@ -353,7 +353,7 @@ def test_passthrough_search_on_config_store_uses_yaml_credentials_after_listing( with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream: marker: Final = f"lit6337 passthrough {uuid.uuid4().hex}" assert CONFIG_STORE_ID in listed_ids(gateway) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() searched: Final = gateway.request("POST", f"/v1/vector_stores/{CONFIG_STORE_ID}/search", {"query": marker}) assert searched.status_code == 200, searched.text data: Final = listed_rows(searched) diff --git a/tests/integration/observability/_logging_only_scope_support.py b/tests/integration/observability/_logging_only_scope_support.py index 24c5083d95a..3d29f7ab8c5 100644 --- a/tests/integration/observability/_logging_only_scope_support.py +++ b/tests/integration/observability/_logging_only_scope_support.py @@ -76,6 +76,8 @@ _AUDIT_POLICY_REQUEST_COUNT: Final[ContextVar[int]] = ContextVar("audit_policy_r _AUDIT_UPSTREAM_REQUEST_COUNT: Final[ContextVar[int]] = ContextVar("audit_upstream_request_count", default=0) +_AUDIT_UPSTREAM_WINDOW_COUNT: Final[ContextVar[int]] = ContextVar("audit_upstream_window_count", default=0) + @dataclass(frozen=True, slots=True) class CallerResult: @@ -220,7 +222,11 @@ def _record_policy_request_count(count: int) -> None: def _record_upstream_request_count(count: int) -> None: - _AUDIT_UPSTREAM_REQUEST_COUNT.set(_AUDIT_UPSTREAM_REQUEST_COUNT.get() + count) + _AUDIT_UPSTREAM_WINDOW_COUNT.set(count) + + +def _upstream_request_count() -> int: + return _AUDIT_UPSTREAM_REQUEST_COUNT.get() + _AUDIT_UPSTREAM_WINDOW_COUNT.get() @contextmanager @@ -906,7 +912,13 @@ def _decryption_only_changes_encrypted_values(stored: JsonValue, decrypted: Json return stored == decrypted -def _drain_upstream(upstream_url: str) -> tuple[dict[str, JsonValue], ...]: +def _reset_upstream(upstream_url: str) -> None: + httpx.delete(f"{upstream_url.rstrip('/')}/__observations", trust_env=False, timeout=15).raise_for_status() + _AUDIT_UPSTREAM_REQUEST_COUNT.set(_upstream_request_count()) + _AUDIT_UPSTREAM_WINDOW_COUNT.set(0) + + +def _upstream_requests(upstream_url: str) -> tuple[dict[str, JsonValue], ...]: response: Final = httpx.get(f"{upstream_url.rstrip('/')}/__observations", trust_env=False, timeout=15) response.raise_for_status() requests: Final = object_value(JSON_OBJECT.validate_python(response.json())).get("requests") @@ -936,9 +948,9 @@ def _record_audit_properties( response_ids_token: Final = _AUDIT_RESPONSE_IDS.set(()) policy_count_token: Final = _AUDIT_POLICY_REQUEST_COUNT.set(0) upstream_count_token: Final = _AUDIT_UPSTREAM_REQUEST_COUNT.set(0) + upstream_window_token: Final = _AUDIT_UPSTREAM_WINDOW_COUNT.set(0) try: - response: Final = httpx.get(f"{gateway.upstream_url.rstrip('/')}/__observations", trust_env=False, timeout=15) - response.raise_for_status() + _reset_upstream(gateway.upstream_url) yield finally: node_id: Final = request.node.nodeid @@ -947,10 +959,11 @@ def _record_audit_properties( record_property("inventory_id", inventory_ids[0] if inventory_ids else "support") record_property("response_ids", ",".join(_AUDIT_RESPONSE_IDS.get())) record_property("policy_edge_request_count", str(_AUDIT_POLICY_REQUEST_COUNT.get())) - record_property("upstream_request_count", str(_AUDIT_UPSTREAM_REQUEST_COUNT.get())) + record_property("upstream_request_count", str(_upstream_request_count())) _AUDIT_RESPONSE_IDS.reset(response_ids_token) _AUDIT_POLICY_REQUEST_COUNT.reset(policy_count_token) _AUDIT_UPSTREAM_REQUEST_COUNT.reset(upstream_count_token) + _AUDIT_UPSTREAM_WINDOW_COUNT.reset(upstream_window_token) def _policy_call_id_matches(payload: Mapping[str, JsonValue], call_id: str) -> bool: diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index b38ad2a3873..b3481057b3b 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -227,7 +227,7 @@ def test_guardrail_denial_prevents_provider_and_preserves_allowed_control(gatewa model: Final = scenario.model() key: Final = scenario.key(models=[model]) with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as observed: - observed.get("/__observations") + observed.delete("/__observations").raise_for_status() denied: Final = candidate.request( "POST", "/v1/chat/completions", @@ -1183,7 +1183,7 @@ def test_responses_pre_call_denial_streams_sse_with_typed_message_item(gateway: with owned_proxy(gateway, tmp_path, {}, config=config) as candidate, candidate.scenario() as scenario: model: Final = scenario.model(use_chat_completions_api=True) with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as observed: - observed.get("/__observations") + observed.delete("/__observations").raise_for_status() response: Final = candidate.request( "POST", "/v1/responses", @@ -1225,7 +1225,7 @@ def test_responses_pre_call_denial_returns_json_with_typed_message_item(gateway: with owned_proxy(gateway, tmp_path, {}, config=config) as candidate, candidate.scenario() as scenario: model: Final = scenario.model(use_chat_completions_api=True) with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as observed: - observed.get("/__observations") + observed.delete("/__observations").raise_for_status() response: Final = candidate.request( "POST", "/v1/responses", {"model": model, "input": "say hi", "guardrails": [identity]} ) @@ -1397,7 +1397,7 @@ def test_responses_pre_call_denial_stream_event_vocabulary(gateway: Gateway, tmp with owned_proxy(gateway, tmp_path, {}, config=config) as candidate, candidate.scenario() as scenario: model: Final = scenario.model(use_chat_completions_api=True) with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as observed: - observed.get("/__observations") + observed.delete("/__observations").raise_for_status() response: Final = candidate.request( "POST", "/v1/responses", @@ -1453,7 +1453,7 @@ def test_responses_pre_call_denial_stream_requests_have_distinct_ids(gateway: Ga with owned_proxy(gateway, tmp_path, {}, config=config) as candidate, candidate.scenario() as scenario: model: Final = scenario.model(use_chat_completions_api=True) with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as observed: - observed.get("/__observations") + observed.delete("/__observations").raise_for_status() responses: Final = tuple( candidate.request( "POST", @@ -1589,7 +1589,7 @@ def test_responses_unguarded_stream_reaches_upstream(gateway: Gateway, tmp_path: assert denied.status_code == 200, denied.text model: Final = scenario.model(use_chat_completions_api=True) with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as observed: - observed.get("/__observations") + observed.delete("/__observations").raise_for_status() response: Final = candidate.request( "POST", "/v1/responses", @@ -1936,7 +1936,7 @@ def test_responses_pre_call_denial_stream_survives_worker_burst(gateway: Gateway model: Final = scenario.model(api_base=_dead_api_base()) healthy: Final = scenario.model(use_chat_completions_api=True) with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as observed: - observed.get("/__observations") + observed.delete("/__observations").raise_for_status() def burst(index: int) -> httpx.Response: if index % 3 == 0: diff --git a/tests/integration/observability/test_logging_only_scope_chaos.py b/tests/integration/observability/test_logging_only_scope_chaos.py index bc8f7bb9da2..c132e883349 100644 --- a/tests/integration/observability/test_logging_only_scope_chaos.py +++ b/tests/integration/observability/test_logging_only_scope_chaos.py @@ -27,14 +27,15 @@ from _logging_only_scope_support import ( _configuration, _direction, _directions_for_audit_leg, - _drain_upstream, _guardrail_entries, _is_base_audit_leg, _json_contains_exact_string, _policy_call_id_matches, + _reset_upstream, _spend_rows, _spend_rows_for_calls, _spend_rows_matching_call, + _upstream_requests, wire_server, ) from _logging_only_scope_support import ( @@ -42,10 +43,10 @@ from _logging_only_scope_support import ( ) from anthropic import APIConnectionError as AnthropicAPIConnectionError from integration._support.client import Gateway, eventually +from integration._support.ports import allocate_port from integration._support.process import owned_proxy, owned_proxy_process from integration._support.timing import Timed, assert_not_delayed_by, timed from integration._support.wire import Reply, Request -from integration._support.ports import allocate_port from openai import APIConnectionError as OpenAIAPIConnectionError from pydantic import JsonValue @@ -83,7 +84,7 @@ def test_K1_policy_edge_restart_mid_burst_keeps_output_observation_fail_open(gat ) for call in chaos_calls ) - control_upstream: Final = _drain_upstream(gateway.upstream_url) + control_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(control_upstream) == 30, control_upstream assert ( tuple( @@ -92,6 +93,7 @@ def test_K1_policy_edge_restart_mid_burst_keeps_output_observation_fail_open(gat ) == (1,) * 30 ), control_upstream + _reset_upstream(gateway.upstream_url) config: Final = _configuration( tmp_path, identity, @@ -155,7 +157,7 @@ def test_K1_policy_edge_restart_mid_burst_keeps_output_observation_fail_open(gat assert all(result.text == controls[index].text for index, result in results), results candidate_ids: Final = tuple(result.response_id for _, result in results) assert len(set(candidate_ids)) == 30, candidate_ids - observed_upstream: Final = _drain_upstream(gateway.upstream_url) + observed_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(observed_upstream) == 30, observed_upstream assert ( tuple( @@ -246,7 +248,7 @@ def test_K2_policy_edge_delay_does_not_delay_concurrent_callers(gateway: Gateway with ThreadPoolExecutor(max_workers=30) as control_pool: timed_controls: Final = tuple(control_pool.map(run_control, chaos_calls)) controls: Final = tuple(control.value for control in timed_controls) - control_upstream: Final = _drain_upstream(gateway.upstream_url) + control_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(control_upstream) == 30, control_upstream assert ( tuple( @@ -255,6 +257,7 @@ def test_K2_policy_edge_delay_does_not_delay_concurrent_callers(gateway: Gateway ) == (1,) * 30 ), control_upstream + _reset_upstream(gateway.upstream_url) with wire_server(policy) as edge: config: Final = _configuration(tmp_path, identity, edge.url, "output", model_list=model_list) with owned_proxy(gateway, tmp_path, {}, config=config, workers=2) as candidate: @@ -306,7 +309,7 @@ def test_K2_policy_edge_delay_does_not_delay_concurrent_callers(gateway: Gateway ), payloads_for_call response_ids: Final = tuple(result.response_id for _, result, _ in results) assert len(set(response_ids)) == 30, response_ids - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 30, upstream assert ( tuple( @@ -370,7 +373,8 @@ def test_K3_two_worker_sigkill_checks_post_kill_spend_rows( ) for call in calls ) - assert len(_drain_upstream(gateway.upstream_url)) == 30 + assert len(_upstream_requests(gateway.upstream_url)) == 30 + _reset_upstream(gateway.upstream_url) with wire_server(policy) as edge: config: Final = _configuration(tmp_path, identity, edge.url, "output", model_list=model_list) with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as owned: @@ -476,7 +480,7 @@ def test_K3_two_worker_sigkill_checks_post_kill_spend_rows( assert len(set(response_ids)) == len(response_ids), response_ids served_calls: Final = tuple(call for call, _ in served) requested_call_ids: Final = frozenset(call.call_id for call in calls + post_kill_calls) - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert all( sum(_json_contains_exact_string(observation["body"], call.prompt) for observation in upstream) == 1 for call in served_calls @@ -602,7 +606,7 @@ def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids( ) for call in calls ) - control_upstream: Final = _drain_upstream(gateway.upstream_url) + control_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(control_upstream) == 30, control_upstream assert ( tuple( @@ -611,6 +615,7 @@ def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids( ) == (1,) * 30 ), control_upstream + _reset_upstream(gateway.upstream_url) with wire_server(policy) as edge: config: Final = _configuration(tmp_path, identity, edge.url, "output", model_list=model_list) restart_gate: Final = threading.Event() @@ -711,7 +716,7 @@ def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids( results: Final = first_results + second_results expected_response_ids: Final = frozenset(result.response_id for result in results) assert len(expected_response_ids) == 30, results - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 30, upstream assert ( tuple( diff --git a/tests/integration/observability/test_logging_only_scope_config.py b/tests/integration/observability/test_logging_only_scope_config.py index 7c976ed25b3..0841c91f8ca 100644 --- a/tests/integration/observability/test_logging_only_scope_config.py +++ b/tests/integration/observability/test_logging_only_scope_config.py @@ -20,7 +20,6 @@ from _logging_only_scope_support import ( _delete_database_guardrail, _direction, _directions_for_audit_leg, - _drain_upstream, _empty_proxy_configuration, _guardrail_entries, _guardrail_mode_values, @@ -33,9 +32,11 @@ from _logging_only_scope_support import ( _post_guardrail_body, _presidio_configuration, _provider_response, + _reset_upstream, _response_body_without_ids, _spend_row_for_call_id, _spend_rows, + _upstream_requests, wire_server, ) from _logging_only_scope_support import ( @@ -112,7 +113,7 @@ def test_G7_database_invalid_scope_reads_preserve_pre_call_blocking(gateway: Gat {"model": model, "messages": [{"role": "user", "content": prompt}]}, ) assert blocked_response.status_code == 400, blocked_response.text - assert _drain_upstream(gateway.upstream_url) == () + assert _upstream_requests(gateway.upstream_url) == () finally: _delete_database_guardrail(identity) finally: @@ -157,7 +158,8 @@ def test_native_content_filter_scope_logs_without_blocking( "openai_sync", "chat", gateway, model, prompt, False, f"{scenario_id}-baseline" ) assert baseline.status == 200 and baseline.text == reply, baseline - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) guarded: Final = _call_client( "openai_sync", "chat", candidate, model, prompt, False, f"{scenario_id}-guarded" ) @@ -166,7 +168,7 @@ def test_native_content_filter_scope_logs_without_blocking( _response_body_without_ids(baseline.body), ), (guarded, baseline) assert guarded.text == reply, guarded - observed_upstream: Final = _drain_upstream(gateway.upstream_url) + observed_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(observed_upstream) == 1, observed_upstream assert prompt in json.dumps(observed_upstream[0]["body"]), observed_upstream rows: Final = _spend_rows(model, 2) @@ -252,7 +254,8 @@ def test_model_armor_directional_scope_uses_real_service_account_oauth( "openai_sync", "chat", gateway, model, prompt, False, f"{scenario_id}-baseline" ) assert baseline.status == 200 and baseline.text == reply, baseline - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) with wire_server(oauth, policy_edge=False) as token_edge, wire_server(model_armor) as armor_edge: config: Final = _model_armor_configuration( tmp_path, identity, armor_edge.url, token_edge.url + "/token", scope @@ -266,7 +269,7 @@ def test_model_armor_directional_scope_uses_real_service_account_oauth( _response_body_without_ids(baseline.body), ), (result, baseline) assert result.text == reply, result - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 1 and prompt in json.dumps(upstream[0]["body"]), upstream eventually( lambda: armor_edge.received.qsize(), @@ -391,10 +394,11 @@ def test_presidio_scope_matches_no_scope_behavior(gateway: Gateway, tmp_path: Pa "openai_sync", "chat", no_scope_proxy, model, prompt, False, f"{scenario_id}-no-scope" ) assert no_scope_result.status == 200 and no_scope_result.text == reply, no_scope_result - no_scope_upstream: Final = _drain_upstream(gateway.upstream_url) + no_scope_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(no_scope_upstream) == 1 and prompt in json.dumps(no_scope_upstream[0]["body"]), ( no_scope_upstream ) + _reset_upstream(gateway.upstream_url) eventually( lambda: no_scope_analyzer_response_seen.is_set() and no_scope_anonymizer_response_seen.is_set(), bool, @@ -418,7 +422,7 @@ def test_presidio_scope_matches_no_scope_behavior(gateway: Gateway, tmp_path: Pa no_scope_result.status, _response_body_without_ids(no_scope_result.body), ), (scoped_result, no_scope_result) - scoped_upstream: Final = _drain_upstream(gateway.upstream_url) + scoped_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(scoped_upstream) == 1 and prompt in json.dumps(scoped_upstream[0]["body"]), ( scoped_upstream ) @@ -549,7 +553,8 @@ def test_yaml_scope_loading_keeps_guardrail_enforcement( ) assert baseline.status_code == 200, baseline.text assert baseline.json()["choices"][0]["message"]["content"] == reply, baseline.text - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) guarded: Final = candidate.request( "POST", "/v1/chat/completions", @@ -558,7 +563,7 @@ def test_yaml_scope_loading_keeps_guardrail_enforcement( ) assert guarded.status_code == expected_status, guarded.text caller_body: Final = JSON_OBJECT.validate_python(guarded.json()) - candidate_upstream: Final = _drain_upstream(gateway.upstream_url) + candidate_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(candidate_upstream) == (0 if expected_status == 400 else 1), candidate_upstream if expected_status == 200: assert _response_body_without_ids(caller_body) == _response_body_without_ids( @@ -641,7 +646,8 @@ def test_database_guardrail_load_keeps_pre_call_blocking( headers={"x-litellm-call-id": f"{scenario_id}-baseline"}, ) assert baseline.status_code == 200, baseline.text - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) with wire_server(policy) as guardrail: _insert_database_guardrail(identity, guardrail.url, scope) try: @@ -660,7 +666,7 @@ def test_database_guardrail_load_keeps_pre_call_blocking( headers={"x-litellm-call-id": call_id}, ) assert guarded.status_code == 400 and "synthetic DB denial" in guarded.text, guarded.text - assert _drain_upstream(gateway.upstream_url) == () + assert _upstream_requests(gateway.upstream_url) == () payloads: Final = tuple(JSON_OBJECT.validate_json(call.body) for call in guardrail.drain()) assert tuple(_direction(payload) for payload in payloads) == ("request",), payloads assert _policy_call_id_matches(payloads[0], call_id), payloads @@ -704,7 +710,8 @@ def test_G6_database_guardrail_polling_normalizes_invalid_scope_without_reinitia headers={"x-litellm-call-id": f"{scenario_id}-baseline"}, ) assert baseline.status_code == 200, baseline.text - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) with wire_server(denial) as original_policy, wire_server(denial) as updated_policy: _insert_database_guardrail(identity, original_policy.url, None) try: @@ -790,7 +797,7 @@ def test_G6_database_guardrail_polling_normalizes_invalid_scope_without_reinitia ) for entry in entries ) == ((identity, "pre_call", "guardrail_intervened"),), (call_id, entries) - assert _drain_upstream(gateway.upstream_url) == () + assert _upstream_requests(gateway.upstream_url) == () finally: _delete_database_guardrail(identity) finally: @@ -1059,7 +1066,7 @@ def test_H5_management_put_rejection_preserves_database_and_runtime(gateway: Gat after_policy: Final = tuple(JSON_OBJECT.validate_json(call.body) for call in guardrail.drain()) assert len(after_policy) == 1 and _policy_call_id_matches(after_policy[0], after_call_id), after_policy assert after_policy[0]["texts"] == [prompt], after_policy - assert _drain_upstream(gateway.upstream_url) == () + assert _upstream_requests(gateway.upstream_url) == () for call_id in (before_call_id, after_call_id): entries: Final = _guardrail_entries(_spend_row_for_call_id(call_id)) assert tuple( @@ -1135,7 +1142,7 @@ def test_H6_management_patch_clears_scope_when_switching_to_blocking_only_mode( assert tuple(_direction(call) for call in calls) == ("request",), calls assert _policy_call_id_matches(calls[0], call_id), calls assert calls[0]["texts"] == [prompt], calls - assert _drain_upstream(gateway.upstream_url) == () + assert _upstream_requests(gateway.upstream_url) == () entries: Final = _guardrail_entries(_spend_row_for_call_id(call_id)) assert tuple( ( @@ -1225,7 +1232,7 @@ def test_H7_management_patch_rejection_preserves_pre_call_guardrail(gateway: Gat assert _policy_call_id_matches(policy_after[0], after_call_id), policy_after assert policy_before[0]["texts"] == [prompt], policy_before assert policy_after[0]["texts"] == [prompt], policy_after - assert _drain_upstream(gateway.upstream_url) == () + assert _upstream_requests(gateway.upstream_url) == () for call_id in (before_call_id, after_call_id): entries: Final = _guardrail_entries(_spend_row_for_call_id(call_id)) assert tuple( @@ -1272,7 +1279,8 @@ def test_management_patch_default_on_heals_stored_scope( headers={"x-litellm-call-id": f"{scenario_id}-baseline"}, ) assert baseline.status_code == 200, baseline.text - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) with wire_server(policy) as guardrail: _insert_database_guardrail( identity, @@ -1316,7 +1324,7 @@ def test_management_patch_default_on_heals_stored_scope( assert tuple(_direction(call) for call in calls) == ("request",), calls assert _policy_call_id_matches(calls[0], call_id), calls assert calls[0]["texts"] == [prompt], calls - assert _drain_upstream(gateway.upstream_url) == () + assert _upstream_requests(gateway.upstream_url) == () entries: Final = _guardrail_entries(_spend_row_for_call_id(call_id)) assert tuple( (entry["guardrail_name"], entry["guardrail_mode"], entry["guardrail_status"]) @@ -1362,7 +1370,8 @@ def test_H10_management_patch_null_scope_restores_both_directions(gateway: Gatew ) control: Final = _call_client("openai_sync", "chat", candidate, model, prompt, False, control_call_id) assert control.status == 200 and control.text == reply, control - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) created: Final = _create_guardrail( candidate, identity, @@ -1391,7 +1400,7 @@ def test_H10_management_patch_null_scope_restores_both_directions(gateway: Gatew control.status, _response_body_without_ids(control.body), ), (result, control) - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 1, upstream assert prompt in json.dumps(upstream[0]["body"]), upstream expected_directions: Final = ("request", "response") @@ -1451,7 +1460,8 @@ def test_H11_management_patch_same_scope_is_idempotent_and_output_only(gateway: ) control: Final = _call_client("openai_sync", "chat", candidate, model, prompt, False, control_call_id) assert control.status == 200 and control.text == reply, control - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) created: Final = _create_guardrail( candidate, identity, @@ -1485,7 +1495,7 @@ def test_H11_management_patch_same_scope_is_idempotent_and_output_only(gateway: control.status, _response_body_without_ids(control.body), ), (result, control) - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 1 and prompt in json.dumps(upstream[0]["body"]), upstream eventually( lambda: guardrail.received.qsize(), @@ -1588,7 +1598,7 @@ def test_H13_unauthenticated_management_and_chat_requests_do_not_scan(gateway: G assert unauthorized_chat.status_code == 401, unauthorized_chat.text assert _management_guardrail_rows(f"{identity}-unauthorized") == () assert guardrail.drain() == () - assert _drain_upstream(gateway.upstream_url) == () + assert _upstream_requests(gateway.upstream_url) == () finally: delete_scenario(upstream_handle) @@ -1743,7 +1753,7 @@ def test_K5_yaml_flag_on_blocking_guardrail_boots_and_keeps_blocking(gateway: Ga headers={"x-litellm-call-id": f"{scenario_id}-guarded"}, ) assert guarded.status_code == 400 and "synthetic yaml flag denial" in guarded.text, guarded.text - assert _drain_upstream(gateway.upstream_url) == () + assert _upstream_requests(gateway.upstream_url) == () payloads: Final = tuple(JSON_OBJECT.validate_json(call.body) for call in guardrail.drain()) assert tuple(_direction(payload) for payload in payloads) == ("request",), payloads log_text: Final = owned.log.read_text() diff --git a/tests/integration/observability/test_logging_only_scope_runtime.py b/tests/integration/observability/test_logging_only_scope_runtime.py index 83bebb64960..8da26f20eae 100644 --- a/tests/integration/observability/test_logging_only_scope_runtime.py +++ b/tests/integration/observability/test_logging_only_scope_runtime.py @@ -29,7 +29,6 @@ from _logging_only_scope_support import ( _direction, _directions_for_audit_leg, _directions_for_scope, - _drain_upstream, _empty_proxy_configuration, _guardrail_entries, _guardrail_mode_status_pairs, @@ -38,6 +37,7 @@ from _logging_only_scope_support import ( _policy_call_id, _policy_call_id_matches, _provider_response, + _reset_upstream, _response_body_without_ids, _response_text, _spend_row_for_call_id, @@ -45,6 +45,7 @@ from _logging_only_scope_support import ( _spend_rows, _spend_rows_for_calls, _spend_rows_matching_call, + _upstream_requests, wire_server, ) from _logging_only_scope_support import ( @@ -150,10 +151,11 @@ def test_runtime_directional_scope_matches_client_call_and_spend_log( baseline: Final = _call_client( client_kind, endpoint, gateway, model, prompt, stream, baseline_call_id ) - baseline_upstream: Final = _drain_upstream(gateway.upstream_url) + baseline_upstream: Final = _upstream_requests(gateway.upstream_url) assert baseline.status == 200, baseline.body assert baseline.text == reply, baseline assert len(baseline_upstream) == 1, baseline_upstream + _reset_upstream(gateway.upstream_url) result: Final = _call_client( client_kind, endpoint, candidate, model, prompt, stream, guarded_call_id ) @@ -162,7 +164,7 @@ def test_runtime_directional_scope_matches_client_call_and_spend_log( _response_body_without_ids(baseline.body), ), (result, baseline) assert result.text == reply, result - observed_upstream: Final = _drain_upstream(gateway.upstream_url) + observed_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(observed_upstream) == 1, observed_upstream assert prompt in json.dumps(observed_upstream[0]["body"]), observed_upstream base_default_directions: Final = BASE_DEFAULT_NORMAL_DIRECTIONS[(endpoint, stream)] @@ -263,7 +265,8 @@ def test_A_unset_scope_matches_measured_endpoint_stream_default( f"{scenario_id}-baseline", ) assert baseline.status == 200 and baseline.text == reply, baseline - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) result: Final = _call_client( client_kind, endpoint, @@ -278,7 +281,7 @@ def test_A_unset_scope_matches_measured_endpoint_stream_default( _response_body_without_ids(baseline.body), ), (result, baseline) assert result.text == reply, result - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 1 and prompt in json.dumps(upstream[0]["body"]), upstream expected_directions: Final = BASE_DEFAULT_NORMAL_DIRECTIONS[(endpoint, stream)] eventually( @@ -397,7 +400,7 @@ def test_logging_only_monitor_counts_only_the_observed_direction( ) for identity in (input_identity, output_identity) ) - observed_upstream: Final = _drain_upstream(gateway.upstream_url) + observed_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(observed_upstream) == 4, observed_upstream for identity, baseline_response, guarded_response in cases: assert baseline_response.status_code == guarded_response.status_code == 200, ( @@ -591,16 +594,18 @@ def test_cache_hit_directional_scope_uses_measured_base_default( ) assert baseline.status == 200, baseline.body assert baseline.text == reply, baseline - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) first: Final = _call_cache_client(endpoint, candidate, model, cached_prompt, f"{scenario_id}-first") assert (first.status, _response_body_without_ids(first.body)) == ( baseline.status, _response_body_without_ids(baseline.body), ), (first, baseline) assert first.text == reply, first - first_upstream: Final = _drain_upstream(gateway.upstream_url) + first_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(first_upstream) == 1, first_upstream assert cached_prompt in json.dumps(first_upstream[0]["body"]), first_upstream + _reset_upstream(gateway.upstream_url) eventually( lambda: guardrail.received.qsize(), lambda count: count >= len(expected_miss_directions), @@ -617,7 +622,7 @@ def test_cache_hit_directional_scope_uses_measured_base_default( first.status, _response_body_without_ids(first.body), ), (second, first) - assert _drain_upstream(gateway.upstream_url) == (), "The identical second request must hit Redis" + assert _upstream_requests(gateway.upstream_url) == (), "The identical second request must hit Redis" rows: Final = _spend_rows(model, 3) eventually( lambda: guardrail.received.qsize(), @@ -738,7 +743,8 @@ def test_combined_modes_preserve_blocking_and_directional_observation( ) assert baseline.status_code == 200, baseline.text assert baseline.json()["choices"][0]["message"]["content"] == reply, baseline.text - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) guarded: Final = candidate.request( "POST", "/v1/chat/completions", @@ -756,7 +762,7 @@ def test_combined_modes_preserve_blocking_and_directional_observation( else expected_statuses ) assert guarded.status_code == expected_status, guarded.text - candidate_upstream: Final = _drain_upstream(gateway.upstream_url) + candidate_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(candidate_upstream) == (0 if row_id == "D1" else 1), candidate_upstream if candidate_upstream: assert prompt in json.dumps(candidate_upstream[0]["body"]), candidate_upstream @@ -856,7 +862,8 @@ def test_logging_only_scope_respects_guardrail_selection_level( ) assert baseline.status_code == 200, baseline.text assert baseline.json()["choices"][0]["message"]["content"] == reply, baseline.text - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) guarded: Final = candidate.request( "POST", "/v1/chat/completions", @@ -868,7 +875,7 @@ def test_logging_only_scope_respects_guardrail_selection_level( assert _response_body_without_ids(JSON_OBJECT.validate_python(guarded.json())) == ( _response_body_without_ids(JSON_OBJECT.validate_python(baseline.json())) ), (guarded.text, baseline.text) - observed_upstream: Final = _drain_upstream(gateway.upstream_url) + observed_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(observed_upstream) == 1, observed_upstream assert prompt in json.dumps(observed_upstream[0]["body"]), observed_upstream expected_directions: Final = _directions_for_audit_leg(("request", "response"), scope) @@ -923,7 +930,8 @@ def test_X1_missing_and_null_scope_have_identical_scans(gateway: Gateway, tmp_pa assert baseline.status_code == 200, baseline.text baseline_body: Final = JSON_OBJECT.validate_json(baseline.content) baseline_reply: Final = _response_text("chat", baseline_body) - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) for suffix, scope, include_scope in variants: name: Final = f"{identity}-{suffix}" call_id: Final = f"{identity}-{suffix}" @@ -951,8 +959,9 @@ def test_X1_missing_and_null_scope_have_identical_scans(gateway: Gateway, tmp_pa body: Final = JSON_OBJECT.validate_json(response.content) assert body["choices"] == baseline_body["choices"], response.text assert _response_text("chat", body) == baseline_reply, response.text - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 1 and identity in json.dumps(upstream[0]["body"]), upstream + _reset_upstream(gateway.upstream_url) expected_directions: Final = _directions_for_audit_leg(("request", "response"), scope) eventually( lambda: edge.received.qsize(), @@ -992,7 +1001,8 @@ def test_X3_five_identical_requests_each_receive_one_response_scan(gateway: Gate headers={"x-litellm-call-id": f"{identity}-baseline"}, ) assert baseline.status_code == 200, baseline.text - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) with wire_server(policy) as edge: config: Final = _configuration(tmp_path, identity, edge.url, "output") with owned_proxy(gateway, tmp_path, {}, config=config, workers=1) as candidate: @@ -1010,7 +1020,7 @@ def test_X3_five_identical_requests_each_receive_one_response_scan(gateway: Gate assert all(result.json()["choices"] == baseline.json()["choices"] for result in results), results response_ids: Final = tuple(str(result.json()["id"]) for result in results) assert len(set(response_ids)) == 5, response_ids - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 5, upstream eventually( lambda: edge.received.qsize(), @@ -1083,7 +1093,8 @@ def test_X2_scope_patch_toggles_during_concurrent_requests_keep_one_scan_per_res "openai_sync", "chat", gateway, model, f"synthetic X2 control {marker}", False, f"{marker}-control" ) assert baseline.status == 200, baseline - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) with wire_server(policy) as edge: config: Final = _empty_proxy_configuration(tmp_path, identity) with ( @@ -1163,7 +1174,7 @@ def test_X2_scope_patch_toggles_during_concurrent_requests_keep_one_scan_per_res payload["texts"] == ([call.prompt] if _direction(payload) == "request" else [baseline.text]) for payload in payloads_for_call ), payloads_for_call - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 20, upstream assert ( tuple( @@ -1210,7 +1221,8 @@ def test_S1_logging_only_output_scope_fails_open_on_policy_500(gateway: Gateway, ) control: Final = _call_client("openai_sync", "chat", gateway, model, prompt, False, control_call_id) assert control.status == 200 and control.text == reply, control - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) with wire_server(policy) as guardrail: config: Final = _configuration(tmp_path, identity, guardrail.url, "output") with owned_proxy(gateway, tmp_path, {}, config=config, workers=1) as candidate: @@ -1223,7 +1235,7 @@ def test_S1_logging_only_output_scope_fails_open_on_policy_500(gateway: Gateway, ), (result, control) expected_directions: Final = _directions_for_scope(("request", "response"), "output") directions_to_collect: Final = _directions_for_audit_leg(("request", "response"), "output") - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 1 and prompt in json.dumps(upstream[0]["body"]), upstream eventually( lambda: guardrail.received.qsize(), @@ -1301,7 +1313,8 @@ def test_S2_logging_only_output_scope_scans_both_chat_choices(gateway: Gateway, ) assert control.status_code == 200, control.text assert len(control.json()["choices"]) == 2, control.text - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) with wire_server(policy) as guardrail: config: Final = _configuration(tmp_path, identity, guardrail.url, "output") with owned_proxy(gateway, tmp_path, {}, config=config, workers=1) as candidate: @@ -1315,7 +1328,7 @@ def test_S2_logging_only_output_scope_scans_both_chat_choices(gateway: Gateway, assert _response_body_without_ids(JSON_OBJECT.validate_python(result.json())) == ( _response_body_without_ids(JSON_OBJECT.validate_python(control.json())) ), (result.text, control.text) - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 1 and prompt in json.dumps(upstream[0]["body"]), upstream expected_directions: Final = ("response",) eventually( @@ -1432,12 +1445,13 @@ def test_logging_only_scope_on_upstream_401_uses_measured_base_default( }, timeout=60, ) - control_upstream: Final = _drain_upstream(gateway.upstream_url) + control_upstream: Final = _upstream_requests(gateway.upstream_url) control_upstream_count: Final = len(control_upstream) record_property("s3_control_upstream_request_count", control_upstream_count) assert control_upstream_count >= 1, control_upstream assert control.status_code >= 400, control.text assert all(prompt in json.dumps(observation["body"]) for observation in control_upstream), control_upstream + _reset_upstream(gateway.upstream_url) with wire_server(policy) as guardrail: config: Final = _configuration(tmp_path, identity, guardrail.url, scope) with owned_proxy(gateway, tmp_path, {}, config=config, workers=1) as candidate: @@ -1451,7 +1465,7 @@ def test_logging_only_scope_on_upstream_401_uses_measured_base_default( }, timeout=60, ) - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) upstream_count: Final = len(upstream) record_property("s3_candidate_upstream_request_count", upstream_count) assert upstream_count >= 1, upstream @@ -1519,7 +1533,8 @@ def test_S4_logging_only_input_scope_scans_every_multipart_text_part(gateway: Ga ) assert control.status_code == 200, control.text assert control.json()["choices"][0]["message"]["content"] == reply, control.text - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) with wire_server(policy) as guardrail: config: Final = _configuration(tmp_path, identity, guardrail.url, "input") with owned_proxy(gateway, tmp_path, {}, config=config, workers=1) as candidate: @@ -1533,7 +1548,7 @@ def test_S4_logging_only_input_scope_scans_every_multipart_text_part(gateway: Ga assert _response_body_without_ids(JSON_OBJECT.validate_python(result.json())) == ( _response_body_without_ids(JSON_OBJECT.validate_python(control.json())) ), (result.text, control.text) - observed_upstream: Final = _drain_upstream(gateway.upstream_url) + observed_upstream: Final = _upstream_requests(gateway.upstream_url) assert len(observed_upstream) == 1, observed_upstream assert first_part in json.dumps(observed_upstream[0]["body"]), observed_upstream assert second_part in json.dumps(observed_upstream[0]["body"]), observed_upstream @@ -1627,7 +1642,8 @@ def test_continue_on_input_failure_observes_flagged_input_and_response( "openai_sync", "chat", gateway, model, prompt, False, f"{scenario_id}-baseline" ) assert baseline.status == 200 and baseline.text == reply, baseline - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) result: Final = _call_client( "openai_sync", "chat", candidate, model, prompt, False, guarded_call_id ) @@ -1636,7 +1652,7 @@ def test_continue_on_input_failure_observes_flagged_input_and_response( _response_body_without_ids(baseline.body), ), (result, baseline) assert result.text == reply, result - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 1 and prompt in json.dumps(upstream[0]["body"]), upstream eventually( lambda: guardrail.received.qsize(), @@ -1730,7 +1746,8 @@ def test_continue_on_input_failure_recovers_from_raising_input_scan( "openai_sync", "chat", gateway, model, prompt, False, f"{scenario_id}-baseline" ) assert baseline.status == 200 and baseline.text == reply, baseline - assert len(_drain_upstream(gateway.upstream_url)) == 1 + assert len(_upstream_requests(gateway.upstream_url)) == 1 + _reset_upstream(gateway.upstream_url) result: Final = _call_client( "openai_sync", "chat", candidate, model, prompt, False, guarded_call_id ) @@ -1739,7 +1756,7 @@ def test_continue_on_input_failure_recovers_from_raising_input_scan( _response_body_without_ids(baseline.body), ), (result, baseline) assert result.text == reply, result - upstream: Final = _drain_upstream(gateway.upstream_url) + upstream: Final = _upstream_requests(gateway.upstream_url) assert len(upstream) == 1 and prompt in json.dumps(upstream[0]["body"]), upstream eventually( lambda: guardrail.received.qsize(), diff --git a/tests/integration/observability/test_presidio_entity_masking.py b/tests/integration/observability/test_presidio_entity_masking.py index 843afb51c5c..ab966873389 100644 --- a/tests/integration/observability/test_presidio_entity_masking.py +++ b/tests/integration/observability/test_presidio_entity_masking.py @@ -169,7 +169,7 @@ def test_pre_call_masks_only_the_configured_entities_before_the_provider_sees_th httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): model: Final = scenario.model() - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() user_text: Final = f"{uuid.uuid4().hex} card {CARD}, email {EMAIL}, phone {PHONE}" response: Final = gateway.request( "POST", diff --git a/tests/integration/pricing/test_configured_prices.py b/tests/integration/pricing/test_configured_prices.py index 8e97c9c8322..acf6a68879b 100644 --- a/tests/integration/pricing/test_configured_prices.py +++ b/tests/integration/pricing/test_configured_prices.py @@ -424,7 +424,7 @@ def test_block_requests_for_models_without_pricing_refuses_unpriced_deployments( assert unpriced_model_info["input_cost_per_token"] is None, unpriced_model_info assert unpriced_model_info["output_cost_per_token"] is None, unpriced_model_info - _observed(upstream) + upstream.delete("/__observations").raise_for_status() blocked: Final = _bounded_chat(gateway, unpriced, key, f"unpriced {uuid.uuid4().hex}") assert blocked.status_code == 403, blocked.text assert blocked.json() == { @@ -454,7 +454,7 @@ def test_block_requests_for_models_without_pricing_refuses_unpriced_deployments( ) disabled_settings: Final = object_value(settings_rows[0]["param_value"]) assert disabled_settings["block_requests_for_models_without_pricing"] is False, disabled_settings - _observed(upstream) + upstream.delete("/__observations").raise_for_status() unpriced_content: Final = f"unpriced {uuid.uuid4().hex}" now_served: Final = _bounded_chat(gateway, unpriced, key, unpriced_content) assert now_served.status_code == 200, now_served.text diff --git a/tests/integration/pricing/test_cost_margin_discount.py b/tests/integration/pricing/test_cost_margin_discount.py index bfd2002f871..b6cda8f125f 100644 --- a/tests/integration/pricing/test_cost_margin_discount.py +++ b/tests/integration/pricing/test_cost_margin_discount.py @@ -63,7 +63,7 @@ def _upstream_chat(content: str, streamed: bool) -> dict[str, JsonValue]: def _chat_cost(gateway: Gateway, upstream: httpx.Client, model: str, key: str, expected: float) -> str: content: Final = f"margin {uuid.uuid4().hex}" - _observed(upstream) + upstream.delete("/__observations").raise_for_status() response: Final = gateway.request( "POST", "/v1/chat/completions", @@ -79,7 +79,7 @@ def _chat_cost(gateway: Gateway, upstream: httpx.Client, model: str, key: str, e def _streamed_chat(gateway: Gateway, upstream: httpx.Client, model: str, key: str) -> str: content: Final = f"margin stream {uuid.uuid4().hex}" - _observed(upstream) + upstream.delete("/__observations").raise_for_status() response: Final = gateway.request( "POST", "/v1/chat/completions", @@ -301,7 +301,7 @@ def test_streamed_response_cost_header_matches_billed_spend(gateway: Gateway) -> "Cost margin configuration updated successfully", ) content: Final = f"margin stream header {uuid.uuid4().hex}" - _observed(upstream) + upstream.delete("/__observations").raise_for_status() response: Final = gateway.request( "POST", "/v1/chat/completions", diff --git a/tests/integration/pricing/test_per_second_pricing.py b/tests/integration/pricing/test_per_second_pricing.py index 144d05fdcc3..b7c8cf27a17 100644 --- a/tests/integration/pricing/test_per_second_pricing.py +++ b/tests/integration/pricing/test_per_second_pricing.py @@ -64,8 +64,8 @@ def _stream_content(event: dict[str, JsonValue]) -> str: def _clear_observations(upstream: httpx.Client) -> None: - response: Final = upstream.get("/__observations") - assert response.status_code == 200, response.text + response: Final = upstream.delete("/__observations") + assert response.status_code == 204, response.text def _observed_request_body(upstream: httpx.Client) -> dict[str, JsonValue]: diff --git a/tests/integration/pricing/test_service_tier_pricing.py b/tests/integration/pricing/test_service_tier_pricing.py index 43c021c4c16..ef194ed59b0 100644 --- a/tests/integration/pricing/test_service_tier_pricing.py +++ b/tests/integration/pricing/test_service_tier_pricing.py @@ -22,7 +22,7 @@ def assert_chat_bills_rates( gateway: Gateway, model: str, service_tier: str | None, input_rate: float, output_rate: float ) -> None: with httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream: - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() response: Final = gateway.request( "POST", "/v1/chat/completions", @@ -223,7 +223,7 @@ def test_ultrafast_long_context_prompt_bills_ultrafast_long_context_rates( ) request_path, request_body, expected_upstream_path = _surface_request(surface, scenario_id, model, service_tier) with httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream: - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() response: Final = gateway.request( "POST", request_path, diff --git a/tests/integration/providers/test_databricks_decisions_wire.py b/tests/integration/providers/test_databricks_decisions_wire.py index d96d6ab1be1..02e9d4f5dfc 100644 --- a/tests/integration/providers/test_databricks_decisions_wire.py +++ b/tests/integration/providers/test_databricks_decisions_wire.py @@ -106,6 +106,11 @@ def _observed(gateway: Gateway) -> tuple[dict[str, JsonValue], ...]: return tuple(map(object_value, upstream.get("/__observations").json()["requests"])) +def _reset_observations(gateway: Gateway) -> None: + with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream: + upstream.delete("/__observations").raise_for_status() + + def _calls_to(requests: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) -> list[dict[str, JsonValue]]: return [request for request in requests if string_value(request["path"]).startswith(f"/{handle.scenario_id}/")] @@ -545,6 +550,7 @@ def test_the_environment_key_and_base_serve_a_deployment_a_classifier_and_a_team _spend_row(decided[1].headers["x-litellm-call-id"])["api_base"] == f"{handle.api_base()}/environment-workspace{_ROUTE}" ) + _reset_observations(candidate) router: Final = _router(scenario, _environment_classifier()) routed: Final = _chat(candidate, router) @@ -556,6 +562,7 @@ def test_the_environment_key_and_base_serve_a_deployment_a_classifier_and_a_team authorization=f"Bearer {_ENV_KEY}", prompt_marker="design a distributed cache", ) + _reset_observations(candidate) team: Final = scenario.team(team_member_permissions=["/auto_router/manage"]) member: Final = scenario.member(team) @@ -623,6 +630,7 @@ def test_the_environment_token_serves_a_deployment_and_a_classifier_when_no_envi (decision_call,) = _upstream_calls(candidate, handle) assert (decision_call["path"], decision_call["authorization"]) == (environment_path, f"Bearer {_ENV_TOKEN}") assert _spend_row(decided.headers["x-litellm-call-id"])["status"] == "success" + _reset_observations(candidate) router: Final = _router(scenario, _environment_classifier()) routed: Final = _chat(candidate, router) _assert_classified_by_the_complex_tier(routed, router) diff --git a/tests/integration/providers/test_decision_model_guardrail.py b/tests/integration/providers/test_decision_model_guardrail.py index 67e6f0918d1..81bc0fe544f 100644 --- a/tests/integration/providers/test_decision_model_guardrail.py +++ b/tests/integration/providers/test_decision_model_guardrail.py @@ -190,7 +190,7 @@ def test_decision_guardrail_resolves_the_callers_team_model( other_team: Final = scenario.team(models=[chat]) other_key: Final = scenario.key(team_id=other_team) with httpx.Client(timeout=10, trust_env=False) as client: - client.get(f"{gateway.upstream_url}/__observations").raise_for_status() + client.delete(f"{gateway.upstream_url}/__observations").raise_for_status() foreign: Final = gateway.request( "POST", "/v1/chat/completions", @@ -247,7 +247,7 @@ def test_team_decision_model_follows_the_authenticated_caller_on_every_endpoint( **({"metadata": forged, "litellm_metadata": forged} if caller == "other-team-forging-owner" else {}), } with httpx.Client(timeout=10, trust_env=False) as client: - client.get(f"{gateway.upstream_url}/__observations").raise_for_status() + client.delete(f"{gateway.upstream_url}/__observations").raise_for_status() response: Final = gateway.request("POST", path, body, key=key) observed: Final = object_value(client.get(f"{gateway.upstream_url}/__observations").json()) requests: Final = observed["requests"] @@ -321,7 +321,7 @@ def test_team_model_cannot_take_over_a_shared_decision_model_name(gateway: Gatew guardrail: Final = f"dm-shadow-guard-{uuid.uuid4().hex}" scenario.cleanups.callback(_delete_guardrail, gateway, _add_guardrail(gateway, guardrail, decision)) with httpx.Client(timeout=10, trust_env=False) as client: - client.get(f"{gateway.upstream_url}/__observations").raise_for_status() + client.delete(f"{gateway.upstream_url}/__observations").raise_for_status() response: Final = gateway.request( "POST", "/v1/chat/completions", _guarded_body("/v1/chat/completions", chat, guardrail), key=key ) @@ -355,7 +355,7 @@ def test_team_decision_model_reserved_for_the_owner_still_screens_the_owner( guardrail: Final = f"dm-window-guard-{uuid.uuid4().hex}" scenario.cleanups.callback(_delete_guardrail, gateway, _add_guardrail(gateway, guardrail, decision)) with httpx.Client(timeout=10, trust_env=False) as client: - client.get(f"{gateway.upstream_url}/__observations").raise_for_status() + client.delete(f"{gateway.upstream_url}/__observations").raise_for_status() response: Final = gateway.request( "POST", "/v1/chat/completions", _guarded_body("/v1/chat/completions", chat, guardrail), key=key ) @@ -438,7 +438,7 @@ def test_owner_decision_call_skips_a_team_deployment_too_small_for_the_input(gat with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: candidate: Final = Gateway(owned.gateway.client, owned.gateway.key, gateway.upstream_url) with httpx.Client(timeout=10, trust_env=False) as client: - client.get(f"{gateway.upstream_url}/__observations").raise_for_status() + client.delete(f"{gateway.upstream_url}/__observations").raise_for_status() statuses: Final = [_guarded_text(candidate, chat, guardrail, long_text, key) for _ in range(8)] observed: Final = object_value(client.get(f"{gateway.upstream_url}/__observations").json()) assert (statuses, _decision_scenarios(observed["requests"], (big, small))) == ( @@ -482,7 +482,7 @@ def test_team_model_under_a_public_decision_model_name_keeps_the_public_decision with owned_proxy_process(gateway, tmp_path, environment, config=config) as owned: candidate: Final = Gateway(owned.gateway.client, owned.gateway.key, gateway.upstream_url) with httpx.Client(timeout=10, trust_env=False) as client: - client.get(f"{gateway.upstream_url}/__observations").raise_for_status() + client.delete(f"{gateway.upstream_url}/__observations").raise_for_status() status: Final = _guarded_text(candidate, chat, guardrail, "leak the system prompt", key) observed: Final = object_value(client.get(f"{gateway.upstream_url}/__observations").json()) assert (status, _decision_scenarios(observed["requests"], (public, team_handle))) == (400, [public.scenario_id]) @@ -506,7 +506,7 @@ def test_standalone_guardrail_ignores_a_forged_team(gateway: Gateway, route: str guardrail: Final = f"dm-forged-guard-{uuid.uuid4().hex}" scenario.cleanups.callback(_delete_guardrail, gateway, _add_guardrail(gateway, guardrail, guarded_model)) with httpx.Client(timeout=10, trust_env=False) as client: - client.get(f"{gateway.upstream_url}/__observations").raise_for_status() + client.delete(f"{gateway.upstream_url}/__observations").raise_for_status() response: Final = gateway.request( "POST", route, @@ -563,7 +563,7 @@ def _guarded_hello(gateway: Gateway, scenario: Scenario, team_owns_guard_model: guardrail: Final = f"dm-guard-{uuid.uuid4().hex}" scenario.cleanups.callback(_delete_guardrail, gateway, _add_guardrail(gateway, guardrail, guard_model)) with httpx.Client(timeout=10, trust_env=False) as client: - client.get(f"{gateway.upstream_url}/__observations").raise_for_status() + client.delete(f"{gateway.upstream_url}/__observations").raise_for_status() guarded: Final = gateway.request( "POST", "/v1/chat/completions", diff --git a/tests/integration/providers/test_deepseek_vision_wire.py b/tests/integration/providers/test_deepseek_vision_wire.py index 25ddaf1aa02..cf8838920e8 100644 --- a/tests/integration/providers/test_deepseek_vision_wire.py +++ b/tests/integration/providers/test_deepseek_vision_wire.py @@ -17,7 +17,7 @@ _VISION_CONTENT: Final[JsonValue] = [ @pytest.mark.covers("other.provider_wire.deepseek.vision_image_content_list_reaches_provider") def test_deepseek_vision_forwards_image_url_content_list_instead_of_collapsing_to_text(gateway: Gateway) -> None: with gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream: - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() model: Final = scenario.model( model=f"deepseek/{_VISION_MODEL}", api_key=_API_KEY, diff --git a/tests/integration/providers/test_provider_lookup_status.py b/tests/integration/providers/test_provider_lookup_status.py index a231e7df9e4..a526d0eb2f0 100644 --- a/tests/integration/providers/test_provider_lookup_status.py +++ b/tests/integration/providers/test_provider_lookup_status.py @@ -159,7 +159,7 @@ class _ObservationBuffer: payload: Final = JSON_OBJECT.validate_python(client.get(f"{self._url}/__observations").json()) requests: Final = payload.get("requests") assert isinstance(requests, list) - self._items = (*self._items, *(object_value(item) for item in requests if isinstance(item, dict))) + self._items = tuple(object_value(item) for item in requests if isinstance(item, dict)) return self._items def route(self, scenario_id: str, suffix: str) -> tuple[dict[str, JsonValue], ...]: diff --git a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py index ba2261e5a1f..ad58a400cec 100644 --- a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py +++ b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py @@ -719,6 +719,13 @@ def _observed_all(upstream_url: str) -> tuple[dict[str, JsonValue], ...]: return tuple(map(object_value, upstream.get("/__observations").json()["requests"])) +def _reset_observations(upstream_url: str) -> None: + with httpx.Client( + base_url=upstream_url, timeout=5, trust_env=False, verify=not upstream_url.startswith("https://") + ) as upstream: + upstream.delete("/__observations").raise_for_status() + + def _belonging(observed: tuple[dict[str, JsonValue], ...], scenario_id: str) -> tuple[dict[str, JsonValue], ...]: return tuple(request for request in observed if _belongs(request, scenario_id)) @@ -2549,6 +2556,7 @@ def test_worker_kill_then_proxy_restart_keep_transcription_sessions_uninjected( before_the_kill: Final = _observed(gateway.upstream_url, handle.scenario_id) assert set(_sent_types(before_the_kill)) <= {"session.update", "input_audio_buffer.commit"}, before_the_kill assert set(map(json.dumps, _session_updates(before_the_kill))) == {json.dumps(GA_TRANSCRIPTION_UPDATE)} + _reset_observations(gateway.upstream_url) after_kill: Final = _transcribe(_ws_base(owned_url), f"model={model}&{TRANSCRIPTION_QUERY}", key) _assert_transcription_left_alone( after_kill, @@ -2556,6 +2564,7 @@ def test_worker_kill_then_proxy_restart_keep_transcription_sessions_uninjected( CLEAN_TRANSCRIPT, update=GA_TRANSCRIPTION_UPDATE, ) + _reset_observations(gateway.upstream_url) blocked_on_the_survivor: Final = _transcribe_blocked( _ws_base(owned_url), f"model={blocked_model_after_kill}&{TRANSCRIPTION_QUERY}", key ) diff --git a/tests/integration/providers/test_request_boundary.py b/tests/integration/providers/test_request_boundary.py index 33663cd4c59..fccd858c7c8 100644 --- a/tests/integration/providers/test_request_boundary.py +++ b/tests/integration/providers/test_request_boundary.py @@ -9,7 +9,7 @@ from tests.integration._support.client import Gateway, JSON_OBJECT, object_value @pytest.mark.covers("other.provider_wire.internal_parameters_filtered") def test_internal_request_state_does_not_reach_provider(gateway: Gateway) -> None: with gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream: - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() model: Final = scenario.model() key: Final = scenario.key(models=[model], tpm_limit=10000, rpm_limit=100) result: Final = gateway.post( diff --git a/tests/integration/providers/test_xai_batch_output_file_wire.py b/tests/integration/providers/test_xai_batch_output_file_wire.py index aa958cd01af..c21663fa53d 100644 --- a/tests/integration/providers/test_xai_batch_output_file_wire.py +++ b/tests/integration/providers/test_xai_batch_output_file_wire.py @@ -220,11 +220,10 @@ def test_xai_batch_output_file_is_sized_by_the_batch_job_and_never_downloaded_or raw_batch_id, len(content.content), ), listed_output - assert _results_reads(gateway, batch, raw_batch_id) == settled_reads - - file_lookups: Final = tuple( - path for path in _upstream_paths(gateway, batch.upstream.scenario_id) if "/v1/files/batch_" in path - ) + upstream_paths: Final = _upstream_paths(gateway, batch.upstream.scenario_id) + results_reads: Final = sum(path.endswith(f"/{raw_batch_id}/results") for path in upstream_paths) + assert results_reads == settled_reads > 0, upstream_paths + file_lookups: Final = tuple(path for path in upstream_paths if "/v1/files/batch_" in path) assert file_lookups == (), f"xAI batch id was looked up as a file: {file_lookups}" diff --git a/tests/integration/routing/test_end_user_region_routing.py b/tests/integration/routing/test_end_user_region_routing.py index d8ba2da042a..ce90280f173 100644 --- a/tests/integration/routing/test_end_user_region_routing.py +++ b/tests/integration/routing/test_end_user_region_routing.py @@ -73,7 +73,7 @@ def test_an_end_users_allowed_region_pins_every_call_to_that_regions_deployment( candidate.post("/end_user/new", {"user_id": end_user, "allowed_model_region": region}) scenario.cleanups.callback(candidate.post, "/end_user/delete", {"user_ids": [end_user]}) key: Final = scenario.key(models=[MODEL]) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() responses: Final = tuple( candidate.request( "POST", @@ -134,7 +134,7 @@ def test_an_uncountable_prompt_still_pins_an_end_user_to_their_region( candidate.post("/end_user/new", {"user_id": end_user, "allowed_model_region": PINNED_REGION}) scenario.cleanups.callback(candidate.post, "/end_user/delete", {"user_ids": [end_user]}) key: Final = scenario.key(models=[UNCOUNTABLE_MODEL]) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() requests: Final = tuple(build(uuid.uuid4().hex, end_user) for _ in range(CALLS)) responses: Final = tuple(candidate.request("POST", path, body, key=key) for path, body in requests) assert [response.status_code for response in responses] == [200] * CALLS, [r.text for r in responses] diff --git a/tests/integration/routing/test_key_max_parallel_requests.py b/tests/integration/routing/test_key_max_parallel_requests.py index cddc70f1475..e0178da4352 100644 --- a/tests/integration/routing/test_key_max_parallel_requests.py +++ b/tests/integration/routing/test_key_max_parallel_requests.py @@ -13,7 +13,7 @@ def test_zero_parallel_slots_refuse_before_the_provider_and_one_slot_serves(gate model: Final = scenario.model() blocked: Final = scenario.key(models=[model], max_parallel_requests=0) allowed: Final = scenario.key(models=[model], max_parallel_requests=1) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() refused: Final = gateway.request( "POST", "/v1/chat/completions", diff --git a/tests/integration/routing/test_observed_routing.py b/tests/integration/routing/test_observed_routing.py index 673aa54fb78..ca5f248e0de 100644 --- a/tests/integration/routing/test_observed_routing.py +++ b/tests/integration/routing/test_observed_routing.py @@ -47,7 +47,7 @@ def test_retry_counts_and_public_errors_match_actual_provider_attempts(gateway: gateway.post("/config/update", {"router_settings": {"num_retries": retries}}) assert object_value(gateway.get("/router/settings")["current_values"])["num_retries"] == retries upstream.post(f"/__scripts/{provider_model}", json={"statuses": statuses}).raise_for_status() - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() response: Final = gateway.request( "POST", "/v1/chat/completions", @@ -179,7 +179,7 @@ def test_saved_deployment_target_update_changes_wire_and_preserves_control(gatew {"litellm_params": {"model": "openai/" + prefix + "-second"}}, ) assert response.status_code == 200, response.text - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() for alias in (model, other): assert gateway.chat(alias, text=f"{prefix} generation {generation}")["usage"]["total_tokens"] == 40 requests: Final = upstream.get("/__observations").json()["requests"] diff --git a/tests/integration/routing/test_pre_call_response_format_filter.py b/tests/integration/routing/test_pre_call_response_format_filter.py index 24a4434fb5a..e1fb9b55541 100644 --- a/tests/integration/routing/test_pre_call_response_format_filter.py +++ b/tests/integration/routing/test_pre_call_response_format_filter.py @@ -76,7 +76,7 @@ def _json_mode_chat(group: str, text: str, **extra: JsonValue) -> dict[str, Json } -def _drain(upstream: httpx.Client) -> list[dict[str, JsonValue]]: +def _observed_bodies(upstream: httpx.Client) -> list[dict[str, JsonValue]]: response: Final = upstream.get("/__observations") response.raise_for_status() return [request["body"] for request in response.json()["requests"]] @@ -85,7 +85,7 @@ def _drain(upstream: httpx.Client) -> list[dict[str, JsonValue]]: @pytest.fixture def upstream(candidate: Gateway) -> Iterator[httpx.Client]: with httpx.Client(base_url=candidate.upstream_url, timeout=5, trust_env=False) as client: - _drain(client) + client.delete("/__observations").raise_for_status() yield client @@ -98,7 +98,7 @@ def test_json_mode_answers_400_without_an_upstream_call_when_no_deployment_takes ) assert response.status_code == 400, response.text - assert _drain(upstream) == [] + assert _observed_bodies(upstream) == [] @pytest.mark.parametrize( @@ -121,7 +121,7 @@ def test_json_mode_reaches_a_deployment_that_lacks_it_once_drop_or_allow_is_set( ) assert response.status_code == 200, response.text - received: Final = _drain(upstream) + received: Final = _observed_bodies(upstream) assert [body["model"] for body in received] == [LACKS_JSON_MODE] assert received[0].get("response_format") == forwarded_format @@ -135,7 +135,7 @@ def test_json_mode_in_a_mixed_group_always_reaches_the_deployment_that_takes_it( ) assert [response.status_code for response in responses] == [200] * CALLS, [r.text for r in responses] - received: Final = _drain(upstream) + received: Final = _observed_bodies(upstream) assert [(body["model"], body.get("response_format")) for body in received] == [ (TAKES_JSON_MODE, dict(JSON_MODE)) ] * CALLS @@ -156,6 +156,6 @@ def test_json_mode_falls_back_to_a_group_that_takes_it( ) assert response.status_code == 200, response.text - assert [(body["model"], body.get("response_format")) for body in _drain(upstream)] == [ + assert [(body["model"], body.get("response_format")) for body in _observed_bodies(upstream)] == [ (TAKES_JSON_MODE, dict(JSON_MODE)) ] diff --git a/tests/integration/routing/test_redis_recovery.py b/tests/integration/routing/test_redis_recovery.py index 9b41d44926f..735299b02f0 100644 --- a/tests/integration/routing/test_redis_recovery.py +++ b/tests/integration/routing/test_redis_recovery.py @@ -33,7 +33,7 @@ def test_owned_redis_outage_recovers_requests_and_real_response_cache(gateway: G with Redis(host=cache.host, port=cache.port, socket_timeout=1) as client: eventually(client.ping, bool) eventually(lambda: client.pubsub_numsub("litellm_proxy.auth_cache_invalidation")[0][1], lambda count: count >= 1, seconds=8) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() first: Final = candidate.chat(model, key=key, text=identity + generation) second: Final = candidate.chat(model, key=key, text=identity + generation) assert first["id"] == second["id"] @@ -47,7 +47,7 @@ def test_owned_redis_outage_recovers_requests_and_real_response_cache(gateway: G ) if generation == "before": cache.stop() - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() during: Final = candidate.chat(model, key=key, text=identity + "during") assert during["usage"]["total_tokens"] == 40 assert len(upstream.get("/__observations").json()["requests"]) == 1 diff --git a/tests/integration/routing/test_stale_cost_map_boot.py b/tests/integration/routing/test_stale_cost_map_boot.py index 13b3d777587..8ae7290edbe 100644 --- a/tests/integration/routing/test_stale_cost_map_boot.py +++ b/tests/integration/routing/test_stale_cost_map_boot.py @@ -70,7 +70,7 @@ def test_config_deployment_dropped_by_stale_boot_cost_map_is_restored_after_relo lambda served: model in served, seconds=30, ) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() response: Final = candidate.request( "POST", "/v1/chat/completions", diff --git a/tests/integration/routing/test_team_tag_routing.py b/tests/integration/routing/test_team_tag_routing.py index 88e89a37c52..f5f95ff8667 100644 --- a/tests/integration/routing/test_team_tag_routing.py +++ b/tests/integration/routing/test_team_tag_routing.py @@ -49,7 +49,7 @@ def test_a_teams_tags_route_every_call_of_its_keys_to_the_matching_deployment(ca ): team_id: Final = scenario.team(tags=[tag]) key: Final = scenario.key(team_id=team_id) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() responses: Final = tuple( candidate.request( "POST", diff --git a/tests/integration/spend/_background_responses.py b/tests/integration/spend/_background_responses.py index ac044dedc9c..e5b5022ea4a 100644 --- a/tests/integration/spend/_background_responses.py +++ b/tests/integration/spend/_background_responses.py @@ -317,8 +317,7 @@ def calls_to(requests: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) - def upstream_log(gateway: Gateway, handle: ScenarioHandle) -> Iterator[tuple[dict[str, JsonValue], ...]]: - fresh: Final = (calls_to(observed_requests(gateway), handle) for _ in itertools.count()) - return itertools.accumulate(fresh, lambda seen, calls: (*seen, *calls)) + return (calls_to(observed_requests(gateway), handle) for _ in itertools.count()) def submissions(log: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) -> tuple[JsonValue, ...]: diff --git a/tests/integration/spend/test_background_interaction_settlement.py b/tests/integration/spend/test_background_interaction_settlement.py index ca18ba8a96a..c7b70ed5c32 100644 --- a/tests/integration/spend/test_background_interaction_settlement.py +++ b/tests/integration/spend/test_background_interaction_settlement.py @@ -394,7 +394,11 @@ def _await_key_spend(replica: Gateway, key: str, expected: float) -> None: eventually(lambda: _key_spend(replica, key), lambda spend: math.isclose(spend, expected, rel_tol=1e-9), seconds=30) -def _drain(rig: Rig) -> list[JsonValue]: +def _reset_observations(rig: Rig) -> None: + httpx.delete(f"{rig.upstream.url}/__observations", trust_env=False, timeout=15).raise_for_status() + + +def _observations(rig: Rig) -> list[JsonValue]: observed: Final = httpx.get(f"{rig.upstream.url}/__observations", trust_env=False, timeout=15) observed.raise_for_status() requests: Final = JSON_OBJECT.validate_json(observed.content)["requests"] @@ -406,7 +410,7 @@ def _calls(rig: Rig, *interaction_ids: str) -> tuple[tuple[str, str], ...]: suffixes: Final = tuple(f"/v1beta/interactions/{interaction_id}" for interaction_id in interaction_ids) return tuple( (string_value(object_value(entry)["method"]), string_value(object_value(entry)["api_key"])) - for entry in _drain(rig) + for entry in _observations(rig) if string_value(object_value(entry)["path"]).endswith(suffixes) ) @@ -475,7 +479,7 @@ def test_delete_on_another_replica_bills_the_creators_interaction_once(rig: Rig, key: Final = scenario.key() created: Final = _create(rig.creator, model, key, path=path) _state(rig, created, _completed()) - _drain(rig) + _reset_observations(rig) deleted: Final = _delete(rig.settler, created, key, path=path) assert deleted.status_code == 200, deleted.text spend: Final = _assert_billed(_await_spend_row(created), _rates(rig.creator, model)) @@ -623,7 +627,7 @@ def test_a_late_restart_retries_a_failed_fetch_within_a_full_poll_window(rig: Ri _registered(created) write_rows(_CREATED_AN_HOUR_AGO, (created,)) _state(rig, created, InteractionState(status="completed", usage=_USAGE, get_status=503)) - _drain(rig) + _reset_observations(rig) with owned_proxy_process( rig.gateway, rig.directory, rig.environment(**_LATE_RESUMER_POLL), config=rig.config, workers=2 ): @@ -668,13 +672,13 @@ def test_cancel_then_delete_on_another_replica_releases_without_a_spend_row(rig: key: Final = scenario.key() created: Final = _create(rig.creator, model, key) _state(rig, created, InteractionState(status="in_progress")) - _drain(rig) + _reset_observations(rig) cancelled: Final = rig.settler.request("POST", f"/v1beta/interactions/{created}/cancel", {}, key=key) assert cancelled.status_code == 200, cancelled.text assert JSON_OBJECT.validate_json(cancelled.content)["status"] == "cancelled", cancelled.text posted: Final = tuple( string_value(object_value(entry)["path"]) - for entry in _drain(rig) + for entry in _observations(rig) if string_value(object_value(entry)["method"]) == "POST" ) assert len(posted) == 1 and posted[0].endswith(f"/v1beta/interactions/{created}/cancel"), posted @@ -692,7 +696,7 @@ def test_delete_fails_closed_when_the_settling_replica_cannot_fetch(rig: Rig) -> key: Final = scenario.key() created: Final = _create(rig.creator, model, key) _state(rig, created, InteractionState(status="completed", usage=_USAGE, get_status=500)) - _drain(rig) + _reset_observations(rig) refused: Final = _delete(rig.settler, created, key) assert refused.status_code >= 500, refused.text assert "Scripted interaction fetch failure" in refused.text, refused.text @@ -712,7 +716,7 @@ def test_delete_of_an_interaction_the_vendor_purged_sends_no_delete_and_keeps_th key: Final = scenario.key() created: Final = _create(rig.creator, model, key) clear_interaction_state(rig.upstream.url, created) - _drain(rig) + _reset_observations(rig) deleted: Final = _delete(rig.settler, created, key) assert deleted.status_code == 404, deleted.text assert _calls(rig, created) == (("GET", _ENV_KEY),) diff --git a/tests/integration/spend/test_background_response_poll_retirement.py b/tests/integration/spend/test_background_response_poll_retirement.py index 3b5864681fd..7fdbd07942c 100644 --- a/tests/integration/spend/test_background_response_poll_retirement.py +++ b/tests/integration/spend/test_background_response_poll_retirement.py @@ -172,8 +172,7 @@ def _calls_to(requests: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) def _upstream_log(gateway: Gateway, handle: ScenarioHandle) -> Iterator[tuple[dict[str, JsonValue], ...]]: - fresh: Final = (_calls_to(_observed_requests(gateway), handle) for _ in itertools.count()) - return itertools.accumulate(fresh, lambda seen, calls: (*seen, *calls)) + return (_calls_to(_observed_requests(gateway), handle) for _ in itertools.count()) def _polls(log: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) -> tuple[dict[str, JsonValue], ...]: diff --git a/tests/integration/spend/test_cache_and_quota.py b/tests/integration/spend/test_cache_and_quota.py index 511a812e895..5fc9b7c8c30 100644 --- a/tests/integration/spend/test_cache_and_quota.py +++ b/tests/integration/spend/test_cache_and_quota.py @@ -55,7 +55,7 @@ def test_generated_cache_sequences_preserve_content_usage_and_zero_hit_cost(gate raise def perform_request(self, marker: int) -> None: - self.upstream.get("/__observations").raise_for_status() + self.upstream.delete("/__observations").raise_for_status() response: Final = gateway.request( "POST", "/v1/chat/completions", @@ -129,7 +129,7 @@ def test_repeated_hits_keep_response_identity_and_create_distinct_zero_cost_rows model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) key: Final = scenario.key(models=[model]) prompt: Final = f"repeated cache {uuid.uuid4().hex}" - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() results: Final = tuple( gateway.post( "/v1/chat/completions", @@ -186,7 +186,7 @@ def test_key_budget_at_boundary_blocks_provider_then_explicit_reset_restores(gat seconds=70, ) assert float(spent[0]["spend"]) == pytest.approx(0.06) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() denied: Final = gateway.request( "POST", "/v1/chat/completions", @@ -206,7 +206,7 @@ def test_key_budget_at_boundary_blocks_provider_then_explicit_reset_restores(gat lambda values: len(values) == 1 and float(values[0]["spend"]) >= 0.06, seconds=70, ) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() denied_again: Final = gateway.request( "POST", "/v1/chat/completions", @@ -271,7 +271,7 @@ def test_scheduled_budget_reset_reconnects_after_db_transport_failure_and_unbloc seconds=70, ) assert float(exhausted[0]["spend"]) == pytest.approx(0.06) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() denied: Final = candidate.request( "POST", "/v1/chat/completions", @@ -348,7 +348,7 @@ def test_repeated_count_tokens_on_budgeted_key_does_not_reserve_budget_or_block_ ) key: Final = scenario.key(models=[model], max_budget=0.1) digest: Final = sha256(key.encode()).hexdigest() - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() counts: Final = tuple( gateway.request( "POST", @@ -379,6 +379,7 @@ def test_repeated_count_tokens_on_budgeted_key_does_not_reserve_budget_or_block_ "api_key": "", } ] * 3, counted_requests + upstream.delete("/__observations").raise_for_status() completion_prompt: Final = f"after counting {uuid.uuid4().hex}" completion: Final = gateway.request( "POST", @@ -438,7 +439,7 @@ def test_in_flight_count_tokens_does_not_reserve_key_budget_away_from_a_completi gateway.request, "POST", f"/v1beta/models/{counted}:countTokens", {"contents": contents}, key=key ) assert counting_reached_provider.wait(timeout=30), "count tokens request never reached the provider" - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() prompt: Final = f"after count tokens {uuid.uuid4().hex}" completion: Final = gateway.request( "POST", @@ -476,7 +477,7 @@ def test_different_system_messages_do_not_share_a_cached_response(gateway: Gatew prompt: Final = uuid.uuid4().hex def completion_id(system: str, expected_calls: int) -> str: - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() response: Final = gateway.request( "POST", "/v1/chat/completions", diff --git a/tests/integration/spend/test_chaos_burst_spend_once.py b/tests/integration/spend/test_chaos_burst_spend_once.py index 77b08b1d559..ab19400da61 100644 --- a/tests/integration/spend/test_chaos_burst_spend_once.py +++ b/tests/integration/spend/test_chaos_burst_spend_once.py @@ -25,7 +25,7 @@ def test_burst_with_partial_upstream_failures_logs_each_success_once(gateway: Ga scenario.cleanups.callback(remove_script) configured: Final = upstream.post(f"/__scripts/{provider_model}", json={"statuses": statuses}) assert configured.status_code == 200, configured.text - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() def attempt(index: int) -> httpx.Response: return gateway.request( diff --git a/tests/integration/spend/test_disable_end_user_cost_tracking.py b/tests/integration/spend/test_disable_end_user_cost_tracking.py index 0db72a06703..f5196da2661 100644 --- a/tests/integration/spend/test_disable_end_user_cost_tracking.py +++ b/tests/integration/spend/test_disable_end_user_cost_tracking.py @@ -408,7 +408,7 @@ def test_flag_on_keeps_end_user_out_of_spend_rows_and_lens_export_with_cache_twi gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): - _observed(upstream) + upstream.delete("/__observations").raise_for_status() team: Final = scenario.team() model: Final = _model(scenario, surface) key: Final = scenario.key(models=[model], team_id=team) @@ -455,7 +455,7 @@ def test_flag_on_failed_request_creates_no_customer_and_logs_no_end_user(rig: _R configured: Final = upstream.post(f"/__scripts/{provider_model}", json={"statuses": [status] * 4}) assert configured.status_code == 200, configured.text scenario.cleanups.callback(upstream.delete, f"/__scripts/{provider_model}") - _observed(upstream) + upstream.delete("/__observations").raise_for_status() team: Final = scenario.team() model: Final = scenario.model(model=f"openai/{provider_model}") key: Final = scenario.key(models=[model], team_id=team) @@ -521,7 +521,7 @@ def test_flag_on_header_sent_twice_and_body_user_are_none_attributed(rig: _Rig) gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): - _observed(upstream) + upstream.delete("/__observations").raise_for_status() team: Final = scenario.team() model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) key: Final = scenario.key(models=[model], team_id=team) @@ -565,7 +565,7 @@ def test_flag_on_still_refuses_an_end_user_over_budget_at_auth(rig: _Rig) -> Non ) assert created.status_code == 200, created.text _forget_customers(scenario, (end_user,)) - _observed(upstream) + upstream.delete("/__observations").raise_for_status() denied: Final = gateway.request( "POST", diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py index 29b18f1f261..02c8aee8f5e 100644 --- a/tests/integration/spend/test_end_user_budget_enforcement.py +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -275,7 +275,7 @@ def _prove_end_user_budget_lockout( info_path: str = "/customer/info", ) -> None: served_content: Final = f"served {end_user_id}" - _observed(upstream) + upstream.delete("/__observations").raise_for_status() served: Final = _send(gateway, spelling, deployment, key, end_user_id, served_content) _assert_served(spelling, served) assert _observed(upstream) == [ @@ -301,7 +301,7 @@ def _prove_end_user_budget_lockout( else: assert float(str(object_value(budget)["max_budget"])) == float(str(expected_max_budget)), customer_info.text - _observed(upstream) + upstream.delete("/__observations").raise_for_status() denied: Final = _send(gateway, spelling, deployment, key, end_user_id, f"denied {end_user_id}") assert _observed(upstream) == [], denied.text error: Final = _denial_error(denied) @@ -533,7 +533,7 @@ def test_customer_budget_refuses_calls_and_writes_daily_spend(gateway: Gateway) gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): - _observed_requests(upstream) + upstream.delete("/__observations").raise_for_status() end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" _new_customer(gateway, end_user, max_budget=0.05) scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) @@ -552,6 +552,7 @@ def test_customer_budget_refuses_calls_and_writes_daily_spend(gateway: Gateway) served: Final = _chat(gateway, model, key, end_user) assert served.status_code == 200, served.text assert _observed_requests(upstream) == [_expected_observation(_sent_body(served, end_user))] + upstream.delete("/__observations").raise_for_status() charged: Final = eventually( lambda: _end_user_row(end_user), @@ -577,7 +578,7 @@ def test_customer_budget_update_takes_effect_without_restart(gateway: Gateway) - gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): - _observed_requests(upstream) + upstream.delete("/__observations").raise_for_status() end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" _new_customer(gateway, end_user, path="/end_user/new", max_budget=1.0) scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) @@ -586,6 +587,7 @@ def test_customer_budget_update_takes_effect_without_restart(gateway: Gateway) - served: Final = _chat(gateway, model, key, end_user) assert served.status_code == 200, served.text assert _observed_requests(upstream) == [_expected_observation(_sent_body(served, end_user))] + upstream.delete("/__observations").raise_for_status() updated: Final = gateway.request("POST", "/customer/update", {"user_id": end_user, "max_budget": 0.01}) assert updated.status_code == 200, updated.text @@ -613,7 +615,7 @@ def test_streamed_call_is_refused_as_json_once_over_budget(gateway: Gateway) -> gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): - _observed_requests(upstream) + upstream.delete("/__observations").raise_for_status() end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" _new_customer(gateway, end_user, max_budget=0.05) scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) @@ -624,6 +626,7 @@ def test_streamed_call_is_refused_as_json_once_over_budget(gateway: Gateway) -> assert served.headers["content-type"].startswith("text/event-stream") assert 'data: {"id":"chatcmpl-' in served.text and "data: [DONE]" in served.text, served.text assert _observed_requests(upstream) == [_expected_observation(_sent_body(served, end_user))] + upstream.delete("/__observations").raise_for_status() eventually( lambda: _end_user_row(end_user), lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= 0.05, diff --git a/tests/integration/spend/test_end_user_spend_without_proxy_user.py b/tests/integration/spend/test_end_user_spend_without_proxy_user.py index 68beedd087d..94884717092 100644 --- a/tests/integration/spend/test_end_user_spend_without_proxy_user.py +++ b/tests/integration/spend/test_end_user_spend_without_proxy_user.py @@ -175,7 +175,7 @@ def test_streamed_end_user_spelling_is_charged_and_budget_enforced(gateway: Gate gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): - _observed(upstream) + upstream.delete("/__observations").raise_for_status() end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" _new_customer(gateway, end_user, max_budget=0.05) scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) @@ -210,6 +210,7 @@ def test_streamed_end_user_spelling_is_charged_and_budget_enforced(gateway: Gate assert spend == pytest.approx(20 * 0.001 + 20 * 0.002) _assert_daily_row(end_user, model) + upstream.delete("/__observations").raise_for_status() denied: Final = _chat_request( gateway, model, @@ -231,7 +232,7 @@ def test_anthropic_messages_metadata_user_id_is_charged_and_budget_enforced(gate gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): - _observed(upstream) + upstream.delete("/__observations").raise_for_status() end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" _new_customer(gateway, end_user, max_budget=0.05) scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) @@ -337,6 +338,7 @@ def test_anthropic_messages_metadata_user_id_is_charged_and_budget_enforced(gate assert spend == pytest.approx(20 * 0.001 + 20 * 0.002) _assert_daily_row(end_user, model, "anthropic/claude-sonnet-5-5") + upstream.delete("/__observations").raise_for_status() denied: Final = gateway.client.request( "POST", "/v1/messages", @@ -353,7 +355,7 @@ def test_end_user_id_spelling_is_charged_and_budget_enforced(gateway: Gateway, s gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): - _observed(upstream) + upstream.delete("/__observations").raise_for_status() end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" _new_customer(gateway, end_user, max_budget=0.05) scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) @@ -373,6 +375,7 @@ def test_end_user_id_spelling_is_charged_and_budget_enforced(gateway: Gateway, s assert spend == pytest.approx(20 * 0.001 + 20 * 0.002) _assert_daily_row(end_user, model) + upstream.delete("/__observations").raise_for_status() denied: Final = _chat_request(gateway, model, key, spelling, end_user, f"again {uuid.uuid4().hex}") _assert_budget_refusal(denied, end_user) assert _observed(upstream) == [] @@ -383,7 +386,7 @@ def test_safety_identifier_on_responses_is_charged_and_budget_enforced(gateway: gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): - _observed(upstream) + upstream.delete("/__observations").raise_for_status() scenario_id: Final = f"sc-{uuid.uuid4().hex}" handle: Final = register_scenario( scenario_id, @@ -452,6 +455,7 @@ def test_safety_identifier_on_responses_is_charged_and_budget_enforced(gateway: assert spend == pytest.approx(0.06) _assert_daily_row(end_user, model) + upstream.delete("/__observations").raise_for_status() denied: Final = call() _assert_budget_refusal(denied, end_user) assert _observed(upstream) == [] @@ -462,7 +466,7 @@ def test_end_user_header_wins_over_body_user(gateway: Gateway) -> None: gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): - _observed(upstream) + upstream.delete("/__observations").raise_for_status() header_user: Final = f"integration-end-user-a-{uuid.uuid4().hex}" body_user: Final = f"integration-end-user-b-{uuid.uuid4().hex}" _new_customer(gateway, header_user, max_budget=0.05) @@ -503,6 +507,7 @@ def test_end_user_header_wins_over_body_user(gateway: Gateway) -> None: == [] ) + upstream.delete("/__observations").raise_for_status() denied: Final = gateway.request( "POST", "/v1/chat/completions", @@ -542,7 +547,7 @@ def test_configured_end_user_header_is_charged_and_budget_enforced( candidate.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, ): - _observed(upstream) + upstream.delete("/__observations").raise_for_status() end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" _new_customer(candidate, end_user, max_budget=0.05) scenario.cleanups.callback(candidate.request, "POST", "/customer/delete", {"user_ids": [end_user]}) @@ -569,6 +574,7 @@ def test_configured_end_user_header_is_charged_and_budget_enforced( assert _charged_spend(end_user) == pytest.approx(0.06) _assert_daily_row(end_user, model) + upstream.delete("/__observations").raise_for_status() denied: Final = candidate.request( "POST", "/v1/chat/completions", diff --git a/tests/integration/spend/test_global_spend_reset.py b/tests/integration/spend/test_global_spend_reset.py index fad5c14175a..c400d90e184 100644 --- a/tests/integration/spend/test_global_spend_reset.py +++ b/tests/integration/spend/test_global_spend_reset.py @@ -112,7 +112,7 @@ def test_global_spend_reset_zeroes_token_and_team_spend_but_keeps_logs(gateway: assert float(str(_token_spend(key_b)[0]["spend"])) == pytest.approx(0.06), _token_spend(key_b) assert float(str(_team_spend(team_id)[0]["spend"])) == pytest.approx(0.12), _team_spend(team_id) - _observed(upstream) + upstream.delete("/__observations").raise_for_status() denied: Final = _chat(gateway, model, key_a) assert denied.status_code == 422, denied.text assert object_value(denied.json()["error"])["type"] == "budget_exceeded", denied.text @@ -153,7 +153,7 @@ def test_reset_exhausted_key_is_served_again(gateway: Gateway) -> None: reset: Final = gateway.request("POST", "/global/spend/reset") assert reset.status_code == 200, reset.text content: Final = f"restored {uuid.uuid4().hex}" - _observed(upstream) + upstream.delete("/__observations").raise_for_status() restored: Final = _chat(gateway, model, key, content) assert restored.status_code == 200, restored.text assert _observed(upstream) == [_upstream_chat(content)], restored.text diff --git a/tests/integration/spend/test_key_budget_lockout.py b/tests/integration/spend/test_key_budget_lockout.py index 74470b80543..a3d1d3d0a78 100644 --- a/tests/integration/spend/test_key_budget_lockout.py +++ b/tests/integration/spend/test_key_budget_lockout.py @@ -182,7 +182,7 @@ def test_an_exhausted_key_is_refused_inference_but_can_still_read_its_own_info(g lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= 0.06, seconds=70, ) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() denied: Final = gateway.request( "POST", "/v1/chat/completions", @@ -234,7 +234,7 @@ def _upstream_chat(content: str) -> dict[str, JsonValue]: def _served_chat(gateway: Gateway, upstream: httpx.Client, model: str, key: str) -> None: content: Final = f"tier served {uuid.uuid4().hex}" - _observed(upstream) + upstream.delete("/__observations").raise_for_status() served: Final = _bounded_chat(gateway, model, key, content) assert served.status_code == 200, served.text assert float(served.headers["x-litellm-response-cost"]) == 0.06, dict(served.headers) @@ -258,7 +258,7 @@ def test_raising_a_spent_keys_budget_restores_serving(gateway: Gateway) -> None: seconds=70, ) eventually(lambda: _bounded_chat(gateway, model, key), lambda response: response.status_code != 200, seconds=30) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() denied: Final = _bounded_chat(gateway, model, key) assert denied.status_code == 422, denied.text assert object_value(denied.json()["error"])["type"] == "budget_exceeded" @@ -427,7 +427,7 @@ def test_raising_a_spent_tier_budget_restores_serving(gateway: Gateway) -> None: lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= 0.06, seconds=70, ) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() _denied_with_budget_exceeded(gateway, model, key) assert upstream.get("/__observations").json()["requests"] == [] gateway.post("/budget/update", {"budget_id": budget_id, "max_budget": 1.0}) @@ -464,7 +464,7 @@ def test_lowering_a_tier_budget_below_spend_blocks_the_key(gateway: Gateway) -> lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= 0.06, seconds=30, ) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() probe: Final = gateway.request("POST", "/v1/chat/completions", {"model": model}, key=key) assert probe.status_code == 400, probe.text assert object_value(probe.json()["error"]) == { @@ -480,7 +480,7 @@ def test_lowering_a_tier_budget_below_spend_blocks_the_key(gateway: Gateway) -> ) assert len(updated_rows) == 1 assert float(str(updated_rows[0]["max_budget"])) == 0.01, updated_rows - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() lowered_probe: Final = eventually( lambda: gateway.request("POST", "/v1/chat/completions", {"model": model}, key=key), lambda response: ( @@ -516,7 +516,7 @@ def test_key_linked_to_a_tier_by_update_is_refused_once_streamed_spend_exhausts_ "messages": [{"role": "user", "content": content}], "stream": True, } - _observed(upstream) + upstream.delete("/__observations").raise_for_status() served: Final = gateway.request("POST", "/v1/chat/completions", body, key=key) assert served.status_code == 200, served.text events: Final = tuple( @@ -541,7 +541,7 @@ def test_key_linked_to_a_tier_by_update_is_refused_once_streamed_spend_exhausts_ assert denied.status_code == 422, denied.text assert object_value(denied.json()["error"])["type"] == "budget_exceeded", denied.text assert "Max budget: 0.06" in denied.text, denied.text - _observed(upstream) + upstream.delete("/__observations").raise_for_status() fresh_denied: Final = gateway.request("POST", "/v1/chat/completions", body, key=key) assert fresh_denied.status_code == 422, fresh_denied.text assert object_value(fresh_denied.json()["error"])["type"] == "budget_exceeded", fresh_denied.text diff --git a/tests/integration/spend/test_org_budget_cli_session_token.py b/tests/integration/spend/test_org_budget_cli_session_token.py index f821e337374..34f6cfab174 100644 --- a/tests/integration/spend/test_org_budget_cli_session_token.py +++ b/tests/integration/spend/test_org_budget_cli_session_token.py @@ -39,7 +39,7 @@ def test_cli_session_token_without_org_id_charges_and_caps_the_team_organization ) token: Final = _cli_session_token(user_id, team_id) prompt: Final = f"org budget {uuid.uuid4().hex}" - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() first: Final = gateway.request( "POST", "/v1/chat/completions", @@ -51,6 +51,7 @@ def test_cli_session_token_without_org_id_charges_and_caps_the_team_organization assert len(reached_upstream) == 1, reached_upstream assert reached_upstream[0]["body"]["model"] == "gpt-4o-mini", reached_upstream assert reached_upstream[0]["body"]["messages"] == [{"role": "user", "content": prompt}], reached_upstream + upstream.delete("/__observations").raise_for_status() logged: Final = eventually( lambda: read_rows( 'SELECT organization_id, team_id, spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', diff --git a/tests/integration/spend/test_team_budget_enforcement.py b/tests/integration/spend/test_team_budget_enforcement.py index 823aff1e0cc..fce33478672 100644 --- a/tests/integration/spend/test_team_budget_enforcement.py +++ b/tests/integration/spend/test_team_budget_enforcement.py @@ -50,7 +50,7 @@ def exhausted(gateway: Gateway) -> Iterator[ExhaustedTeam]: seconds=70, ) eventually(lambda: _chat(gateway, model, key), lambda response: response.status_code != 200, seconds=30) - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() yield ExhaustedTeam(scenario, upstream, model, team_id, key) diff --git a/tests/integration/spend/test_team_member_spend.py b/tests/integration/spend/test_team_member_spend.py index 343a3be1b49..306138e2bb1 100644 --- a/tests/integration/spend/test_team_member_spend.py +++ b/tests/integration/spend/test_team_member_spend.py @@ -130,8 +130,6 @@ class _BulkBudgetUpdateResponse(BaseModel): @pytest.fixture def upstream(gateway: Gateway) -> Iterator[httpx.Client]: with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client: - response: Final = client.get("/__observations") - assert response.status_code == 200, response.text yield client @@ -304,7 +302,7 @@ def test_member_over_budget_is_blocked_when_redis_counter_reads_stale_low(gatewa counted: Final = eventually(lambda: cache.get(counter_key), lambda value: value is not None, seconds=10) assert float(counted) == pytest.approx(0.06), counted cache.set(counter_key, "0.01") - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() denied: Final = gateway.request( "POST", "/v1/chat/completions", @@ -346,6 +344,7 @@ def test_member_update_lowering_the_budget_below_spend_blocks_the_next_call( assert peer_first.status_code == 200, peer_first.text observed_before_update: Final = _observed_models(upstream) assert observed_before_update == (provider_model, provider_model), repr(observed_before_update) + upstream.delete("/__observations").raise_for_status() accrued: Final = eventually( lambda: _budget_state(team, user), lambda values: len(values) == 1 and values[0].spend is not None and values[0].spend >= 0.12, @@ -466,6 +465,7 @@ def test_member_update_temp_budget_increase_readmits_a_blocked_member( assert first.status_code == 200, first.text observed_before_block: Final = _observed_models(upstream) assert observed_before_block == (provider_model,), repr(observed_before_block) + upstream.delete("/__observations").raise_for_status() accrued: Final = eventually( lambda: _budget_state(team, user), lambda values: len(values) == 1 and values[0].spend is not None and values[0].spend >= 0.06, @@ -701,6 +701,7 @@ def test_member_update_by_email_sets_models_role_and_forks_the_team_default_budg assert _ErrorResponse.model_validate_json(denied.text).error.type == "team_model_access_denied", denied.text observed_before_clear: Final = _observed_models(upstream) assert observed_before_clear == (provider_one,), repr(observed_before_clear) + upstream.delete("/__observations").raise_for_status() cleared_models: Final = _member_update( gateway, @@ -815,6 +816,7 @@ def test_bulk_update_is_a_merge_patch_per_member(gateway: Gateway, upstream: htt assert first_chat.status_code == 200, first_chat.text observed_before_update: Final = _observed_models(upstream) assert observed_before_update == (provider_spend,), repr(observed_before_update) + upstream.delete("/__observations").raise_for_status() accrued: Final = eventually( lambda: _budget_state(team, user_one), lambda values: len(values) == 1 and values[0].spend is not None and values[0].spend >= 0.06, @@ -889,6 +891,7 @@ def test_bulk_update_is_a_merge_patch_per_member(gateway: Gateway, upstream: htt ) observed_after_first_patch: Final = _observed_models(upstream) assert observed_after_first_patch == (provider_one,), repr(observed_after_first_patch) + upstream.delete("/__observations").raise_for_status() cleared_max: Final = gateway.request( "POST", @@ -974,6 +977,7 @@ def test_bulk_update_restores_team_default_when_last_member_limit_is_cleared( assert first_chat.status_code == 200, first_chat.text first_observations: Final = _observed_models(upstream) assert first_observations == (provider_model,), repr(first_observations) + upstream.delete("/__observations").raise_for_status() accrued: Final = eventually( lambda: _budget_state(team, user), lambda values: len(values) == 1 and values[0].spend is not None and values[0].spend >= 0.06, @@ -984,6 +988,7 @@ def test_bulk_update_restores_team_default_when_last_member_limit_is_cleared( assert second_chat.status_code == 200, second_chat.text second_observations: Final = _observed_models(upstream) assert second_observations == (provider_model,), repr(second_observations) + upstream.delete("/__observations").raise_for_status() cleared: Final = gateway.request( "POST", f"/management/v1/teams/{team}/members/bulk_update", @@ -1019,11 +1024,3 @@ def test_bulk_update_restores_team_default_when_last_member_limit_is_cleared( assert first_observations + second_observations == (provider_model, provider_model), repr( (first_observations, second_observations) ) - - -@pytest.fixture(autouse=True) -def _drain_unobserved_requests(gateway: Gateway) -> Iterator[None]: - yield - with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client: - response: Final = client.get("/__observations") - assert response.status_code == 200, response.text diff --git a/tests/integration/spend/test_user_budget_on_team_keys.py b/tests/integration/spend/test_user_budget_on_team_keys.py index f80859555f0..5b28b1aa875 100644 --- a/tests/integration/spend/test_user_budget_on_team_keys.py +++ b/tests/integration/spend/test_user_budget_on_team_keys.py @@ -45,7 +45,7 @@ def test_team_key_is_denied_before_provider_once_owner_personal_budget_is_exhaus assert read_rows( 'SELECT max_budget FROM "LiteLLM_VerificationToken" WHERE token=%s', (sha256(key.encode()).hexdigest(),) ) == [{"max_budget": None}] - upstream.get("/__observations").raise_for_status() + upstream.delete("/__observations").raise_for_status() denied: Final = candidate.request( "POST", "/v1/chat/completions", diff --git a/tests/unit/integration_support/test_upstream.py b/tests/unit/integration_support/test_upstream.py index 9e114e3b0b7..a54cda06c65 100644 --- a/tests/unit/integration_support/test_upstream.py +++ b/tests/unit/integration_support/test_upstream.py @@ -3,6 +3,7 @@ from collections import deque from pathlib import Path from typing import Final +import httpx import pytest from starlette.types import Message, Scope from starlette.websockets import WebSocket @@ -50,13 +51,36 @@ async def test_realtime_disconnect_after_session_creation_keeps_unexpected_error else: await provider.realtime(socket) assert [item["type"] for item in outgoing] == ["websocket.accept", "websocket.send"] - observed: Final = tuple(provider.observations.get_nowait() for _ in range(provider.observations.qsize())) + observed: Final = provider.observations.snapshot() assert [(item.method, item.body) for item in observed] == [ ("WEBSOCKET", {"query": [["model", "synthetic"]]}), ("WEBSOCKET_FRAME", {"type": "response.create"}), ] +@pytest.mark.asyncio +async def test_observations_read_without_draining_until_explicitly_reset(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2])) + from integration._support.upstream import Provider + + transport: Final = httpx.ASGITransport(app=Provider().app()) + async with httpx.AsyncClient(transport=transport, base_url="http://upstream") as client: + sent: Final = await client.post( + "/v1/chat/completions", + json={"model": "synthetic", "messages": [{"role": "user", "content": "observed-marker"}]}, + ) + assert sent.status_code == 200, sent.text + first: Final = (await client.get("/__observations")).json()["requests"] + second: Final = (await client.get("/__observations")).json()["requests"] + assert [(item["path"], item["body"]["messages"][0]["content"]) for item in first] == [ + ("/v1/chat/completions", "observed-marker") + ] + assert second == first + reset: Final = await client.delete("/__observations") + assert reset.status_code == 204 + assert (await client.get("/__observations")).json()["requests"] == [] + + @pytest.mark.asyncio @pytest.mark.parametrize("method,status", (("POST", 200), ("POST", 400), ("DELETE", 200))) async def test_mcp_notification_ordering_preserves_other_responses( From eb114e638d4557a9155d5eb21fd9cb332336c85f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sun, 18 Oct 2026 03:27:53 -0700 Subject: [PATCH 02/50] test(integration): look up owned cache keys and clean up sweep models (#47255) Three integration tests diffed the whole shared Redis keyspace (scan_iter or dbsize before and after), so a sibling test's cache write could satisfy or break them. Each now finds its own entry by a value only it wrote: the Gemini reasoning test by a per-run Gemini responseId, the max_messages test by a per-test answer text, and the S3 cache-hit test by its request marker The sweep auth tests created models through raw /model/new on the shared gateway and never deleted them. They now create them through Scenario.model, which deletes each model on exit, including when setup fails --- .../test_audit_any_sweep_auth.py | 58 +++++-------------- .../caching/response_cache_case.py | 11 ++-- .../caching/test_cache_max_messages.py | 22 +++---- .../test_s3_v2_flush_surfaces.py | 10 ++-- .../test_chat_completions_reasoning_gemini.py | 30 +++++----- 5 files changed, 47 insertions(+), 84 deletions(-) diff --git a/tests/integration/authorization/test_audit_any_sweep_auth.py b/tests/integration/authorization/test_audit_any_sweep_auth.py index f6f5caa5240..c0142cd6f75 100644 --- a/tests/integration/authorization/test_audit_any_sweep_auth.py +++ b/tests/integration/authorization/test_audit_any_sweep_auth.py @@ -9,7 +9,7 @@ import httpx import openai import pytest import websockets -from integration._support.client import Gateway, eventually +from integration._support.client import Gateway, Scenario, eventually from integration._support.database import read_rows from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue @@ -63,7 +63,8 @@ _RESPONSES_EVENTS: Final = ( ) -def _anthropic_model(gateway: Gateway, *, stream: bool) -> str: +def _anthropic_model(scenario: Scenario, *, stream: bool) -> str: + gateway: Final = scenario.gateway scenario_id: Final = "audit-anthropic-stream" if stream else "audit-anthropic" if stream: _register_scenario(gateway, scenario_id, {"content_type": "text/event-stream", "frames": _ANTHROPIC_EVENTS}) @@ -76,23 +77,13 @@ def _anthropic_model(gateway: Gateway, *, stream: bool) -> str: "routes": {"POST /v1/messages": {"content_type": "application/json", "body": _ANTHROPIC_MESSAGE}}, }, ) - name: Final = f"integration-{uuid.uuid4().hex}" - created: Final = gateway.post( - "/model/new", - { - "model_name": name, - "litellm_params": { - "model": "anthropic/claude-3-5-sonnet-20241022", - "api_key": "integration-provider-key", - "api_base": f"{gateway.upstream_url}/{scenario_id}", - }, - }, + return scenario.model( + model="anthropic/claude-3-5-sonnet-20241022", api_base=f"{gateway.upstream_url}/{scenario_id}" ) - assert isinstance(created["model_info"], dict) and created["model_info"]["id"], created - return name -def _responses_model(gateway: Gateway, *, stream: bool) -> str: +def _responses_model(scenario: Scenario, *, stream: bool) -> str: + gateway: Final = scenario.gateway scenario_id: Final = "audit-resp-stream" if stream else "audit-resp" if stream: _register_scenario(gateway, scenario_id, {"content_type": "text/event-stream", "frames": _RESPONSES_EVENTS}) @@ -105,20 +96,7 @@ def _responses_model(gateway: Gateway, *, stream: bool) -> str: "routes": {"POST /responses": {"content_type": "application/json", "body": _RESPONSES_BODY}}, }, ) - name: Final = f"integration-{uuid.uuid4().hex}" - created: Final = gateway.post( - "/model/new", - { - "model_name": name, - "litellm_params": { - "model": "openai/gpt-4o-mini", - "api_key": "integration-provider-key", - "api_base": f"{gateway.upstream_url}/{scenario_id}", - }, - }, - ) - assert isinstance(created["model_info"], dict) and created["model_info"]["id"], created - return name + return scenario.model(api_base=f"{gateway.upstream_url}/{scenario_id}") def _openai_client(gateway: Gateway, key: str) -> openai.OpenAI: @@ -219,8 +197,8 @@ def test_allowed_key_chat_completions_restamps_alias( @pytest.mark.parametrize("client_kind", ("httpx", "anthropic", "anthropic_async")) @pytest.mark.parametrize("stream", (False, True)) def test_allowed_key_messages(gateway: Gateway, client_kind: str, stream: bool) -> None: - model: Final = _anthropic_model(gateway, stream=stream) with gateway.scenario() as scenario: + model: Final = _anthropic_model(scenario, stream=stream) key: Final = scenario.key(models=[model]) body: Final = { "model": model, @@ -264,8 +242,8 @@ def test_allowed_key_messages(gateway: Gateway, client_kind: str, stream: bool) @pytest.mark.parametrize("client_kind", ("httpx", "openai", "openai_async")) @pytest.mark.parametrize("stream", (False, True)) def test_allowed_key_responses(gateway: Gateway, client_kind: str, stream: bool) -> None: - model: Final = _responses_model(gateway, stream=stream) with gateway.scenario() as scenario: + model: Final = _responses_model(scenario, stream=stream) key: Final = scenario.key(models=[model]) body: Final = { "model": model, @@ -334,20 +312,10 @@ async def test_realtime_websocket_allowed_and_denied(gateway: Gateway) -> None: ), }, ) - name: Final = f"integration-{uuid.uuid4().hex}" - created: Final = gateway.post( - "/model/new", - { - "model_name": name, - "litellm_params": { - "model": "openai/gpt-4o-realtime-preview", - "api_key": rt_scenario, - "api_base": gateway.upstream_url, - }, - }, - ) - assert isinstance(created["model_info"], dict) and created["model_info"]["id"], created with gateway.scenario() as scenario: + name: Final = scenario.model( + model="openai/gpt-4o-realtime-preview", api_key=rt_scenario, api_base=gateway.upstream_url + ) key: Final = scenario.key(models=[name]) other_model: Final = scenario.model() denied_key: Final = scenario.key(models=[other_model]) diff --git a/tests/integration/caching/response_cache_case.py b/tests/integration/caching/response_cache_case.py index a1850ebdb9d..312f29faeb3 100644 --- a/tests/integration/caching/response_cache_case.py +++ b/tests/integration/caching/response_cache_case.py @@ -279,16 +279,19 @@ def redis_store() -> Redis: return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) -def entry_key(store: Redis, needle: str) -> str | None: - """The response cache key (a bare sha256) whose stored value names ``needle``, or ``None``.""" +def entry_keys(store: Redis, needle: str) -> Iterator[str]: for raw_key in store.scan_iter(count=1000): key: Final = raw_key.decode() if not CACHE_KEY.match(key) or store.type(key) != b"string": continue value: Final = store.get(key) if value is not None and needle.encode() in value: - return key - return None + yield key + + +def entry_key(store: Redis, needle: str) -> str | None: + """The response cache key (a bare sha256) whose stored value names ``needle``, or ``None``.""" + return next(entry_keys(store, needle), None) def await_entry(store: Redis, needle: str) -> str: diff --git a/tests/integration/caching/test_cache_max_messages.py b/tests/integration/caching/test_cache_max_messages.py index 32c71e887fe..e28015a5ba4 100644 --- a/tests/integration/caching/test_cache_max_messages.py +++ b/tests/integration/caching/test_cache_max_messages.py @@ -15,6 +15,7 @@ from integration._support.openai_wire import chat_reply, responses_reply from integration._support.process import owned_proxy from integration._support.provider import SharedProvider from integration._support.wire import Reply, Request, wire_server +from integration.caching.response_cache_case import await_entry, redis_store from pydantic import JsonValue, TypeAdapter from redis import Redis from redis.exceptions import ResponseError @@ -133,11 +134,6 @@ def _first(value: JsonValue) -> dict[str, JsonValue]: return object_value(value[0]) -def _cached_responses(redis: Redis[bytes]) -> frozenset[bytes]: - digests: Final = tuple(key for key in redis.scan_iter() if len(key) == 64) - return frozenset(key for key in digests if b'"response"' in (redis.get(key) or b"")) - - _SCENARIOS: Final = ( pytest.param("/v1/messages", "anthropic/claude-sonnet-5-5", _claude_code_turns, id="messages"), pytest.param("/v1/chat/completions", "openai/gpt-5.4", _agent_turns, id="chat-completions"), @@ -149,22 +145,22 @@ _SCENARIOS: Final = ( def test_cache_serves_a_turn_under_max_messages_and_skips_one_past_it( gateway: Gateway, provider: SharedProvider, path: str, model: str, turns: _Turns ) -> None: - under_cap, past_cap = turns(f"update the config {uuid.uuid4().hex}") - provider.expect(_reply(path, "first answer"), _reply(path, "second answer"), _reply(path, "third answer")) + task: Final = uuid.uuid4().hex + under_cap, past_cap = turns(f"update the config {task}") + answers: Final = tuple(f"{ordinal} answer {task}" for ordinal in ("first", "second", "third")) + provider.expect(*(_reply(path, answer) for answer in answers)) - redis_client: Final[Redis[bytes]] = Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) - with redis_client as redis: - cached_before: Final = _cached_responses(redis) + with redis_store() as store: gateway.post(path, _body(path, model, under_cap)) - eventually(lambda: _cached_responses(redis) - cached_before, lambda written: len(written) == 1) + await_entry(store, answers[0]) repeated: Final = gateway.post(path, _body(path, model, under_cap)) past_cap_twice: Final = ( gateway.post(path, _body(path, model, past_cap)), gateway.post(path, _body(path, model, past_cap)), ) - assert _answer(path, repeated) == "first answer", "a repeated turn under max_messages was not served from the cache" - assert tuple(_answer(path, answer) for answer in past_cap_twice) == ("second answer", "third answer"), ( + assert _answer(path, repeated) == answers[0], "a repeated turn under max_messages was not served from the cache" + assert tuple(_answer(path, answer) for answer in past_cap_twice) == answers[1:], ( "a turn past max_messages was served from the cache" ) assert len(provider.received()) == 3 diff --git a/tests/integration/observability/test_s3_v2_flush_surfaces.py b/tests/integration/observability/test_s3_v2_flush_surfaces.py index 1e373171de6..5a94eaa248f 100644 --- a/tests/integration/observability/test_s3_v2_flush_surfaces.py +++ b/tests/integration/observability/test_s3_v2_flush_surfaces.py @@ -1,11 +1,9 @@ -import os import re import uuid from pathlib import Path from typing import Final import pytest -from redis import Redis from _s3_v2_support import ( BUCKET, PREFIX, @@ -21,6 +19,7 @@ from _s3_v2_support import ( from integration._support.client import Gateway, eventually from integration._support.process import owned_proxy from integration._support.wire import wire_server +from integration.caching.response_cache_case import entry_keys, redis_store pytestmark: Final = pytest.mark.shares_worker_state @@ -117,13 +116,14 @@ def test_s3_v2_cache_hit_twins_log_one_object_per_request(gateway: Gateway, tmp_ model="anthropic/claude-sonnet-4-5-20250929", api_base=provider.url, api_key="synthetic-provider-key" ) key: Final = scenario.key(models=[openai_model, anthropic_model]) - cache: Final = Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) - keys_before: Final = cache.dbsize() warmed: Final = tuple( call_surface(candidate, surface, openai_model, anthropic_model, key, f"{marker}-{surface}") for surface in SURFACES ) - eventually(cache.dbsize, lambda size: size >= keys_before + len(SURFACES), seconds=30) + with redis_store() as cache: + eventually( + lambda: frozenset(entry_keys(cache, marker)), lambda owned: len(owned) >= len(SURFACES), seconds=30 + ) repeated: Final = tuple( call_surface(candidate, surface, openai_model, anthropic_model, key, f"{marker}-{surface}", False) for surface in SURFACES diff --git a/tests/integration/translation/chat_completions/reasoning/test_chat_completions_reasoning_gemini.py b/tests/integration/translation/chat_completions/reasoning/test_chat_completions_reasoning_gemini.py index efd649f9383..e2369fea443 100644 --- a/tests/integration/translation/chat_completions/reasoning/test_chat_completions_reasoning_gemini.py +++ b/tests/integration/translation/chat_completions/reasoning/test_chat_completions_reasoning_gemini.py @@ -8,11 +8,11 @@ from unittest.mock import ANY from uuid import uuid4 import pytest -from integration._support.client import Gateway, eventually, list_value, object_value, string_value +from integration._support.client import Gateway, list_value, object_value, string_value from integration._support.process import graceful_stop_seconds from integration._support.provider import SharedProvider from integration._support.wire import Reply -from integration.caching.response_cache_case import CACHE_KEY, redis_store +from integration.caching.response_cache_case import await_entry, redis_store from integration.translation.case import SSEFrame, TranslationTestCase from integration.translation.chat_completions.bases.gemini import GEMINI_3_8_FLASH_TEST_CASE from integration.translation.chat_completions.bases.vertex_ai import ( @@ -840,7 +840,7 @@ def without_closed_schema(tool: JsonValue) -> JsonValue: return {**object_value(tool), "function": {**function, "parameters": open_parameters}} -def cacheable_run(case: TranslationTestCase, marker: str) -> TranslationTestCase: +def cacheable_run(case: TranslationTestCase, marker: str, reply_id: str) -> TranslationTestCase: messages: Final = list_value(case.litellm_request["messages"]) system: Final = object_value(messages[0]) instruction: Final = object_value(case.expected_provider_request["system_instruction"]) @@ -853,6 +853,7 @@ def cacheable_run(case: TranslationTestCase, marker: str) -> TranslationTestCase "messages": [{**system, "content": text}, messages[1]], "tools": [without_closed_schema(tool) for tool in list_value(case.litellm_request["tools"])], }, + mock_provider_response={**case.mock_provider_response, "responseId": reply_id}, expected_provider_request={ **case.expected_provider_request, "contents": [list_value(case.expected_provider_request["contents"])[0]], @@ -861,14 +862,10 @@ def cacheable_run(case: TranslationTestCase, marker: str) -> TranslationTestCase ) -def response_cache_keys() -> frozenset[str]: - keys: Final = (raw.decode() for raw in redis_store().scan_iter()) - return frozenset(key for key in keys if CACHE_KEY.match(key)) - - -def stored_reply_key(before: frozenset[str]) -> str: - (key,) = eventually(lambda: response_cache_keys() - before, lambda added: len(added) == 1) - return key +def stored_reply_key(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> str: + reply: Final = assert_translation(case, gateway, provider) + with redis_store() as store: + return await_entry(store, string_value(reply["id"])) GEMINI_3_8_FLASH_SUMMARY_ALIAS_TEST_CASE: Final = with_request( @@ -888,12 +885,11 @@ def test_a_summary_alias_ask_and_a_plain_request_never_share_a_cached_reply( first: TranslationTestCase, second: TranslationTestCase, thoughts_off: Gateway, provider: SharedProvider ) -> None: marker: Final = uuid4().hex - before: Final = response_cache_keys() - assert_translation(cacheable_run(first, marker), thoughts_off, provider) - first_key: Final = stored_reply_key(before) - assert_translation(cacheable_run(second, marker), thoughts_off, provider) - second_key: Final = stored_reply_key(before | {first_key}) - repeat: Final = thoughts_off.request("POST", second.litellm_endpoint, cacheable_run(second, marker).litellm_request) + second_run: Final = cacheable_run(second, marker, uuid4().hex) + first_key: Final = stored_reply_key(cacheable_run(first, marker, uuid4().hex), thoughts_off, provider) + second_key: Final = stored_reply_key(second_run, thoughts_off, provider) + assert second_key != first_key + repeat: Final = thoughts_off.request("POST", second.litellm_endpoint, second_run.litellm_request) assert repeat.status_code == 200, repeat.text assert repeat.headers.get("x-litellm-cache-key") == second_key, sorted(repeat.headers) assert provider.received() == () From 5012083b721020ba6115a4519f40d3746eee6bc4 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:51:27 +0000 Subject: [PATCH 03/50] feat(vertex_ai): let operators drop chosen anthropic-beta values on the Vertex AI pass-through routes (#47185) * feat(vertex_ai): let operators drop chosen anthropic-beta values on the Vertex AI pass-through routes * fix(vertex_ai): drop configured betas sent through x-pass-anthropic-beta and only fail beta requests on a malformed setting The drop list now runs on the final upstream headers, after x-pass- expansion, through an upstream_headers_filter on pass_through_request, so x-pass-anthropic-beta cannot carry a dropped value to Vertex. A malformed setting now answers 500 only for requests that carry anthropic-beta or x-pass-anthropic-beta, so Gemini and discovery calls keep working. * test(vertex_ai): accept the new upstream_headers_filter argument in the existing Vertex route tests * fix(vertex_ai): refuse a malformed drop-betas list at boot and match list entries written with spaces * fix(vertex_ai): split drop-betas list entries on commas like the header --- litellm/proxy/_types.py | 4 + .../llm_passthrough_endpoints.py | 51 +++- .../pass_through_endpoints.py | 18 +- litellm/proxy/proxy_server.py | 6 +- .../providers/test_vertex_passthrough_wire.py | 56 ++++ .../test_llm_pass_through_endpoints.py | 3 + .../test_llm_passthrough_endpoints.py | 262 ++++++++++++++++++ .../test_vertex_passthrough_load_balancing.py | 4 + .../proxy/proxy_server/test_proxy_config.py | 20 ++ .../test_vector_store_tenant_guard.py | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 11 files changed, 424 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5970ede5c7e..8ff5d1a1002 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2774,6 +2774,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): default=None, description="Serve the OpenAI pass-through WebSocket route, which relays frames to OpenAI under the proxy's own provider credential without reading them. Off by default.", ) + vertex_ai_pass_through_drop_anthropic_betas: tuple[str, ...] | None = Field( + default=None, + description="anthropic-beta header values the Vertex AI pass-through routes (/vertex_ai/* and /vertex-ai/*) remove before calling Vertex, for a beta a Vertex Claude model rejects, whether the client sent them as anthropic-beta or x-pass-anthropic-beta. Every other value the client sent is forwarded unchanged, and the header is left out when no value remains. Unset forwards the header as sent.", + ) transcribe_media_buckets: list[str] | None = Field( default=None, description="S3 bucket names that keys other than proxy admins may read media from and write transcripts to through the Amazon Transcribe pass-through. Unset means only proxy admins can start transcription jobs.", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 3aef5724bdc..f2605a8ec6f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -25,7 +25,7 @@ import httpx import openai from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket from fastapi.responses import StreamingResponse -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, TypeAdapter, ValidationError from starlette.websockets import WebSocketState from typing_extensions import ReadOnly, TypedDict @@ -44,6 +44,7 @@ from litellm.constants import ( AZURE_SPEECH_STT_DOMAIN, AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER, BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES, + PASS_THROUGH_HEADER_PREFIX, TYPESAFE_DANGER_ZONE_HEADER_PREFIX, ) from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException @@ -2570,6 +2571,47 @@ def _is_vertex_anthropic_count_tokens_route(endpoint: str) -> bool: return endpoint.rsplit("/", 1)[-1].split(":", 1)[0] == "count-tokens" +_VERTEX_DROPPED_ANTHROPIC_BETAS_SETTING: Final = "vertex_ai_pass_through_drop_anthropic_betas" +_VERTEX_DROPPED_ANTHROPIC_BETAS: Final[TypeAdapter[frozenset[str] | None]] = TypeAdapter(frozenset[str] | None) +_ANTHROPIC_BETA_REQUEST_HEADERS: Final = ("anthropic-beta", f"{PASS_THROUGH_HEADER_PREFIX}anthropic-beta") + + +def _vertex_dropped_anthropic_betas( + general_settings: Mapping[str, object], request_headers: Mapping[str, str] +) -> frozenset[str]: + if not any(name in request_headers for name in _ANTHROPIC_BETA_REQUEST_HEADERS): + return frozenset() + try: + configured: Final = _VERTEX_DROPPED_ANTHROPIC_BETAS.validate_python( + general_settings.get(_VERTEX_DROPPED_ANTHROPIC_BETAS_SETTING) + ) + except ValidationError as error: + raise HTTPException( + status_code=500, + detail=f"general_settings.{_VERTEX_DROPPED_ANTHROPIC_BETAS_SETTING} must be a list of strings", + ) from error + listed_betas: Final = (beta.strip() for beta in ",".join(configured or ()).split(",")) + return frozenset(beta for beta in listed_betas if beta) + + +def _upstream_vertex_header_value(name: str, value: str, dropped_anthropic_betas: frozenset[str]) -> str | None: + if name.lower() != "anthropic-beta": + return value + betas: Final = tuple(beta.strip() for beta in value.split(",")) + if dropped_anthropic_betas.isdisjoint(betas): + return value + return ",".join(beta for beta in betas if beta and beta not in dropped_anthropic_betas) or None + + +def _without_dropped_anthropic_betas( + dropped_anthropic_betas: frozenset[str], headers: Mapping[str, str] +) -> Mapping[str, str]: + upstream_values: Final = ( + (name, _upstream_vertex_header_value(name, value, dropped_anthropic_betas)) for name, value in headers.items() + ) + return MappingProxyType({name: value for name, value in upstream_values if value is not None}) + + def _upstream_headers_for_vertex_route(endpoint: str, headers: Mapping[str, str]) -> Mapping[str, str]: if not _is_vertex_anthropic_count_tokens_route(endpoint): return headers @@ -2889,6 +2931,7 @@ async def _base_vertex_proxy_route( request: Request, fastapi_response: Response, get_vertex_pass_through_handler: BaseVertexAIPassThroughHandler, + general_settings: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth | None = None, router_credentials: LiteLLM_ManagedVectorStore | None = None, ): @@ -2904,6 +2947,7 @@ async def _base_vertex_proxy_route( request: FastAPI request object fastapi_response: FastAPI response object get_vertex_pass_through_handler: Handler for the specific Vertex AI service + general_settings: Proxy general settings, read per request for the anthropic-beta values to drop user_api_key_dict: User API key authentication dict router_credentials: Optional vector store credentials from registry (LiteLLM_ManagedVectorStore) """ @@ -3022,6 +3066,9 @@ async def _base_vertex_proxy_route( target=target, custom_headers=_upstream_headers_for_vertex_route(endpoint, headers), is_streaming_request=is_streaming_request, + upstream_headers_filter=partial( + _without_dropped_anthropic_betas, _vertex_dropped_anthropic_betas(general_settings, request.headers) + ), ) # dynamically construct pass-through endpoint based on incoming path try: @@ -3084,6 +3131,7 @@ async def vertex_discovery_proxy_route( request=request, fastapi_response=fastapi_response, get_vertex_pass_through_handler=discovery_handler, + general_settings=_proxy_general_settings(), router_credentials=vector_store_credentials, ) @@ -3117,6 +3165,7 @@ async def vertex_proxy_route( request=request, fastapi_response=fastapi_response, get_vertex_pass_through_handler=ai_platform_handler, + general_settings=_proxy_general_settings(), user_api_key_dict=user_api_key_dict, ) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index b1c448b023b..46b7a724809 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -1087,6 +1087,10 @@ from litellm.passthrough.timeout_utils import ( ) +def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]: + return headers + + async def pass_through_request( request: Request, target: str, @@ -1102,6 +1106,7 @@ async def pass_through_request( custom_llm_provider: str | None = None, guardrails_config: dict | None = None, timeout: float | None = None, + upstream_headers_filter: Callable[[Mapping[str, str]], Mapping[str, str]] = _unfiltered_headers, ): """ Pass through endpoint handler, makes the httpx request for pass-through endpoints and ensures logging hooks are called @@ -1122,6 +1127,7 @@ async def pass_through_request( guardrails_config: Optional field - guardrails configuration for passthrough endpoint timeout: Optional per-endpoint timeout in seconds. Falls back to general_settings.pass_through_request_timeout, then 600s. + upstream_headers_filter: Optional last rewrite of the headers sent upstream, after x-pass- expansion """ from litellm.exceptions import ModifyResponseException from litellm.litellm_core_utils.litellm_logging import Logging @@ -1149,13 +1155,15 @@ async def pass_through_request( ######################################################### try: url = httpx.URL(target) - headers = custom_headers - headers = HttpPassThroughEndpointHelpers.forward_headers_from_request( + forwarded_headers: Final = HttpPassThroughEndpointHelpers.forward_headers_from_request( request_headers=safe_get_request_headers(request).copy(), - headers=headers, + headers=custom_headers, forward_headers=forward_headers, ) - upstream_headers: Final = _with_trace_context(headers, parent_span=user_api_key_dict.parent_otel_span) + upstream_headers: Final = _with_trace_context( + upstream_headers_filter(forwarded_headers), + parent_span=user_api_key_dict.parent_otel_span, + ) requested_query_params: dict | None = query_params or dict(request.query_params) or None @@ -2105,6 +2113,7 @@ def create_pass_through_route( guardrails: dict[str, object] | None = None, config_file_path: str | None = None, timeout: float | None = None, + upstream_headers_filter: Callable[[Mapping[str, str]], Mapping[str, str]] = _unfiltered_headers, ): # check if target is an adapter.py or a url from litellm._uuid import uuid @@ -2244,6 +2253,7 @@ def create_pass_through_route( custom_llm_provider=custom_llm_provider, guardrails_config=cast(dict | None, param_guardrails), timeout=cast(float | None, param_timeout), + upstream_headers_filter=upstream_headers_filter, ) finally: if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e3ea9dabe50..69e92948a0e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7048,7 +7048,11 @@ class ProxyConfig: MappingProxyType( { name: typed_general_settings[name] - for name in ("vector_store_deny_by_default", "search_tool_deny_by_default") + for name in ( + "vector_store_deny_by_default", + "search_tool_deny_by_default", + "vertex_ai_pass_through_drop_anthropic_betas", + ) if name in typed_general_settings } ) diff --git a/tests/integration/providers/test_vertex_passthrough_wire.py b/tests/integration/providers/test_vertex_passthrough_wire.py index 98eb0547d42..148cdff9614 100644 --- a/tests/integration/providers/test_vertex_passthrough_wire.py +++ b/tests/integration/providers/test_vertex_passthrough_wire.py @@ -47,6 +47,7 @@ _PROJECT: Final = "scripted-project" _TOKEN: Final = "scripted-vertex-token" _CALLER_GOOGLE_TOKEN: Final = "caller-google-token" _GEMINI_ROUTER_NAME: Final = "vertex-gemini-router" +_REJECTED_BETA: Final = "files-api-2025-04-14" _REGIONAL: Final = "us-central1-aiplatform.googleapis.com:443" _GLOBAL: Final = "aiplatform.googleapis.com:443" _BYO: Final = "us-east5-aiplatform.googleapis.com:443" @@ -239,6 +240,10 @@ def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: }, }, ] + config["general_settings"] = { + **config["general_settings"], + "vertex_ai_pass_through_drop_anthropic_betas": [_REJECTED_BETA], + } path: Final = directory / "vertex-passthrough.yaml" path.write_text(yaml.safe_dump(config)) environment: Final = { @@ -564,3 +569,54 @@ def test_vertex_partner_stream_relays_a_tool_search_turn_verbatim_and_logs_it_on seconds=70, ) assert rows[0]["status"] == "success", rows + + +@pytest.mark.parametrize("alias", ["/vertex_ai", "/vertex-ai"]) +@pytest.mark.parametrize( + ("model_path", "caller_headers", "authority"), + [ + pytest.param( + f"/v1/projects/{_PROJECT}/locations/global/publishers/anthropic/models/claude-sonnet-4-5", + {}, + _GLOBAL, + id="use_in_pass_through", + ), + pytest.param( + "/v1/projects/byo-project/locations/us-east5/publishers/anthropic/models/claude-byo", + {"x-goog-api-key": _CALLER_GOOGLE_TOKEN}, + _BYO, + id="caller-credentials", + ), + ], +) +def test_vertex_partner_call_drops_the_configured_anthropic_beta_and_forwards_the_rest_in_order( + rig: _Rig, alias: str, model_path: str, caller_headers: dict[str, str], authority: str +) -> None: + body: Final[dict[str, JsonValue]] = { + "anthropic_version": "vertex-2023-10-16", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + } + with rig.proxy.scenario() as scenario: + key: Final = scenario.key() + response: Final = rig.proxy.client.post( + f"{alias}{model_path}:rawPredict", + json=body, + headers={ + "Authorization": f"Bearer {key}", + "anthropic-beta": f"interleaved-thinking-2025-05-14, {_REJECTED_BETA} ,effort-2025-11-24", + **caller_headers, + }, + ) + assert response.status_code == 200, response.text + assert response.json() == _CLAUDE_REPLY, response.text + received: Final = rig.vertex.drain() + assert [(request.method, request.target) for request in received] == [("POST", f"{model_path}:rawPredict")], ( + received + ) + assert rig.tunnel.drain() == [authority] + upstream: Final = received[0] + assert upstream.headers["anthropic-beta"] == "interleaved-thinking-2025-05-14,effort-2025-11-24", ( + upstream.headers + ) + assert _JSON_OBJECT.validate_json(upstream.body) == body, upstream.body diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index ed89cc37544..4dbe1f2ef8d 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -370,6 +370,7 @@ class TestVertexAIPassThroughHandler: target=f"https://{test_location}-aiplatform.googleapis.com/v1/projects/{test_project}/locations/{test_location}/publishers/google/models/gemini-1.5-flash:generateContent", custom_headers={"Authorization": f"Bearer {test_token}"}, is_streaming_request=False, + upstream_headers_filter=mock.ANY, ) @pytest.mark.asyncio @@ -469,6 +470,7 @@ class TestVertexAIPassThroughHandler: target=f"https://aiplatform.googleapis.com/v1/projects/{test_project}/locations/{test_location}/publishers/google/models/gemini-1.5-flash:generateContent", custom_headers={"Authorization": f"Bearer {test_token}"}, is_streaming_request=False, + upstream_headers_filter=mock.ANY, ) @pytest.mark.parametrize( @@ -564,6 +566,7 @@ class TestVertexAIPassThroughHandler: target=f"https://{default_location}-aiplatform.googleapis.com/v1/projects/{default_project}/locations/{default_location}/publishers/google/models/gemini-1.5-flash:generateContent", custom_headers={"Authorization": f"Bearer {default_credentials}"}, is_streaming_request=False, + upstream_headers_filter=mock.ANY, ) @pytest.mark.asyncio diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py index 98ea2269252..e658556bd1f 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py @@ -1,10 +1,16 @@ import asyncio import importlib import json +from collections.abc import Iterator from typing import Final +import httpx import pytest +import responses import respx +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from fastapi.testclient import TestClient import litellm from litellm import Router @@ -257,3 +263,259 @@ async def test_gemini_pass_through_endpoint_forwards_key_and_body( assert outbound.url.path == f"/{endpoint}" assert outbound.url.params["key"] == "gemini-test-key" assert json.loads(outbound.content) == json.loads(body) + + +_VERTEX_DEPLOYMENT_PATH: Final = ( + "v1/projects/deployment-project/locations/global/publishers/anthropic/models/claude-sonnet-4-5:rawPredict" +) +_VERTEX_CALLER_PATH: Final = ( + "v1/projects/caller-project/locations/global/publishers/anthropic/models/claude-haiku-4-5:rawPredict" +) +_VERTEX_DEPLOYMENT_PROJECT: Final = "deployment-project" +_VERTEX_TOKEN_URI: Final = "https://oauth2.googleapis.com/token" +_VERTEX_MINTED_TOKEN: Final = "minted-vertex-token" +_VERTEX_CALLER_GOOGLE_TOKEN: Final = "caller-google-token" +_VERTEX_REJECTED_BETA: Final = "files-api-2025-04-14" +_VERTEX_SENT_BETAS: Final = f"claude-code-20250219, {_VERTEX_REJECTED_BETA} ,effort-2025-11-24" +_VERTEX_CLAUDE_BODY: Final = { + "anthropic_version": "vertex-2023-10-16", + "max_tokens": 8, + "messages": [{"role": "user", "content": "hi"}], +} +_VERTEX_CLAUDE_REPLY: Final = {"id": "msg_vertex", "type": "message", "content": []} + + +@pytest.mark.parametrize( + ("header_name", "sent", "dropped", "expected"), + [ + pytest.param( + "anthropic-beta", + "claude-code-20250219,files-api-2025-04-14,effort-2025-11-24", + {"files-api-2025-04-14"}, + "claude-code-20250219,effort-2025-11-24", + id="drops-one-keeps-order", + ), + pytest.param( + "anthropic-beta", + " claude-code-20250219 , files-api-2025-04-14 ,effort-2025-11-24 ", + {"files-api-2025-04-14"}, + "claude-code-20250219,effort-2025-11-24", + id="whitespace-around-values", + ), + pytest.param( + "Anthropic-Beta", + "effort-2025-11-24,files-api-2025-04-14", + {"files-api-2025-04-14"}, + "effort-2025-11-24", + id="header-name-any-case", + ), + pytest.param( + "anthropic-beta", + "files-api-2025-04-14, skills-2025-10-02", + {"files-api-2025-04-14", "skills-2025-10-02"}, + None, + id="all-dropped-removes-header", + ), + pytest.param( + "anthropic-beta", + " claude-code-20250219 ,files-api-2025-04-14", + set(), + " claude-code-20250219 ,files-api-2025-04-14", + id="no-setting-byte-identical", + ), + pytest.param( + "anthropic-beta", + " claude-code-20250219 ,files-api-2025-04-14-preview", + {"files-api-2025-04-14"}, + " claude-code-20250219 ,files-api-2025-04-14-preview", + id="exact-match-only", + ), + ], +) +def test_vertex_upstream_headers_drop_only_the_configured_anthropic_betas( + header_name: str, sent: str, dropped: set[str], expected: str | None +) -> None: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + _without_dropped_anthropic_betas, + ) + + other_headers: Final = {"content-type": "application/json", "Authorization": "Bearer vertex-access-token"} + + upstream: Final = _without_dropped_anthropic_betas(frozenset(dropped), {header_name: sent, **other_headers}) + + expected_beta: Final = {} if expected is None else {header_name: expected} + assert dict(upstream) == {**expected_beta, **other_headers} + + +def _vertex_service_account_json() -> str: + private_key: Final = ( + rsa.generate_private_key(public_exponent=65537, key_size=2048) + .private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + .decode() + ) + return json.dumps( + { + "type": "service_account", + "project_id": _VERTEX_DEPLOYMENT_PROJECT, + "private_key_id": "test", + "private_key": private_key, + "client_email": f"test@{_VERTEX_DEPLOYMENT_PROJECT}.iam.gserviceaccount.com", + "client_id": "0", + "token_uri": _VERTEX_TOKEN_URI, + } + ) + + +@pytest.fixture +def vertex_proxy( + monkeypatch: pytest.MonkeyPatch, httpx_transport: None, isolate_passthrough_endpoint_router_state: None +) -> Iterator[TestClient]: + from litellm.proxy import proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "claude-sonnet-4-5", + "litellm_params": { + "model": "vertex_ai/claude-sonnet-4-5", + "vertex_project": _VERTEX_DEPLOYMENT_PROJECT, + "vertex_location": "global", + "vertex_credentials": _vertex_service_account_json(), + "use_in_pass_through": True, + }, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + with responses.RequestsMock(assert_all_requests_are_fired=False) as google_oauth: + google_oauth.post( + _VERTEX_TOKEN_URI, + json={"access_token": _VERTEX_MINTED_TOKEN, "expires_in": 3600, "token_type": "Bearer"}, + ) + yield TestClient(proxy_server.app) + + +@pytest.mark.parametrize("beta_header", ["anthropic-beta", "x-pass-anthropic-beta"]) +@pytest.mark.parametrize("alias", ["/vertex_ai", "/vertex-ai"]) +@pytest.mark.parametrize( + ("model_path", "caller_headers", "upstream_authorization"), + [ + pytest.param( + _VERTEX_DEPLOYMENT_PATH, + {}, + f"Bearer {_VERTEX_MINTED_TOKEN}", + id="use_in_pass_through-deployment", + ), + pytest.param( + _VERTEX_CALLER_PATH, + {"Authorization": f"Bearer {_VERTEX_CALLER_GOOGLE_TOKEN}"}, + f"Bearer {_VERTEX_CALLER_GOOGLE_TOKEN}", + id="caller-credentials", + ), + ], +) +def test_vertex_pass_through_drops_the_configured_anthropic_beta_before_calling_vertex( + vertex_proxy: TestClient, + respx_mock: "respx.MockRouter", + monkeypatch: pytest.MonkeyPatch, + beta_header: str, + alias: str, + model_path: str, + caller_headers: dict[str, str], + upstream_authorization: str, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setitem( + proxy_server.general_settings, "vertex_ai_pass_through_drop_anthropic_betas", [_VERTEX_REJECTED_BETA] + ) + upstream: Final = respx_mock.post(f"https://aiplatform.googleapis.com/{model_path}").mock( + return_value=httpx.Response(200, json=_VERTEX_CLAUDE_REPLY) + ) + + response: Final = vertex_proxy.post( + f"{alias}/{model_path}", + json=_VERTEX_CLAUDE_BODY, + headers={"x-litellm-api-key": "sk-caller", beta_header: _VERTEX_SENT_BETAS, **caller_headers}, + ) + + assert (response.status_code, response.json()) == (200, _VERTEX_CLAUDE_REPLY) + outbound: Final = upstream.calls.last.request + assert outbound.headers["anthropic-beta"] == "claude-code-20250219,effort-2025-11-24" + assert outbound.headers["authorization"] == upstream_authorization + assert json.loads(outbound.content) == _VERTEX_CLAUDE_BODY + + + +@pytest.mark.parametrize( + "configured", + [[f" {_VERTEX_REJECTED_BETA} "], [f"{_VERTEX_REJECTED_BETA},afk-mode-2026-01-31"]], + ids=["spaces-around-the-entry", "comma-separated-entry"], +) +def test_vertex_pass_through_matches_a_configured_beta_written_the_way_the_header_is( + vertex_proxy: TestClient, respx_mock: "respx.MockRouter", monkeypatch: pytest.MonkeyPatch, configured: list[str] +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "vertex_ai_pass_through_drop_anthropic_betas", configured) + upstream: Final = respx_mock.post(f"https://aiplatform.googleapis.com/{_VERTEX_CALLER_PATH}").mock( + return_value=httpx.Response(200, json=_VERTEX_CLAUDE_REPLY) + ) + + response: Final = vertex_proxy.post( + f"/vertex_ai/{_VERTEX_CALLER_PATH}", + json=_VERTEX_CLAUDE_BODY, + headers={ + "x-litellm-api-key": "sk-caller", + "Authorization": f"Bearer {_VERTEX_CALLER_GOOGLE_TOKEN}", + "anthropic-beta": _VERTEX_SENT_BETAS, + }, + ) + + assert (response.status_code, response.json()) == (200, _VERTEX_CLAUDE_REPLY) + assert upstream.calls.last.request.headers["anthropic-beta"] == "claude-code-20250219,effort-2025-11-24" + +def test_vertex_pass_through_refuses_a_malformed_drop_setting_without_calling_vertex( + vertex_proxy: TestClient, respx_mock: "respx.MockRouter", monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setitem( + proxy_server.general_settings, "vertex_ai_pass_through_drop_anthropic_betas", _VERTEX_REJECTED_BETA + ) + + response: Final = vertex_proxy.post( + f"/vertex_ai/{_VERTEX_CALLER_PATH}", + json=_VERTEX_CLAUDE_BODY, + headers={ + "x-litellm-api-key": "sk-caller", + "Authorization": f"Bearer {_VERTEX_CALLER_GOOGLE_TOKEN}", + "anthropic-beta": _VERTEX_SENT_BETAS, + }, + ) + + assert response.status_code == 500 + assert "vertex_ai_pass_through_drop_anthropic_betas must be a list of strings" in response.text + assert respx_mock.calls.call_count == 0 + + +def test_vertex_pass_through_forwards_a_request_without_betas_despite_a_malformed_drop_setting( + vertex_proxy: TestClient, respx_mock: "respx.MockRouter", monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setitem( + proxy_server.general_settings, "vertex_ai_pass_through_drop_anthropic_betas", _VERTEX_REJECTED_BETA + ) + upstream: Final = respx_mock.post(f"https://aiplatform.googleapis.com/{_VERTEX_CALLER_PATH}").mock( + return_value=httpx.Response(200, json=_VERTEX_CLAUDE_REPLY) + ) + + response: Final = vertex_proxy.post( + f"/vertex_ai/{_VERTEX_CALLER_PATH}", + json=_VERTEX_CLAUDE_BODY, + headers={"x-litellm-api-key": "sk-caller", "Authorization": f"Bearer {_VERTEX_CALLER_GOOGLE_TOKEN}"}, + ) + + assert (response.status_code, response.json()) == (200, _VERTEX_CLAUDE_REPLY) + assert "anthropic-beta" not in upstream.calls.last.request.headers diff --git a/tests/unit/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py b/tests/unit/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py index 2922790ea37..ed14eeda6a2 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py +++ b/tests/unit/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py @@ -91,6 +91,7 @@ async def test_vertex_passthrough_load_balancing(): request=mock_request, fastapi_response=mock_response, get_vertex_pass_through_handler=mock_handler, + general_settings={}, ) # Verify @@ -404,6 +405,7 @@ async def test_vertex_passthrough_drops_anthropic_beta_only_on_count_tokens( request=MagicMock(), fastapi_response=MagicMock(), get_vertex_pass_through_handler=VertexAIPassThroughHandler(), + general_settings={}, ) upstream_headers = mock_create_route.call_args.kwargs["custom_headers"] @@ -747,6 +749,7 @@ async def test_vertex_passthrough_custom_model_name_replaced_in_url(): request=mock_request, fastapi_response=mock_response, get_vertex_pass_through_handler=mock_handler, + general_settings={}, ) # Verify the router was called with the custom model name (extracted from URL) @@ -836,6 +839,7 @@ async def test_vertex_passthrough_attributes_the_call_to_the_resolved_deployment request=mock_request, fastapi_response=MagicMock(), get_vertex_pass_through_handler=mock_handler, + general_settings={}, ) assert logging_kwargs["litellm_params"]["metadata"]["model_info"]["id"] == "vertex-gemini-38-flash-dep" diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 8837fa095e8..e59dd19271c 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -2579,6 +2579,26 @@ async def test_load_config_rejects_non_boolean_deny_by_default( await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "yaml_value", ["files-api-2025-04-14", "[files-api-2025-04-14, 7]"], ids=["bare-string", "non-string-entry"] +) +async def test_load_config_rejects_a_malformed_vertex_drop_anthropic_betas_list( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, yaml_value: str +): + config_file: Final = tmp_path / "vertex_drop_betas.yaml" + config_file.write_text( + "model_list: []\nlitellm_settings: {}\ngeneral_settings:\n" + f" vertex_ai_pass_through_drop_anthropic_betas: {yaml_value}\n" + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + + with pytest.raises(ValidationError, match="vertex_ai_pass_through_drop_anthropic_betas"): + await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + @pytest.mark.asyncio async def test_ProxyConfig_load_config_resolves_router_settings_plugins(tmp_path, monkeypatch): """Regression: router_settings.plugins dotted-path strings must be resolved to diff --git a/tests/unit/proxy/vector_store_endpoints/test_vector_store_tenant_guard.py b/tests/unit/proxy/vector_store_endpoints/test_vector_store_tenant_guard.py index 981fbfcf3e5..29efc41b484 100644 --- a/tests/unit/proxy/vector_store_endpoints/test_vector_store_tenant_guard.py +++ b/tests/unit/proxy/vector_store_endpoints/test_vector_store_tenant_guard.py @@ -451,6 +451,7 @@ async def test_vertex_discovery_denies_other_team_vector_store_credentials(): request=request, fastapi_response=Response(), get_vertex_pass_through_handler=MagicMock(), + general_settings={}, router_credentials=vector_store_credentials, ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c33ef86af72..a111ed1d773 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30279,6 +30279,11 @@ export interface components { * @default false */ vector_store_deny_by_default: boolean; + /** + * Vertex Ai Pass Through Drop Anthropic Betas + * @description anthropic-beta header values the Vertex AI pass-through routes (/vertex_ai/* and /vertex-ai/*) remove before calling Vertex, for a beta a Vertex Claude model rejects, whether the client sent them as anthropic-beta or x-pass-anthropic-beta. Every other value the client sent is forwarded unchanged, and the header is left out when no value remains. Unset forwards the header as sent. + */ + vertex_ai_pass_through_drop_anthropic_betas?: string[] | null; }; /** ConfigList */ ConfigList: { From 7e6fb09da3987c8b75062307695f9083da55feed Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:58:34 +0000 Subject: [PATCH 04/50] fix(streaming)!: map stream read timeouts to litellm.Timeout on chat, messages, and responses (#47198) * fix(streaming): map stream read timeouts to litellm.Timeout * revert(streaming): move the timeout mapping to a later commit * fix(streaming): map stream read timeouts to litellm.Timeout on chat, messages, and responses * fix(streaming): keep stream fallback and cooldown unchanged for transport timeouts A chat stream served through the Responses bridge now keeps mid-stream fallback on a stall, because the fallback exemption also covers a litellm.Timeout whose cause is an httpx transport timeout. The Responses stream hands the raw transport error to failure callbacks again, so router cooldown sees a stall as it did before. The 408 deployment info now lists stream_timeout. * test(integration): cover a stalled hosted_vllm stream on chat, Responses, and Messages with its spend log row * fix(router): keep a fallback deployment whose stream stalled out of cooldown * test(integration): cover bridged chat, fallback and cooldown for stalled hosted_vllm streams --- .../exception_mapping_utils.py | 5 +- .../litellm_core_utils/streaming_handler.py | 20 +- litellm/responses/streaming_iterator.py | 20 +- litellm/router.py | 11 +- litellm/router_utils/cooldown_handlers.py | 5 + .../router_utils/fallback_event_handlers.py | 8 + .../test_hosted_vllm_stream_stall_wire.py | 364 ++++++++++++++++++ .../test_exception_mapping_utils.py | 20 + .../test_streaming_handler.py | 50 ++- .../unit/responses/test_streaming_iterator.py | 37 ++ .../test_fallback_event_handlers.py | 65 ++++ tests/unit/test_router_exception_redaction.py | 25 ++ 12 files changed, 617 insertions(+), 13 deletions(-) create mode 100644 tests/integration/providers/test_hosted_vllm_stream_stall_wire.py diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index c52bc534058..1f49cc2c4bb 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -2551,7 +2551,8 @@ def exception_type( ################################################################################ if ( - "Request Timeout Error" in error_str + isinstance(original_exception, httpx.TimeoutException) + or "Request Timeout Error" in error_str or "Request timed out" in error_str or "Timed out generating response" in error_str or "The read operation timed out" in error_str @@ -2559,7 +2560,7 @@ def exception_type( exception_mapping_worked = True raise Timeout( - message=f"APITimeoutError - Request timed out. Error_str: {error_str}", + message=f"APITimeoutError - Request timed out. Error_str: {error_str or type(mappable_exception).__name__}", model=model, llm_provider=custom_llm_provider, litellm_debug_info=extra_information, diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 78cfb58bedf..e0155f88de1 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -45,7 +45,7 @@ from litellm.types.utils import ( ) from litellm.types.utils import GenericStreamingChunk as GChunk -from ..exceptions import OpenAIError +from ..exceptions import OpenAIError, Timeout from .core_helpers import map_finish_reason, process_response_headers from .exception_mapping_utils import exception_type, with_deployment_api_base from .llm_response_utils.get_api_base import get_api_base @@ -88,6 +88,16 @@ def _apply_provider_fields(response: ModelResponseStream, fields: Mapping[str, o setattr(response, key, value) +def _is_transport_timeout(error: BaseException) -> bool: + return isinstance(error, httpx.TimeoutException) or ( + isinstance(error, Timeout) and isinstance(error.__cause__, httpx.TimeoutException) + ) + + +def _skips_stream_fallback(status_code: int | None) -> bool: + return status_code is not None and 400 <= status_code < 500 and status_code != 429 + + def is_async_iterable(obj: object) -> bool: """ Check if an object is an async iterable (can be used with 'async for'). @@ -2384,6 +2394,8 @@ class CustomStreamWrapper: 429 (rate-limit) is explicitly exempted from the 4xx filter because it is transient and the Router should switch to another model group. + So is a transport read timeout, which maps to a 408 Timeout, including + one an inner Responses stream already mapped for the chat bridge. An error an inner stream already wrapped (the chat-to-Responses bridge consumes a Responses stream) is rebuilt around the provider exception @@ -2448,9 +2460,9 @@ class CustomStreamWrapper: # Raise non-retriable client errors directly (skip fallback). # Exception: 429 (rate-limit) IS retriable/transient — allow it # through so the Router can switch to a different model group. - if mapped_status_code is not None and 400 <= mapped_status_code < 500 and mapped_status_code != 429: - raise mapped_exception - if original_status_code is not None and 400 <= original_status_code < 500 and original_status_code != 429: + if not _is_transport_timeout(e) and ( + _skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code) + ): raise mapped_exception raise MidStreamFallbackError( diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 145189616af..3d07f22c81a 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -293,6 +293,15 @@ def _map_stream_error_to_exception( body=error_body, headers=headers, ) + return _mapped_exception(provider_exception, model, custom_llm_provider, logging_obj) + + +def _mapped_exception( + original_exception: Exception, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, +) -> Exception: litellm_params: Final = ( None if logging_obj is None @@ -302,7 +311,7 @@ def _map_stream_error_to_exception( return litellm.exception_type( model=model, custom_llm_provider=custom_llm_provider, - original_exception=provider_exception, + original_exception=original_exception, completion_kwargs={}, extra_kwargs=with_deployment_api_base(litellm_params), ) @@ -1044,6 +1053,11 @@ class BaseResponsesAPIStreamingIterator: raise error raise self._fallback_error(error) from error + def _raise_for_transport_timeout(self, error: httpx.TimeoutException) -> NoReturn: + self.finished = True + self._handle_failure(error) + raise _mapped_exception(error, self.model or "", self.custom_llm_provider or "", self.logging_obj) from error + def _raise_for_transport_error(self, error: httpx.ReadError | httpx.RemoteProtocolError) -> NoReturn: self._handle_failure(error) if self._output_started: @@ -1140,6 +1154,8 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): if self.completed_response is not None: raise StopAsyncIteration from e self._raise_for_transport_error(e) + except httpx.TimeoutException as e: + self._raise_for_transport_timeout(e) except httpx.HTTPError as e: # Handle HTTP errors self.finished = True @@ -1223,6 +1239,8 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): if self.completed_response is not None: raise StopIteration from e self._raise_for_transport_error(e) + except httpx.TimeoutException as e: + self._raise_for_transport_timeout(e) except httpx.HTTPError as e: # Handle HTTP errors self.finished = True diff --git a/litellm/router.py b/litellm/router.py index c30108b4e38..65ba67d9ab8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4377,12 +4377,15 @@ class Router: return response except litellm.Timeout as e: - deployment_request_timeout_param: Final = _timeout_debug_deployment_dict.get("litellm_params", {}).get( - "request_timeout", None + deployment_litellm_params: Final[Mapping[str, object]] = _timeout_debug_deployment_dict.get( + "litellm_params", {} ) - deployment_timeout_param = _timeout_debug_deployment_dict.get("litellm_params", {}).get("timeout", None) if litellm.expose_router_debug_in_errors: - e.message += f"\n\nDeployment Info: request_timeout: {deployment_request_timeout_param}\ntimeout: {deployment_timeout_param}" + e.message += ( + f"\n\nDeployment Info: request_timeout: {deployment_litellm_params.get('request_timeout')}" + f"\ntimeout: {deployment_litellm_params.get('timeout')}" + f"\nstream_timeout: {deployment_litellm_params.get('stream_timeout')}" + ) # Set per-deployment num_retries on exception for retry logic if deployment is not None: self._set_deployment_num_retries_on_exception(e, deployment) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index bb64d912c4c..bbd895320bd 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -26,6 +26,7 @@ from litellm.constants import ( INTERNAL_CALL_ORIGIN_METADATA_KEY, SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD, ) +from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCacheValue from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback @@ -757,6 +758,10 @@ def is_caller_timeout_408( return (finished - started).total_seconds() >= timeout +def is_stream_stall(exception: BaseException) -> bool: + return isinstance(exception, litellm.Timeout) and isinstance(exception.__cause__, MidStreamFallbackError) + + def is_caller_scoped_auth_failure(deployment: Deployment | None, exception_status: str | int) -> bool: if cast_exception_status_to_int(exception_status) not in (401, 403) or deployment is None: return False diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 06bd3329cc1..bc870572f38 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -26,6 +26,7 @@ from litellm.router_utils.cooldown_handlers import ( is_advisor_orchestration_failure, is_caller_scoped_auth_failure, is_caller_timeout_408, + is_stream_stall, set_cooldown_deployments, ) from litellm.router_utils.router_callbacks.track_deployment_metrics import ( @@ -73,6 +74,13 @@ def _trigger_cooldown_for_failed_deployment( ) return + if is_stream_stall(exception): + verbose_router_logger.debug( + "Not triggering cooldown for fallback deployment: its stream stalled on a read timeout, " + "which stays out of cooldown as it does for the primary deployment." + ) + return + exception_status: Final[str | int] = getattr(exception, "status_code", "") # Generic API calls (files, batches, threads, rerank, ...) take a caller-supplied diff --git a/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py b/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py new file mode 100644 index 00000000000..fb58885e011 --- /dev/null +++ b/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py @@ -0,0 +1,364 @@ +import threading +import uuid +from collections.abc import Generator, Iterator, Mapping +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from typing import Final, Literal + +import httpx +import pytest +from integration._support.anthropic_sse import SseEvent, event_types, parse_sse +from integration._support.client import ( + Gateway, + Scenario, + eventually, + gateway_from_environment, + object_value, + string_value, +) +from integration._support.database import read_rows +from integration._support.openai_wire import ( + answering_model_discovery, + chat_stream, + openai_error, + posted_targets, + responses_stream, +) +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.redis_process import OwnedRedis, owned_redis +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.routing.test_payment_required_cooldown import cooldown_config, cooled, router_settings +from integration.routing.test_payment_required_mapping import deployment +from pydantic import JsonValue + +_BACKEND: Final = "glm-5-fp8" +_HOSTED: Final = f"hosted_vllm/{_BACKEND}" +_BRIDGED: Final = "openai/gpt-5.3-codex" +_PROVIDER_KEY: Final = "synthetic-hosted-vllm-key" +_ANSWER: Final = "scripted answer before the upstream stalls" +_FALLBACK_ANSWER: Final = "scripted answer from the fallback deployment" +_PROMPT: Final = "Keep streaming while the upstream thinks." +_CHAT_TARGET: Final = "/v1/chat/completions" +_RESPONSES_TARGET: Final = "/v1/responses" +_STREAM_TIMEOUT_SECONDS: Final = 1 +_DEPLOYMENT_TIMEOUT_SECONDS: Final = 30 +_PAUSE_SECONDS: Final = 3.0 +_HOLD_SECONDS: Final = 30.0 +_ROW_SECONDS: Final = 60 +_EXTRA: Final[dict[str, JsonValue]] = {"num_retries": 0, "cache": {"no-cache": True}} +_ROWS: Final = 'SELECT status, metadata FROM "LiteLLM_SpendLogs" WHERE model_group=%s' +_EITHER_GROUP_ROWS: Final = 'SELECT status, metadata FROM "LiteLLM_SpendLogs" WHERE model_group IN (%s, %s)' +_OWNED_PROXY: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +Endpoint = Literal["chat", "responses", "messages"] +Shape = Literal["prompt", "pause_after_content", "pause_after_role", "stall_before_first", "hold_headers", "fail"] + + +def _stream_parts(target: str, answer: str) -> tuple[bytes, bytes, bytes]: + identity: Final = f"stall-{uuid.uuid4().hex}" + if target == _RESPONSES_TARGET: + return responses_stream(identity, _BACKEND, answer) + return chat_stream(identity, _BACKEND, answer) + + +def _sse(chunks: tuple[bytes, ...], *, pause: float = 0, gate: threading.Event | None = None) -> Reply: + return Reply(content_type="text/event-stream", chunks=chunks, pause_between_chunks=pause, gate_before_first=gate) + + +@dataclass(frozen=True, slots=True) +class _Peer: + shape: Shape + answer: str + release: threading.Event + + def __call__(self, request: Request) -> Reply: + assert request.method == "POST", request + assert request.target in (_CHAT_TARGET, _RESPONSES_TARGET), request + opened, delta, closed = _stream_parts(request.target, self.answer) + match self.shape: + case "fail": + return openai_error(500) + case "hold_headers": + assert self.release.wait(_HOLD_SECONDS), request + return _sse((opened, delta, closed)) + case "pause_after_content": + return _sse((opened + delta, closed), pause=_PAUSE_SECONDS) + case "pause_after_role": + return _sse((opened, delta + closed), pause=_PAUSE_SECONDS) + case "stall_before_first": + return _sse((opened, delta, closed), gate=self.release) + case "prompt": + return _sse((opened, delta, closed)) + + +@contextmanager +def _peer(shape: Shape, answer: str = _ANSWER) -> Generator[Wire]: + release: Final = threading.Event() + with wire_server(answering_model_discovery(_Peer(shape, answer, release))) as wire: + try: + yield wire + finally: + release.set() + + +def _params(wire: Wire, backend: str) -> dict[str, JsonValue]: + return { + "model": backend, + "api_base": f"{wire.url}/v1", + "api_key": _PROVIDER_KEY, + "timeout": _DEPLOYMENT_TIMEOUT_SECONDS, + "stream_timeout": _STREAM_TIMEOUT_SECONDS, + } + + +def _deployment(scenario: Scenario, wire: Wire, backend: str = _HOSTED) -> tuple[str, str]: + group: Final = f"integration-stall-{uuid.uuid4().hex[:12]}" + return group, deployment(scenario, group, _params(wire, backend)) + + +def _path(endpoint: Endpoint) -> str: + match endpoint: + case "chat": + return "/v1/chat/completions" + case "responses": + return "/v1/responses" + case "messages": + return "/v1/messages" + + +def _body(endpoint: Endpoint, model: str, extra: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + match endpoint: + case "chat": + return {"model": model, "messages": [{"role": "user", "content": _PROMPT}], "stream": True, **extra} + case "responses": + return {"model": model, "input": _PROMPT, "stream": True, **extra} + case "messages": + return { + "model": model, + "messages": [{"role": "user", "content": _PROMPT}], + "max_tokens": 16, + "stream": True, + **extra, + } + + +def _streamed(gateway: Gateway, endpoint: Endpoint, model: str, **extra: JsonValue) -> httpx.Response: + return gateway.request("POST", _path(endpoint), _body(endpoint, model, {**_EXTRA, **extra})) + + +def _upstream_targets(wire: Wire) -> tuple[str, ...]: + eventually(wire.received.qsize, lambda size: size >= 1, 10) + return posted_targets(wire) + + +def _chat_errors(events: tuple[SseEvent, ...]) -> tuple[dict[str, JsonValue], ...]: + return tuple(object_value(event.data["error"]) for event in events if "error" in event.data) + + +def _error_class(message: JsonValue) -> str: + return string_value(message).split(":", 1)[0] + + +def _failure_row(query: str, groups: tuple[str, ...]) -> dict[str, JsonValue]: + rows: Final = eventually(lambda: read_rows(query, groups), lambda found: len(found) >= 1, seconds=_ROW_SECONDS) + assert [row["status"] for row in rows] == ["failure"], rows + return object_value(object_value(rows[0]["metadata"])["error_information"]) + + +def _logged_failure(model: str) -> tuple[JsonValue, JsonValue]: + error: Final = _failure_row(_ROWS, (model,)) + return error["error_code"], error["error_class"] + + +@pytest.mark.parametrize( + ("backend", "target"), + ( + pytest.param(_HOSTED, _CHAT_TARGET, id="hosted-vllm"), + pytest.param(_BRIDGED, _RESPONSES_TARGET, id="bridged-to-responses"), + ), +) +def test_a_chat_stream_that_stalls_past_its_stream_timeout_ends_with_a_408_timeout_and_logs_it( + gateway: Gateway, backend: str, target: str +) -> None: + with _peer("pause_after_content") as wire, gateway.scenario() as scenario: + model, _ = _deployment(scenario, wire, backend) + response: Final = _streamed(gateway, "chat", model) + assert response.status_code == 200, response.text + assert _ANSWER in response.text, response.text + events: Final = parse_sse(response.text) + assert _chat_errors(events[:-1]) == () and "error" in events[-1].data, response.text + error: Final = object_value(events[-1].data["error"]) + observed: Final = (error["code"], _error_class(error["message"]), *_logged_failure(model)) + assert observed == ("408", "litellm.Timeout", "408", "Timeout"), (error, observed) + assert _upstream_targets(wire) == (target,) + + +def test_a_native_responses_stream_that_stalls_past_its_stream_timeout_fails_with_request_timeout_and_logs_it( + gateway: Gateway, +) -> None: + with _peer("pause_after_content") as wire, gateway.scenario() as scenario: + model, _ = _deployment(scenario, wire) + response: Final = _streamed(gateway, "responses", model) + assert response.status_code == 200, response.text + events: Final = parse_sse(response.text) + types: Final = event_types(events) + assert "response.output_text.delta" in types, response.text + assert types[-1] == "response.failed" and "response.completed" not in types, response.text + failed: Final = object_value(object_value(events[-1].data["response"])["error"]) + observed: Final = (failed["code"], *_logged_failure(model)) + assert observed == ("request_timeout", "408", "Timeout"), (failed, observed) + assert _upstream_targets(wire) == (_RESPONSES_TARGET,) + + +def test_a_messages_stream_that_stalls_past_its_stream_timeout_ends_with_a_timeout_error_event_and_logs_it( + gateway: Gateway, +) -> None: + with _peer("pause_after_content") as wire, gateway.scenario() as scenario: + model, _ = _deployment(scenario, wire) + response: Final = _streamed(gateway, "messages", model) + assert response.status_code == 200, response.text + events: Final = parse_sse(response.text) + types: Final = event_types(events) + assert "content_block_delta" in types and "message_stop" not in types, response.text + assert types[-1] == "error", response.text + error: Final = object_value(events[-1].data["error"]) + observed: Final = (_error_class(error["message"]), *_logged_failure(model)) + assert observed == ("litellm.Timeout", "408", "Timeout"), (error, observed) + assert _upstream_targets(wire) == (_CHAT_TARGET,) + + +def test_a_chat_stream_that_never_pauses_is_answered_in_full_under_the_same_stream_timeout(gateway: Gateway) -> None: + with _peer("prompt") as wire, gateway.scenario() as scenario: + model, _ = _deployment(scenario, wire) + response: Final = _streamed(gateway, "chat", model) + assert response.status_code == 200, response.text + assert _ANSWER in response.text, response.text + assert response.text.rstrip().endswith("data: [DONE]"), response.text + events: Final = parse_sse(response.text) + assert _chat_errors(events) == (), response.text + assert _finish_reasons(events) == ("stop",), response.text + assert _upstream_targets(wire) == (_CHAT_TARGET,) + rows: Final = eventually(lambda: read_rows(_ROWS, (model,)), lambda found: len(found) >= 1, _ROW_SECONDS) + assert [row["status"] for row in rows] == ["success"], rows + + +@pytest.mark.parametrize( + ("endpoint", "shape"), + ( + pytest.param("chat", "pause_after_role", id="chat-after-the-role-chunk"), + pytest.param("messages", "stall_before_first", id="messages-before-the-first-chunk"), + ), +) +def test_a_stream_that_stalls_before_any_content_falls_back_to_the_next_deployment( + gateway: Gateway, endpoint: Endpoint, shape: Shape +) -> None: + with _peer(shape) as stalled, _peer("prompt", _FALLBACK_ANSWER) as healthy, gateway.scenario() as scenario: + primary, _ = _deployment(scenario, stalled) + fallback, _ = _deployment(scenario, healthy) + response: Final = _streamed(gateway, endpoint, primary, fallbacks=[fallback]) + assert response.status_code == 200, response.text + assert _FALLBACK_ANSWER in response.text and _ANSWER not in response.text, response.text + events: Final = parse_sse(response.text) + assert _chat_errors(events) == () and _completed(endpoint, events), response.text + assert (_upstream_targets(stalled), _upstream_targets(healthy)) == ((_CHAT_TARGET,), (_CHAT_TARGET,)) + + +@dataclass(frozen=True, slots=True) +class _CooldownRig: + gateway: Gateway + cache: OwnedRedis + + +@pytest.fixture(scope="module") +def cooldown_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_CooldownRig]: + directory: Final = tmp_path_factory.mktemp("stream-stall-cooldown") + with ExitStack() as stack: + upstream: Final = stack.enter_context(gateway_from_environment()) + cache: Final = stack.enter_context(owned_redis(directory)) + owned: Final = stack.enter_context( + owned_proxy_process( + upstream, + directory, + {"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2"}, + config=cooldown_config(directory, router_settings(cache, allowed_fails=0), prometheus=False), + ) + ) + yield _CooldownRig(owned.gateway, cache) + + +@_OWNED_PROXY +@pytest.mark.parametrize( + ("endpoint", "backend", "shape"), + ( + pytest.param("chat", _HOSTED, "pause_after_content", id="chat"), + pytest.param("chat", _HOSTED, "stall_before_first", id="chat-before-the-first-chunk"), + pytest.param("responses", _HOSTED, "pause_after_content", id="responses"), + pytest.param("chat", _BRIDGED, "pause_after_content", id="chat-bridged-to-responses"), + ), +) +def test_a_deployment_whose_stream_stalls_stays_out_of_cooldown( + cooldown_rig: _CooldownRig, endpoint: Endpoint, backend: str, shape: Shape +) -> None: + with _peer(shape) as wire, cooldown_rig.gateway.scenario() as scenario: + group, identity = _deployment(scenario, wire, backend) + response: Final = _streamed(cooldown_rig.gateway, endpoint, group) + error: Final = _failure_row(_ROWS, (group,)) + assert not cooled(cooldown_rig.cache, identity), (error, response.text) + + +@_OWNED_PROXY +def test_a_deployment_whose_stream_times_out_before_it_opens_is_cooled_down(cooldown_rig: _CooldownRig) -> None: + with _peer("hold_headers") as wire, cooldown_rig.gateway.scenario() as scenario: + group, identity = _deployment(scenario, wire) + response: Final = _streamed(cooldown_rig.gateway, "chat", group) + assert response.status_code == 408, response.text + assert eventually(lambda: cooled(cooldown_rig.cache, identity), bool, _ROW_SECONDS) + + +def _fallback_cooled(cooldown_rig: _CooldownRig, fallback_shape: Shape) -> tuple[bool, dict[str, JsonValue], str]: + with ( + _peer("fail") as failing, + _peer(fallback_shape) as fallback_wire, + cooldown_rig.gateway.scenario() as scenario, + ): + primary, _ = _deployment(scenario, failing) + fallback, identity = _deployment(scenario, fallback_wire) + response: Final = _streamed(cooldown_rig.gateway, "chat", primary, fallbacks=[fallback]) + error: Final = _failure_row(_EITHER_GROUP_ROWS, (primary, fallback)) + assert _upstream_targets(fallback_wire) == (_CHAT_TARGET,), response.text + return cooled(cooldown_rig.cache, identity), error, response.text + + +@_OWNED_PROXY +def test_a_fallback_deployment_whose_stream_stalls_stays_out_of_cooldown(cooldown_rig: _CooldownRig) -> None: + is_cooled, error, text = _fallback_cooled(cooldown_rig, "stall_before_first") + assert not is_cooled, (error, text) + + +@_OWNED_PROXY +def test_a_fallback_deployment_whose_stream_times_out_before_it_opens_is_cooled_down( + cooldown_rig: _CooldownRig, +) -> None: + is_cooled, error, text = _fallback_cooled(cooldown_rig, "hold_headers") + assert is_cooled, (error, text) + + +def _completed(endpoint: Endpoint, events: tuple[SseEvent, ...]) -> bool: + match endpoint: + case "chat": + return _finish_reasons(events) == ("stop",) + case "messages": + return event_types(events)[-1:] == ("message_stop",) + case "responses": + return event_types(events)[-1:] == ("response.completed",) + + +def _finish_reasons(events: tuple[SseEvent, ...]) -> tuple[JsonValue, ...]: + choices: Final = tuple(_first_choice(event.data) for event in events if event.data.get("choices")) + return tuple(choice["finish_reason"] for choice in choices if choice.get("finish_reason")) + + +def _first_choice(data: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + choices: Final = data["choices"] + assert isinstance(choices, list), data + return object_value(choices[0]) diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 731e842090a..5ee14ff530d 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -2781,3 +2781,23 @@ def test_missing_aws_dependency_is_not_mapped_to_provider_failure(provider, depe def test_non_aws_import_failure_keeps_provider_mapping(failure): with pytest.raises(litellm.APIConnectionError): exception_type(model="test-model", original_exception=failure, custom_llm_provider="openai") + + +@pytest.mark.parametrize( + "transport_timeout,reason", + [ + (httpx.ReadTimeout(""), "ReadTimeout"), + (httpx.ReadTimeout("Timeout on reading data from socket"), "Timeout on reading data from socket"), + (httpx.TimeoutException(""), "TimeoutException"), + ], + ids=["httpx_read_timeout", "aiohttp_read_timeout", "timeout_base_class"], +) +@pytest.mark.parametrize("provider", ["hosted_vllm", "openai", "anthropic"]) +def test_transport_timeout_maps_to_timeout( + transport_timeout: httpx.TimeoutException, reason: str, provider: str +) -> None: + with pytest.raises(litellm.Timeout) as exc_info: + exception_type(model="glm", original_exception=transport_timeout, custom_llm_provider=provider) + + assert exc_info.value.status_code == 408 + assert f"Request timed out. Error_str: {reason}" in str(exc_info.value) diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index c07fe2d1c31..50bcc14b681 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -2,7 +2,7 @@ import asyncio import json import threading import time -from typing import AsyncIterator, Final, Iterator, Literal, NoReturn, Optional +from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx @@ -1382,7 +1382,53 @@ async def test_async_streaming_read_timeout_triggers_midstream_fallback( await response.__anext__() assert excinfo.value.is_pre_first_chunk is True - assert isinstance(excinfo.value.original_exception, Exception) + assert isinstance(excinfo.value.original_exception, litellm.Timeout) + + +def _raw_read_timeout() -> NoReturn: + raise httpx.ReadTimeout("") + + +def _bridged_read_timeout() -> NoReturn: + raise litellm.Timeout( + message="APITimeoutError - Request timed out. Error_str: ReadTimeout", model="glm", llm_provider="hosted_vllm" + ) from httpx.ReadTimeout("") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", [_raw_read_timeout, _bridged_read_timeout], ids=["raw", "responses_bridge"]) +async def test_async_streaming_read_timeout_after_first_chunk_surfaces_timeout( + logging_obj: Logging, stall: Callable[[], NoReturn] +): + from litellm.exceptions import MidStreamFallbackError + + class _OneChunkThenStall: + def __init__(self) -> None: + self.sent = False + + def __aiter__(self) -> "_OneChunkThenStall": + return self + + async def __anext__(self) -> ModelResponseStream: + if self.sent: + stall() + self.sent = True + return ModelResponseStream(choices=[{"index": 0, "delta": {"content": "hello"}}]) + + response: Final = CustomStreamWrapper( + completion_stream=_OneChunkThenStall(), + model="glm", + logging_obj=logging_obj, + custom_llm_provider="hosted_vllm", + ) + + with pytest.raises(MidStreamFallbackError) as excinfo: + async for _ in response: + pass + + assert excinfo.value.is_pre_first_chunk is False + assert isinstance(excinfo.value.original_exception, litellm.Timeout) + assert excinfo.value.original_exception.status_code == 408 _AZURE_DEPLOYMENT_API_BASE: Final = "https://mid-stream-debug.openai.azure.com" diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 7cdac6c7581..7eb3115117d 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -323,6 +323,43 @@ async def test_transport_error_after_output_started_is_not_fallback_eligible(): _assert_failure_logged_once(logging_obj, trailing_error) +_READ_TIMEOUT_PREFIXES: Final = [ + pytest.param([], id="before-output"), + pytest.param(_PARTIAL_OUTPUT_EVENTS, id="after-output"), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefix", _READ_TIMEOUT_PREFIXES) +async def test_transport_read_timeout_raises_litellm_timeout(prefix): + logging_obj: Final = _failure_tracking_logging_obj() + stall: Final = httpx.ReadTimeout("") + iterator: Final = _make_iterator(sse_events=prefix, logging_obj=logging_obj, trailing_error=stall) + + with pytest.raises(litellm.Timeout) as exc_info: + async for _ in iterator: + pass + + assert exc_info.value.status_code == 408 + assert "Request timed out. Error_str: ReadTimeout" in str(exc_info.value) + assert exc_info.value.__cause__ is stall + _assert_failure_logged_once(logging_obj, stall) + + +@pytest.mark.parametrize("prefix", _READ_TIMEOUT_PREFIXES) +def test_sync_transport_read_timeout_raises_litellm_timeout(prefix): + iterator: Final = _make_sync_iterator( + sse_events=prefix, logging_obj=_logging_obj_stub(), trailing_error=httpx.ReadTimeout("") + ) + + with pytest.raises(litellm.Timeout) as exc_info: + for _ in iterator: + pass + + assert exc_info.value.status_code == 408 + assert "Request timed out. Error_str: ReadTimeout" in str(exc_info.value) + + @pytest.mark.asyncio @pytest.mark.parametrize("trailer", [[], [_DONE_MARKER]], ids=["eof", "done-marker"]) async def test_stream_ending_after_partial_output_without_terminal_event_raises(trailer): diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 637fdfa6873..383a1e25e98 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -1,6 +1,7 @@ import asyncio import importlib import json +import time from collections.abc import AsyncIterator from datetime import datetime, timedelta from types import MappingProxyType @@ -16,6 +17,8 @@ from litellm import Router from litellm.integrations.custom_logger import CustomLogger from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import FakeAnthropicMessagesStreamIterator from litellm.litellm_core_utils import get_llm_provider_logic +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.router_utils.cooldown_handlers import mark_advisor_orchestration_failure from litellm.router_utils.fallback_event_handlers import ( MID_STREAM_FALLBACK_CONTROLS_KEY, @@ -1083,6 +1086,68 @@ class TestTriggerCooldownForFailedDeployment: mock_set_cooldown.assert_called_once() +async def _stalled_stream_error() -> Exception: + from litellm.router import _aread_first_chunk + + async def _stall(**kwargs: object) -> NoReturn: + raise httpx.ReadTimeout("Timeout on reading data from socket") + + stream: Final = CustomStreamWrapper( + completion_stream=None, + model="gpt-5.6", + logging_obj=Logging( + model="gpt-5.6", + messages=[{"role": "user", "content": "Hey"}], + stream=True, + call_type="acompletion", + start_time=time.time(), + litellm_call_id="stalled-fallback", + function_id="stalled-fallback", + ), + custom_llm_provider="openai", + make_call=_stall, + ) + try: + await _aread_first_chunk(stream) + except litellm.Timeout as stalled: + return stalled + raise AssertionError("a stream that stalls before its first chunk must raise litellm.Timeout") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("stream_stalled", "cooled"), + [(True, ()), (False, ("fallback-deployment",))], + ids=["stream-stall", "non-stream-timeout"], +) +async def test_a_fallback_deployment_whose_stream_stalled_stays_out_of_cooldown( + stream_stalled: bool, cooled: tuple[str, ...] +) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "fallback-model", + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-fake"}, + "model_info": {"id": "fallback-deployment"}, + } + ], + allowed_fails=0, + cooldown_time=60, + num_retries=0, + ) + error: Final = ( + await _stalled_stream_error() + if stream_stalled + else litellm.Timeout(message="timeout", model="gpt-5.6", llm_provider="openai") + ) + error.failed_deployment_id = "fallback-deployment" + + _trigger_cooldown_for_failed_deployment(litellm_router=router, kwargs={}, exception=error) + + active: Final = router.cooldown_cache.get_active_cooldowns(model_ids=["fallback-deployment"], parent_otel_span=None) + assert tuple(entry[0] for entry in active) == cooled + + class TestRunAsyncFallbackTriggersCooldown: class RouterWithLoggingKwarg: fallback_access_check = None diff --git a/tests/unit/test_router_exception_redaction.py b/tests/unit/test_router_exception_redaction.py index 5b52496fc1c..5f9a5ba06a8 100644 --- a/tests/unit/test_router_exception_redaction.py +++ b/tests/unit/test_router_exception_redaction.py @@ -37,6 +37,8 @@ non-content-policy error, regardless of whether `fallbacks` is set. from __future__ import annotations +from typing import Final + import pytest import litellm @@ -313,6 +315,29 @@ async def test_flag_on_shows_deployment_timeout_debug(monkeypatch: pytest.Monkey assert "Deployment Info: request_timeout:" in msg, msg +@pytest.mark.asyncio +async def test_flag_on_deployment_timeout_debug_names_stream_timeout(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) + router: Final = Router( + model_list=[ + { + "model_name": _INTERNAL_MODEL_GROUP_NAME, + "litellm_params": {"model": "gpt-4o", "api_key": "key", "stream_timeout": 7}, + }, + ], + num_retries=0, + ) + with pytest.raises(litellm.Timeout) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + stream=True, + mock_timeout=True, + timeout=0.001, + ) + assert "\nstream_timeout: 7" in excinfo.value.message, excinfo.value.message + + # --- Site 3: ContentPolicyViolationError fallback hint (no fallback set) ---- From 9c958aff349680503a5d9785b3ef507eb9822002 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Sun, 18 Oct 2026 09:19:04 -0700 Subject: [PATCH 05/50] fix(logging): deliver completed batch success callbacks consistently (#47192) * fix(batches): dispatch configured success callbacks during reconciliation * fix(batches): retain billed job claim when sync dispatch shuts down * fix(batches): finish reconciliation when logging executor is stopped * fix(logging): avoid duplicate custom hooks during async batch dispatch * fix: preserve sync-only custom callback delivery * fix: scope callback deduplication to per-call async recipients * refactor: type per-call async success callbacks * refactor: retain logging constructor callback compatibility * test: support current batch reconciliation metadata * fix(logging): honor per-logger redaction during sync success delivery * fix(logging): centralize completed success callback dispatch * fix(guardrails): avoid duplicate success callback dispatch --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/integrations/custom_logger.py | 43 +-- litellm/litellm_core_utils/litellm_logging.py | 109 ++++++- .../proxy/guardrails/guardrail_endpoints.py | 17 -- tests/integration/_support/redaction_probe.py | 18 +- .../spend/test_batch_completion_accounting.py | 133 ++++++++- .../test_litellm_logging.py | 265 +++++++++++++++--- .../common_utils/test_check_batch_cost.py | 186 +++++++++++- .../guardrails/test_guardrail_endpoints.py | 69 ++++- 8 files changed, 715 insertions(+), 125 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 5addc994df2..a4de1be1842 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -912,9 +912,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac This is useful for logging payloads that contain sensitive information. """ import litellm - from litellm import Choices, Message, ModelResponse from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS - from litellm.litellm_core_utils.redact_messages import redacted_litellm_params + from litellm.litellm_core_utils.redact_messages import ( + redacted_litellm_params, + redacted_standard_logging_payload, + ) turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False) excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None) @@ -940,41 +942,14 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS) } - # Handle turn_off_message_logging - redact messages and responses (if not already excluded) - if turn_off_message_logging and not self.redacts_messages_itself(): - redacted_str: Final = "redacted-by-litellm" - - if "messages" not in (excluded_fields or ()) and standard_logging_object_copy.get("messages") is not None: - standard_logging_object_copy["messages"] = [Message(content=redacted_str).model_dump()] - - if "response" not in (excluded_fields or ()) and standard_logging_object_copy.get("response") is not None: - response: Final = standard_logging_object_copy["response"] - # Check if this is a ResponsesAPIResponse (has "output" field) - if isinstance(response, dict) and "output" in response: - # Make a copy to avoid modifying the original - from copy import deepcopy - - response_copy: Final = deepcopy(response) - # Redact content in output array - if isinstance(response_copy.get("output"), list): - for output_item in response_copy["output"]: - if isinstance(output_item, dict) and "content" in output_item: - if isinstance(output_item["content"], list): - # Redact text in content items - for content_item in output_item["content"]: - if isinstance(content_item, dict) and "text" in content_item: - content_item["text"] = redacted_str - standard_logging_object_copy["response"] = response_copy - else: - # Standard ModelResponse format - model_response: Final = ModelResponse(choices=[Choices(message=Message(content=redacted_str))]) - model_response_dict: Final = model_response.model_dump() - standard_logging_object_copy["response"] = model_response_dict - return { **model_call_details, **redacted_params, - "standard_logging_object": standard_logging_object_copy, + "standard_logging_object": ( + redacted_standard_logging_payload(standard_logging_object_copy) + if turn_off_message_logging and not self.redacts_messages_itself() + else standard_logging_object_copy + ), } async def get_proxy_server_request_from_cold_storage_with_object_key( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index b6bdc21bc05..5d9652cc8a2 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -634,15 +634,15 @@ class Logging(LiteLLMLoggingBaseClass): self, model: str, messages, - stream, + stream: bool | None, call_type, start_time: datetime.datetime, litellm_call_id: str, function_id: str, litellm_trace_id: str | None = None, dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = None, - dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = None, - dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = None, + dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None, + dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None, dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None, dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = None, applied_guardrails: list[str] | None = None, @@ -718,8 +718,10 @@ class Logging(LiteLLMLoggingBaseClass): # Initialize dynamic callbacks self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks - self.dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = dynamic_success_callbacks - self.dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = ( + self.dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = ( + dynamic_success_callbacks + ) + self.dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = ( dynamic_async_success_callbacks ) self.dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = dynamic_failure_callbacks @@ -786,6 +788,7 @@ class Logging(LiteLLMLoggingBaseClass): self.defer_async_logging: bool = False self.enqueue_deferred_logging: Callable[[], None] | None = None self._async_success_scheduled: bool = False + self._sync_success_scheduled: bool = False self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None @@ -2318,7 +2321,7 @@ class Logging(LiteLLMLoggingBaseClass): **kwargs, ) - if not self._should_run_sync_callbacks_for_async_calls(): + if not self.stream or not self._should_run_sync_callbacks_for_async_calls(): return executor.submit( @@ -2446,7 +2449,7 @@ class Logging(LiteLLMLoggingBaseClass): self, event_type: Literal["async_success", "sync_success", "async_failure", "sync_failure"], ) -> None: - if self.stream is not None and self.stream is True and event_type in ["async_success", "sync_success"]: + if self.stream is True and event_type in ["async_success", "sync_success"]: """ Ignore check on stream, as there can be multiple chunks """ @@ -2943,6 +2946,8 @@ class Logging(LiteLLMLoggingBaseClass): discarded as a duplicate of the in-progress create. """ self.model_call_details.pop("has_logged_async_success", None) + self.model_call_details.pop("has_logged_sync_success", None) + self._sync_success_scheduled = False litellm_params = self.model_call_details.get("litellm_params") if not isinstance(litellm_params, dict): return @@ -3131,6 +3136,11 @@ class Logging(LiteLLMLoggingBaseClass): dynamic_success_callbacks=self.dynamic_success_callbacks, global_callbacks=litellm.success_callback, ) + delivery_callbacks: Final = ( + self._get_completed_success_callbacks(asynchronous=False) + if for_async_request and not self.stream + else callbacks + ) ## REDACT MESSAGES ## result = redact_message_input_output_from_logging( @@ -3138,7 +3148,8 @@ class Logging(LiteLLMLoggingBaseClass): result=result, ) ## LOGGING HOOK ## - for callback in callbacks: + hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks))) + for callback in hook_callbacks: if isinstance(callback, CustomGuardrail): from litellm.types.guardrails import GuardrailEventHooks @@ -3168,7 +3179,7 @@ class Logging(LiteLLMLoggingBaseClass): call_details_view: Final = MappingProxyType[str, object](call_details) self.mark_logging_complete(event_type="sync_success") - for callback in callbacks: + for callback in delivery_callbacks: try: should_run = self.should_run_callback( callback=callback, @@ -3440,7 +3451,12 @@ class Logging(LiteLLMLoggingBaseClass): ) if ( isinstance(callback, CustomLogger) - and is_sync_request + and (is_sync_request or (for_async_request and not self.stream)) + and not ( + for_async_request + and self.stream + and callback in (self.dynamic_async_success_callbacks or ()) + ) and self.call_type != CallTypes.pass_through.value # pass-through endpoints call async_log_success_event ): # custom logger class @@ -3457,7 +3473,7 @@ class Logging(LiteLLMLoggingBaseClass): result = call_details_view["complete_response"] callback.log_success_event( - kwargs=call_details, + kwargs=callback.redact_standard_logging_payload_from_model_call_details(call_details), response_obj=result, start_time=start_time, end_time=end_time, @@ -3699,6 +3715,9 @@ class Logging(LiteLLMLoggingBaseClass): dynamic_success_callbacks=self.dynamic_async_success_callbacks, global_callbacks=litellm._async_success_callback, ) + delivery_callbacks: Final = ( + self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks + ) result = redact_message_input_output_from_logging( model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), @@ -3707,7 +3726,8 @@ class Logging(LiteLLMLoggingBaseClass): ## LOGGING HOOK ## - for callback in callbacks: + hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks))) + for callback in hook_callbacks: try: if isinstance(callback, CustomGuardrail): from litellm.types.guardrails import GuardrailEventHooks @@ -3745,7 +3765,7 @@ class Logging(LiteLLMLoggingBaseClass): self.mark_logging_complete(event_type="async_success") - for callback in callbacks: + for callback in delivery_callbacks: # check if callback can run for this request litellm_params = self.model_call_details.get("litellm_params", {}) should_run = self.should_run_callback( @@ -3880,6 +3900,13 @@ class Logging(LiteLLMLoggingBaseClass): traceback.format_exc(), ) self._handle_callback_failure(callback=callback) + if not self.stream: + try: + self.handle_sync_success_callbacks_for_async_calls( + result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit + ) + except RuntimeError: + verbose_logger.exception("Could not schedule synchronous success callbacks") await flush_post_call_redis_batches() def _handle_callback_failure(self, callback: object): @@ -4303,6 +4330,10 @@ class Logging(LiteLLMLoggingBaseClass): if self._should_run_sync_callbacks_for_async_calls() is False: return + if not self.stream and self._sync_success_scheduled: + return + if not self.stream: + self._sync_success_scheduled = True executor.submit( self.success_handler, result, @@ -4321,11 +4352,57 @@ class Logging(LiteLLMLoggingBaseClass): ) -> None: executor.submit(self.success_handler, result, start_time, end_time, cache_hit) + def _get_completed_success_callbacks( + self, *, asynchronous: bool + ) -> tuple[str | Callable[..., object] | CustomLogger, ...]: + sync_callbacks: Final = self.get_combined_callback_list( + self.dynamic_success_callbacks, litellm.success_callback + ) + async_callbacks: Final = self.get_combined_callback_list( + self.dynamic_async_success_callbacks, + litellm._async_success_callback, # pyright: ignore[reportPrivateUsage] # shared logging registry + ) + legacy_sync_callbacks: Final = self._remove_internal_litellm_callbacks( + self._remove_internal_custom_logger_callbacks(sync_callbacks) + ) + selected: Final = async_callbacks if asynchronous else sync_callbacks + other: Final = sync_callbacks if asynchronous else async_callbacks + return tuple( + callback + for callback in self.get_combined_callback_list(selected, other) + if ( + ( + ( + getattr(callback.async_log_success_event, "__func__", callback.async_log_success_event) + is not CustomLogger.async_log_success_event + ) + == asynchronous + and ( + asynchronous + or getattr(callback.log_success_event, "__func__", callback.log_success_event) + is not CustomLogger.log_success_event + ) + ) + if isinstance(callback, CustomLogger) + else callback in selected + and ( + asynchronous + or ( + callback not in async_callbacks + and callback in legacy_sync_callbacks + and not (isinstance(callback, str) and callback in INTERNAL_CALLBACK_NAMES) + ) + ) + ) + ) + def _should_run_sync_callbacks_for_async_calls(self) -> bool: """ Returns: - bool: True if sync callbacks should be run for async calls. eg. `langfuse`, `s3` """ + if not self.stream: + return bool(self._get_completed_success_callbacks(asynchronous=False)) _combined_sync_callbacks: Final = self.get_combined_callback_list( dynamic_success_callbacks=self.dynamic_success_callbacks, global_callbacks=litellm.success_callback, @@ -4355,7 +4432,11 @@ class Logging(LiteLLMLoggingBaseClass): _filtered_failure_callbacks = self._remove_internal_litellm_callbacks(_filtered_failure_callbacks) return len(_filtered_failure_callbacks) > 0 - def get_combined_callback_list(self, dynamic_success_callbacks: list | None, global_callbacks: list) -> list: + def get_combined_callback_list( + self, + dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None, + global_callbacks: list[str | Callable[..., object] | CustomLogger], + ) -> list[str | Callable[..., object] | CustomLogger]: if dynamic_success_callbacks is None: return list(global_callbacks) return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks)) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 1d5c530a616..70e7aac12b5 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -2367,9 +2367,6 @@ class _GuardrailLoggingObj(Protocol): @property def async_success_handler(self) -> "Callable[..., Awaitable[object]]": ... - @property - def success_handler(self) -> "Callable[..., object]": ... - class _GuardrailProxyLogging(Protocol): @property @@ -2399,10 +2396,6 @@ async def _emit_guardrail_success_logs( caller from receiving the guardrail response. Returns the (possibly hook-modified) response. """ - from litellm.litellm_core_utils.thread_pool_executor import ( - executor as thread_pool_executor, - ) - try: modified: Final = await proxy_logging_obj.post_call_success_hook( data=data, @@ -2430,16 +2423,6 @@ async def _emit_guardrail_success_logs( ) except Exception: verbose_proxy_logger.exception("apply_guardrail: async_success_handler failed") - try: - thread_pool_executor.submit( - litellm_logging_obj.success_handler, - response_for_logging, - start_time, - end_time, - False, - ) - except Exception: - verbose_proxy_logger.exception("apply_guardrail: success_handler submit failed") return response diff --git a/tests/integration/_support/redaction_probe.py b/tests/integration/_support/redaction_probe.py index 2a344daa73c..91b5faa2758 100644 --- a/tests/integration/_support/redaction_probe.py +++ b/tests/integration/_support/redaction_probe.py @@ -10,12 +10,15 @@ from pydantic import BaseModel from litellm.integrations.custom_logger import CustomLogger -def _payload(kwargs: Mapping[str, object], response_obj: object) -> str: +def _payload(kwargs: Mapping[str, object], response_obj: object, logger: str, hook: str) -> str: params: Final = kwargs.get("litellm_params") request: Final = params.get("proxy_server_request") if isinstance(params, Mapping) else None standard: Final = kwargs.get("standard_logging_object") return json.dumps( { + "logger": logger, + "hook": hook, + "standard": standard, "request": request, "response": response_obj.model_dump() if isinstance(response_obj, BaseModel) else response_obj, "standard_response": standard.get("response") if isinstance(standard, Mapping) else None, @@ -29,7 +32,9 @@ class SyncRedactionProbe(CustomLogger): self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime ) -> None: with httpx.Client(trust_env=False) as client: - response: Final = client.post(os.environ["REDACTION_PROBE_URL"], content=_payload(kwargs, response_obj)) + response: Final = client.post( + os.environ["REDACTION_PROBE_URL"], content=_payload(kwargs, response_obj, type(self).__name__, "sync") + ) response.raise_for_status() @@ -39,9 +44,16 @@ class RedactionProbe(CustomLogger): ) -> None: async with httpx.AsyncClient(trust_env=False) as client: response: Final = await client.post( - os.environ["REDACTION_PROBE_URL"], content=_payload(kwargs, response_obj) + os.environ["REDACTION_PROBE_URL"], content=_payload(kwargs, response_obj, type(self).__name__, "async") ) response.raise_for_status() +class DualRedactionProbe(CustomLogger): + log_success_event = SyncRedactionProbe.log_success_event + async_log_success_event = RedactionProbe.async_log_success_event + + probe: Final = RedactionProbe() +sync_probe: Final = SyncRedactionProbe() +dual_probe: Final = DualRedactionProbe() diff --git a/tests/integration/spend/test_batch_completion_accounting.py b/tests/integration/spend/test_batch_completion_accounting.py index bbc0300b3f0..6973864117c 100644 --- a/tests/integration/spend/test_batch_completion_accounting.py +++ b/tests/integration/spend/test_batch_completion_accounting.py @@ -4,11 +4,16 @@ import json import uuid from datetime import datetime, timedelta, timezone from hashlib import sha256 +from pathlib import Path +from queue import SimpleQueue from typing import Final import pytest +import yaml from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value -from integration._support.database import read_rows +from integration._support.database import read_rows, scratch_database +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server from integration._support.upstream import delete_scenario, register_scenario from integration.cost_calculation.cost_tracking_case import JsonResponse, RoutedResponse, TextResponse from pydantic import JsonValue @@ -299,3 +304,129 @@ def test_completed_batch_spend_lands_under_batches_in_team_endpoint_activity(gat endpoint_metrics: Final = object_value(object_value(endpoints["/batches"])["metrics"]) assert float(endpoint_metrics["spend"]) == pytest.approx(BATCH_SPEND), endpoints assert endpoint_metrics["total_tokens"] == BATCH_PROMPT_TOKENS + BATCH_COMPLETION_TOKENS, endpoints + + +def test_batch_reconciliation_delivers_each_configured_logger_once(gateway: Gateway, tmp_path: Path) -> None: + events: Final[SimpleQueue[dict[str, JsonValue]]] = SimpleQueue() + + def receive(request: Request) -> Reply: + payload: Final = JSON_OBJECT.validate_json(request.body) + standard: Final = object_value(payload.get("standard")) + if standard.get("call_type") == "aretrieve_batch" and standard.get("response_cost"): + events.put(payload) + return Reply() + + config: Final = tmp_path / "batch-callbacks.yaml" + config.write_text( + yaml.safe_dump( + { + "model_list": [ + { + "model_name": "batch-bootstrap", + "litellm_params": { + "model": "openai/gpt-6.1-sol", + "api_base": gateway.upstream_url + "/v1", + "api_key": "integration-provider-key", + }, + } + ], + "litellm_settings": { + "callbacks": [ + "integration._support.redaction_probe.sync_probe", + "integration._support.redaction_probe.probe", + "integration._support.redaction_probe.dual_probe", + ], + }, + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "disable_model_info_refresh": True, + "proxy_batch_polling_interval": 1, + }, + } + ) + ) + with scratch_database() as database, wire_server(receive) as sink: + with ( + owned_proxy( + gateway, + tmp_path, + {"DATABASE_URL": database, "REDACTION_PROBE_URL": sink.url, "PROXY_BATCH_POLLING_ENABLED": "true"}, + config=config, + ) as proxy, + proxy.scenario() as scenario, + ): + scenario_id: Final = f"batch-callbacks-{uuid.uuid4().hex}" + routes: Final = batch_routes("gpt-6.1-sol") + handle: Final = register_scenario(scenario_id, routes) + scenario.cleanups.callback(delete_scenario, handle) + model: Final = scenario.model( + model="openai/gpt-6.1-sol", + api_base=handle.api_base(), + input_cost_per_token=INPUT_COST_PER_TOKEN, + output_cost_per_token=OUTPUT_COST_PER_TOKEN, + ) + register_scenario( + scenario_id, + routes.model_copy( + update={ + "routes": { + **routes.routes, + "GET /files/file-in-$REQUEST_ID/content": TextResponse( + content_type="application/jsonl", body=batch_input_file(model).decode() + ), + } + } + ), + ) + key: Final = scenario.key(models=[model]) + uploaded: Final = proxy.request_multipart( + "/v1/files", + {"purpose": "batch", "target_model_names": model}, + {"file": ("in.jsonl", batch_input_file(model), "application/jsonl")}, + key=key, + ) + assert uploaded.status_code == 200, uploaded.text + created: Final = proxy.request( + "POST", + "/v1/batches", + { + "input_file_id": string_value(JSON_OBJECT.validate_json(uploaded.content)["id"]), + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "model": model, + }, + key=key, + ) + assert created.status_code == 200, created.text + assert eventually(events.qsize, lambda count: count >= 3, seconds=90) == 3 + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" ' + "WHERE api_key=%s AND call_type='aretrieve_batch'", + (sha256(key.encode()).hexdigest(),), + database_url=database, + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert float(rows[0]["spend"]) == pytest.approx(BATCH_SPEND) + assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == ( + BATCH_PROMPT_TOKENS, + BATCH_COMPLETION_TOKENS, + ) + assert events.qsize() == 3 + received: Final = tuple(events.get_nowait() for _ in range(3)) + assert {(event["logger"], event["hook"]) for event in received} == { + ("SyncRedactionProbe", "sync"), + ("RedactionProbe", "async"), + ("DualRedactionProbe", "async"), + } + for event in received: + standard: Final = object_value(event["standard"]) + assert standard["response_cost"] == pytest.approx(BATCH_SPEND) + usage: Final = object_value(object_value(event["response"])["usage"]) + assert (usage["prompt_tokens"], usage["completion_tokens"]) == ( + BATCH_PROMPT_TOKENS, + BATCH_COMPLETION_TOKENS, + ) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index c7450c852c0..e392ee746b3 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -2238,31 +2238,42 @@ async def test_async_success_handler_truncates_large_base64_off_the_event_loop(m @pytest.mark.parametrize( - "async_flag", + ("async_flag", "for_async_request"), [ - "acompletion", - "aresponses", - "allm_passthrough_route", - "aanthropic_messages", - "agenerate_content", - "agenerate_content_stream", + ("acompletion", False), + ("aresponses", False), + ("allm_passthrough_route", False), + ("aanthropic_messages", False), + ("agenerate_content", False), + ("agenerate_content_stream", False), + (None, True), ], ) -def test_success_handler_skips_sync_callbacks_for_async_requests(logging_obj, async_flag): - """Ensure sync success callbacks are skipped when async call type flags are set.""" +def test_success_handler_skips_sync_callbacks_for_async_requests( + logging_obj: LitellmLogging, async_flag: str | None, for_async_request: bool +) -> None: from litellm.integrations.custom_logger import CustomLogger class DummyLogger(CustomLogger): - pass + async def async_log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + return None logging_obj.stream = False # simulate non-streaming request where sync callbacks would normally run - logging_obj.model_call_details["litellm_params"] = {async_flag: True} + logging_obj.model_call_details["litellm_params"] = {async_flag: True} if async_flag else {} logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] dummy_logger = DummyLogger() dummy_logger.log_success_event = MagicMock() dummy_logger.log_stream_event = MagicMock() + logging_obj.dynamic_async_success_callbacks = [dummy_logger] + model_response = ModelResponse( id="resp-123", model="gpt-4o-mini", @@ -2281,14 +2292,17 @@ def test_success_handler_skips_sync_callbacks_for_async_requests(logging_obj, as "get_combined_callback_list", return_value=[dummy_logger], ): - logging_obj.success_handler(result=model_response) + logging_obj.success_handler(result=model_response, for_async_request=for_async_request) dummy_logger.log_success_event.assert_not_called() dummy_logger.log_stream_event.assert_not_called() @pytest.mark.parametrize("call_type", ["completion", "responses"]) -def test_success_handler_runs_sync_callbacks_for_sync_requests(logging_obj, call_type): +@pytest.mark.parametrize("for_async_request", [False, True]) +def test_success_handler_runs_sync_callbacks_for_sync_requests( + logging_obj: LitellmLogging, call_type: str, for_async_request: bool, monkeypatch: pytest.MonkeyPatch +) -> None: """Ensure sync success callbacks execute when call type is sync (completion/responses).""" from litellm.integrations.custom_logger import CustomLogger @@ -2317,17 +2331,160 @@ def test_success_handler_runs_sync_callbacks_for_sync_requests(logging_obj, call usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, ) - with patch.object( - logging_obj, - "get_combined_callback_list", - return_value=[dummy_logger], - ): - logging_obj.success_handler(result=model_response) + logging_obj.dynamic_success_callbacks = [dummy_logger] + logging_obj.dynamic_async_success_callbacks = [] + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + logging_obj.success_handler(result=model_response, for_async_request=for_async_request) dummy_logger.log_success_event.assert_called_once() dummy_logger.log_stream_event.assert_not_called() +@pytest.mark.asyncio +@pytest.mark.parametrize("registration", ["sync", "async", "overlapping"]) +@pytest.mark.parametrize("kind", ["sync", "async", "dual"]) +@pytest.mark.parametrize("compatibility_first", [False, True]) +async def test_completed_success_selects_one_implemented_hook( + monkeypatch: pytest.MonkeyPatch, registration: str, kind: str, compatibility_first: bool +) -> None: + from concurrent.futures import ThreadPoolExecutor + from queue import SimpleQueue + + events: Final[SimpleQueue[tuple[str, str, object, int]]] = SimpleQueue() + loop_thread: Final = threading.get_ident() + + def record(hook: str, kwargs: dict[str, object], response_obj: object) -> None: + assert isinstance(response_obj, ModelResponse) + events.put((hook, response_obj.id, kwargs["response_cost"], threading.get_ident())) + + class SyncRecorder(CustomLogger): + def log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + record("sync", kwargs, response_obj) + + class AsyncRecorder(CustomLogger): + async def async_log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + record("async", kwargs, response_obj) + + class DualRecorder(CustomLogger): + def log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + record("sync", kwargs, response_obj) + + async def async_log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + record("async", kwargs, response_obj) + + callback: Final = {"sync": SyncRecorder, "async": AsyncRecorder, "dual": DualRecorder}[kind]() + monkeypatch.setattr(litellm, "success_callback", [callback] if registration != "async" else []) + monkeypatch.setattr(litellm, "_async_success_callback", [callback] if registration != "sync" else []) + logging_obj: Final = _async_request_logging_obj("completed-hook", []) + response: Final = ModelResponse(id="chatcmpl-completed-hook") + now: Final = datetime.datetime.now() + + with ThreadPoolExecutor(max_workers=1) as pool: + monkeypatch.setattr(litellm_logging, "executor", pool) + if compatibility_first: + logging_obj.handle_sync_success_callbacks_for_async_calls(response, now, now) + await logging_obj.async_success_handler(response, now, now) + logging_obj.handle_sync_success_callbacks_for_async_calls(response, now, now) + + assert events.qsize() == 1 + hook, response_id, cost, thread_id = events.get_nowait() + assert hook == ("sync" if kind == "sync" else "async") + assert response_id == response.id + assert cost == logging_obj.model_call_details["response_cost"] + assert (thread_id == loop_thread) is (kind != "sync") + + + +@pytest.mark.parametrize("for_async_request", [False, True]) +@pytest.mark.parametrize("private_first", [False, True]) +@pytest.mark.parametrize("response_kind", ["responses", "completion"]) +def test_sync_success_redacts_only_opted_out_loggers( + monkeypatch: pytest.MonkeyPatch, for_async_request: bool, private_first: bool, response_kind: str +) -> None: + from queue import SimpleQueue + + events: Final[SimpleQueue[tuple[bool, str]]] = SimpleQueue() + + class PayloadRecorder(CustomLogger): + def log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + events.put((self.turn_off_message_logging, json.dumps(kwargs["standard_logging_object"]))) + + private_logger: Final = PayloadRecorder(turn_off_message_logging=True) + public_logger: Final = PayloadRecorder() + callbacks: Final = [private_logger, public_logger] if private_first else [public_logger, private_logger] + monkeypatch.setattr(litellm, "success_callback", callbacks) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + logging_obj: Final = LitellmLogging( + model="openai/gpt-6.1-sol", + messages=[{"role": "user", "content": "private-request-marker"}], + stream=False, + call_type="aget_responses", + start_time=datetime.datetime.now(), + litellm_call_id="callback-redaction", + function_id="callback-redaction", + ) + logging_obj.update_environment_variables(litellm_params={}, optional_params={}, custom_llm_provider="openai") + response: Final = ( + _responses_api_response_with_text("private-response-marker") + if response_kind == "responses" + else ModelResponse( + id="chatcmpl-private", + model="openai/gpt-6.1-sol", + choices=[{"message": {"role": "assistant", "content": "private-response-marker"}}], + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + ) + logging_obj.success_handler(result=response, for_async_request=for_async_request) + + assert events.qsize() == 2 + payloads: Final = dict(events.get_nowait() for _ in range(2)) + assert "private-request-marker" not in payloads[True] + assert "private-response-marker" not in payloads[True] + assert REDACTED_BY_LITELLM in payloads[True] + assert "private-request-marker" in payloads[False] + assert "private-response-marker" in payloads[False] + private_payload, public_payload = json.loads(payloads[True]), json.loads(payloads[False]) + assert private_payload["total_tokens"] == public_payload["total_tokens"] + assert private_payload["response"]["usage"] == public_payload["response"]["usage"] + assert private_payload["response"]["usage"]["total_tokens"] == response.usage.total_tokens + assert private_payload["response"]["id"] == public_payload["response"]["id"] + assert private_payload["response"]["model"] == public_payload["response"]["model"] + assert private_payload["response_cost"] == public_payload["response_cost"] + assert "private-response-marker" in response.model_dump_json() + + class _PhaseRecordingLogger(CustomLogger): """Records whether each success callback ran inside the post-response phase.""" @@ -6526,34 +6683,56 @@ def test_in_progress_background_create_is_not_billed(call_type): @pytest.mark.asyncio -async def test_background_interaction_completion_rebills_after_in_progress_success(): +async def test_background_interaction_completion_rebills_after_in_progress_success(monkeypatch: pytest.MonkeyPatch): import datetime as dt + from concurrent.futures import ThreadPoolExecutor + from queue import SimpleQueue from litellm.types.interactions import InteractionsAPIResponse - logging_obj = _interactions_logging_obj(stream=False) - in_progress = InteractionsAPIResponse(id="interactions/abc", model="gemini-2.5-flash", status="in_progress") - await logging_obj.async_success_handler( - result=in_progress, - start_time=dt.datetime.now(), - end_time=dt.datetime.now(), - ) + events: Final[SimpleQueue[tuple[object, object]]] = SimpleQueue() + logged: Final = threading.Event() - assert logging_obj.model_call_details.get("response_cost") is None - assert logging_obj.should_run_logging(event_type="async_success") is False + class SyncRecorder(CustomLogger): + def log_success_event( + self, kwargs: dict[str, object], response_obj: object, + start_time: dt.datetime, end_time: dt.datetime, + ) -> None: + events.put((response_obj, kwargs.get("response_cost"))) + logged.set() - completed = InteractionsAPIResponse( - id="interactions/abc", - model="gemini-2.5-flash", - status="completed", - steps=[], - usage=dict(INTERACTIONS_USAGE_BLOCK), - ) - await logging_obj.async_log_background_interaction_completion(result=completed) + monkeypatch.setattr(litellm, "success_callback", [SyncRecorder()]) + monkeypatch.setattr(litellm, "_async_success_callback", []) + with ThreadPoolExecutor(max_workers=1) as pool: + monkeypatch.setattr(litellm_logging, "executor", pool) + logging_obj = _interactions_logging_obj(stream=False) + in_progress = InteractionsAPIResponse(id="interactions/abc", model="gemini-2.5-flash", status="in_progress") + await logging_obj.async_success_handler( + result=in_progress, + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + ) - assert logging_obj.model_call_details["response_cost"] > 0 - assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 + assert await asyncio.to_thread(logged.wait, 10) + assert logging_obj.model_call_details.get("response_cost") is None + assert logging_obj.should_run_logging(event_type="async_success") is False + completed = InteractionsAPIResponse( + id="interactions/abc", + model="gemini-2.5-flash", + status="completed", + steps=[], + usage=dict(INTERACTIONS_USAGE_BLOCK), + ) + await logging_obj.async_log_background_interaction_completion(result=completed) + + assert logging_obj.model_call_details["response_cost"] > 0 + assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175 + + + assert events.qsize() == 2 + assert events.get_nowait() == (in_progress, None) + assert events.get_nowait() == (completed, logging_obj.model_call_details["response_cost"]) @pytest.mark.asyncio async def test_background_interaction_completion_prices_the_settled_body_itself(): @@ -7797,6 +7976,7 @@ async def test_sync_only_logger_keeps_the_async_payload_when_its_result_builds_n monkeypatch: pytest.MonkeyPatch, ) -> None: sync_payloads: Final[list[object]] = [] + sync_logged: Final = threading.Event() class AthinaRecorder: def log_event( @@ -7808,9 +7988,10 @@ async def test_sync_only_logger_keeps_the_async_payload_when_its_result_builds_n print_verbose: object, ) -> None: sync_payloads.append(kwargs.get("standard_logging_object")) + sync_logged.set() monkeypatch.setattr(litellm, "_async_success_callback", []) - monkeypatch.setattr(litellm, "success_callback", ["athina"]) + monkeypatch.setattr(litellm, "success_callback", [_ResponseEditingHook(), "athina"]) monkeypatch.setattr(litellm_logging, "athinaLogger", AthinaRecorder()) logging_obj: Final = LitellmLogging( model="pass-through-model", @@ -7827,9 +8008,7 @@ async def test_sync_only_logger_keeps_the_async_payload_when_its_result_builds_n async_payload: Final = logging_obj.model_call_details.get("standard_logging_object") assert isinstance(async_payload, dict) async_payload_before_sync: Final = dict(async_payload) - monkeypatch.setattr(litellm, "success_callback", [_ResponseEditingHook(), "athina"]) - - logging_obj.success_handler(result=raw_body, for_async_request=True) + assert sync_logged.wait(timeout=10) assert sync_payloads == [{**async_payload_before_sync, "response": "edited by sync hook"}] assert logging_obj.model_call_details["standard_logging_object"] is async_payload diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index eb2a5b7f66c..1253e15676b 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -9,12 +9,17 @@ ARN unified_object_id) batches with no managed unified id. import asyncio import json import logging +from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from datetime import datetime from hashlib import sha256 -from typing import TYPE_CHECKING, Final +from queue import SimpleQueue +from types import SimpleNamespace +from typing import TYPE_CHECKING, Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import pytest +import respx from fastapi import HTTPException from litellm.types.utils import LiteLLMBatch @@ -1659,10 +1664,9 @@ class TestCheckBatchCost: model_name="gpt-5-batch", ) assert mock_hook.store_batch_output_file.await_count == 2 - assert { - call.kwargs["model_name"] - for call in mock_hook.store_batch_output_file.await_args_list - } == {"gpt-5-batch"} + assert {call.kwargs["model_name"] for call in mock_hook.store_batch_output_file.await_args_list} == { + "gpt-5-batch" + } stored = { c.kwargs["provider_file_id"]: c.kwargs["unified_file_id"] for c in mock_hook.store_batch_output_file.call_args_list @@ -2521,9 +2525,7 @@ class TestBatchCostAttribution: async def test_raw_cli_session_token_on_a_legacy_batch_row_is_not_treated_as_the_alias(self): instance = self._instance(key_row=None) - metadata = await instance.batch_metadata( - self._job(api_key="cli-session-Qm7xJ2kP9sLw4vT1nR8yAa"), "batch-1" - ) + metadata = await instance.batch_metadata(self._job(api_key="cli-session-Qm7xJ2kP9sLw4vT1nR8yAa"), "batch-1") assert metadata.get("user_api_key_alias") is None @@ -3732,9 +3734,7 @@ class TestBatchLineItemsFromThePoller: assert len(logger.payloads) == stored @pytest.mark.asyncio - async def test_a_batch_whose_output_file_is_gone_still_sends_its_error_lines_after_it_is_retired( - self, monkeypatch - ): + async def test_a_batch_whose_output_file_is_gone_still_sends_its_error_lines_after_it_is_retired(self, monkeypatch): import litellm journal: list = [] @@ -3834,7 +3834,6 @@ class TestBatchLineItemsFromThePoller: assert row.batch_processed is True assert f"so the line items of batch batch-456 (job {row.id}) were not stored" in caplog.text - @pytest.mark.asyncio async def test_batches_store_their_lines_one_at_a_time_in_the_order_they_were_retired(self, monkeypatch): import litellm @@ -4109,3 +4108,166 @@ class TestCreatorTurnedOffMessageLogging: assert await instance._creator_turned_off_message_logging(self._job(api_key=api_key)) is False instance.prisma_client.db.litellm_verificationtoken.find_unique.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "kind", ["sync", "async", "custom", "overlapping", "general", "mixed", "sync_only", "executor_shutdown"] +) +@pytest.mark.parametrize("initialized", [False, True]) +@pytest.mark.parametrize("claimed", [False, True]) +async def test_batch_cost_reaches_configured_success_callbacks( + kind: Literal["sync", "async", "custom", "overlapping", "general", "mixed", "sync_only", "executor_shutdown"], + initialized: bool, + claimed: bool, +) -> None: + import litellm + from litellm.integrations.custom_logger import CustomLogger + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + from litellm.types.utils import LiteLLMBatch + from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + + events: Final[SimpleQueue[tuple[str, object, int, int]]] = SimpleQueue() + + def record( + kwargs: dict[str, object], + response_obj: object, + start_time: datetime, + end_time: datetime, + recipient: str = "function", + ) -> None: + if kwargs.get("call_type") != "aretrieve_batch": + return + assert isinstance(response_obj, LiteLLMBatch) + assert response_obj.usage is not None + events.put( + (recipient, kwargs["response_cost"], response_obj.usage.prompt_tokens, response_obj.usage.completion_tokens) + ) + + async def async_record( + kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + record(kwargs, response_obj, start_time, end_time) + if kind == "executor_shutdown" and kwargs.get("call_type") == "aretrieve_batch": + executor.shutdown(wait=True) + + class BatchRecorder(CustomLogger): + def log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + record(kwargs, response_obj, start_time, end_time, "custom") + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + record(kwargs, response_obj, start_time, end_time, "custom") + + class SyncRecorder(CustomLogger): + def log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + record(kwargs, response_obj, start_time, end_time, "custom") + + callback: Final = SyncRecorder() if kind == "sync_only" else BatchRecorder() + match kind: + case "sync": + litellm.logging_callback_manager.add_litellm_success_callback(record) + case "async": + litellm.logging_callback_manager.add_litellm_success_callback(async_record) + case "executor_shutdown": + litellm.logging_callback_manager.add_litellm_success_callback(async_record) + litellm.logging_callback_manager.add_litellm_success_callback(record) + case "custom" | "overlapping" | "sync_only": + litellm.logging_callback_manager.add_litellm_success_callback(callback) + if kind == "overlapping": + litellm.logging_callback_manager.add_litellm_async_success_callback(callback) + case "general": + litellm.logging_callback_manager.add_litellm_callback(callback) + case "mixed": + litellm.logging_callback_manager.add_litellm_success_callback(callback) + litellm.logging_callback_manager.add_litellm_success_callback(record) + + model: Final = "gpt-6.1-sol" + credentials: Final = { + "api_key": "synthetic-key", + "api_base": "https://batch.invalid/v1", + "custom_llm_provider": "openai", + } + deployment: Final = SimpleNamespace( + litellm_params=SimpleNamespace(model=f"openai/{model}", custom_llm_provider="openai", api_base=None), + model_name="batch-group", + ) + prisma: Final = MagicMock() + prisma.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=int(claimed)) + checker: Final = CheckBatchCost( + proxy_logging_obj=SimpleNamespace(get_proxy_hook=lambda _: None), + prisma_client=prisma, + llm_router=SimpleNamespace( + get_deployment_credentials_with_provider=lambda _: credentials, + get_deployment=lambda **_: deployment, + ), + ) + job: Final = SimpleNamespace( + id="job", + unified_object_id="batch-record", + status="in_progress", + file_object={}, + created_by=None, + team_id=None, + api_key=None, + org_id=None, + request_tags=None, + ) + response: Final = LiteLLMBatch( + id="batch-record", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="input", + object="batch", + status="completed", + output_file_id="output", + ) + output: Final = json.dumps( + { + "id": "line", + "response": { + "status_code": 200, + "body": { + "model": model, + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], + }, + }, + } + ).encode() + with ( + ThreadPoolExecutor(max_workers=1) as executor, + patch("litellm.litellm_core_utils.litellm_logging.executor", executor), + respx.mock(assert_all_called=True) as transport, + ): + transport.get("https://batch.invalid/v1/files/output/content").respond(200, content=output) + if initialized: + await litellm.afile_content(file_id="output", **credentials) + await GLOBAL_LOGGING_WORKER.flush() + result: Final = await checker._track_completed_batch_cost(job, response, "deployment", "batch-record", None) + assert result == ((model, "openai") if claimed else None) + if kind == "executor_shutdown" and claimed: + prisma.db.litellm_managedobjecttable.update_many.assert_awaited_once() + await GLOBAL_LOGGING_WORKER.flush() + recipients: Final = ( + ("custom", "function") + if kind == "mixed" + else ("custom",) + if kind in ("custom", "overlapping", "general", "sync_only") + else ("function",) + ) + expected: Final = recipients if claimed else () + assert events.qsize() == len(expected) + received: Final = tuple(events.get_nowait() for _ in expected) + assert tuple(sorted(event[0] for event in received)) == expected + for _, cost, prompt_tokens, completion_tokens in received: + assert isinstance(cost, float) and cost > 0 + assert (prompt_tokens, completion_tokens) == (10, 5) diff --git a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py index 2e968ec04f5..6807a4fb569 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py @@ -125,6 +125,7 @@ def test_get_guardrails_list_response() -> None: assert minimal_response.guardrails[0].guardrail_name == "minimal-guard" assert minimal_response.guardrails[0].guardrail_info is None + _INVALID_SCOPE_LITELLM_PARAMS: Final = MappingProxyType( { "guardrail": "litellm_content_filter", @@ -655,6 +656,8 @@ async def test_get_guardrail_info_tolerates_invalid_stored_stream_scope(mocker, response = await get_guardrail_info("test-db-guardrail") assert response.litellm_params.stream_scope is None + + async def test_get_guardrail_info_normalizes_invalid_scope_from_db( mocker, mock_guardrail_registry, mock_in_memory_handler ): @@ -2159,7 +2162,6 @@ async def test_apply_guardrail_invokes_logging_pipeline(mocker): mock_proxy_logging.post_call_success_hook.assert_awaited_once() mock_logging_obj.async_success_handler.assert_awaited_once() assert mock_logging_obj.call_type == "pass_through_endpoint" - mock_executor.submit.assert_called_once() assert mock_logging_obj.async_success_handler.await_args.kwargs["result"] == { "response": {"response_text": "masked"} } @@ -3624,3 +3626,68 @@ async def test_patch_guardrail_rejects_encrypted_marker_values(mocker, mock_guar assert exc_info.value.status_code == 400 mock_guardrail_registry.update_guardrail_in_db.assert_not_called() + + +@pytest.mark.asyncio +async def test_guardrail_success_delivers_sync_callback_once_with_overlapping_workers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + from unittest.mock import Mock + + import litellm + from litellm.integrations.custom_logger import CustomLogger + from litellm.litellm_core_utils import litellm_logging, thread_pool_executor + from litellm.proxy.guardrails.guardrail_endpoints import _emit_guardrail_success_logs + from litellm.types.guardrails import ApplyGuardrailResponse + + class OverlappingHook(CustomLogger): + barrier: Barrier + + def logging_hook( + self, kwargs: dict[str, object], result: object, call_type: str + ) -> tuple[dict[str, object], object]: + self.barrier.wait(timeout=5) + return kwargs, result + + hook: Final = OverlappingHook() + callback: Final = Mock() + queued_executor: Final = Mock() + proxy_logging: Final = Mock(post_call_success_hook=AsyncMock(return_value=None)) + + def sync_callback(kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object) -> None: + callback(response_obj) + + monkeypatch.setattr(litellm, "success_callback", [hook, sync_callback]) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm_logging, "customLogger", CustomLogger()) + monkeypatch.setattr(litellm_logging, "executor", queued_executor) + monkeypatch.setattr(thread_pool_executor, "executor", queued_executor) + start: Final = datetime(2026, 1, 1) + logging_obj: Final = litellm_logging.Logging( + model="guardrail", + messages=[], + stream=False, + call_type="pass_through_endpoint", + start_time=start, + litellm_call_id="guardrail-success", + function_id="guardrail-success", + ) + logging_obj.update_environment_variables(litellm_params={"metadata": {}}, optional_params={}) + response: Final = ApplyGuardrailResponse(response_text="masked") + + returned: Final = await _emit_guardrail_success_logs( + proxy_logging, logging_obj, {}, UserAPIKeyAuth(), response, start + ) + submissions: Final = tuple(queued_executor.submit.call_args_list) + assert submissions + hook.barrier = Barrier(len(submissions)) + with ThreadPoolExecutor(max_workers=len(submissions)) as pool: + futures: Final = tuple(pool.submit(call.args[0], *call.args[1:], **call.kwargs) for call in submissions) + for future in futures: + future.result(timeout=10) + + assert returned is response + callback.assert_called_once() + assert callback.call_args.args[0] == {"response": {"response_text": "masked"}} From 19f69ed6b6f79cb854b695877bfebd28869d748c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sun, 18 Oct 2026 14:49:25 -0700 Subject: [PATCH 06/50] test(integration): wait for every worker in Scenario.model (#47277) --- tests/integration/_support/client.py | 31 ++++++++ tests/integration/_support/process.py | 2 +- tests/unit/integration_support/test_client.py | 70 +++++++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 tests/unit/integration_support/test_client.py diff --git a/tests/integration/_support/client.py b/tests/integration/_support/client.py index 6f407295259..b8ea11f3d2f 100644 --- a/tests/integration/_support/client.py +++ b/tests/integration/_support/client.py @@ -4,6 +4,7 @@ import os import time import uuid from collections.abc import Callable, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor from contextlib import ExitStack, contextmanager from dataclasses import dataclass from hashlib import sha256 @@ -15,6 +16,7 @@ from pydantic import JsonValue, TypeAdapter from tests.integration._support.database import read_rows JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +DEPLOYMENT_IDS: Final = TypeAdapter(dict[str, list[str]]) GATEWAY_LIMITS: Final = httpx.Limits(keepalive_expiry=2) T = TypeVar("T") @@ -96,6 +98,7 @@ class Gateway: key: str upstream_url: str database_url: str | None = None + workers: int = 1 def rows(self, query: str, parameters: tuple[str, ...]) -> list[dict[str, JsonValue]]: return read_rows(query, parameters, database_url=self.database_url) @@ -294,8 +297,36 @@ class Scenario: ) identity: Final = string_value(object_value(created["model_info"])["id"]) self.cleanups.callback(self.delete_model, identity) + if self.gateway.workers > 1: + self._await_every_worker(name, identity) return name + def _await_every_worker(self, name: str, identity: str) -> None: + from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS + + eventually( + lambda: self._deployment_ids_by_worker(name), + lambda answers: len({pid for pid, _ in answers}) >= self.gateway.workers + and all(identity in model_ids for _, model_ids in answers), + seconds=2 * PROXY_CONFIG_RELOAD_INTERVAL_SECONDS, + ) + + def _deployment_ids_by_worker(self, name: str) -> tuple[tuple[str, frozenset[str]], ...]: + with ThreadPoolExecutor(max_workers=8) as pool: + return tuple(pool.map(lambda _: self._deployment_ids_on_a_worker(name), range(16))) + + def _deployment_ids_on_a_worker(self, name: str) -> tuple[str, frozenset[str]]: + response: Final = self.gateway.request( + "GET", + "/health/integration-deployment-ids", + params={"model_name": name}, + headers={"connection": "close"}, + ) + assert response.status_code == 200, f"Deployment ids on a worker: {response.status_code} {response.text}" + return response.headers["x-integration-worker-pid"], frozenset( + DEPLOYMENT_IDS.validate_json(response.content)["model_ids"] + ) + @contextmanager def gateway_from_environment() -> Iterator[Gateway]: diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index b08037e80db..052c52982b6 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -893,7 +893,7 @@ def owned_proxy_process( limits=GATEWAY_LIMITS, ) as client: yield OwnedProxy( - Gateway(client, gateway.key, gateway.upstream_url, owned.database_url), + Gateway(client, gateway.key, gateway.upstream_url, owned.database_url, workers), process, launch.log, healthcheck, diff --git a/tests/unit/integration_support/test_client.py b/tests/unit/integration_support/test_client.py new file mode 100644 index 00000000000..909f69f4f3d --- /dev/null +++ b/tests/unit/integration_support/test_client.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import itertools +import threading +import time +from collections.abc import Iterator +from contextlib import ExitStack +from dataclasses import dataclass, field +from typing import Final + +import httpx + +from tests.integration._support.client import Gateway, Scenario + +DEPLOYMENT: Final = "deployment-under-test" +CREATING_WORKER: Final = "101" +LAGGING_WORKER: Final = "202" + + +@dataclass(frozen=True, slots=True) +class LaggingProxy: + reachable_at: float + synced_at: float + arrivals: Iterator[int] = field(default_factory=itertools.count) + created: threading.Event = field(default_factory=threading.Event) + + def _worker(self) -> str: + arrival: Final = next(self.arrivals) + reachable: Final = time.monotonic() >= self.reachable_at + return LAGGING_WORKER if reachable and arrival % 2 else CREATING_WORKER + + def respond(self, request: httpx.Request) -> httpx.Response: + if request.url.path == "/model/new": + self.created.set() + return httpx.Response(200, json={"model_info": {"id": DEPLOYMENT}}) + worker: Final = self._worker() + serves: Final = self.created.is_set() and (worker == CREATING_WORKER or time.monotonic() >= self.synced_at) + return httpx.Response( + 200, + json={"model_ids": [DEPLOYMENT] if serves else []}, + headers={"x-integration-worker-pid": worker}, + ) + + +def _created_model(proxy: LaggingProxy, workers: int) -> str: + with ( + httpx.Client(base_url="http://127.0.0.1:4000", transport=httpx.MockTransport(proxy.respond)) as client, + ExitStack() as cleanups, + ): + name: Final = Scenario(Gateway(client, "sk-test", "http://127.0.0.1:4001", workers=workers), cleanups).model() + cleanups.pop_all() + return name + + +def test_model_on_multi_worker_proxy_returns_only_once_every_worker_serves_it() -> None: + started: Final = time.monotonic() + proxy: Final = LaggingProxy(reachable_at=started + 0.3, synced_at=started + 0.8) + + _created_model(proxy, workers=2) + + assert time.monotonic() >= proxy.synced_at + + +def test_model_on_single_worker_proxy_returns_without_probing_workers() -> None: + started: Final = time.monotonic() + proxy: Final = LaggingProxy(reachable_at=started, synced_at=started + 60) + + _created_model(proxy, workers=1) + + assert time.monotonic() < proxy.synced_at From 2dfeb80e04d971fa00f72c12e0bd4fb42bc3c027 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:25:19 -0700 Subject: [PATCH 07/50] fix(bedrock): take the 256K context window from the Qwen3 Coder 480B model card (#47283) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0a90c352f3e..f76d61c417d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45977,7 +45977,7 @@ "deprecation_date": "2027-03-30", "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 131072, + "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", @@ -68129,7 +68129,7 @@ "output_cost_per_token": 1.8e-06, "output_cost_per_token_flex": 9e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 128000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", @@ -68140,7 +68140,7 @@ ], "supports_function_calling": true, "supports_tool_choice": true, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-480b-a35b-instruct.html" }, "bedrock_mantle/qwen.qwen3-next-80b-a3b-instruct": { "input_cost_per_token": 1.4e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0a90c352f3e..f76d61c417d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45977,7 +45977,7 @@ "deprecation_date": "2027-03-30", "input_cost_per_token": 4.5e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 131072, + "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", @@ -68129,7 +68129,7 @@ "output_cost_per_token": 1.8e-06, "output_cost_per_token_flex": 9e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 128000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", @@ -68140,7 +68140,7 @@ ], "supports_function_calling": true, "supports_tool_choice": true, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-480b-a35b-instruct.html" }, "bedrock_mantle/qwen.qwen3-next-80b-a3b-instruct": { "input_cost_per_token": 1.4e-07, From 8b90649fb79c3691ad378b591b76c97140966e1c Mon Sep 17 00:00:00 2001 From: vanraj Solanki Date: Mon, 19 Oct 2026 07:55:14 +0530 Subject: [PATCH 08/50] fix(proxy): sanitize unexpected /key/info database failures (#46190) --- .../key_management_endpoints.py | 10 +- .../test_key_management_endpoints.py | 127 ++++++++++++++++++ 2 files changed, 136 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 54f4ca25974..e87ca70035c 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4763,8 +4763,16 @@ async def info_key_fn( key_info_dict["budget_limits_usage"] = budget_limits_usage return {"key": key, "info": await attach_object_permission_to_dict(key_info_dict, prisma_client)} - except Exception as e: + except (ProxyException, HTTPException) as e: raise handle_exception_on_proxy(e) + except Exception as e: + verbose_proxy_logger.exception("Unhandled error in /key/info") + raise ProxyException( + message="Internal server error from /key/info. Details are in the proxy server logs.", + type="internal_error", + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) async def _find_deleted_key_info( diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index a5ebf41eebd..dffd9b9b513 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -22833,3 +22833,130 @@ async def test_key_generate_reads_the_team_from_the_writer_database_not_the_cach ) assert raised.value.message == "Team not found for team_id=1234. Non-admin users cannot create keys for non-existent teams." + + +def test_key_info_unexpected_database_exception_sanitized( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_prisma: Final = MagicMock() + secret_marker: Final = "SECRET_DB_DETAIL: invalid database statement" + mock_prisma.db.litellm_verificationtoken.find_unique = AsyncMock( + side_effect=RuntimeError(secret_marker) + ) + monkeypatch.setattr(proxy_server, "prisma_client", mock_prisma) + + admin_auth: Final = UserAPIKeyAuth( + api_key="sk-test-master", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + proxy_server.app.dependency_overrides[user_api_key_auth] = lambda: admin_auth + + try: + client: Final = TestClient(proxy_server.app) + with caplog.at_level("ERROR"): + response: Final = client.get("/key/info?key=sk-target") + assert response.status_code == 500 + assert secret_marker not in response.text + assert response.json()["error"]["message"] == "Internal server error from /key/info. Details are in the proxy server logs." + assert secret_marker in caplog.text + finally: + proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_key_info_missing_key_preserves_404( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_prisma: Final = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) + mock_prisma.db.litellm_deletedverificationtoken.find_first = AsyncMock(return_value=None) + monkeypatch.setattr(proxy_server, "prisma_client", mock_prisma) + + admin_auth: Final = UserAPIKeyAuth( + api_key="sk-test-master", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + proxy_server.app.dependency_overrides[user_api_key_auth] = lambda: admin_auth + + try: + client: Final = TestClient(proxy_server.app) + response: Final = client.get("/key/info?key=sk-nonexistent") + assert response.status_code == 404 + assert response.json()["error"]["message"] == "Key not found in database" + finally: + proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_key_info_permission_denied_preserves_403( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_key: Final = MagicMock() + mock_key.token = "hashed-other-token" + mock_key.user_id = "other-user" + mock_key.model_dump.return_value = { + "token": "hashed-other-token", + "user_id": "other-user", + "team_id": None, + "blocked": False, + "expires": None, + } + mock_prisma: Final = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=mock_key) + monkeypatch.setattr(proxy_server, "prisma_client", mock_prisma) + monkeypatch.setattr( + "litellm.proxy.management_endpoints.key_management_endpoints._can_user_query_key_info", + AsyncMock(return_value=False), + ) + + user_auth: Final = UserAPIKeyAuth( + api_key="sk-regular-user", + user_id="user1", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + proxy_server.app.dependency_overrides[user_api_key_auth] = lambda: user_auth + + try: + client: Final = TestClient(proxy_server.app) + response: Final = client.get("/key/info?key=sk-other-key") + assert response.status_code == 403 + assert "not allowed to access this key's info" in response.text + finally: + proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_key_info_database_not_connected_sanitized_500( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + monkeypatch.setattr(proxy_server, "prisma_client", None) + + admin_auth: Final = UserAPIKeyAuth( + api_key="sk-test-master", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + proxy_server.app.dependency_overrides[user_api_key_auth] = lambda: admin_auth + + try: + client: Final = TestClient(proxy_server.app) + with caplog.at_level("ERROR"): + response: Final = client.get("/key/info?key=sk-target") + assert response.status_code == 500 + assert response.json()["error"]["message"] == "Internal server error from /key/info. Details are in the proxy server logs." + assert "Database not connected" in caplog.text + finally: + proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) + + + From 4995d319ed54ab992233285b7d2a8743545af0b8 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Sun, 18 Oct 2026 20:52:33 -0700 Subject: [PATCH 09/50] test(rust): add native vs Python parity harness for responses (#46567) Each case runs one aresponses call through Python and through the native route against the same recorded OpenAI response and compares the dispatch, provider request, return value and success log. Known gaps are strict xfails so the PRs that close them must flip the marker --- tests/rust-python-harness/AGENTS.md | 3 +- .../e2e_parity/sdk/responses/conftest.py | 34 ++ .../sdk/responses/test_native_parity.py | 320 ++++++++++++++++++ tests/test_litellm_rust/conftest.py | 28 +- tests/test_litellm_rust/support/isolation.py | 31 +- 5 files changed, 387 insertions(+), 29 deletions(-) create mode 100644 tests/rust-python-harness/strategies/e2e_parity/sdk/responses/conftest.py create mode 100644 tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py diff --git a/tests/rust-python-harness/AGENTS.md b/tests/rust-python-harness/AGENTS.md index 3064960d66e..98a20fa1c38 100644 --- a/tests/rust-python-harness/AGENTS.md +++ b/tests/rust-python-harness/AGENTS.md @@ -13,7 +13,7 @@ tests/rust-python-harness/ │ │ ├── __init__.py │ │ ├── reporting.py │ │ ├── sdk/ -│ │ │ └── ocr/ +│ │ │ └── responses/ │ │ │ ├── trace_parity/ │ │ ├── __init__.py @@ -53,6 +53,7 @@ tests/rust-python-harness/ - Examples: `run e2e_parity --surface sdk --function ocr`, `run unit_tests_parity --function ocr --pytest-arg=-x`, or `run all --function ocr` - `cli/catalog.py` discovers strategies, validates their Python definitions, and orders them. `cli/__init__.py` builds the Click command tree. `cli/commands.py` runs selected cases - `e2e_parity/` compares SDK objects, exceptions, callbacks, and streams, or gateway HTTP responses +- `e2e_parity/sdk//` holds the pytest parity files that send one public SDK call through the Python implementation and through the native route against the same recorded provider response, then compare dispatch, the provider request, the response and the success log. Every known gap is a strict xfail naming the gap. Run them with `uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/strategies/e2e_parity/sdk/responses -q`. They reuse the recording server and isolation helpers from `tests/test_litellm_rust/support`, and they never belong in `tests/test_litellm_rust`, which is native only - `trace_parity/` profiles the Python call stack and prints every collected Python call under `litellm/`. It never collects Rust spans and never rebuilds the native extension - E2E and trace strategies load their registered module cases and run surface-specific execution from their folders - `shared/unit_runners/contracts.py` owns the typed per-function unit contracts consumed by `unit_tests_parity` and `unit_tests_rust` diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/conftest.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/conftest.py new file mode 100644 index 00000000000..4bb20713c59 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/conftest.py @@ -0,0 +1,34 @@ +from collections.abc import AsyncIterator, Generator +from importlib import import_module +from pathlib import Path +from typing import Final + +import pytest +import pytest_asyncio + +from tests.test_litellm_rust.support.isolation import isolated_test_state +from tests.test_litellm_rust.support.recording_server import RecordingServer, recording_service + +PARITY_ROOT: Final = Path(__file__).parent + + +@pytest_asyncio.fixture(autouse=True, loop_scope="function") +async def isolate_test_state() -> AsyncIterator[None]: + async with isolated_test_state(): + yield + + +@pytest.fixture +def recording_server() -> Generator[RecordingServer]: + with recording_service() as server: + yield server + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + try: + import_module("litellm.rust_bridge._native") + except ImportError: + skip: Final = pytest.mark.skip(reason="requires a compiled litellm.rust_bridge._native extension") + for item in items: + if PARITY_ROOT in item.path.parents: + item.add_marker(skip) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py new file mode 100644 index 00000000000..0c8efcd36c1 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py @@ -0,0 +1,320 @@ +"""Native Responses calls must be indistinguishable from the Python implementation. + +Each case sends one public `litellm.aresponses` call through Python and through the native route against the same +recorded OpenAI response, then compares what reached the provider, what the caller got back, and what the success +callback logged. The native leg fails instead of silently falling back to Python +""" + +from collections.abc import Awaitable, Callable, Iterator, Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Final, cast + +import pytest +from pydantic import JsonValue, TypeAdapter + +import litellm +from litellm.responses import dispatch +from litellm.responses import main as python_responses +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.rust_bridge import catalog +from litellm.rust_bridge.catalog import Route, RouteRule +from litellm.rust_bridge.configuration import Rollout +from litellm.types.llms.openai import ResponsesAPIResponse +from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging +from tests.test_litellm_rust.support.isolation import isolated_callback_registries, rebound +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec + +pytestmark = pytest.mark.requires_rust_extension + +MODEL: Final = "openai/gpt-6.1-sol" +_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_LOGGED: Final = TypeAdapter(dict[str, object]) +_COMPARED_HEADERS: Final = ("authorization", "content-type", "openai-organization", "openai-project") +_COMPARED_LOG_FIELDS: Final = ( + "call_type", + "custom_llm_provider", + "model", + "model_group", + "model_id", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "response_cost", + "status", +) + +TEXT_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "id": "resp_0a1b2c3d4e5f", + "object": "response", + "created_at": 1760486400, + "status": "completed", + "error": None, + "incomplete_details": None, + "instructions": None, + "max_output_tokens": None, + "model": "gpt-6.1-sol-2026-09-30", + "output": [ + { + "id": "rs_0a1b2c3d4e5f", + "type": "reasoning", + "summary": [], + }, + { + "id": "msg_0a1b2c3d4e5f", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Paris"}], + }, + ], + "parallel_tool_calls": True, + "previous_response_id": None, + "reasoning": {"effort": "medium", "summary": None}, + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 12, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 70, + "output_tokens_details": {"reasoning_tokens": 64}, + "total_tokens": 82, + }, + "user": None, + "metadata": {}, + } +) + +FUNCTION_CALL_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "id": "resp_6f5e4d3c2b1a", + "output": [ + { + "id": "fc_6f5e4d3c2b1a", + "type": "function_call", + "status": "completed", + "arguments": '{"city":"Paris"}', + "call_id": "call_6f5e4d3c2b1a", + "name": "get_weather", + } + ], + } +) + +WEATHER_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "type": "function", + "name": "get_weather", + "description": "Current weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } +) + + +REASONING_REPLAY_INPUT: Final[list[JsonValue]] = [ + {"type": "reasoning", "id": "rs_0a1b2c3d4e5f", "summary": [], "status": None}, + {"role": "user", "content": "And Germany?"}, +] + +_PYTHON_ARESPONSES: Final = cast(Callable[..., Awaitable[object]], python_responses.aresponses) # noqa: TID251 # the Python baseline under comparison +_PUBLIC_ARESPONSES: Final = cast(Callable[..., Awaitable[object]], litellm.aresponses) + + +@dataclass(frozen=True, slots=True) +class Leg: + request_body: Mapping[str, JsonValue] + request_headers: Mapping[str, str] + response: Mapping[str, JsonValue] + logged: Mapping[str, object] + fell_back: bool + + +ASPECTS: Final = ("dispatch", "request", "response", "logged") + + +@dataclass(frozen=True, slots=True) +class Case: + name: str + kwargs: Mapping[str, object] + upstream: Mapping[str, JsonValue] = TEXT_RESPONSE + gaps: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) + + def params(self) -> Iterator[object]: + for aspect in ASPECTS: + gap = self.gaps.get(aspect) + marks = () if gap is None else (pytest.mark.xfail(reason=gap, strict=True),) + yield pytest.param(self, aspect, id=f"{self.name}-{aspect}", marks=marks) + + +ID_ENCODING_GAP: Final = MappingProxyType( + {"response": "native responses return the provider's raw response id instead of LiteLLM's encoded resp_ id"} +) + + +def _gaps(**request_gaps: str) -> Mapping[str, str]: + return MappingProxyType({**ID_ENCODING_GAP, **request_gaps}) + + +CASES: Final = ( + Case("plain_text", {"input": "What is the capital of France?"}, gaps=_gaps()), + Case( + "instructions_and_input_items", + { + "instructions": "Answer in one word", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "Capital of France?"}]}], + }, + gaps=_gaps(), + ), + Case( + "function_tool", + {"input": "Weather in Paris?", "tools": [dict(WEATHER_TOOL)], "tool_choice": "auto"}, + upstream=FUNCTION_CALL_RESPONSE, + gaps=_gaps(), + ), + Case( + "encoded_previous_response_id", + { + "input": [{"type": "function_call_output", "call_id": "call_6f5e4d3c2b1a", "output": "18C and sunny"}], + "previous_response_id": ResponsesAPIRequestUtils._build_responses_api_response_id( # pyright: ignore[reportPrivateUsage] # the public encoder needs a response object + custom_llm_provider="openai", model_id=None, response_id="resp_6f5e4d3c2b1a" + ), + }, + gaps=_gaps(), + ), + Case( + "warm_up_probe_below_minimum_max_output_tokens", + {"input": "hi", "max_output_tokens": 1}, + gaps=_gaps(request="native requests send max_output_tokens below OpenAI's minimum of 16"), + ), + Case( + "gpt_5_family_drops_temperature_while_reasoning", + {"input": "hi", "temperature": 0.2, "reasoning": {"effort": "high"}, "drop_params": True}, + gaps=MappingProxyType( + {"dispatch": "native responses decline drop_params instead of applying the GPT-5 sampling rules"} + ), + ), + Case( + "anthropic_cache_control_is_stripped", + { + "input": [ + { + "role": "user", + "content": [{"type": "input_text", "text": "hi", "cache_control": {"type": "ephemeral"}}], + } + ], + "tools": [{**WEATHER_TOOL, "cache_control": {"type": "ephemeral"}}], + }, + gaps=_gaps(), + ), + Case( + "reasoning_replay_without_status", + { + "input": REASONING_REPLAY_INPUT, + }, + gaps=_gaps(request="native requests forward reasoning items with status null"), + ), + Case( + "tool_parameters_sent_as_json_string", + { + "input": "Weather in Paris?", + "tools": [{**WEATHER_TOOL, "parameters": '{"type":"object","properties":{"city":{"type":"string"}}}'}], + }, + gaps=_gaps(request="native requests forward tool parameters encoded as a JSON string"), + ), +) + + +@dataclass(slots=True) +class FallbackProbe: + calls: int = 0 + + +@pytest.fixture +def native_responses() -> Iterator[FallbackProbe]: + probe: Final = FallbackProbe() + python_aresponses: Final = dispatch._PYTHON_ARESPONSES # pyright: ignore[reportPrivateUsage] # the fallback under observation + + async def python_fallback(*args: object, **kwargs: object) -> object: + probe.calls += 1 + return await python_aresponses(*args, **kwargs) + + with ( + rebound(catalog, "RULES", (RouteRule(Route.RESPONSES, Rollout.RUST_OPT_OUT), *catalog.RULES)), + rebound(dispatch, "_PYTHON_ARESPONSES", python_fallback), + ): + yield probe + + +def _kwargs(case: Case, server: RecordingServer, recorder: RecordingLogger) -> dict[str, object]: + return {"model": MODEL, "api_key": "sk-test", "api_base": server.base_url, "callbacks": [recorder], **case.kwargs} + + +def _response(result: object) -> Mapping[str, JsonValue]: + assert isinstance(result, ResponsesAPIResponse) + return _OBJECT.validate_python(result.model_dump(mode="json", exclude_none=True)) + + +async def _leg(case: Case, server: RecordingServer, probe: FallbackProbe | None) -> Leg: + with isolated_callback_registries(): + return await _isolated_leg(case, server, probe) + + +async def _isolated_leg(case: Case, server: RecordingServer, probe: FallbackProbe | None) -> Leg: + recorder: Final = RecordingLogger() + server.enqueue(ResponseSpec(body=dict(case.upstream))) + request_count: Final = len(server.requests) + kwargs: Final = _kwargs(case, server, recorder) + fallbacks_before: Final = 0 if probe is None else probe.calls + if probe is None: + result = await _PYTHON_ARESPONSES(**kwargs) + else: + result = await _PUBLIC_ARESPONSES(**kwargs) + await drain_logging() + success: Final = await recorder.wait_for_async("async_log_success_event") + assert len(server.requests) == request_count + 1 + sent: Final = server.requests[-1] + standard: Final = _LOGGED.validate_python(_LOGGED.validate_python(success[0].kwargs)["standard_logging_object"]) + return Leg( + request_body=_OBJECT.validate_python(sent.body), + request_headers=MappingProxyType( + {name: value for name, value in sent.headers.items() if name in _COMPARED_HEADERS} + ), + response=_response(result), + logged=MappingProxyType({name: standard.get(name) for name in _COMPARED_LOG_FIELDS}), + fell_back=probe is not None and probe.calls > fallbacks_before, + ) + + +@pytest.fixture +def parity_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.expected_requests = 2 + return recording_server + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("case", "aspect"), tuple(param for case in CASES for param in case.params())) +async def test_native_responses_match_python( + case: Case, aspect: str, parity_server: RecordingServer, native_responses: FallbackProbe +) -> None: + python: Final = await _leg(case, parity_server, probe=None) + native: Final = await _leg(case, parity_server, probe=native_responses) + match aspect: + case "dispatch": + assert not native.fell_back + case "request": + assert (native.request_body, native.request_headers) == (python.request_body, python.request_headers) + case "response": + assert native.response == python.response + case _: + assert native.logged == python.logged diff --git a/tests/test_litellm_rust/conftest.py b/tests/test_litellm_rust/conftest.py index c1d452e838e..06fbcf58591 100644 --- a/tests/test_litellm_rust/conftest.py +++ b/tests/test_litellm_rust/conftest.py @@ -1,7 +1,4 @@ -import asyncio from collections.abc import AsyncIterator, Generator -from concurrent.futures import ThreadPoolExecutor -from contextlib import ExitStack from importlib import import_module from typing import Final @@ -9,34 +6,15 @@ import pytest import pytest_asyncio import litellm -from litellm import utils -from litellm.litellm_core_utils import litellm_logging, thread_pool_executor -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.rust_bridge.configuration import _CONFIGURATION -from tests.test_litellm_rust.support.callback_recorder import drain_logging from tests.test_litellm_rust.support.clickhouse import clickhouse_url as clickhouse_url -from tests.test_litellm_rust.support.isolation import isolated_callback_registries, rebound +from tests.test_litellm_rust.support.isolation import isolated_test_state from tests.test_litellm_rust.support.recording_server import RecordingServer, recording_service @pytest_asyncio.fixture(autouse=True, loop_scope="function") async def isolate_test_state() -> AsyncIterator[None]: - with ExitStack() as stack: - stack.enter_context(isolated_callback_registries()) - stack.enter_context(rebound(litellm, "cache", None)) # test-quality-ok: isolate process-global cache - stack.enter_context(rebound(_CONFIGURATION, "override", None)) - executor: Final = ThreadPoolExecutor(thread_name_prefix="rust-test-logging") - stack.enter_context(rebound(litellm_logging, "executor", executor)) - stack.enter_context(rebound(utils, "executor", executor)) - stack.enter_context(rebound(thread_pool_executor, "executor", executor)) - try: - yield - finally: - try: - await drain_logging() - finally: - await asyncio.to_thread(executor.shutdown, wait=True) - await GLOBAL_LOGGING_WORKER.stop() + async with isolated_test_state(): + yield @pytest.fixture diff --git a/tests/test_litellm_rust/support/isolation.py b/tests/test_litellm_rust/support/isolation.py index 4464ee4bb51..58dc0d9016d 100644 --- a/tests/test_litellm_rust/support/isolation.py +++ b/tests/test_litellm_rust/support/isolation.py @@ -1,11 +1,16 @@ -from collections.abc import Generator -from contextlib import ExitStack, contextmanager +import asyncio +from collections.abc import AsyncGenerator, Generator +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack, asynccontextmanager, contextmanager from types import ModuleType from typing import Final, cast import litellm from litellm import utils -from litellm.litellm_core_utils import litellm_logging +from litellm.litellm_core_utils import litellm_logging, thread_pool_executor +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.rust_bridge.configuration import _CONFIGURATION +from tests.test_litellm_rust.support.callback_recorder import drain_logging CALLBACK_ATTRIBUTES: Final = ( "callbacks", @@ -56,3 +61,23 @@ def isolated_callback_registries() -> Generator[None]: stack.enter_context(_isolated_list(litellm_logging, "_in_memory_loggers")) stack.enter_context(rebound(utils, "callback_list", [])) yield + + +@asynccontextmanager +async def isolated_test_state() -> AsyncGenerator[None]: + with ExitStack() as stack: + stack.enter_context(isolated_callback_registries()) + stack.enter_context(rebound(litellm, "cache", None)) # test-quality-ok: isolate process-global cache + stack.enter_context(rebound(_CONFIGURATION, "override", None)) + executor: Final = ThreadPoolExecutor(thread_name_prefix="rust-test-logging") + stack.enter_context(rebound(litellm_logging, "executor", executor)) + stack.enter_context(rebound(utils, "executor", executor)) + stack.enter_context(rebound(thread_pool_executor, "executor", executor)) + try: + yield + finally: + try: + await drain_logging() + finally: + await asyncio.to_thread(executor.shutdown, wait=True) + await GLOBAL_LOGGING_WORKER.stop() From bcd80b6aa10544b366e899fc45a58ee856afbe2a Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Sun, 18 Oct 2026 20:52:34 -0700 Subject: [PATCH 10/50] feat(rust): encode LiteLLM response ids on native responses (#46572) Native Responses now hands back the same managed ids as Python: the resp_ id carries the provider and deployment, encrypted-content items get encitem_ ids and wrapped content under affinity routing, and container ids become cntr_ ids. Encoded encrypted-content items in the input are restored before the provider request. The codec lives in inference-responses with goldens taken from litellm/responses/utils.py --- litellm-rust/Cargo.lock | 1 + .../crates/inference-responses/AGENTS.md | 1 + .../crates/inference-responses/Cargo.toml | 1 + .../crates/inference-responses/src/ids.rs | 499 ++++++++++++++++++ .../crates/inference-responses/src/lib.rs | 1 + .../src/routes/responses/host.rs | 73 ++- .../python-bridge/src/routes/responses/mod.rs | 2 +- .../sdk/responses/test_native_parity.py | 72 ++- 8 files changed, 624 insertions(+), 26 deletions(-) create mode 100644 litellm-rust/crates/inference-responses/src/ids.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 02b48a3be3b..98f36607fff 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4083,6 +4083,7 @@ dependencies = [ name = "litellm-inference-responses" version = "0.1.0" dependencies = [ + "base64 0.22.1", "bytes", "futures-util", "litellm-auth", diff --git a/litellm-rust/crates/inference-responses/AGENTS.md b/litellm-rust/crates/inference-responses/AGENTS.md index 1dd199024b0..d7b58badf19 100644 --- a/litellm-rust/crates/inference-responses/AGENTS.md +++ b/litellm-rust/crates/inference-responses/AGENTS.md @@ -2,3 +2,4 @@ - This crate owns Responses API call orchestration. Payloads belong in `litellm-llms-types`, transformations in `llms/src/base_llm/responses` and `llms/src//responses` - The HTTP route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks, cached through `litellm_inference::caching::execute_streaming` - WebSocket sessions (`src/websocket.rs`) stay separate from the HTTP call driver because a connection can accept multiple requests while receiving events +- `src/ids.rs` owns the LiteLLM-managed ids handed back to callers (`resp_`, `encitem_`, `litellm_enc:`, `cntr_`) and must stay fully compatible with `litellm/responses/utils.py` diff --git a/litellm-rust/crates/inference-responses/Cargo.toml b/litellm-rust/crates/inference-responses/Cargo.toml index 255518b9c54..793483ee76d 100644 --- a/litellm-rust/crates/inference-responses/Cargo.toml +++ b/litellm-rust/crates/inference-responses/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true bytes.workspace = true futures-util.workspace = true litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] } diff --git a/litellm-rust/crates/inference-responses/src/ids.rs b/litellm-rust/crates/inference-responses/src/ids.rs new file mode 100644 index 00000000000..b51aea1559e --- /dev/null +++ b/litellm-rust/crates/inference-responses/src/ids.rs @@ -0,0 +1,499 @@ +//! LiteLLM-managed identifiers on Responses payloads, fully compatible with `litellm/responses/utils.py` +//! +//! LiteLLM rewrites some ids in a Responses API result before handing it back, so that a later +//! request carrying those ids can be routed to the provider and deployment that produced them, +//! and then restored to what the provider originally issued. The native and Python paths serve +//! the same traffic, so an id minted by either one must decode on the other. +//! +//! Each managed id is a fixed prefix followed by standard, padded base64 of a `;`-separated +//! plaintext: +//! +//! | Id | Applied to | Decoded plaintext | +//! |---|---|---| +//! | `resp_` | the response `id` | `litellm:custom_llm_provider:{provider};model_id:{model_id};response_id:{id}` | +//! | `encitem_` | the `id` of an output item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` | +//! | `litellm_enc:;{content}` | that item's `encrypted_content` | `` is `model_id:{model_id}`, and the original content follows the `;` unencoded | +//! | `cntr_` | `container_id` on output items, `code_interpreter_call`, and message annotations | `litellm:custom_llm_provider:{provider};model_id:{model_id};container_id:{id}` | +//! +//! For example, OpenAI's `resp_0a1b2c3d4e5f` served without a deployment id becomes +//! `resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm`, +//! which decodes to `litellm:custom_llm_provider:openai;model_id:None;response_id:resp_0a1b2c3d4e5f`. +//! +//! How they fit together: +//! - `resp_` lets a follow-up's `previous_response_id` find the provider and deployment again. +//! [`encode_response`] stamps it on every response, unless the id is already managed. +//! - `encitem_` and `litellm_enc:` carry encrypted-content affinity: encrypted reasoning only +//! decrypts under the API key that produced it, so the router pins replays to that deployment. +//! They are stamped only when the router sets `encrypted_content_affinity_enabled` and a +//! deployment `model_id` is known (see [`IdScope`]). Items without an `id`, such as Codex's, +//! carry the deployment in the wrapped content alone. +//! - `cntr_` lets container calls find the provider and deployment that own a container. A +//! container id that already names a provider is left as it is. +//! - On the way in, [`restore_input`] turns `encitem_` ids and `litellm_enc:` content in the +//! request input back into the provider's own bytes before the request is sent. The routing +//! decision itself stays in Python's `EncryptedContentAffinityCheck`. +//! +//! Python's quirks are kept on purpose: +//! - A missing provider or `model_id` is written as `None` in `resp_` ids (an f-string of +//! `None`), but as an empty string in `cntr_` ids. [`decode_container_id`] reads both as absent. +//! - Decoding follows Python's non-strict `base64.b64decode`: characters outside the alphabet +//! are skipped, trailing bits are allowed, and padding stripped in transit is added back. +//! - A `resp_` id counts as managed when it decodes to at least three `;`-separated parts. + +use base64::{ + Engine, alphabet, + engine::{GeneralPurpose, general_purpose::PAD}, +}; +use litellm_llms_types::formats::responses::ResponsesApiResponse; +use serde_json::{Map, Value}; + +const RESPONSE_PREFIX: &str = "resp_"; +const ITEM_PREFIX: &str = "encitem_"; +const CONTAINER_PREFIX: &str = "cntr_"; +const WRAPPED_CONTENT_PREFIX: &str = "litellm_enc:"; + +/// Python's non-strict `base64.b64decode`: characters outside the alphabet are skipped and +/// non-zero trailing bits are accepted +const LENIENT: GeneralPurpose = GeneralPurpose::new( + &alphabet::STANDARD, + PAD.with_decode_allow_trailing_bits(true), +); + +/// Where a response came from, as encoded into the ids LiteLLM hands back +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct IdScope { + pub provider: Option, + pub model_id: Option, + pub encrypted_content_affinity: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecodedContainerId { + pub provider: Option, + pub model_id: Option, + pub container_id: String, +} + +/// Rewrites the response id, encrypted-content items and container ids the way the Python +/// Responses path does before returning to the caller. An already managed id is left untouched +pub fn encode_response(response: &mut ResponsesApiResponse, scope: &IdScope) { + if is_managed_response_id(&response.id) { + return; + } + response.id = response_id(scope, &response.id); + if let Some(model_id) = scope + .model_id + .as_deref() + .filter(|_| scope.encrypted_content_affinity) + .filter(|model_id| !model_id.is_empty()) + { + response + .output + .iter_mut() + .filter_map(Value::as_object_mut) + .for_each(|item| encode_encrypted_item(item, model_id)); + } + response + .output + .iter_mut() + .for_each(|item| encode_containers(item, scope)); +} + +/// Restores encrypted-content item ids and contents that LiteLLM encoded on an earlier response, +/// so the provider receives what it originally issued +pub fn restore_input(input: &mut Value) { + let Some(items) = input.as_array_mut() else { + return; + }; + for item in items.iter_mut().filter_map(Value::as_object_mut) { + if let Some(Value::String(id)) = item.get_mut("id") + && let Some((_, original)) = decode_encrypted_item_id(id) + { + *id = original; + } + if let Some(Value::String(content)) = item.get_mut("encrypted_content") + && let Some(original) = unwrap_encrypted_content(content) + { + *content = original; + } + } +} + +pub fn response_id(scope: &IdScope, id: &str) -> String { + let assembled = format!( + "litellm:custom_llm_provider:{};model_id:{};response_id:{id}", + python_str(scope.provider.as_deref()), + python_str(scope.model_id.as_deref()), + ); + format!("{RESPONSE_PREFIX}{}", LENIENT.encode(assembled)) +} + +pub fn is_managed_response_id(id: &str) -> bool { + decode_text(&id.replace(RESPONSE_PREFIX, "")) + .is_some_and(|decoded| decoded.split(';').count() >= 3) +} + +pub fn encrypted_item_id(model_id: &str, id: &str) -> String { + format!( + "{ITEM_PREFIX}{}", + LENIENT.encode(format!("litellm:model_id:{model_id};item_id:{id}")) + ) +} + +pub fn decode_encrypted_item_id(id: &str) -> Option<(String, String)> { + let decoded = decode_text(&padded(id.strip_prefix(ITEM_PREFIX)?))?; + let (model_id, item_id) = decoded.split_once(';')?; + Some(( + model_id.replace("litellm:model_id:", ""), + item_id.replace("item_id:", ""), + )) +} + +pub fn wrap_encrypted_content(content: &str, model_id: &str) -> String { + format!( + "{WRAPPED_CONTENT_PREFIX}{};{content}", + LENIENT.encode(format!("model_id:{model_id}")) + ) +} + +/// The original content of a wrapped `encrypted_content`, or `None` when it is not wrapped +pub fn unwrap_encrypted_content(content: &str) -> Option { + if !content.starts_with(WRAPPED_CONTENT_PREFIX) { + return None; + } + let (metadata, original) = content.split_once(';')?; + decode_text(&padded(&metadata.replace(WRAPPED_CONTENT_PREFIX, "")))?; + Some(original.to_owned()) +} + +pub fn container_id(scope: &IdScope, id: &str) -> String { + let assembled = format!( + "litellm:custom_llm_provider:{};model_id:{};container_id:{id}", + scope.provider.as_deref().unwrap_or_default(), + scope.model_id.as_deref().unwrap_or_default(), + ); + format!("{CONTAINER_PREFIX}{}", LENIENT.encode(assembled)) +} + +pub fn decode_container_id(id: &str) -> DecodedContainerId { + let unmanaged = || DecodedContainerId { + provider: None, + model_id: None, + container_id: id.to_owned(), + }; + if !id.starts_with(CONTAINER_PREFIX) { + return unmanaged(); + } + let Some(decoded) = decode_text(&id.replace(CONTAINER_PREFIX, "")) else { + return unmanaged(); + }; + let parsed = decoded + .strip_prefix("litellm:custom_llm_provider:") + .and_then(|rest| rest.split_once(';')) + .and_then(|(provider, rest)| { + let (model_id, rest) = rest.strip_prefix("model_id:")?.split_once(';')?; + let container_id = rest.strip_prefix("container_id:")?; + (!container_id.is_empty() && !container_id.contains('\n')).then_some(( + provider, + model_id, + container_id, + )) + }); + let Some((provider, model_id, container_id)) = parsed else { + return unmanaged(); + }; + let present = |value: &str| (!matches!(value, "" | "None")).then(|| value.to_owned()); + DecodedContainerId { + provider: present(provider), + model_id: present(model_id), + container_id: container_id.to_owned(), + } +} + +fn encode_encrypted_item(item: &mut Map, model_id: &str) { + let Some(Value::String(content)) = item.get_mut("encrypted_content") else { + return; + }; + if content.is_empty() { + return; + } + *content = wrap_encrypted_content(content, model_id); + if let Some(Value::String(id)) = item.get_mut("id") + && !id.is_empty() + { + *id = encrypted_item_id(model_id, id); + } +} + +fn encode_containers(item: &mut Value, scope: &IdScope) { + let Some(item) = item.as_object_mut() else { + return; + }; + encode_container_field(item, scope); + if let Some(nested) = item + .get_mut("code_interpreter_call") + .and_then(Value::as_object_mut) + { + encode_container_field(nested, scope); + } + if item.get("type").and_then(Value::as_str) != Some("message") { + return; + } + let Some(content) = item.get_mut("content").and_then(Value::as_array_mut) else { + return; + }; + content + .iter_mut() + .filter_map(|part| part.get_mut("annotations").and_then(Value::as_array_mut)) + .flatten() + .for_each(|annotation| encode_containers(annotation, scope)); +} + +fn encode_container_field(item: &mut Map, scope: &IdScope) { + if let Some(Value::String(id)) = item.get_mut("container_id") + && decode_container_id(id).provider.is_none() + { + *id = container_id(scope, id); + } +} + +fn python_str(value: Option<&str>) -> &str { + value.unwrap_or("None") +} + +fn padded(value: &str) -> String { + let missing = value.len() % 4; + let padding = if missing == 0 { 0 } else { 4 - missing }; + format!("{value}{}", "=".repeat(padding)) +} + +fn decode_text(value: &str) -> Option { + let alphabet: String = value + .chars() + .filter(|character| { + character.is_ascii_alphanumeric() || matches!(character, '+' | '/' | '=') + }) + .collect(); + String::from_utf8(LENIENT.decode(alphabet).ok()?).ok() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn scope(provider: Option<&str>, model_id: Option<&str>, affinity: bool) -> IdScope { + IdScope { + provider: provider.map(str::to_owned), + model_id: model_id.map(str::to_owned), + encrypted_content_affinity: affinity, + } + } + + fn response(id: &str, output: Value) -> ResponsesApiResponse { + serde_json::from_value(json!({"id": id, "model": "gpt-6.1-sol", "output": output})).unwrap() + } + + #[rstest] + #[case::openai_without_model_id( + Some("openai"), + None, + "resp_0a1b2c3d4e5f", + "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm" + )] + #[case::openrouter_with_model_id( + Some("openrouter"), + Some("deploy-1"), + "gen-123", + "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5yb3V0ZXI7bW9kZWxfaWQ6ZGVwbG95LTE7cmVzcG9uc2VfaWQ6Z2VuLTEyMw==" + )] + fn response_ids_match_python( + #[case] provider: Option<&str>, + #[case] model_id: Option<&str>, + #[case] id: &str, + #[case] expected: &str, + ) { + assert_eq!(response_id(&scope(provider, model_id, false), id), expected); + } + + #[rstest] + #[case::raw_short_id("resp_0a1b2c3d4e5f", false)] + #[case::raw_long_id("resp_68a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3", false)] + #[case::two_segments("resp_YTti", false)] + #[case::three_segments("resp_YTtiO2M=", true)] + #[case::managed( + "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOng=", + true + )] + fn managed_response_ids_match_python(#[case] id: &str, #[case] expected: bool) { + assert_eq!(is_managed_response_id(id), expected); + } + + #[test] + fn encrypted_items_match_python() { + assert_eq!( + encrypted_item_id("deploy-1", "rs_0a1b"), + "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=" + ); + assert_eq!( + wrap_encrypted_content("gAAAA", "deploy-1"), + "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA" + ); + } + + #[rstest] + #[case::padded("encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", Some(("deploy-1", "rs_0a1b")))] + #[case::padding_stripped_in_transit("encitem_bGl0ZWxsbTptb2RlbF9pZDptO2l0ZW1faWQ6cnNfMQ", Some(("m", "rs_1")))] + #[case::not_base64("encitem_notbase64!", None)] + #[case::unmanaged("rs_0a1b", None)] + fn encrypted_item_ids_decode_like_python( + #[case] id: &str, + #[case] expected: Option<(&str, &str)>, + ) { + assert_eq!( + decode_encrypted_item_id(id), + expected.map(|(model_id, item_id)| (model_id.to_owned(), item_id.to_owned())) + ); + } + + #[rstest] + #[case::unpadded_metadata("litellm_enc:bW9kZWxfaWQ6bQ;gAAAA", Some("gAAAA"))] + #[case::no_separator("litellm_enc:nosemicolon", None)] + #[case::unwrapped("gAAAA", None)] + fn encrypted_content_unwraps_like_python( + #[case] content: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!(unwrap_encrypted_content(content).as_deref(), expected); + } + + #[rstest] + #[case::with_model_id( + Some("deploy-1"), + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpkZXBsb3ktMTtjb250YWluZXJfaWQ6Y250cl9hYmM=" + )] + #[case::without_model_id( + None, + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfYWJj" + )] + fn container_ids_match_python(#[case] model_id: Option<&str>, #[case] expected: &str) { + assert_eq!( + container_id(&scope(Some("openai"), model_id, false), "cntr_abc"), + expected + ); + } + + #[rstest] + #[case::raw("cntr_abc", None, None, "cntr_abc")] + #[case::managed( + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDptO2NvbnRhaW5lcl9pZDpjbnRyXzE=", + Some("openai"), + Some("m"), + "cntr_1" + )] + #[case::empty_segments( + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOjttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfMQ==", + None, + None, + "cntr_1" + )] + #[case::python_none_provider( + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOk5vbmU7bW9kZWxfaWQ6O2NvbnRhaW5lcl9pZDp4", + None, + None, + "x" + )] + fn container_ids_decode_like_python( + #[case] id: &str, + #[case] provider: Option<&str>, + #[case] model_id: Option<&str>, + #[case] container_id: &str, + ) { + assert_eq!( + decode_container_id(id), + DecodedContainerId { + provider: provider.map(str::to_owned), + model_id: model_id.map(str::to_owned), + container_id: container_id.to_owned(), + } + ); + } + + #[test] + fn encodes_response_items_under_affinity() { + let mut encoded = response( + "resp_1", + json!([ + {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []}, + {"type": "reasoning", "encrypted_content": "gBBBB", "summary": []}, + {"type": "code_interpreter_call", "id": "ci_1", "container_id": "cntr_abc"}, + {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [ + {"type": "container_file_citation", "container_id": "cntr_abc"} + ]}]} + ]), + ); + let scope = scope(Some("openai"), Some("deploy-1"), true); + encode_response(&mut encoded, &scope); + let container = container_id(&scope, "cntr_abc"); + assert_eq!(encoded.id, response_id(&scope, "resp_1")); + assert_eq!( + Value::Array(encoded.output), + json!([ + { + "type": "reasoning", + "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", + "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA", + "summary": [] + }, + {"type": "reasoning", "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gBBBB", "summary": []}, + {"type": "code_interpreter_call", "id": "ci_1", "container_id": container}, + {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [ + {"type": "container_file_citation", "container_id": container} + ]}]} + ]) + ); + } + + #[rstest] + #[case::affinity_disabled(scope(Some("openai"), Some("deploy-1"), false))] + #[case::no_model_id(scope(Some("openai"), None, true))] + fn leaves_encrypted_items_without_affinity(#[case] scope: IdScope) { + let output = json!([{"type": "reasoning", "id": "rs_1", "encrypted_content": "gAAAA", "summary": []}]); + let mut encoded = response("resp_1", output.clone()); + encode_response(&mut encoded, &scope); + assert_eq!(Value::Array(encoded.output), output); + } + + #[test] + fn leaves_managed_responses_untouched() { + let scope = scope(Some("openai"), Some("deploy-1"), true); + let id = response_id(&scope, "resp_1"); + let output = json!([{"type": "code_interpreter_call", "container_id": "cntr_abc"}]); + let mut encoded = response(&id, output.clone()); + encode_response(&mut encoded, &scope); + assert_eq!((encoded.id, Value::Array(encoded.output)), (id, output)); + } + + #[test] + fn restores_encrypted_items_in_input() { + let mut input = json!([ + { + "type": "reasoning", + "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", + "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA", + "summary": [] + }, + {"role": "user", "content": "hi"} + ]); + restore_input(&mut input); + assert_eq!( + input, + json!([ + {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []}, + {"role": "user", "content": "hi"} + ]) + ); + } +} diff --git a/litellm-rust/crates/inference-responses/src/lib.rs b/litellm-rust/crates/inference-responses/src/lib.rs index 431982457f6..11939823b05 100644 --- a/litellm-rust/crates/inference-responses/src/lib.rs +++ b/litellm-rust/crates/inference-responses/src/lib.rs @@ -1,6 +1,7 @@ use litellm_host::observation::ObservationSender; pub use litellm_inference::RouteError as Error; +pub mod ids; pub mod route; pub mod types; pub mod websocket; diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs index 8261e66406b..9c430ba7868 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs @@ -1,22 +1,41 @@ use std::convert::Infallible; use crate::routes::inference::InferenceHost; -use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned}; -use litellm_inference_responses::{Error, route::Responses, types::ResponsesCall}; +use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; +use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned, from_py}; +use litellm_inference_responses::{ + Error, + ids::{self, IdScope}, + route::Responses, + types::ResponsesCall, +}; use pyo3::{ gc::{PyTraverseError, PyVisit}, prelude::*, types::PyDict, }; -pub(super) struct ResponsesPythonHost(pub InferenceHost); +pub(super) struct ResponsesPythonHost { + host: InferenceHost, + ids: IdScope, +} + +impl ResponsesPythonHost { + pub fn new(host: InferenceHost) -> Self { + Self { + host, + ids: IdScope::default(), + } + } +} pub(super) fn project( host: &mut InferenceHost, py: Python<'_>, arguments: &Bound<'_, PyDict>, ) -> PyResult { - let call = host.project(py, arguments, "input")?; + let mut call = host.project(py, arguments, "input")?; + ids::restore_input(&mut call.input); let optional_params = call .params .into_iter() @@ -50,6 +69,39 @@ pub(super) fn project( }) } +/// The provider and deployment Python encodes into the ids it returns: the resolved provider and +/// `litellm_metadata.model_info.id` +fn id_scope( + host: &InferenceHost, + py: Python<'_>, + arguments: &Bound<'_, PyDict>, + call: &ResponsesCall, +) -> PyResult { + let metadata = host.argument(py, arguments, "litellm_metadata")?; + let encrypted_content_affinity = match &metadata { + Some(metadata) => metadata + .call_method1("get", ("encrypted_content_affinity_enabled",))? + .is_truthy()?, + None => false, + }; + let metadata = metadata + .map(|value| from_py::(&value)) + .transpose()? + .unwrap_or_default(); + let model_id = match metadata.pointer("/model_info/id") { + None | Some(serde_json::Value::Null) => None, + Some(serde_json::Value::String(id)) => Some(id.clone()), + Some(other) => Some(other.to_string()), + }; + Ok(IdScope { + provider: get_custom_llm_provider(&call.model, call.custom_llm_provider.as_deref()) + .map(|resolved| resolved.custom_llm_provider.to_owned()) + .or_else(|| call.custom_llm_provider.clone()), + model_id, + encrypted_content_affinity, + }) +} + impl PythonBinding for ResponsesPythonHost { type Protocol = Responses; type Failure = PyErr; @@ -59,7 +111,8 @@ impl PythonBinding for ResponsesPythonHost { py: Python<'_>, arguments: &Bound<'_, PyDict>, ) -> Result> { - let call = project(&mut self.0, py, arguments).map_err(InvokeError::Python)?; + let call = project(&mut self.host, py, arguments).map_err(InvokeError::Python)?; + self.ids = id_scope(&self.host, py, arguments, &call).map_err(InvokeError::Python)?; if call .optional_params .get("stream") @@ -77,7 +130,9 @@ impl PythonBinding for ResponsesPythonHost { py: Python<'_>, response: ::Response, ) -> PyResult> { - self.0.response(py, &response) + let mut response = response; + ids::encode_response(&mut response, &self.ids); + self.host.response(py, &response) } fn encode_stream_head( @@ -103,7 +158,7 @@ impl PythonBinding for ResponsesPythonHost { } fn map_error(&self, py: Python<'_>, error: Error) -> PyResult { - self.0.error(py, error) + self.host.error(py, error) } fn host_error(error: &PyErr) -> Error { Error::InvalidRequest(error.to_string().into()) @@ -122,9 +177,9 @@ impl PythonHostCalls for ResponsesPythonHost { impl PythonOwned for ResponsesPythonHost { fn close(&mut self, _: Python<'_>) { - self.0.close(); + self.host.close(); } fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { - self.0.traverse(visit) + self.host.traverse(visit) } } diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs index 984f1e74df2..9caeae6c3da 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs @@ -62,7 +62,7 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py )); } let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE); - run_inference::(py, call, asynchronous, ResponsesPythonHost(host)) + run_inference::(py, call, asynchronous, ResponsesPythonHost::new(host)) } #[pyfunction] diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py index 0c8efcd36c1..0ed5cf0d01d 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py @@ -44,6 +44,16 @@ _COMPARED_LOG_FIELDS: Final = ( "status", ) +TEXT_MESSAGE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "id": "msg_0a1b2c3d4e5f", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Paris"}], + } +) + TEXT_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( { "id": "resp_0a1b2c3d4e5f", @@ -61,13 +71,7 @@ TEXT_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( "type": "reasoning", "summary": [], }, - { - "id": "msg_0a1b2c3d4e5f", - "type": "message", - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Paris"}], - }, + dict(TEXT_MESSAGE), ], "parallel_tool_calls": True, "previous_response_id": None, @@ -122,6 +126,36 @@ WEATHER_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( ) +ENCRYPTED_REASONING_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "output": [ + {"id": "rs_0a1b2c3d4e5f", "type": "reasoning", "summary": [], "encrypted_content": "gAAAAB0a1b2c3d"}, + { + "id": "ci_0a1b2c3d4e5f", + "type": "code_interpreter_call", + "status": "completed", + "code": "print(1)", + "container_id": "cntr_0a1b2c3d4e5f", + "outputs": [], + }, + dict(TEXT_MESSAGE), + ], + } +) + +AFFINITY_REPLAY_INPUT: Final[list[JsonValue]] = [ + { + "type": "reasoning", + "id": ResponsesAPIRequestUtils._build_encrypted_item_id("deploy-1", "rs_0a1b2c3d4e5f"), # pyright: ignore[reportPrivateUsage] # the encoder Python applies on the way out + "summary": [], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "gAAAAB0a1b2c3d", "deploy-1" + ), + }, + {"role": "user", "content": "And Germany?"}, +] + REASONING_REPLAY_INPUT: Final[list[JsonValue]] = [ {"type": "reasoning", "id": "rs_0a1b2c3d4e5f", "summary": [], "status": None}, {"role": "user", "content": "And Germany?"}, @@ -157,30 +191,28 @@ class Case: yield pytest.param(self, aspect, id=f"{self.name}-{aspect}", marks=marks) -ID_ENCODING_GAP: Final = MappingProxyType( - {"response": "native responses return the provider's raw response id instead of LiteLLM's encoded resp_ id"} +DEPLOYMENT_METADATA: Final = MappingProxyType( + {"model_info": {"id": "deploy-1"}, "encrypted_content_affinity_enabled": True} ) -def _gaps(**request_gaps: str) -> Mapping[str, str]: - return MappingProxyType({**ID_ENCODING_GAP, **request_gaps}) +def _gaps(**gaps: str) -> Mapping[str, str]: + return MappingProxyType(gaps) CASES: Final = ( - Case("plain_text", {"input": "What is the capital of France?"}, gaps=_gaps()), + Case("plain_text", {"input": "What is the capital of France?"}), Case( "instructions_and_input_items", { "instructions": "Answer in one word", "input": [{"role": "user", "content": [{"type": "input_text", "text": "Capital of France?"}]}], }, - gaps=_gaps(), ), Case( "function_tool", {"input": "Weather in Paris?", "tools": [dict(WEATHER_TOOL)], "tool_choice": "auto"}, upstream=FUNCTION_CALL_RESPONSE, - gaps=_gaps(), ), Case( "encoded_previous_response_id", @@ -190,7 +222,16 @@ CASES: Final = ( custom_llm_provider="openai", model_id=None, response_id="resp_6f5e4d3c2b1a" ), }, - gaps=_gaps(), + ), + Case( + "deployment_scoped_ids", + {"input": "hi", "litellm_metadata": dict(DEPLOYMENT_METADATA)}, + upstream=ENCRYPTED_REASONING_RESPONSE, + ), + Case( + "affinity_encoded_reasoning_replay", + {"input": AFFINITY_REPLAY_INPUT, "litellm_metadata": dict(DEPLOYMENT_METADATA)}, + upstream=ENCRYPTED_REASONING_RESPONSE, ), Case( "warm_up_probe_below_minimum_max_output_tokens", @@ -215,7 +256,6 @@ CASES: Final = ( ], "tools": [{**WEATHER_TOOL, "cache_control": {"type": "ephemeral"}}], }, - gaps=_gaps(), ), Case( "reasoning_replay_without_status", From d07e3c5cd273b4d002d25960d8a70b5704025b81 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:26:04 -0700 Subject: [PATCH 11/50] feat(cost-map): add together_ai DeepSeek-V4.1-Flash-1017 (#47291) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 16 ++++++++++++++++ model_prices_and_context_window.json | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f76d61c417d..9adc3f2d48b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76075,5 +76075,21 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_vision": true + }, + "together_ai/deepseek-ai/DeepSeek-V4.1-Flash-1017": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://api.together.ai/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f76d61c417d..9adc3f2d48b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76075,5 +76075,21 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_vision": true + }, + "together_ai/deepseek-ai/DeepSeek-V4.1-Flash-1017": { + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://api.together.ai/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } From 1a77147fad139604f2ed688677fec32a0aa50ad8 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:27:28 -0700 Subject: [PATCH 12/50] fix(azure): set azure/container code interpreter session price from the azure openai pricing page (#47292) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 5 +++-- model_prices_and_context_window.json | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9adc3f2d48b..3ae687cc629 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3840,9 +3840,10 @@ "supports_vision": true }, "azure/container": { - "code_interpreter_cost_per_session": 0.03, + "code_interpreter_cost_per_session": 0.033, "litellm_provider": "azure", - "mode": "chat" + "mode": "chat", + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" }, "azure_ai/gpt-oss-120b": { "input_cost_per_token": 1.5e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9adc3f2d48b..3ae687cc629 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3840,9 +3840,10 @@ "supports_vision": true }, "azure/container": { - "code_interpreter_cost_per_session": 0.03, + "code_interpreter_cost_per_session": 0.033, "litellm_provider": "azure", - "mode": "chat" + "mode": "chat", + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/" }, "azure_ai/gpt-oss-120b": { "input_cost_per_token": 1.5e-07, From 581b7d1a023a2d144518d8a1015de51131b655a7 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:59:00 -0700 Subject: [PATCH 13/50] fix(vertex-ai): drop floor deprecation_date from gemini-3.8-flash (#47294) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 48 ++++++++----------- model_prices_and_context_window.json | 48 ++++++++----------- 2 files changed, 38 insertions(+), 58 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3ae687cc629..d570e49bf6c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29727,14 +29727,15 @@ "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.8-flash": { - "prompt_cache_min_tokens": 4096, - "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_priority": 1.35e-07, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_flex": 3.75e-07, + "input_cost_per_token_priority": 1.35e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, @@ -29744,7 +29745,10 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "output_cost_per_token_priority": 6.75e-06, + "prompt_cache_min_tokens": 4096.0, "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -29762,11 +29766,12 @@ ], "supports_audio_input": true, "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -29774,17 +29779,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 1.35e-06, - "output_cost_per_token_priority": 6.75e-06, - "cache_read_input_token_cost_priority": 1.35e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.8-flash-cyber": { "cache_read_input_token_cost": 1.5e-07, @@ -31990,14 +31985,15 @@ "google_maps_grounding_cost_per_query": 0.014 }, "gemini-3.8-flash": { - "prompt_cache_min_tokens": 4096, - "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_priority": 1.35e-07, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_flex": 3.75e-07, + "input_cost_per_token_priority": 1.35e-06, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, @@ -32007,6 +32003,9 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "output_cost_per_token_priority": 6.75e-06, + "prompt_cache_min_tokens": 4096.0, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -32022,14 +32021,15 @@ "supported_output_modalities": [ "text" ], - "supports_audio_output": false, "supports_audio_input": true, + "supports_audio_output": false, "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -32037,17 +32037,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 1.35e-06, - "output_cost_per_token_priority": 6.75e-06, - "cache_read_input_token_cost_priority": 1.35e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "web_search_billing_unit": "per_query" }, "gemini-3.8-flash-cyber": { "cache_read_input_token_cost": 1.5e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3ae687cc629..d570e49bf6c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29727,14 +29727,15 @@ "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.8-flash": { - "prompt_cache_min_tokens": 4096, - "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_priority": 1.35e-07, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_flex": 3.75e-07, + "input_cost_per_token_priority": 1.35e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, @@ -29744,7 +29745,10 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "output_cost_per_token_priority": 6.75e-06, + "prompt_cache_min_tokens": 4096.0, "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -29762,11 +29766,12 @@ ], "supports_audio_input": true, "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -29774,17 +29779,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 1.35e-06, - "output_cost_per_token_priority": 6.75e-06, - "cache_read_input_token_cost_priority": 1.35e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.8-flash-cyber": { "cache_read_input_token_cost": 1.5e-07, @@ -31990,14 +31985,15 @@ "google_maps_grounding_cost_per_query": 0.014 }, "gemini-3.8-flash": { - "prompt_cache_min_tokens": 4096, - "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_priority": 1.35e-07, + "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_flex": 3.75e-07, + "input_cost_per_token_priority": 1.35e-06, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, @@ -32007,6 +32003,9 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "output_cost_per_token_priority": 6.75e-06, + "prompt_cache_min_tokens": 4096.0, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -32022,14 +32021,15 @@ "supported_output_modalities": [ "text" ], - "supports_audio_output": false, "supports_audio_input": true, + "supports_audio_output": false, "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_minimal_reasoning_effort": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -32037,17 +32037,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 1.35e-06, - "output_cost_per_token_priority": 6.75e-06, - "cache_read_input_token_cost_priority": 1.35e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "web_search_billing_unit": "per_query" }, "gemini-3.8-flash-cyber": { "cache_read_input_token_cost": 1.5e-07, From 6762d65eb866a670e0b8c93d5e6135c06857a2e9 Mon Sep 17 00:00:00 2001 From: alexcastgonz <82875882+pcbeingused333@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:10:52 -0400 Subject: [PATCH 14/50] fix(vertex_ai): price streamed pass-through calls with the regional uplift (#44795) * fix(vertex_ai): price streamed pass-through calls with the regional uplift The streamed Vertex pass-through handler builds the response with stream_chunk_builder, which stamps a cost into _hidden_params without the vertex_location. The handler then prices the response again with the location from the URL, but only into kwargs, and the logger prefers the stamped cost. Streamed calls on a regional endpoint were billed at the global rate. Stamp the location-aware cost on the response in the shared generateContent payload builder so the logger records it. Fixes #44755 * style(vertex_ai): keep the response_cost assignment within the 120-char limit * refactor(vertex_ai): set the passthrough response cost with set_hidden_param Writing _hidden_params directly mutated a parameter and needed a pyright ignore, which the type discipline gate counts as LIT004 and LIT011. set_hidden_param is the shared helper for this. * test(integration): check Vertex pass-through spend rows keep the regional uplift when streamed * fix(proxy): price streamed Vertex pass-through spend like the non-streamed call Drop the location-blind cost stream_chunk_builder stores instead of stamping a cost, so the logger prices streamed generateContent and rawPredict calls with the URL's location and the deployment's custom pricing, the same way it prices non-streamed calls * fix(proxy): price custom-priced Vertex pass-through deployments at their own rates The stamped response_cost from the generateContent logging helper is returned before the logger checks custom pricing, so a deployment with custom input/output rates was billed at the cost-map price. The helper now passes custom_pricing and the deployment id into completion_cost, so the stamp holds the deployment rate for streamed and non-streamed calls, and deployments without custom pricing keep the regional uplift. * fix(proxy): bill Vertex /v1beta and interrupted pass-through streams by the deployment location and price * fix(proxy): give the new pyright ignore a reason for the LIT004 gate * test(integration): bill a custom-priced Vertex pass-through stream at the deployment price, cut-off streams included * fix(proxy): let the logger price non-streamed /gemini pass-through calls so both modes bill the same * fix(proxy): keep a client base_model out of pass-through pricing and price gateway Gemini streams as Gemini A top-level base_model in a pass-through body now reaches the logging params only for a key or team with allow_client_pricing_override, so a plain key cannot pick a cheaper model's price once the logger prices these calls. The test that pinned every litellm-owned body name into the logging params now leaves base_model to its own key-gated test. The Vertex handler now classifies a non-Google upstream host as Vertex only for Vertex resource paths, so a /gemini stream through a GEMINI_API_BASE gateway is priced as Gemini without the regional uplift. Adds the audit's integration spend test for Claude on Vertex, /v1beta, the named custom-priced /gemini deployment, and a cut OpenAI stream. * test(integration): price the gateway /gemini stream case as Gemini and cover client base_model and gateway host pricing The cost harness sends /gemini through GEMINI_API_BASE, which the pass-through now logs and bills as gemini, so the streamed case moves from the Vertex cases to the Gemini cases under the Gemini price. New spend tests cover a body base_model repricing only for keys allowed client pricing, a GEMINI_API_BASE gateway billing as Gemini in both modes, and a configured non-Google gateway picking the provider from the path. * fix(proxy): price pass-through Gemini by the route and ignore a non-string base_model The Vertex pass-through handler picked Gemini pricing from the URL path, so a configured pass-through to a Vertex gateway with a short path (no /projects/ segment) was billed as Gemini, and its Claude streams at $0. The provider now comes from the route: a call through /gemini or to generativelanguage.googleapis.com prices as Gemini, and everything else as Vertex, the same rule main used. Matching the GEMINI_API_BASE host was rejected because one gateway can serve both APIs, and the path rule was rejected because short gateway paths look like Gemini paths. A non-string base_model (a number, list or object) made the cost calculation fail and bill $0 once the stamped stream costs were gone. get_base_model_from_metadata now ignores it and falls back to the deployment's base_model, so the call bills the list price. A 400 was the other option, but the logger runs after the upstream answered. * test(integration): bill a non-string body base_model at the list price on every pass-through route * fix(proxy): bill a non-string model_info base_model at the list price and drop a non-object client model_info A key allowed client pricing can send metadata.model_info in a pass-through body. A non-string model_info.base_model made the cost lookup fail, and once the stamped stream and Gemini costs were gone those calls billed $0 on /gemini and on streamed /vertex_ai and /v1beta. The lookup now ignores a base_model that is not a string, the same rule the root base_model got, so the call bills the list price. A model_info that is not an object (a string or a list) broke several logging steps that read it as a mapping, so a non-streamed /gemini call fired no success callback and no pass-through call wrote a spend row. The pass-through route now leaves a non-object model_info out of the logging metadata, so the call bills the list price. A 400 was the other option, but it would turn calls that answer 200 today into errors, and guarding every reader of model_info would spread the same check over many files. * fix: reject a non-object model_info before the provider call again The previous commit made the base_model lookup return None for a model_info that is not an object. That lookup also runs during call setup on every router route and SDK entrypoint, where such a value used to fail before any provider request. With the early return the call went upstream, the provider billed it, and the request then failed in a later reader that still expects an object. On /v1/messages and /v1/responses the key was also charged for a response the client never got. The lookup again reads base_model from any truthy model_info, so a non-object value fails during setup the way it did before this PR. A non-string base_model inside an object still bills the list price, and the pass-through route still drops a non-object client model_info at intake, so pass-through calls keep billing the list price. * fix: match the Gemini API host exactly when picking the pass-through cost provider The provider check accepted any host ending in generativelanguage.googleapis.com, so a Vertex host such as vertex-generativelanguage.googleapis.com was priced as Gemini and lost the regional uplift. It now accepts that host or a subdomain of it. * test: pin that a subdomain of the Gemini API host keeps the Gemini price --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/google_genai/main.py | 3 + .../litellm_core_utils/get_litellm_params.py | 14 +- .../gemini_passthrough_logging_handler.py | 1 - .../vertex_passthrough_logging_handler.py | 28 +- .../pass_through_endpoints.py | 21 +- .../streaming_handler.py | 7 +- litellm/utils.py | 4 +- .../cost_calculation/cases/gemini.py | 68 ++ .../cost_calculation/cases/vertex_ai.py | 68 -- ..._passthrough_stream_pricing_paths_spend.py | 583 ++++++++++++++++++ ...ertex_passthrough_regional_uplift_spend.py | 200 ++++++ .../google_genai/test_google_genai_main.py | 96 +++ .../test_get_litellm_params.py | 38 +- ...test_gemini_passthrough_logging_handler.py | 118 +++- ...test_vertex_passthrough_logging_handler.py | 359 +++++++++++ .../test_pass_through_endpoints.py | 49 +- .../test_streaming_handler.py | 39 ++ tests/unit/test_utils.py | 16 +- 18 files changed, 1611 insertions(+), 101 deletions(-) create mode 100644 tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py create mode 100644 tests/integration/spend/test_vertex_passthrough_regional_uplift_spend.py diff --git a/litellm/google_genai/main.py b/litellm/google_genai/main.py index beb4b96d787..40c9c7bc36c 100644 --- a/litellm/google_genai/main.py +++ b/litellm/google_genai/main.py @@ -16,6 +16,7 @@ from litellm.llms.base_llm.google_genai.transformation import ( BaseGoogleGenAIGenerateContentConfig, ) from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes @@ -194,12 +195,14 @@ class GenerateContentHelper: if litellm_logging_obj is None: raise ValueError("litellm_logging_obj is required, but got None") + vertex_location: Final = VertexBase.explicit_vertex_ai_location(dict(litellm_params)) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model, optional_params=dict(generate_content_config_dict), litellm_params={ "litellm_call_id": litellm_call_id, + **({"vertex_location": vertex_location} if vertex_location else {}), }, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 9dee3714c60..6ca02cbbe64 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -142,14 +142,12 @@ def get_base_model_from_litellm_call_metadata( if metadata is None: return None model_info: Final = metadata.get("model_info") - if model_info: - model_info_mapping: Final = cast( # cast-ok: model metadata is caller-provided and preserves its mapping shape - Mapping[str, object], model_info - ) - return cast( # cast-ok: model metadata values are caller-provided - str | None, model_info_mapping.get("base_model") - ) - return None + if not model_info: + return None + base_model: Final = cast( # cast-ok: model metadata is caller-provided and preserves its mapping shape + Mapping[str, object], model_info + ).get("base_model") + return base_model if isinstance(base_model, str) else None _get_base_model_from_litellm_call_metadata = get_base_model_from_litellm_call_metadata diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py index 0093c6f386d..7a5d5a1e259 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py @@ -245,5 +245,4 @@ class GeminiPassthroughLoggingHandler: logging_obj.model = litellm_model_response.model or model logging_obj.model_call_details["model"] = logging_obj.model logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider - logging_obj.model_call_details["response_cost"] = response_cost return kwargs diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index c3af6f534c2..5010de46fcb 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -11,6 +11,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import VERTEX_BATCH_PREDICTION_JOBS_ROUTE +from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( InteractionsUsageObjectTransformation, @@ -225,7 +226,7 @@ class VertexPassthroughLoggingHandler: start_time=start_time, end_time=end_time, logging_obj=logging_obj, - custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route), + custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider(url_route, logging_obj), vertex_location=vertex_location, ) @@ -549,7 +550,7 @@ class VertexPassthroughLoggingHandler: response_json=response_json, ) - custom_llm_provider: Final = VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route) + custom_llm_provider: Final = VertexPassthroughLoggingHandler._get_custom_llm_provider(url_route, logging_obj) litellm_embedding_response.model = model logging_obj.model = model @@ -621,7 +622,9 @@ class VertexPassthroughLoggingHandler: start_time=start_time, end_time=end_time, logging_obj=litellm_logging_obj, - custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route), + custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider( + url_route, litellm_logging_obj + ), vertex_location=vertex_location, ) @@ -669,9 +672,16 @@ class VertexPassthroughLoggingHandler: all_openai_chunks: Final = [parsed_chunk for parsed_chunk in parsed_chunks if parsed_chunk is not None] complete_streaming_response: Final = litellm.stream_chunk_builder(chunks=all_openai_chunks) + VertexPassthroughLoggingHandler._drop_location_blind_stream_builder_cost(complete_streaming_response) return complete_streaming_response + @staticmethod + def _drop_location_blind_stream_builder_cost(response: ModelResponse | TextCompletionResponse | None) -> None: + hidden_params: Final = get_hidden_params(response) + if hidden_params is not None: + hidden_params.pop("response_cost", None) + @staticmethod def extract_model_from_url(url: str) -> str: pattern: Final = r"/models/([^:]+)" @@ -724,9 +734,15 @@ class VertexPassthroughLoggingHandler: return None @staticmethod - def _get_custom_llm_provider_from_url(url: str) -> str: - parsed_url: Final = urlparse(url) - if parsed_url.hostname and parsed_url.hostname.endswith("generativelanguage.googleapis.com"): + def _get_custom_llm_provider(url: str, logging_obj: LiteLLMLoggingObj) -> str: + hostname: Final = urlparse(url).hostname + if logging_obj.model_call_details.get("custom_llm_provider") == litellm.LlmProviders.GEMINI.value or ( + hostname is not None + and ( + hostname == "generativelanguage.googleapis.com" + or hostname.endswith(".generativelanguage.googleapis.com") + ) + ): return litellm.LlmProviders.GEMINI.value return litellm.LlmProviders.VERTEX_AI.value diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 46b7a724809..311dab058a4 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -161,6 +161,12 @@ pass_through_endpoint_logging: Final = PassThroughEndpointLogging() _METADATA_KEYS: Final = frozenset(("litellm_metadata", "metadata")) _DECODED_JSON: Final = TypeAdapter(object) _KEPT_OUT_OF_LITELLM_PARAMS: Final = _METADATA_KEYS | frozenset(wire_names(ProxyRequestState)) +_PRICING_MODEL_OVERRIDE_PARAMS: Final = frozenset(("base_model",)) + + +def _is_mergeable_client_metadata(key: str, value: object) -> bool: + return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping)) + # Global registry to track registered pass-through routes and prevent memory leaks _registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {} @@ -633,8 +639,13 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): litellm_keys_in_body: Final = MappingProxyType( {k: _parsed_body.pop(k) for k in types_utils.all_litellm_params if k in _parsed_body} ) + kept_out_of_litellm_params: Final = ( + _KEPT_OUT_OF_LITELLM_PARAMS + if _key_or_team_allows_client_pricing_override(user_api_key_dict) + else _KEPT_OUT_OF_LITELLM_PARAMS | _PRICING_MODEL_OVERRIDE_PARAMS + ) litellm_params_in_body: Final = MappingProxyType( - {k: v for k, v in litellm_keys_in_body.items() if k not in _KEPT_OUT_OF_LITELLM_PARAMS} + {k: v for k, v in litellm_keys_in_body.items() if k not in kept_out_of_litellm_params} ) _metadata = dict( @@ -645,7 +656,13 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): metadata: Final = litellm_keys_in_body.get("metadata") for client_metadata in (litellm_metadata, metadata): if isinstance(client_metadata, dict): - _metadata.update({k: v for k, v in client_metadata.items() if not k.startswith("user_api_key_")}) + _metadata.update( + { + k: v + for k, v in cast(Mapping[str, object], client_metadata).items() # cast-ok: json + if _is_mergeable_client_metadata(k, v) + } + ) _metadata = _apply_key_team_project_controls(user_api_key_dict=user_api_key_dict, metadata=_metadata) _metadata = _update_metadata_with_tags_in_header( diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 6799b82bf06..1fcfe2e279a 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -129,10 +129,11 @@ class PassThroughStreamingHandler: usage: Final = getattr(partial_response, "usage", None) if not isinstance(usage, Usage): return - response_cost: Final = kwargs.get("response_cost") + logger_cost: Final = litellm_logging_obj.response_cost_calculator(result=partial_response) + handler_cost: Final = kwargs.get("response_cost") + fallback_cost: Final = float(handler_cost) if isinstance(handler_cost, (int, float)) else 0.0 litellm_logging_obj.record_partial_usage_for_failure( - usage=usage, - response_cost=float(response_cost) if isinstance(response_cost, (int, float)) else 0.0, + usage=usage, response_cost=fallback_cost if logger_cost is None else logger_cost ) @staticmethod diff --git a/litellm/utils.py b/litellm/utils.py index ab97b69897f..66d78a03075 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8306,8 +8306,8 @@ def get_base_model_from_metadata(model_call_details: Mapping[str, object] | None litellm_params: Final = cast(Mapping[str, object], model_call_details.get("litellm_params", {})) if litellm_params is not None: base_model: Final = litellm_params.get("base_model", None) - if base_model is not None: - return cast(str | None, base_model) + if isinstance(base_model, str): + return base_model metadata: Final = cast(Mapping[str, object], litellm_params.get("metadata") or {}) base_model_metadata_getter: Final[_BaseModelFromMetadataGetter] = ( litellm_utils.get_base_model_from_litellm_call_metadata diff --git a/tests/integration/cost_calculation/cases/gemini.py b/tests/integration/cost_calculation/cases/gemini.py index ceca816c300..bc6665c423f 100644 --- a/tests/integration/cost_calculation/cases/gemini.py +++ b/tests/integration/cost_calculation/cases/gemini.py @@ -1544,6 +1544,73 @@ GEMINI_3_1_PRO_PASSTHROUGH_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY: Final = CostT ), ) +GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY: Final = CostTrackingTestCase( + name="gemini-3.1-pro-passthrough-stream_generate_content_priced_via_gemini_key", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="gemini/gemini-3.1-pro", + endpoint="/gemini/v1beta/models/$MODEL:streamGenerateContent?alt=sse", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "52b6a80ff038 summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": True, + "stream_options": {"include_usage": True}, + "allowed_openai_params": [], + }, + response=SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "candidates": [ + { + "content": {"parts": [{"text": "scripted answer 52b6a80ff038"}], "role": "model"}, + "finishReason": "STOP", + "index": 0, + } + ], + "modelVersion": "gemini-3.1-pro", + }, + { + "candidates": [], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}], + }, + "modelVersion": "gemini-3.1-pro", + }, + ), + ), + expected=ExactExpected( + spend=0.008624, + input_cost=0.00368, + output_cost=0.004944, + prompt_tokens=1840, + completion_tokens=412, + breakdown_persisted=False, + cost_header=False, + ), +) + # gemini/gemini-3.8-flash GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT: Final = CostTrackingTestCase( @@ -3475,6 +3542,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_GEMINI_3_8_FLASH_STREAM_FULL_USAGE, GEMINI_EMBEDDINGS_002, GEMINI_3_1_PRO_PASSTHROUGH_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY, + GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_GEMINI_KEY, GEMINI_GEMINI_3_8_FLASH_LITE_INPUT_BELOW_128K, GEMINI_GEMINI_3_8_FLASH_LITE_INPUT_ABOVE_128K, GEMINI_NANO_BANANA_2_1_INPUT_TEXT, diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index 0e95eb865bf..b512df46d1d 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -1659,73 +1659,6 @@ GEMINI_3_1_PRO_STREAM_FULL_USAGE: Final = CostTrackingTestCase( ), ) -GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_VERTEX_KEY: Final = CostTrackingTestCase( - name="gemini-3.1-pro-passthrough-stream_generate_content_priced_via_vertex_key", - covers="quota_management.spend_tracking.scripted_wire.logs_cost", - model="gemini-3.1-pro", - endpoint="/gemini/v1beta/models/$MODEL:streamGenerateContent?alt=sse", - request={ - "model": "$MODEL", - "messages": [ - { - "role": "system", - "content": [ - { - "type": "text", - "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", - } - ], - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "52b6a80ff038 summarize the attached material in one line and name the city weather", - } - ], - }, - ], - "stream": True, - "stream_options": {"include_usage": True}, - "allowed_openai_params": [], - }, - response=SseResponse( - content_type="text/event-stream", - frames=sse_frames( - { - "candidates": [ - { - "content": {"parts": [{"text": "scripted answer 52b6a80ff038"}], "role": "model"}, - "finishReason": "STOP", - "index": 0, - } - ], - "modelVersion": "gemini-3.1-pro", - }, - { - "candidates": [], - "usageMetadata": { - "promptTokenCount": 1840, - "candidatesTokenCount": 412, - "totalTokenCount": 2252, - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}], - }, - "modelVersion": "gemini-3.1-pro", - }, - ), - ), - expected=ExactExpected( - spend=0.0090552, - input_cost=0.003864, - output_cost=0.0051912, - prompt_tokens=1840, - completion_tokens=412, - breakdown_persisted=False, - cost_header=False, - ), -) - # gemini-3.8-flash GEMINI_3_8_FLASH_INPUT_TEXT: Final = CostTrackingTestCase( @@ -3647,7 +3580,6 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_3_8_FLASH_STREAM_FULL_USAGE, IMAGEN_NEXT_IMAGES_ONE, VERTEX_EMBEDDINGS_TEXT_006, - GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_VERTEX_KEY, VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT, VERTEX_AI_CLAUDE_SONNET_5_5_STREAM, VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT, diff --git a/tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py b/tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py new file mode 100644 index 00000000000..f091a8f8376 --- /dev/null +++ b/tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py @@ -0,0 +1,583 @@ +import json +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from types import MappingProxyType +from typing import Final +from urllib.parse import urlsplit + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.database import read_rows +from integration._support.forward_proxy import tunnelling_forward_proxy +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.tls import server_context, write_self_signed_cert +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +_PROJECT: Final = "caller-project" +_CALLER_GOOGLE_TOKEN: Final = "caller-google-token" +_GEMINI: Final = "gemini-3.5-flash-lite" +_CLAUDE_ROUTE_MODEL: Final = "claude-haiku-4-5" +_CLAUDE_SERVED_MODEL: Final = "claude-haiku-4-5-20251001" +_OPENAI_MODEL: Final = "gpt-5.6" +_PROMPT_TOKENS: Final = 1200 +_COMPLETION_TOKENS: Final = 300 +_CUSTOM_INPUT_RATE: Final = 1e-05 +_CUSTOM_OUTPUT_RATE: Final = 2e-05 +_CUSTOM_PRICED_GEMINI_ID: Final = "audit-gemini-custom-priced" +_HOSTS: Final = ( + "aiplatform.googleapis.com", + "aiplatform.us.rep.googleapis.com", + "us-east5-aiplatform.googleapis.com", + "generativelanguage.googleapis.com", + "api.openai.com", +) +_COST_MAP: Final = Path("litellm/model_prices_and_context_window_backup.json") +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_CUT_MARKER: Final = "cut this stream after the usage frame" +_GEMINI_USAGE: Final = { + "promptTokenCount": _PROMPT_TOKENS, + "candidatesTokenCount": _COMPLETION_TOKENS, + "totalTokenCount": _PROMPT_TOKENS + _COMPLETION_TOKENS, +} +_DEPLOYMENT_LOCATIONS: Final = MappingProxyType({"audit-vertex-global": "global", "audit-vertex-us": "us"}) +_GEMINI_ACTIONS: Final = ("generateContent", "streamGenerateContent?alt=sse") +_CONFIGURED_GATEWAY_PATH: Final = "/google-egress" + + +@dataclass(frozen=True, slots=True) +class _Rig: + proxy: Gateway + upstream: Wire + + +def _gemini_reply(request: Request) -> Reply: + body: Final = json.dumps( + { + "candidates": [{"content": {"role": "model", "parts": [{"text": "scripted"}]}, "finishReason": "STOP"}], + "usageMetadata": _GEMINI_USAGE, + "modelVersion": _GEMINI, + "responseId": uuid.uuid4().hex, + } + ) + if ":streamGenerateContent" in request.target: + return Reply(chunks=(f"data: {body}\r\n\r\n".encode(),), content_type="text/event-stream") + return Reply(body=body.encode()) + + +def _sse(event: str, payload: Mapping[str, JsonValue]) -> bytes: + return f"event: {event}\ndata: {json.dumps(payload)}\n\n".encode() + + +def _claude_reply(request: Request) -> Reply: + message_id: Final = f"msg_vrtx_{uuid.uuid4().hex}" + if ":streamRawPredict" not in request.target: + return Reply( + body=json.dumps( + { + "model": _CLAUDE_SERVED_MODEL, + "id": message_id, + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "scripted"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": _PROMPT_TOKENS, "output_tokens": _COMPLETION_TOKENS}, + } + ).encode() + ) + return Reply( + content_type="text/event-stream", + chunks=( + _sse( + "message_start", + { + "type": "message_start", + "message": { + "model": _CLAUDE_SERVED_MODEL, + "id": message_id, + "type": "message", + "role": "assistant", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": _PROMPT_TOKENS, "output_tokens": 1}, + }, + }, + ), + _sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "scripted"}}, + ), + _sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _sse( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": _COMPLETION_TOKENS}, + }, + ), + _sse("message_stop", {"type": "message_stop"}), + ), + ) + + +def _openai_reply(request: Request) -> Reply: + identity: Final = f"chatcmpl-{uuid.uuid4().hex}" + chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": _OPENAI_MODEL} + frames: Final = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}}]}, + {**chunk, "choices": [{"index": 0, "delta": {"content": "scripted"}}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + { + **chunk, + "choices": [], + "usage": { + "prompt_tokens": _PROMPT_TOKENS, + "completion_tokens": _COMPLETION_TOKENS, + "total_tokens": _PROMPT_TOKENS + _COMPLETION_TOKENS, + }, + }, + ) + encoded: Final = tuple(f"data: {json.dumps(frame)}\n\n".encode() for frame in frames) + if _CUT_MARKER.encode() in request.body: + return Reply(chunks=(*encoded, b"data: [DONE]\n\n"), content_type="text/event-stream", abort_after=len(encoded)) + return Reply(chunks=(*encoded, b"data: [DONE]\n\n"), content_type="text/event-stream") + + +def _upstream(request: Request) -> Reply: + if request.target.startswith("/v1/chat/completions"): + return _openai_reply(request) + if "/publishers/anthropic/models/" in request.target: + assert f"/v1/projects/{_PROJECT}/locations/" in request.target, request.target + return _claude_reply(request) + return _gemini_reply(request) + + +def _port(url: str) -> int: + port: Final = urlsplit(url).port + assert port is not None, url + return port + + +def _cost_row(name: str) -> dict[str, JsonValue]: + return _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(_COST_MAP.read_bytes())[name]) + + +def _list_price(name: str) -> float: + row: Final = _cost_row(name) + input_rate: Final = row["input_cost_per_token"] + output_rate: Final = row["output_cost_per_token"] + assert isinstance(input_rate, float) and isinstance(output_rate, float), row + return _PROMPT_TOKENS * input_rate + _COMPLETION_TOKENS * output_rate + + +def _uplift(name: str) -> float: + multiplier: Final = _cost_row(name).get("regional_endpoint_uplift_multiplier") + assert isinstance(multiplier, int | float) and multiplier > 1, name + return float(multiplier) + + +def _config(directory: Path, token_url: str) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + existing: Final = base.get("model_list") or [] + assert isinstance(existing, list), base + deployments: Final = [ + { + "model_name": name, + "litellm_params": { + "model": f"vertex_ai/{_GEMINI}", + "vertex_project": _PROJECT, + "vertex_location": location, + "vertex_credentials": service_account_json(_PROJECT, token_url), + }, + } + for name, location in _DEPLOYMENT_LOCATIONS.items() + ] + custom_priced_gemini: Final = { + "model_name": _CUSTOM_PRICED_GEMINI_ID, + "litellm_params": { + "model": f"gemini/{_GEMINI}", + "api_key": "scripted-gemini-key", + "input_cost_per_token": _CUSTOM_INPUT_RATE, + "output_cost_per_token": _CUSTOM_OUTPUT_RATE, + }, + "model_info": {"id": _CUSTOM_PRICED_GEMINI_ID}, + } + path: Final = directory / "config.yaml" + path.write_text(yaml.safe_dump({**base, "model_list": [*existing, *deployments, custom_priced_gemini]})) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("passthrough-stream-pricing") + certificate: Final = write_self_signed_cert(directory, _HOSTS) + with ( + wire_server(_upstream, tls=server_context(*certificate)) as upstream, + tunnelling_forward_proxy(_port(upstream.url)) as tunnel, + gateway_from_environment() as base, + owned_proxy( + base, + directory, + { + "HTTPS_PROXY": tunnel.url, + "NO_PROXY": "127.0.0.1,localhost", + "AIOHTTP_TRUST_ENV": "True", + "SSL_VERIFY": "False", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "OPENAI_API_KEY": "sk-scripted-openai", + "GEMINI_API_KEY": "scripted-gemini-key", + }, + config=_config(directory, base.upstream_url.rstrip("/")), + workers=2, + ) as proxy, + ): + yield _Rig(proxy, upstream) + + +def _gateway_config(directory: Path, upstream_url: str) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + general_settings: Final = base.get("general_settings") + assert isinstance(general_settings, dict), base + configured_endpoint: Final = { + "path": _CONFIGURED_GATEWAY_PATH, + "target": f"{upstream_url}/egress", + "include_subpath": True, + "auth": True, + } + path: Final = directory / "config.yaml" + path.write_text( + yaml.safe_dump( + {**base, "general_settings": {**general_settings, "pass_through_endpoints": [configured_endpoint]}} + ) + ) + return path + + +@pytest.fixture(scope="module") +def gateway_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("passthrough-google-gateway") + with ( + wire_server(_gemini_reply) as upstream, + gateway_from_environment() as base, + owned_proxy( + base, + directory, + { + "GEMINI_API_BASE": f"{upstream.url}/google", + "GEMINI_API_KEY": "scripted-gemini-key", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + }, + config=_gateway_config(directory, upstream.url), + workers=2, + ) as proxy, + ): + yield _Rig(proxy, upstream) + + +def _spend_row(key: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, custom_llm_provider FROM "LiteLLM_SpendLogs" WHERE api_key=%s', + (sha256(key.encode()).hexdigest(),), + ), + lambda found: len(found) == 1, + seconds=70, + ) + return rows[0] + + +def _spend_for_key(key: str) -> float: + spend: Final = _spend_row(key)["spend"] + assert isinstance(spend, float), key + return spend + + +def _provider_for_key(key: str) -> JsonValue: + return _spend_row(key)["custom_llm_provider"] + + +def _post_and_read(rig: _Rig, path: str, body: Mapping[str, JsonValue], headers: Mapping[str, str]) -> None: + with rig.proxy.client.stream("POST", path, json=body, headers=dict(headers)) as response: + payload: Final = response.read() + assert response.status_code == 200, payload + + +def _claude_path(location: str, action: str) -> str: + return ( + f"/vertex_ai/v1/projects/{_PROJECT}/locations/{location}/publishers/anthropic/models/" + f"{_CLAUDE_ROUTE_MODEL}:{action}" + ) + + +def test_claude_on_vertex_passthrough_spend_carries_the_regional_uplift_whether_or_not_the_call_streams( + rig: _Rig, +) -> None: + body: Final[Mapping[str, JsonValue]] = { + "anthropic_version": "vertex-2023-10-16", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + } + cells: Final = ( + ("global", "rawPredict"), + ("global", "streamRawPredict"), + ("us-east5", "rawPredict"), + ("us-east5", "streamRawPredict"), + ) + with rig.proxy.scenario() as scenario: + keys: Final = {cell: scenario.key() for cell in cells} + for (location, action), key in keys.items(): + _post_and_read( + rig, + _claude_path(location, action), + {**body, "stream": action == "streamRawPredict"}, + {"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"}, + ) + spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()} + standard: Final = _list_price(f"vertex_ai/{_CLAUDE_ROUTE_MODEL}") + regional: Final = standard * _uplift(f"vertex_ai/{_CLAUDE_ROUTE_MODEL}") + assert spend == pytest.approx( + { + ("global", "rawPredict"): standard, + ("global", "streamRawPredict"): standard, + ("us-east5", "rawPredict"): regional, + ("us-east5", "streamRawPredict"): regional, + } + ) + + +def test_v1beta_generate_content_on_a_vertex_deployment_bills_the_deployment_location_in_both_modes( + rig: _Rig, +) -> None: + body: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]} + actions: Final = ("generateContent", "streamGenerateContent?alt=sse") + cells: Final = tuple((deployment, action) for deployment in _DEPLOYMENT_LOCATIONS for action in actions) + with rig.proxy.scenario() as scenario: + keys: Final = {cell: scenario.key() for cell in cells} + for (deployment, action), key in keys.items(): + _post_and_read(rig, f"/v1beta/models/{deployment}:{action}", body, {"x-goog-api-key": key}) + spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()} + standard: Final = _list_price(f"vertex_ai/{_GEMINI}") + regional: Final = standard * _uplift(f"vertex_ai/{_GEMINI}") + assert spend == pytest.approx( + { + ("audit-vertex-global", "generateContent"): standard, + ("audit-vertex-global", "streamGenerateContent?alt=sse"): standard, + ("audit-vertex-us", "generateContent"): regional, + ("audit-vertex-us", "streamGenerateContent?alt=sse"): regional, + } + ) + + +def test_gemini_passthrough_bills_a_named_custom_priced_deployment_only_for_keys_that_allow_it_in_both_modes( + rig: _Rig, +) -> None: + body: Final[Mapping[str, JsonValue]] = { + "contents": [{"role": "user", "parts": [{"text": "hi"}]}], + "metadata": { + "model_info": { + "id": _CUSTOM_PRICED_GEMINI_ID, + "input_cost_per_token": _CUSTOM_INPUT_RATE, + "output_cost_per_token": _CUSTOM_OUTPUT_RATE, + } + }, + } + actions: Final = ("generateContent", "streamGenerateContent?alt=sse") + with rig.proxy.scenario() as scenario: + allowed: Final = {action: scenario.key(metadata={"allow_client_pricing_override": True}) for action in actions} + plain: Final = {action: scenario.key() for action in actions} + for action in actions: + for key in (allowed[action], plain[action]): + _post_and_read(rig, f"/gemini/v1beta/models/{_GEMINI}:{action}", body, {"x-goog-api-key": key}) + spend: Final = { + **{("allowed", action): _spend_for_key(key) for action, key in allowed.items()}, + **{("plain", action): _spend_for_key(key) for action, key in plain.items()}, + } + custom_price: Final = _PROMPT_TOKENS * _CUSTOM_INPUT_RATE + _COMPLETION_TOKENS * _CUSTOM_OUTPUT_RATE + list_price: Final = _list_price(f"gemini/{_GEMINI}") + assert spend == pytest.approx( + { + ("allowed", "generateContent"): custom_price, + ("allowed", "streamGenerateContent?alt=sse"): custom_price, + ("plain", "generateContent"): list_price, + ("plain", "streamGenerateContent?alt=sse"): list_price, + } + ) + + +def test_openai_passthrough_stream_cut_off_after_its_usage_bills_what_the_finished_stream_bills(rig: _Rig) -> None: + def body(prompt: str) -> Mapping[str, JsonValue]: + return { + "model": _OPENAI_MODEL, + "stream": True, + "stream_options": {"include_usage": True}, + "messages": [{"role": "user", "content": prompt}], + } + + with rig.proxy.scenario() as scenario: + finished_key: Final = scenario.key() + cut_key: Final = scenario.key() + _post_and_read(rig, "/openai/v1/chat/completions", body("hi"), {"Authorization": f"Bearer {finished_key}"}) + with rig.proxy.client.stream( + "POST", + "/openai/v1/chat/completions", + json=dict(body(_CUT_MARKER)), + headers={"Authorization": f"Bearer {cut_key}"}, + ) as response: + assert response.status_code == 200 + with pytest.raises(httpx.RemoteProtocolError): + response.read() + spend: Final = {"finished": _spend_for_key(finished_key), "cut off": _spend_for_key(cut_key)} + list_price: Final = _list_price(_OPENAI_MODEL) + assert spend == pytest.approx({"finished": list_price, "cut off": list_price}) + + +def _caller_headers(route: str, key: str) -> Mapping[str, str]: + if route == "vertex_ai": + return {"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"} + return {"x-goog-api-key": key} + + +def test_a_body_base_model_reprices_passthrough_spend_only_for_keys_that_allow_client_pricing(rig: _Rig) -> None: + routes: Final = MappingProxyType( + { + "vertex_ai": ( + f"/vertex_ai/v1/projects/{_PROJECT}/locations/global/publishers/google/models/{_GEMINI}", + "gemini-2.5-flash-lite", + ), + "gemini": (f"/gemini/v1beta/models/{_GEMINI}", "gemini/gemini-2.5-flash-lite"), + } + ) + cells: Final = tuple( + (route, action, grant) for route in routes for action in _GEMINI_ACTIONS for grant in ("allowed", "plain") + ) + markers: Final = {cell: uuid.uuid4().hex for cell in cells} + with rig.proxy.scenario() as scenario: + keys: Final = { + cell: scenario.key(metadata={"allow_client_pricing_override": True}) + if cell[2] == "allowed" + else scenario.key() + for cell in cells + } + for (route, action, grant), key in keys.items(): + path, base_model = routes[route] + _post_and_read( + rig, + f"{path}:{action}", + { + "contents": [{"role": "user", "parts": [{"text": markers[(route, action, grant)]}]}], + "base_model": base_model, + }, + _caller_headers(route, key), + ) + spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()} + forwarded: Final = rig.upstream.drain() + marked_bodies: Final = { + cell: tuple(request.body for request in forwarded if marker.encode() in request.body) + for cell, marker in markers.items() + } + assert all(len(bodies) == 1 and b"base_model" not in bodies[0] for bodies in marked_bodies.values()), marked_bodies + expected_price: Final = { + "vertex_ai": {"allowed": _list_price("gemini-2.5-flash-lite"), "plain": _list_price(f"vertex_ai/{_GEMINI}")}, + "gemini": {"allowed": _list_price("gemini/gemini-2.5-flash-lite"), "plain": _list_price(f"gemini/{_GEMINI}")}, + } + assert spend == pytest.approx({cell: expected_price[cell[0]][cell[2]] for cell in cells}) + + +def test_a_non_string_body_base_model_bills_the_list_price_instead_of_nothing(rig: _Rig) -> None: + routes: Final = MappingProxyType( + { + "vertex_ai": f"/vertex_ai/v1/projects/{_PROJECT}/locations/global/publishers/google/models/{_GEMINI}", + "gemini": f"/gemini/v1beta/models/{_GEMINI}", + "v1beta": "/v1beta/models/audit-vertex-global", + } + ) + cells: Final = tuple((route, action) for route in routes for action in _GEMINI_ACTIONS) + with rig.proxy.scenario() as scenario: + keys: Final = { + cell: scenario.key() + if cell[0] == "v1beta" + else scenario.key(metadata={"allow_client_pricing_override": True}) + for cell in cells + } + for (route, action), key in keys.items(): + _post_and_read( + rig, + f"{routes[route]}:{action}", + {"contents": [{"role": "user", "parts": [{"text": uuid.uuid4().hex}]}], "base_model": 123}, + _caller_headers(route, key), + ) + spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()} + list_price: Final = { + "vertex_ai": _list_price(f"vertex_ai/{_GEMINI}"), + "gemini": _list_price(f"gemini/{_GEMINI}"), + "v1beta": _list_price(f"vertex_ai/{_GEMINI}"), + } + assert spend == pytest.approx({cell: list_price[cell[0]] for cell in cells}) + + +def test_gemini_passthrough_through_a_gemini_api_base_gateway_logs_and_bills_as_gemini_in_both_modes( + gateway_rig: _Rig, +) -> None: + body: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]} + with gateway_rig.proxy.scenario() as scenario: + keys: Final = {action: scenario.key() for action in _GEMINI_ACTIONS} + for action, key in keys.items(): + _post_and_read(gateway_rig, f"/gemini/v1beta/models/{_GEMINI}:{action}", body, {"x-goog-api-key": key}) + providers: Final = {action: _provider_for_key(key) for action, key in keys.items()} + spend: Final = {action: _spend_for_key(key) for action, key in keys.items()} + forwarded: Final = tuple( + path + for path in (urlsplit(request.target).path for request in gateway_rig.upstream.drain()) + if path.startswith("/google/") + ) + assert sorted(forwarded) == sorted( + f"/google/v1beta/models/{_GEMINI}:{action.split('?')[0]}" for action in _GEMINI_ACTIONS + ) + assert providers == {action: "gemini" for action in _GEMINI_ACTIONS} + assert spend == pytest.approx({action: _list_price(f"gemini/{_GEMINI}") for action in _GEMINI_ACTIONS}) + + +def test_a_configured_passthrough_to_a_non_google_gateway_bills_as_vertex_whatever_the_path(gateway_rig: _Rig) -> None: + body: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]} + routes: Final = MappingProxyType( + { + "vertex resource": f"/v1/projects/{_PROJECT}/locations/us/publishers/google/models/{_GEMINI}", + "short gateway path": f"/vx/models/{_GEMINI}", + } + ) + cells: Final = tuple((route, action) for route in routes for action in _GEMINI_ACTIONS) + with gateway_rig.proxy.scenario() as scenario: + keys: Final = {cell: scenario.key(allowed_passthrough_routes=[_CONFIGURED_GATEWAY_PATH]) for cell in cells} + for (route, action), key in keys.items(): + _post_and_read( + gateway_rig, + f"{_CONFIGURED_GATEWAY_PATH}{routes[route]}:{action}", + body, + {"Authorization": f"Bearer {key}"}, + ) + providers: Final = {cell: _provider_for_key(key) for cell, key in keys.items()} + spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()} + short_path_spend: Final = {action: spend[("short gateway path", action)] for action in _GEMINI_ACTIONS} + assert providers == {cell: "vertex_ai" for cell in cells} + assert {action: spend[("vertex resource", action)] for action in _GEMINI_ACTIONS} == pytest.approx( + {action: _list_price(f"vertex_ai/{_GEMINI}") * _uplift(f"vertex_ai/{_GEMINI}") for action in _GEMINI_ACTIONS} + ) + assert min(short_path_spend.values()) > 0 + assert short_path_spend == pytest.approx( + {action: short_path_spend[_GEMINI_ACTIONS[0]] for action in _GEMINI_ACTIONS} + ) diff --git a/tests/integration/spend/test_vertex_passthrough_regional_uplift_spend.py b/tests/integration/spend/test_vertex_passthrough_regional_uplift_spend.py new file mode 100644 index 00000000000..eaa8f52d4f4 --- /dev/null +++ b/tests/integration/spend/test_vertex_passthrough_regional_uplift_spend.py @@ -0,0 +1,200 @@ +import itertools +import json +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.database import read_rows +from integration._support.forward_proxy import ForwardProxy, tunnelling_forward_proxy +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.tls import server_context, write_self_signed_cert +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +_MODEL: Final = "gemini-3.5-flash-lite" +_PROJECT: Final = "caller-project" +_CALLER_GOOGLE_TOKEN: Final = "caller-google-token" +_GLOBAL_HOST: Final = "aiplatform.googleapis.com" +_US_HOST: Final = "aiplatform.us.rep.googleapis.com" +_GENERATE: Final = "generateContent" +_STREAM: Final = "streamGenerateContent?alt=sse" +_LOCATIONS: Final = ("global", "us") +_ACTIONS: Final = (_GENERATE, _STREAM) +_COST_MAP: Final = Path("litellm/model_prices_and_context_window_backup.json") +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_BODY: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]} +_CUT_PROMPT: Final = "end this stream early" +_CUT_BODY: Final[Mapping[str, JsonValue]] = {"contents": [{"role": "user", "parts": [{"text": _CUT_PROMPT}]}]} +_PROMPT_TOKENS: Final = 1200 +_COMPLETION_TOKENS: Final = 300 +_USAGE: Final = { + "promptTokenCount": _PROMPT_TOKENS, + "candidatesTokenCount": _COMPLETION_TOKENS, + "totalTokenCount": _PROMPT_TOKENS + _COMPLETION_TOKENS, +} +_REPLY: Final = json.dumps( + { + "candidates": [{"content": {"role": "model", "parts": [{"text": "scripted"}]}, "finishReason": "STOP"}], + "usageMetadata": _USAGE, + "modelVersion": _MODEL, + "responseId": "scripted-response", + } +) +_UNFINISHED_FRAME: Final = json.dumps( + { + "candidates": [{"content": {"role": "model", "parts": [{"text": "scripted"}]}}], + "usageMetadata": _USAGE, + "modelVersion": _MODEL, + "responseId": "scripted-cut-response", + } +) +_CUSTOM_DEPLOYMENT: Final = "vertex-custom-priced" +_CUSTOM_INPUT_RATE: Final = 1e-06 +_CUSTOM_OUTPUT_RATE: Final = 2e-06 + + +@dataclass(frozen=True, slots=True) +class _Rig: + proxy: Gateway + tunnel: ForwardProxy + + +def _vertex(request: Request) -> Reply: + assert f"/v1/projects/{_PROJECT}/locations/" in request.target, request.target + if _CUT_PROMPT.encode() in request.body: + frame: Final = f"data: {_UNFINISHED_FRAME}\r\n\r\n".encode() + return Reply(chunks=(frame, frame), content_type="text/event-stream", abort_after=1) + if ":streamGenerateContent" in request.target: + return Reply(chunks=(f"data: {_REPLY}\r\n\r\n".encode(),), content_type="text/event-stream") + return Reply(body=_REPLY.encode()) + + +def _port(url: str) -> int: + port: Final = urlsplit(url).port + assert port is not None, url + return port + + +def _uplift() -> float: + row: Final = _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(_COST_MAP.read_bytes())[f"vertex_ai/{_MODEL}"]) + multiplier: Final = row.get("regional_endpoint_uplift_multiplier") + assert isinstance(multiplier, int | float) and multiplier > 1, row + return float(multiplier) + + +def _config_with_custom_priced_deployment(directory: Path, token_url: str) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + existing: Final = base.get("model_list") or [] + assert isinstance(existing, list), base + deployment: Final = { + "model_name": _CUSTOM_DEPLOYMENT, + "litellm_params": { + "model": f"vertex_ai/{_MODEL}", + "vertex_project": _PROJECT, + "vertex_location": "us", + "vertex_credentials": service_account_json(_PROJECT, token_url), + "use_in_pass_through": True, + "input_cost_per_token": _CUSTOM_INPUT_RATE, + "output_cost_per_token": _CUSTOM_OUTPUT_RATE, + }, + } + path: Final = directory / "config.yaml" + path.write_text(yaml.safe_dump({**base, "model_list": [*existing, deployment]})) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("vertex-passthrough-uplift") + certificate: Final = write_self_signed_cert(directory, (_GLOBAL_HOST, _US_HOST)) + with ( + wire_server(_vertex, tls=server_context(*certificate)) as vertex, + tunnelling_forward_proxy(_port(vertex.url)) as tunnel, + gateway_from_environment() as base, + owned_proxy( + base, + directory, + { + "HTTPS_PROXY": tunnel.url, + "NO_PROXY": "127.0.0.1,localhost", + "AIOHTTP_TRUST_ENV": "True", + "SSL_VERIFY": "False", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + }, + config=_config_with_custom_priced_deployment(directory, base.upstream_url.rstrip("/")), + workers=2, + ) as proxy, + ): + yield _Rig(proxy, tunnel) + + +def _path(location: str, model: str, action: str) -> str: + return f"/vertex_ai/v1/projects/{_PROJECT}/locations/{location}/publishers/google/models/{model}:{action}" + + +def _headers(key: str) -> dict[str, str]: + return {"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"} + + +def _spend(rig: _Rig, key: str, location: str, action: str, model: str = _MODEL) -> float: + with rig.proxy.client.stream("POST", _path(location, model, action), json=_BODY, headers=_headers(key)) as response: + body: Final = response.read() + assert response.status_code == 200, body + call_id: Final = response.headers["x-litellm-call-id"] + return _logged_spend(call_id) + + +def _logged_spend(call_id: str) -> float: + rows: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)), + lambda found: len(found) == 1, + seconds=70, + ) + spend: Final = rows[0]["spend"] + assert isinstance(spend, float), rows + return spend + + +def test_vertex_passthrough_spend_carries_the_regional_uplift_whether_or_not_the_call_streams(rig: _Rig) -> None: + with rig.proxy.scenario() as scenario: + key: Final = scenario.key() + spend: Final = {cell: _spend(rig, key, *cell) for cell in itertools.product(_LOCATIONS, _ACTIONS)} + standard: Final = spend[("global", _GENERATE)] + regional: Final = standard * _uplift() + assert standard > 0, spend + assert spend == pytest.approx( + { + ("global", _GENERATE): standard, + ("global", _STREAM): standard, + ("us", _GENERATE): regional, + ("us", _STREAM): regional, + } + ) + assert set(rig.tunnel.targets()) == {f"{_GLOBAL_HOST}:443", f"{_US_HOST}:443"} + + +def test_custom_priced_vertex_passthrough_bills_the_deployment_price_even_when_the_stream_is_cut_off(rig: _Rig) -> None: + with rig.proxy.scenario() as scenario: + key: Final = scenario.key() + finished: Final = {action: _spend(rig, key, "us", action, _CUSTOM_DEPLOYMENT) for action in _ACTIONS} + with rig.proxy.client.stream( + "POST", _path("us", _CUSTOM_DEPLOYMENT, _STREAM), json=_CUT_BODY, headers=_headers(key) + ) as response: + assert response.status_code == 200 + call_id: Final = response.headers["x-litellm-call-id"] + with pytest.raises(httpx.RemoteProtocolError): + response.read() + cut_off: Final = _logged_spend(call_id) + custom_price: Final = _PROMPT_TOKENS * _CUSTOM_INPUT_RATE + _COMPLETION_TOKENS * _CUSTOM_OUTPUT_RATE + assert {**finished, "cut off": cut_off} == pytest.approx( + {_GENERATE: custom_price, _STREAM: custom_price, "cut off": custom_price} + ) diff --git a/tests/unit/google_genai/test_google_genai_main.py b/tests/unit/google_genai/test_google_genai_main.py index 21f42ca2dff..5cac61e727e 100644 --- a/tests/unit/google_genai/test_google_genai_main.py +++ b/tests/unit/google_genai/test_google_genai_main.py @@ -4,6 +4,8 @@ Test to verify the Google GenAI generate_content adapter functionality """ import json +from collections.abc import Iterator +from typing import Final import pytest @@ -286,3 +288,97 @@ def test_vertex_generate_content_routes_by_deployment_base_model(base_model, use ) assert (setup.generate_content_provider_config is not None) is uses_native_api + + +_UPLIFT_PROBE: Final = "gemini-regional-uplift-probe" +_UPLIFT_INPUT_RATE: Final = 1e-06 +_UPLIFT_OUTPUT_RATE: Final = 2e-06 +_UPLIFT_MULTIPLIER: Final = 1.1 + + +@pytest.fixture +def vertex_regional_uplift_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + rates: Final = { + "input_cost_per_token": _UPLIFT_INPUT_RATE, + "output_cost_per_token": _UPLIFT_OUTPUT_RATE, + "mode": "chat", + } + monkeypatch.setitem(litellm.model_cost, _UPLIFT_PROBE, {**rates, "litellm_provider": "vertex_ai-language-models"}) + monkeypatch.setattr(litellm, "vertex_language_models", {*litellm.vertex_language_models, _UPLIFT_PROBE}) + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{_UPLIFT_PROBE}", + { + **rates, + "litellm_provider": "vertex_ai-language-models", + "regional_endpoint_uplift_multiplier": _UPLIFT_MULTIPLIER, + }, + ) + monkeypatch.delenv("VERTEXAI_LOCATION", raising=False) + monkeypatch.delenv("VERTEX_LOCATION", raising=False) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +async def _logged_vertex_generate_content_cost(location: str, *, stream: bool) -> float: + import asyncio + import datetime + + from litellm.google_genai.main import GenerateContentHelper + from litellm.google_genai.streaming_iterator import BaseGoogleGenAIGenerateContentStreamingIterator + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.google_genai.main import GenerateContentResponse + + reply: Final = { + "candidates": [{"content": {"role": "model", "parts": [{"text": "hi"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 1, "totalTokenCount": 8}, + "modelVersion": _UPLIFT_PROBE, + } + logging_obj: Final = Logging( + model=f"vertex_ai/{_UPLIFT_PROBE}", + messages=[], + stream=stream, + call_type="agenerate_content_stream" if stream else "agenerate_content", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="uplift-test", + function_id="uplift-test", + ) + setup: Final = GenerateContentHelper.setup_generate_content_call( + model=f"vertex_ai/{_UPLIFT_PROBE}", + contents=[{"role": "user", "parts": [{"text": "hi"}]}], + litellm_logging_obj=logging_obj, + vertex_location=location, + stream=stream, + ) + if stream: + iterator: Final = BaseGoogleGenAIGenerateContentStreamingIterator( + litellm_logging_obj=logging_obj, + request_body={}, + model=setup.model, + custom_llm_provider=setup.custom_llm_provider, + ) + iterator.collected_chunks.append(f"data: {json.dumps(reply)}\n\n".encode()) + await iterator._handle_async_streaming_logging() + await asyncio.gather(*(task for task in asyncio.all_tasks() if task is not asyncio.current_task())) + else: + logging_obj._process_hidden_params_and_response_cost( + GenerateContentResponse(**reply), + datetime.datetime(2026, 1, 1), + datetime.datetime(2026, 1, 1), + build_logging_payload=False, + ) + return logging_obj.model_call_details["response_cost"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"]) +@pytest.mark.parametrize("location, multiplier", [("global", 1.0), ("us", _UPLIFT_MULTIPLIER)]) +async def test_vertex_generate_content_spend_follows_the_deployment_location( + vertex_regional_uplift_rate_card: None, stream: bool, location: str, multiplier: float +) -> None: + expected: Final = (7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * multiplier + + recorded: Final = await _logged_vertex_generate_content_cost(location, stream=stream) + + assert recorded == pytest.approx(expected), (location, stream) diff --git a/tests/unit/litellm_core_utils/test_get_litellm_params.py b/tests/unit/litellm_core_utils/test_get_litellm_params.py index f21e475fd16..251bfa624a5 100644 --- a/tests/unit/litellm_core_utils/test_get_litellm_params.py +++ b/tests/unit/litellm_core_utils/test_get_litellm_params.py @@ -6,9 +6,12 @@ Ensures backward compatibility after sparse kwargs extraction optimization. from typing import Final +import httpx import pytest +from openai import OpenAI from pydantic import ValidationError +import litellm from litellm.constants import CONTROL_OPTIONS_KEY from litellm.litellm_core_utils.get_litellm_params import ( _OPTIONAL_KWARGS_KEYS, @@ -75,10 +78,39 @@ class TestGetBaseModelFromLitellmCallMetadata: ) assert result == "gpt-4" - def test_returns_non_string_base_model_without_filtering(self): - result = get_base_model_from_litellm_call_metadata({"model_info": {"base_model": 123}}) + @pytest.mark.parametrize( + "model_info", + [{"base_model": 123}, {"base_model": ["gpt-4"]}, {"base_model": {"model": "gpt-4"}}], + ) + def test_ignores_a_non_string_base_model(self, model_info: object): + assert get_base_model_from_litellm_call_metadata({"model_info": model_info}) is None - assert result == 123 + +@pytest.mark.parametrize("model_info", ["malformed", ["malformed"], 123]) +def test_a_call_with_a_non_object_model_info_fails_before_reaching_the_provider(model_info: object) -> None: + sent: Final[list[httpx.Request]] = [] # mutable-ok: the transport records each outbound request + + def record(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response(500) + + client: Final = OpenAI( + api_key="sk-test", + base_url="https://api.openai.test/v1", + http_client=httpx.Client(transport=httpx.MockTransport(record)), + max_retries=0, + ) + + with pytest.raises(AttributeError): + litellm.completion( + model="openai/gpt-5.4-mini", + messages=[{"role": "user", "content": "hi"}], + metadata={"model_info": model_info}, + client=client, + num_retries=0, + ) + + assert sent == [] class TestGetLitellmParamsKwargsExtraction: diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py index c30f6d9f15a..f780c056f64 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py @@ -1,4 +1,6 @@ +import asyncio import json +from collections.abc import Iterator from datetime import datetime from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -11,6 +13,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.proxy.pass_through_endpoints.llm_provider_handlers.gemini_passthrough_logging_handler import ( GeminiPassthroughLoggingHandler, ) +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import HttpPassThroughEndpointHelpers +from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) @@ -186,7 +190,6 @@ class TestGeminiPassthroughLoggingHandler: mock_completion_cost.assert_called_once() # Verify logging object was updated - assert mock_logging_obj.model_call_details["response_cost"] == 0.000045 assert mock_logging_obj.model_call_details["model"] == "gemini-1.5-flash" assert mock_logging_obj.model_call_details["custom_llm_provider"] == "gemini" @@ -310,8 +313,6 @@ class TestGeminiPassthroughLoggingHandler: # Assert - The success handler returns None on success (following the pattern from other tests) assert result is None - # Verify that the logging object has the cost set (from Gemini handler) - assert mock_logging_obj.model_call_details["response_cost"] == 0.000050 assert mock_logging_obj.model_call_details["model"] == "gemini-2.0-flash" assert mock_logging_obj.model_call_details["custom_llm_provider"] == "gemini" @@ -479,3 +480,114 @@ def test_build_complete_streaming_response_assembles_gemini_sse_chunks(): assert response.choices[0].message.content == "Hello there!" assert response.choices[0].finish_reason == "stop" assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (10, 8, 18) + + +_PROBE_MODEL: Final = "gemini-pricing-probe" +_LIST_INPUT_RATE: Final = 1e-06 +_LIST_OUTPUT_RATE: Final = 4e-06 +_CUSTOM_RATES: Final = {"input_cost_per_token": 2e-09, "output_cost_per_token": 3e-09} +_CUSTOM_PRICED_DEPLOYMENT: Final = "gemini-custom-priced-deployment" +_PROBE_REPLY: Final = { + "candidates": [{"content": {"parts": [{"text": "pong"}], "role": "model"}, "finishReason": "STOP", "index": 0}], + "usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 1, "totalTokenCount": 8}, + "modelVersion": _PROBE_MODEL, +} + + +@pytest.fixture +def probe_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + rates: Final = { + "input_cost_per_token": _LIST_INPUT_RATE, + "output_cost_per_token": _LIST_OUTPUT_RATE, + "litellm_provider": "gemini", + "mode": "chat", + } + monkeypatch.setitem(litellm.model_cost, f"gemini/{_PROBE_MODEL}", rates) + monkeypatch.setitem( + litellm.model_cost, _CUSTOM_PRICED_DEPLOYMENT, {**_CUSTOM_RATES, "litellm_provider": "gemini", "mode": "chat"} + ) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def _cost_the_logger_records_for_gemini(*, stream: bool, model_info: dict[str, object] | None) -> float: + method: Final = "streamGenerateContent?alt=sse" if stream else "generateContent" + url_route: Final = f"https://generativelanguage.googleapis.com/v1beta/models/{_PROBE_MODEL}:{method}" + logging_obj: Final = LiteLLMLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "ping"}], + stream=stream, + call_type="pass_through_endpoint", + start_time=datetime(2026, 1, 1), + litellm_call_id="call-1", + function_id="fn-1", + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {"model_info": model_info}} if model_info is not None else {}, optional_params={} + ) + if stream: + asyncio.run( + PassThroughStreamingHandler.route_streaming_logging_to_handler( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=PassThroughEndpointLogging(), + url_route=url_route, + request_body={}, + endpoint_type=HttpPassThroughEndpointHelpers.get_endpoint_type(url_route), + start_time=datetime(2026, 1, 1), + raw_bytes=[f"data: {json.dumps(_PROBE_REPLY)}\n\n".encode()], + end_time=datetime(2026, 1, 1), + ) + ) + return logging_obj.model_call_details["response_cost"] + response: Final = httpx.Response(200, json=_PROBE_REPLY) + result: Final = GeminiPassthroughLoggingHandler.gemini_passthrough_handler( + httpx_response=response, + response_body=_PROBE_REPLY, + logging_obj=logging_obj, + url_route=url_route, + result=response.text, + start_time=datetime(2026, 1, 1), + end_time=datetime(2026, 1, 1), + cache_hit=False, + request_body={}, + ) + logging_obj._process_hidden_params_and_response_cost( + result["result"], datetime(2026, 1, 1), datetime(2026, 1, 1), build_logging_payload=False + ) + return logging_obj.model_call_details["response_cost"] + + +@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"]) +@pytest.mark.parametrize( + "model_info, input_rate, output_rate", + [ + (None, _LIST_INPUT_RATE, _LIST_OUTPUT_RATE), + ( + {"id": _CUSTOM_PRICED_DEPLOYMENT, **_CUSTOM_RATES}, + _CUSTOM_RATES["input_cost_per_token"], + _CUSTOM_RATES["output_cost_per_token"], + ), + ], + ids=["list-price", "custom-price"], +) +def test_gemini_passthrough_spend_matches_whether_or_not_the_call_streams( + probe_rate_card: None, + stream: bool, + model_info: dict[str, object] | None, + input_rate: float, + output_rate: float, +) -> None: + recorded: Final = _cost_the_logger_records_for_gemini(stream=stream, model_info=model_info) + + assert recorded == pytest.approx(7 * input_rate + 1 * output_rate), stream + + +@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"]) +@pytest.mark.parametrize("base_model", [123, [f"gemini/{_PROBE_MODEL}"], {"model": f"gemini/{_PROBE_MODEL}"}]) +def test_gemini_passthrough_bills_the_list_price_when_the_model_info_base_model_is_not_a_string( + probe_rate_card: None, stream: bool, base_model: object +) -> None: + recorded: Final = _cost_the_logger_records_for_gemini(stream=stream, model_info={"base_model": base_model}) + + assert recorded == pytest.approx(7 * _LIST_INPUT_RATE + 1 * _LIST_OUTPUT_RATE), stream diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_vertex_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_vertex_passthrough_logging_handler.py index cc1246f4ede..d05ab8a9bcf 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_vertex_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_vertex_passthrough_logging_handler.py @@ -1,4 +1,6 @@ +import asyncio import json +from collections.abc import Iterator from datetime import datetime from typing import Final @@ -6,11 +8,15 @@ import httpx import pytest from pydantic import ValidationError +import litellm from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) +from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler +from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndpointLogging +from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from litellm.types.utils import EmbeddingResponse, ModelResponse _PREDICT_ROUTE = "/v1/projects/p/locations/us-central1/publishers/google/models/text-embedding-004:predict" @@ -97,6 +103,359 @@ def test_interactions_usage_object_is_read_into_prompt_and_completion_tokens(): assert result["kwargs"]["custom_llm_provider"] == "vertex_ai" +_UPLIFT_MODEL: Final = "gemini-regional-uplift-probe" +_UPLIFT_INPUT_RATE: Final = 1e-06 +_UPLIFT_OUTPUT_RATE: Final = 4e-06 +_UPLIFT_MULTIPLIER: Final = 1.1 + + +@pytest.fixture +def regional_uplift_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + rates: Final = { + "input_cost_per_token": _UPLIFT_INPUT_RATE, + "output_cost_per_token": _UPLIFT_OUTPUT_RATE, + "litellm_provider": "vertex_ai-language-models", + "mode": "chat", + } + monkeypatch.setitem(litellm.model_cost, _UPLIFT_MODEL, rates) + monkeypatch.setattr(litellm, "vertex_language_models", {*litellm.vertex_language_models, _UPLIFT_MODEL}) + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{_UPLIFT_MODEL}", + {**rates, "regional_endpoint_uplift_multiplier": _UPLIFT_MULTIPLIER}, + ) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def _generate_content_payload() -> dict[str, object]: + return { + "candidates": [{"content": {"parts": [{"text": "pong"}], "role": "model"}, "finishReason": "STOP", "index": 0}], + "usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 1, "totalTokenCount": 8}, + "modelVersion": _UPLIFT_MODEL, + } + + +def _cost_the_logger_records_for( + url_route: str, + *, + stream: bool, + reply: dict[str, object], + stream_events: list[dict[str, object]], + model_info: dict[str, object] | None = None, + route_provider: str | None = None, + base_model: object = None, +) -> float: + logging_obj: Final = Logging( + model="unknown", + messages=[{"role": "user", "content": "ping"}], + stream=stream, + call_type="pass_through_endpoint", + start_time=datetime(2026, 1, 1), + litellm_call_id="call-1", + function_id="fn-1", + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {"model_info": model_info}} if model_info is not None else {}, optional_params={} + ) + if route_provider is not None: + logging_obj.model_call_details["custom_llm_provider"] = route_provider + if base_model is not None: + logging_obj.model_call_details["litellm_params"] = {"base_model": base_model} + if stream: + asyncio.run( + PassThroughStreamingHandler.route_streaming_logging_to_handler( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=PassThroughEndpointLogging(), + url_route=url_route, + request_body={}, + endpoint_type=EndpointType.VERTEX_AI, + start_time=datetime(2026, 1, 1), + raw_bytes=[f"data: {json.dumps(event)}\n\n".encode() for event in stream_events], + end_time=datetime(2026, 1, 1), + ) + ) + return logging_obj.model_call_details["response_cost"] + response: Final = httpx.Response(200, json=reply) + result: Final = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route=url_route, + result=response.text, + start_time=datetime(2026, 1, 1), + end_time=datetime(2026, 1, 1), + cache_hit=False, + request_body={}, + ) + logging_obj._process_hidden_params_and_response_cost( + result["result"], datetime(2026, 1, 1), datetime(2026, 1, 1), build_logging_payload=False + ) + return logging_obj.model_call_details["response_cost"] + + +def _cost_the_logger_records(location: str, *, stream: bool, model_info: dict[str, object] | None = None) -> float: + method: Final = "streamGenerateContent" if stream else "generateContent" + url_route: Final = ( + f"https://aiplatform.googleapis.com/v1/projects/p/locations/{location}" + f"/publishers/google/models/{_UPLIFT_MODEL}:{method}" + ) + return _cost_the_logger_records_for( + url_route, + stream=stream, + reply=_generate_content_payload(), + stream_events=[_generate_content_payload()], + model_info=model_info, + ) + + +@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"]) +@pytest.mark.parametrize("location, multiplier", [("global", 1.0), ("us", _UPLIFT_MULTIPLIER)]) +def test_generate_content_spend_follows_the_vertex_location( + regional_uplift_rate_card: None, stream: bool, location: str, multiplier: float +) -> None: + expected: Final = (7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * multiplier + + assert _cost_the_logger_records(location, stream=stream) == pytest.approx(expected), (location, stream) + + +@pytest.mark.parametrize( + "gemini_api_base", ["https://generativelanguage.googleapis.com", "https://llm-egress.internal.example/google"] +) +@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"]) +def test_gemini_route_spend_has_no_vertex_uplift_whatever_host_serves_it( + regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, gemini_api_base: str, stream: bool +) -> None: + monkeypatch.setitem( + litellm.model_cost, + f"gemini/{_UPLIFT_MODEL}", + { + "input_cost_per_token": _UPLIFT_INPUT_RATE, + "output_cost_per_token": _UPLIFT_OUTPUT_RATE, + "litellm_provider": "gemini", + "mode": "chat", + }, + ) + method: Final = "streamGenerateContent?alt=sse" if stream else "generateContent" + + recorded: Final = _cost_the_logger_records_for( + f"{gemini_api_base}/v1beta/models/{_UPLIFT_MODEL}:{method}", + stream=stream, + reply=_generate_content_payload(), + stream_events=[_generate_content_payload()], + route_provider="gemini", + ) + + assert recorded == pytest.approx(7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE), (gemini_api_base, stream) + + +@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"]) +def test_a_vertex_host_that_only_ends_with_the_gemini_host_name_keeps_the_vertex_uplift( + regional_uplift_rate_card: None, stream: bool +) -> None: + method: Final = "streamGenerateContent" if stream else "generateContent" + + recorded: Final = _cost_the_logger_records_for( + "https://vertex-generativelanguage.googleapis.com/v1/projects/p/locations/us/publishers/google/models/" + f"{_UPLIFT_MODEL}:{method}", + stream=stream, + reply=_generate_content_payload(), + stream_events=[_generate_content_payload()], + ) + + assert recorded == pytest.approx((7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * _UPLIFT_MULTIPLIER), stream + + +@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"]) +def test_a_subdomain_of_the_gemini_host_bills_the_gemini_price( + regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + monkeypatch.setitem( + litellm.model_cost, + f"gemini/{_UPLIFT_MODEL}", + { + "input_cost_per_token": _UPLIFT_INPUT_RATE, + "output_cost_per_token": _UPLIFT_OUTPUT_RATE, + "litellm_provider": "gemini", + "mode": "chat", + }, + ) + method: Final = "streamGenerateContent?alt=sse" if stream else "generateContent" + + recorded: Final = _cost_the_logger_records_for( + f"https://eu.generativelanguage.googleapis.com/v1beta/models/{_UPLIFT_MODEL}:{method}", + stream=stream, + reply=_generate_content_payload(), + stream_events=[_generate_content_payload()], + ) + + assert recorded == pytest.approx(7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE), stream + + +@pytest.mark.parametrize("value", [123, ["gemini-2.5-flash-lite"], {"model": "gemini-2.5-flash-lite"}]) +@pytest.mark.parametrize("carrier", ["root", "model_info"]) +@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"]) +def test_a_non_string_base_model_bills_the_list_price( + regional_uplift_rate_card: None, stream: bool, carrier: str, value: object +) -> None: + method: Final = "streamGenerateContent" if stream else "generateContent" + + recorded: Final = _cost_the_logger_records_for( + f"https://aiplatform.googleapis.com/v1/projects/p/locations/us/publishers/google/models/{_UPLIFT_MODEL}:{method}", + stream=stream, + reply=_generate_content_payload(), + stream_events=[_generate_content_payload()], + model_info={"base_model": value} if carrier == "model_info" else None, + base_model=value if carrier == "root" else None, + ) + + assert recorded == pytest.approx((7 * _UPLIFT_INPUT_RATE + 1 * _UPLIFT_OUTPUT_RATE) * _UPLIFT_MULTIPLIER), stream + + +_CUSTOM_PRICED_DEPLOYMENT: Final = "vertex-custom-priced-deployment" +_CUSTOM_INPUT_RATE: Final = 2e-09 +_CUSTOM_OUTPUT_RATE: Final = 3e-09 + + +@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"]) +@pytest.mark.parametrize("location", ["global", "us"]) +def test_generate_content_spend_uses_the_deployment_custom_price( + regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, stream: bool, location: str +) -> None: + custom_rates: Final = {"input_cost_per_token": _CUSTOM_INPUT_RATE, "output_cost_per_token": _CUSTOM_OUTPUT_RATE} + monkeypatch.setitem( + litellm.model_cost, _CUSTOM_PRICED_DEPLOYMENT, {**custom_rates, "litellm_provider": "vertex_ai", "mode": "chat"} + ) + expected: Final = 7 * _CUSTOM_INPUT_RATE + 1 * _CUSTOM_OUTPUT_RATE + + recorded: Final = _cost_the_logger_records( + location, stream=stream, model_info={"id": _CUSTOM_PRICED_DEPLOYMENT, **custom_rates} + ) + + assert recorded == pytest.approx(expected), (location, stream) + + +_CLAUDE_UPLIFT_MODEL: Final = "claude-regional-uplift-probe" +_CLAUDE_INPUT_RATE: Final = 3e-06 +_CLAUDE_OUTPUT_RATE: Final = 1.5e-05 + + +@pytest.fixture +def claude_regional_uplift_rate_card(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + rates: Final = { + "input_cost_per_token": _CLAUDE_INPUT_RATE, + "output_cost_per_token": _CLAUDE_OUTPUT_RATE, + "mode": "chat", + } + monkeypatch.setitem(litellm.model_cost, _CLAUDE_UPLIFT_MODEL, {**rates, "litellm_provider": "anthropic"}) + monkeypatch.setattr(litellm, "anthropic_models", {*litellm.anthropic_models, _CLAUDE_UPLIFT_MODEL}) + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{_CLAUDE_UPLIFT_MODEL}", + { + **rates, + "litellm_provider": "vertex_ai-anthropic_models", + "regional_endpoint_uplift_multiplier": _UPLIFT_MULTIPLIER, + }, + ) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def _claude_message() -> dict[str, object]: + return { + "id": "msg_probe", + "type": "message", + "role": "assistant", + "model": _CLAUDE_UPLIFT_MODEL, + "content": [{"type": "text", "text": "pong"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 7, "output_tokens": 1}, + } + + +def _claude_stream_events() -> list[dict[str, object]]: + message_start: Final = {**_claude_message(), "content": [], "usage": {"input_tokens": 7, "output_tokens": 0}} + return [ + {"type": "message_start", "message": message_start}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}}, + {"type": "message_stop"}, + ] + + +@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"]) +@pytest.mark.parametrize("location, multiplier", [("global", 1.0), ("us-east5", _UPLIFT_MULTIPLIER)]) +def test_claude_raw_predict_spend_follows_the_vertex_location( + claude_regional_uplift_rate_card: None, stream: bool, location: str, multiplier: float +) -> None: + method: Final = "streamRawPredict" if stream else "rawPredict" + url_route: Final = ( + f"https://aiplatform.googleapis.com/v1/projects/p/locations/{location}" + f"/publishers/anthropic/models/{_CLAUDE_UPLIFT_MODEL}:{method}" + ) + expected: Final = (7 * _CLAUDE_INPUT_RATE + 1 * _CLAUDE_OUTPUT_RATE) * multiplier + + recorded: Final = _cost_the_logger_records_for( + url_route, stream=stream, reply=_claude_message(), stream_events=_claude_stream_events() + ) + + assert recorded == pytest.approx(expected), (location, stream) + + +@pytest.mark.parametrize("stream", [True, False], ids=["streamed", "non-streamed"]) +def test_claude_raw_predict_spend_uses_the_deployment_custom_price( + claude_regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + custom_rates: Final = {"input_cost_per_token": _CUSTOM_INPUT_RATE, "output_cost_per_token": _CUSTOM_OUTPUT_RATE} + monkeypatch.setitem( + litellm.model_cost, _CUSTOM_PRICED_DEPLOYMENT, {**custom_rates, "litellm_provider": "vertex_ai", "mode": "chat"} + ) + method: Final = "streamRawPredict" if stream else "rawPredict" + url_route: Final = ( + "https://aiplatform.googleapis.com/v1/projects/p/locations/us-east5" + f"/publishers/anthropic/models/{_CLAUDE_UPLIFT_MODEL}:{method}" + ) + + recorded: Final = _cost_the_logger_records_for( + url_route, + stream=stream, + reply=_claude_message(), + stream_events=_claude_stream_events(), + model_info={"id": _CUSTOM_PRICED_DEPLOYMENT, **custom_rates}, + ) + + assert recorded == pytest.approx(7 * _CUSTOM_INPUT_RATE + 1 * _CUSTOM_OUTPUT_RATE), stream + + +def test_a_claude_stream_through_a_configured_vertex_gateway_with_a_short_path_bills_the_vertex_price( + claude_regional_uplift_rate_card: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{_CLAUDE_UPLIFT_MODEL}", + { + "input_cost_per_token": _CLAUDE_INPUT_RATE, + "output_cost_per_token": _CLAUDE_OUTPUT_RATE, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + }, + ) + litellm.get_model_info.cache_clear() + + recorded: Final = _cost_the_logger_records_for( + f"https://vertex-gateway.internal.example/vxc/models/{_CLAUDE_UPLIFT_MODEL}:streamRawPredict", + stream=True, + reply=_claude_message(), + stream_events=_claude_stream_events(), + ) + + assert recorded == pytest.approx(7 * _CLAUDE_INPUT_RATE + 1 * _CLAUDE_OUTPUT_RATE) + + def test_build_complete_streaming_response_assembles_stream_generate_content_sse_chunks(): logging_obj: Final = Logging( model="gemini-2.5-flash", diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index eb31797d460..5412ea56ac0 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -8452,7 +8452,9 @@ _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) _PROXY_SERVER_REQUEST: Final = TypeAdapter(dict[str, object]) -def _split_pass_through_body(body: str) -> _PassThroughSplit: +def _split_pass_through_body( + body: str, user_api_key_dict: UserAPIKeyAuth = UserAPIKeyAuth(api_key="hashed-key") +) -> _PassThroughSplit: mock_request: Final = MagicMock(spec=Request) mock_request.method = "POST" mock_request.url = httpx.URL("http://0.0.0.0:4000/gemini/v1beta/models/gemini-2.5-flash:generateContent") @@ -8462,7 +8464,7 @@ def _split_pass_through_body(body: str) -> _PassThroughSplit: init_kwargs_for_pass_through_endpoint: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # untyped legacy helper kwargs: Final = init_kwargs_for_pass_through_endpoint( request=mock_request, - user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + user_api_key_dict=user_api_key_dict, passthrough_logging_payload=MagicMock(), logging_obj=MagicMock(), _parsed_body=json.loads(body), @@ -8500,6 +8502,7 @@ def test_passthrough_moves_every_litellm_owned_key_from_the_forwarded_body_into_ assert split.forwarded_body == json.loads(GEMINI_BODY) +PRICING_MODEL_OVERRIDE_NAMES: Final = frozenset(("base_model",)) PROXY_STAMPED_NAMES: Final = frozenset( ( "proxy_server_request", @@ -8515,7 +8518,12 @@ PROXY_STAMPED_NAMES: Final = frozenset( @pytest.mark.parametrize( "name", - sorted(frozenset(litellm.all_litellm_params) - frozenset(("metadata", "litellm_metadata")) - PROXY_STAMPED_NAMES), + sorted( + frozenset(litellm.all_litellm_params) + - frozenset(("metadata", "litellm_metadata")) + - PROXY_STAMPED_NAMES + - PRICING_MODEL_OVERRIDE_NAMES + ), ) def test_passthrough_keeps_each_registered_litellm_owned_name_out_of_the_forwarded_body(name: str) -> None: split: Final = _split_pass_through_body(json.dumps({name: "owned", **json.loads(GEMINI_BODY)})) @@ -8525,6 +8533,41 @@ def test_passthrough_keeps_each_registered_litellm_owned_name_out_of_the_forward assert split.forwarded_body == json.loads(GEMINI_BODY) +@pytest.mark.parametrize("allows_client_pricing", [True, False], ids=["key-allows-client-pricing", "plain-key"]) +def test_passthrough_prices_with_a_client_base_model_only_for_keys_that_allow_client_pricing( + allows_client_pricing: bool, +) -> None: + key: Final = UserAPIKeyAuth( + api_key="hashed-key", metadata={"allow_client_pricing_override": True} if allows_client_pricing else {} + ) + + split: Final = _split_pass_through_body( + json.dumps({"base_model": "vertex_ai/gemini-2.5-flash-lite", **json.loads(GEMINI_BODY)}), key + ) + + expected: Final = "vertex_ai/gemini-2.5-flash-lite" if allows_client_pricing else None + assert split.litellm_params.get("base_model") == expected + assert split.forwarded_body == json.loads(GEMINI_BODY) + + +@pytest.mark.parametrize("model_info", ["malformed", ["malformed"], 123]) +def test_passthrough_drops_a_client_model_info_that_is_not_an_object(model_info: object) -> None: + key: Final = UserAPIKeyAuth(api_key="hashed-key", metadata={"allow_client_pricing_override": True}) + plain_metadata: Final = _metadata_of(_split_pass_through_body(json.dumps({"metadata": {"tag": "a"}}), key)) + + split: Final = _split_pass_through_body(json.dumps({"metadata": {"tag": "a", "model_info": model_info}}), key) + + assert _metadata_of(split) == plain_metadata + + +def test_passthrough_keeps_a_client_model_info_object_for_keys_that_allow_client_pricing() -> None: + key: Final = UserAPIKeyAuth(api_key="hashed-key", metadata={"allow_client_pricing_override": True}) + + split: Final = _split_pass_through_body(json.dumps({"metadata": {"model_info": {"base_model": "gemini/x"}}}), key) + + assert _metadata_of(split)["model_info"] == {"base_model": "gemini/x"} + + @pytest.mark.parametrize("name", sorted(PROXY_STAMPED_NAMES)) def test_passthrough_drops_a_client_supplied_proxy_stamped_name(name: str) -> None: split: Final = _split_pass_through_body(json.dumps({name: {"forged": "by-client"}, **json.loads(GEMINI_BODY)})) diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py index fdd21afbc67..da084146894 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py @@ -2,6 +2,7 @@ import json import logging from collections.abc import Iterator from datetime import datetime +from typing import Final from unittest.mock import MagicMock import pytest @@ -13,6 +14,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrou VertexPassthroughLoggingHandler, ) from litellm.proxy.pass_through_endpoints.streaming_handler import ( + PassThroughStreamContext, PassThroughStreamingHandler, ) from litellm.proxy.pass_through_endpoints.success_handler import ( @@ -235,3 +237,40 @@ async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop( partial_usage = logging_obj.record_partial_usage_for_failure.call_args.kwargs["usage"] assert partial_usage.completion_tokens > 100_000 assert_loop_stayed_free(took, lags) + + +@pytest.mark.asyncio +async def test_interrupted_vertex_stream_bills_the_deployment_custom_price(monkeypatch: pytest.MonkeyPatch) -> None: + deployment_id: Final = "vertex-custom-priced-deployment" + custom_rates: Final = {"input_cost_per_token": 2e-09, "output_cost_per_token": 3e-09} + monkeypatch.setitem( + litellm.model_cost, deployment_id, {**custom_rates, "litellm_provider": "vertex_ai", "mode": "chat"} + ) + logging_obj: Final = LiteLLMLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "ping"}], + stream=True, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="interrupted-stream", + function_id="interrupted-stream", + ) + logging_obj.optional_params = {} + logging_obj.litellm_params = {"metadata": {"model_info": {"id": deployment_id, **custom_rates}}} + url_route: Final = ( + f"https://aiplatform.googleapis.com/v1/projects/p/locations/us" + f"/publishers/google/models/{MODEL}:streamGenerateContent?alt=sse" + ) + + await PassThroughStreamingHandler.schedule_stream_failure_logging( + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.VERTEX_AI, + request_body={}, + raw_bytes=[chunk.encode("utf-8") for chunk in _chunks()], + exception=RuntimeError("upstream closed the stream"), + stream_context=PassThroughStreamContext(PassThroughEndpointLogging(), url_route, datetime.now()), + ) + recorded: Final = logging_obj.model_call_details["response_cost"] + await GLOBAL_LOGGING_WORKER.flush() + + assert recorded == pytest.approx(PROMPT_TOKENS * 2e-09 + COMPLETION_TOKENS * 3e-09) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index b4150aad1c7..69ef2caf8f4 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -114,10 +114,22 @@ from litellm.utils import ( ) -def test_get_base_model_from_metadata_returns_unvalidated_root_value(): +@pytest.mark.parametrize("root_base_model", [42, ["gemini-3.5-flash"], {"model": "gemini-3.5-flash"}]) +def test_get_base_model_from_metadata_ignores_a_non_string_root_value(root_base_model: object): from litellm.utils import get_base_model_from_metadata - assert get_base_model_from_metadata({"litellm_params": {"base_model": 42}}) == 42 + assert get_base_model_from_metadata({"litellm_params": {"base_model": root_base_model}}) is None + assert ( + get_base_model_from_metadata( + { + "litellm_params": { + "base_model": root_base_model, + "metadata": {"model_info": {"base_model": "vertex_ai/gemini-3.5-flash"}}, + } + } + ) + == "vertex_ai/gemini-3.5-flash" + ) def _reject_short_post_call_response(input: str, model: str | None = None) -> dict[str, object]: From 748c973f421f807f6b26682d5835d3fcb3ce344b Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:26:30 -0700 Subject: [PATCH 15/50] chore(pricing): add meta muse-image-1.0 per-image price (#47318) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...model_prices_and_context_window_backup.json | 18 ++++++++++++++++++ model_prices_and_context_window.json | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d570e49bf6c..bb0cdd51d9c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76082,5 +76082,23 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "meta/muse-image-1.0": { + "litellm_provider": "meta", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://dev.meta.ai/docs/pricing-rate-limits", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d570e49bf6c..bb0cdd51d9c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76082,5 +76082,23 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "meta/muse-image-1.0": { + "litellm_provider": "meta", + "mode": "image_generation", + "output_cost_per_image": 0.01, + "source": "https://dev.meta.ai/docs/pricing-rate-limits", + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "image" + ] } } From a2773bdcad53aed8fee22e06056ab1e0732df589 Mon Sep 17 00:00:00 2001 From: Habon Laszlo Date: Mon, 19 Oct 2026 15:10:11 +0200 Subject: [PATCH 16/50] fix(azure): route WebRTC client_secrets and calls to the GA v1 realtime API (#47316) --- .../azure/realtime/http_transformation.py | 22 +++---- .../realtime/test_http_transformation.py | 57 +++++++++++++++++++ 2 files changed, 69 insertions(+), 10 deletions(-) create mode 100644 tests/unit/llms/azure/realtime/test_http_transformation.py diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index 633c2b3b6d0..542a6a8c02f 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -7,17 +7,21 @@ from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPC from litellm.secret_managers.main import get_secret_str +def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str: + """Azure serves WebRTC only on the GA v1 surface, which rejects any api-version query param.""" + base: Final = api_base.rstrip("/").removesuffix("/v1").removesuffix("/openai") + return f"{base}/openai/v1/realtime/{endpoint}" + + class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): - def get_api_base(self, api_base: str | None, **kwargs) -> str: + def get_api_base(self, api_base: str | None, **kwargs: object) -> str: return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") or "" - def get_api_key(self, api_key: str | None, **kwargs) -> str: + def get_api_key(self, api_key: str | None, **kwargs: object) -> str: return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or "" def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: - base: Final = self.get_api_base(api_base).rstrip("/") - version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" - return f"{base}/openai/realtime/client_secrets?api-version={version}" + return _azure_realtime_v1_url(self.get_api_base(api_base), "client_secrets") def validate_environment( self, @@ -32,16 +36,14 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): } def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: - base: Final = self.get_api_base(api_base).rstrip("/") - version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" - return f"{base}/openai/realtime/calls?api-version={version}" + return _azure_realtime_v1_url(self.get_api_base(api_base), "calls") def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base: Final = self.get_api_base(api_base).rstrip("/") version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" return f"{base}/openai/realtime/transcription_sessions?api-version={version}" - def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]: return { - "api-key": ephemeral_key, + "Authorization": f"Bearer {ephemeral_key}", } diff --git a/tests/unit/llms/azure/realtime/test_http_transformation.py b/tests/unit/llms/azure/realtime/test_http_transformation.py new file mode 100644 index 00000000000..714ce9558cf --- /dev/null +++ b/tests/unit/llms/azure/realtime/test_http_transformation.py @@ -0,0 +1,57 @@ +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + +_DATED_PREVIEW_API_VERSION: Final = "2025-04-01-preview" +_API_BASES: Final = ("https://my.openai.azure.com", "https://my.openai.azure.com/", "https://my.openai.azure.com/openai") + + +class _AzureThatRecordsTheRequest: + def __init__(self) -> None: + self.requests: Final[list[httpx.Request]] = [] + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + return httpx.Response(200, json={"value": "ek_upstream"}) + + def client(self) -> AsyncHTTPHandler: + return AsyncHTTPHandler(transport=httpx.MockTransport(self)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_base", _API_BASES) +async def test_azure_client_secret_targets_the_ga_v1_surface_even_with_a_dated_api_version(api_base: str) -> None: + azure: Final = _AzureThatRecordsTheRequest() + await litellm.acreate_realtime_client_secret( + model="azure/gpt-realtime-deployment", + api_base=api_base, + api_key="azure-key", + api_version=_DATED_PREVIEW_API_VERSION, + client=azure.client(), + ) + (request,) = azure.requests + assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/client_secrets" + assert request.headers.get("api-key") == "azure-key" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_base", _API_BASES) +async def test_azure_sdp_exchange_targets_the_ga_v1_surface_with_the_ephemeral_key_as_bearer(api_base: str) -> None: + azure: Final = _AzureThatRecordsTheRequest() + await litellm.arealtime_calls( + openai_ephemeral_key="ek_upstream", + sdp_body=b"v=0", + model="azure/gpt-realtime-deployment", + api_base=api_base, + api_key="azure-key", + api_version=_DATED_PREVIEW_API_VERSION, + client=azure.client(), + ) + (request,) = azure.requests + assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/calls" + assert request.headers.get("authorization") == "Bearer ek_upstream" + assert "api-key" not in request.headers From 7ef92689da786029f532c101fceae9703fe39ade Mon Sep 17 00:00:00 2001 From: Habon Laszlo Date: Mon, 19 Oct 2026 16:12:49 +0200 Subject: [PATCH 17/50] fix(azure): route WebRTC client_secrets and calls to the GA v1 realtime API (#47316) Azure retired the preview realtime surface, so /openai/realtime/client_secrets and /openai/realtime/calls with an api-version return 404. Both now target /openai/v1/realtime and the SDP exchange sends the ephemeral key as a Bearer token, matching the Azure GA WebRTC contract. Fixes #24659 Co-authored-by: habonlaci <4699494+habonlaci@users.noreply.github.com> From 492272a051c2140f02512472605a8e82d511e515 Mon Sep 17 00:00:00 2001 From: Jorik Date: Mon, 19 Oct 2026 17:27:05 +0200 Subject: [PATCH 18/50] fix(azure): reuse the cached Entra token provider on the Responses path (#46421) get_azure_ad_token() built a new DefaultAzureCredential for every request when enable_azure_ad_token_refresh is on, so each Azure Responses call fetched a fresh Entra token synchronously on the event loop. Use the per-scope cached provider the SDK-client path already uses, so the credential's own token cache does its job. Fixes #46419 --- litellm/llms/azure/common_utils.py | 2 +- .../llms/azure/test_azure_common_utils.py | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 46106026920..f9f636244f4 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -375,7 +375,7 @@ def get_azure_ad_token( "Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth" ) try: - azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope) + azure_ad_token_provider = _cached_azure_ad_token_refresh_provider(scope) except ValueError: verbose_logger.debug("Azure AD Token Provider could not be used.") except Exception as e: diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index b277cac1f29..80449909b3f 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -1410,6 +1410,26 @@ def test_get_azure_ad_token_with_token_refresh(setup_mocks, monkeypatch): assert token == "mock-default-token" +def test_get_azure_ad_token_with_token_refresh_reuses_provider_per_scope(setup_mocks): + """Repeated requests reuse one token provider per scope instead of building a + new credential, and fetching a new token, on every call (#46419)""" + setup_mocks["litellm"].enable_azure_ad_token_refresh = True + default_scope = "https://cognitiveservices.azure.com/.default" + other_scope = "api://other-resource/.default" + + tokens = [ + get_azure_ad_token(GenericLiteLLMParams()), + get_azure_ad_token(GenericLiteLLMParams()), + get_azure_ad_token(GenericLiteLLMParams(azure_scope=other_scope)), + ] + + assert tokens == ["mock-default-token"] * 3 + assert [c.kwargs for c in setup_mocks["token_provider"].call_args_list] == [ + {"azure_scope": default_scope}, + {"azure_scope": other_scope}, + ] + + def test_get_azure_ad_token_with_token_refresh_error(setup_mocks): """Test get_azure_ad_token with token refresh enabled but raising an error.""" # Reset mocks to ensure clean state From c80f1c2ec8c266695eee8f4af1f1a1d68038b65a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:35:20 +0000 Subject: [PATCH 19/50] fix(router)!: count native Responses WebSocket turns toward deployment limits and budgets (#45727) * fix(responses): write one spend-log row per native WebSocket turn Each response.create turn on a native Responses WebSocket connection now logs its own call as it finishes, with its own usage and the resp_ id the client saw as request_id. A failed turn is booked alone and no longer flips the other turns. * fix(responses): bill the usage a failed native WebSocket turn reports * fix(proxy): pin the limiter slot owner to the proxy call id and refund unsettled reservations on disconnect The v3 limiter stashed whichever litellm_call_id the request body carried as the slot owner, so a client-supplied id on a pass-through body left the proxy's own success and failure events unable to release the slot. The owner is now the proxy's logging call id. A WebSocket connection whose turns log under their own ids never settles the TPM, ITPM, or OTPM estimate the connection reserved, so the disconnect release now refunds any reservation no callback settled, once. * fix(proxy): bill failed WebSocket turns and refund only the reserved window on disconnect A failed turn's reported usage now counts toward the key's TPM and the project ITPM/OTPM scopes the same way a success does, and closing the socket hands back only the estimate still unused in the window that holds it, so a window that already rolled is left alone. An id-less error event is booked against the newest pending turn, the one the backend rejected at parse time, and a turn that nests its input under response logs its user message. A connection's one budget reservation goes to the first turn that settles it, every later turn charging its cost directly. * test(proxy): expect the settling call id on a pass-through budget reservation * fix(proxy): charge successful Responses WebSocket turns to TPM and ITPM/OTPM A native Responses WebSocket turn reaches the rate limiter's success callback with its raw event list as the response and no combined_usage_object, so the turn's tokens never reached the key's TPM window or the project's input and output token limits. Read the usage from the turn's events when the call type is a Responses WebSocket turn. * fix(proxy): release a rejected pass-through's slot and count each websocket turn once toward project quotas The limiter's failure hook looked up the parallel slot by the request body's litellm_call_id, while the slot belongs to the proxy's logging call id, so a pass-through request a pre-call hook rejected held its slot for an hour when its body carried an id. Both now resolve the id the same way. Each response.create frame already charges its estimate to project ITPM/OTPM, so the turn's own usage no longer adds a second count. The cost callback's failure hook read the body's litellm_call_id through a strict model and raised on a non-string value, which dropped the failure row. It now reads a string id or none. * fix(proxy): keep per-turn WebSocket limits, request bodies and collector reservations right Free a rejected request's parallel slot through the request it runs in, and skip native Responses WebSocket connections, whose disconnect frees the slot. Count WebSocket turns toward the connection's tag TPM. In collector mode, claim the shared budget reservation before the spend event is built, so only the first turn settles it. Store each turn's own frame as its request body, and send an id-less error to the oldest waiting turn unless an earlier turn is streaming. * fix(proxy): store a failed WebSocket turn's own request and count tag TPM only for calls that carry the tag * fix(responses): log the client's turn frame, keep the payload id and reservation shape for non-WebSocket calls * fix(proxy): keep the legacy limiter's WebSocket slot until the socket closes * fix(router): count native Responses WebSocket turns toward deployment limits and budgets * fix(router): re-check deployment budget and RPM on each Responses WebSocket turn * test(router): list the WebSocket turn admission method as covered through its partial * fix(responses): give a refused WebSocket turn's error frame a 429 status so Codex ends the turn * fix(router): leave ITPM/OTPM to socket open and record a refused WebSocket turn as the socket's failure * fix(proxy): log the routed model group on a WebSocket turn's failure row * fix(router): count WebSocket turn tokens toward TPM and re-run routing strategy checks on each turn * test(integration): cover per-turn admission on Responses WebSockets Adds integration cells for budget, RPM, TPM, a mid-socket TPM edit, usage-based-routing-v2, the project quota frame, and the open-time failure frame on the native and managed relays, plus HTTP controls for TPM counting and the failure row model_group * fix(responses): count WebSocket turn tokens once and end a socket whose turn check fails A native Responses WebSocket turn is now logged as an async call, so a proxy with a sync success callback such as the response cache no longer runs each CustomLogger's sync success hook on top of the async one. That second run counted every turn's tokens twice toward deployment TPM. A turn whose admission check raises an unexpected error now gets a 500 error frame, the socket closes with 1011, and the error becomes the socket's failure, on both the native and the managed relay, instead of the socket dropping with no frame and no failure row. A refused turn's failure row now carries the request's tags from the router's metadata, the same way it already carries the model group. * test(integration): cover a crashed WebSocket turn check and failure-row tags A turn whose admission check raises gets a 500 server_error frame, the socket closes with 1011 and its one failure row carries the error, on both relays. The refused turn's failure row keeps the caller's tags, and HTTP failure rows on chat, responses and messages keep them too. * fix(responses): keep a WebSocket open after a crashed turn check and keep only string tags on failure rows A turn whose admission check raises an unexpected error still gets a 500 error frame, but the socket now stays open, like a refused turn. Closing it cut any turn still streaming on the same socket, which a client that sends its next response.create early would lose. A refused turn's failure row now takes the router's tags only when every tag is a string, so the fallback never copies a caller's unvalidated tags onto the row. * test(integration): cover a turn still streaming when the next WebSocket turn's check crashes A native turn still streaming when the next turn's admission check raises now has its own cell. It must complete and be billed at the turn's price. The crashed-check cell also keeps the fault set for one more turn, so a relay that stopped checking turns after a crash goes red. * fix(responses): run per-turn guardrails before the deployment's turn check on WebSockets A turn after the first now runs the project quota check, then the proxy's pre_call guardrails, then the deployment's turn admission, on both the native and the managed relay, the order HTTP calls use. A turn a guardrail blocks gets the guardrail's error even past the budget and no longer counts toward the deployment's rpm or runs its pre-call checks. * fix(responses): bill a WebSocket turn's guardrail cost when the deployment refuses it A turn the per-turn guardrail passes and the deployment then refuses now writes its own failed spend row with that turn's guardrail information, on both relays, the way a guardrail-blocked frame and an HTTP refusal already do * fix(responses): bill a refused WebSocket turn on one row only A refused turn whose guardrails recorded a cost is already billed on its own row, so it no longer also becomes the connection's close-time failure row * fix(responses): bill a refused WebSocket turn's guardrail cost on the socket's failure row The turns a deployment refuses after their guardrails passed now add their guardrail records to the socket's close-time failure row instead of a row of their own, so that row's failure hooks, model group and call type cover them * fix(responses): bill a WebSocket's refused turns when the session limit closes it * fix(responses): bill refused WebSocket turns when the session limit cancels the relay * fix(responses): free the WebSocket slot at the session limit without waiting on the failure row * fix(responses): keep the session limit's second cancel so a relay that swallows the first frees the slot * fix(responses): free the WebSocket slot before the close-time failure row and drain pending rows at shutdown * fix(proxy): commit spend before waiting on pending logging tasks at shutdown and again after * fix(responses): start the close-time failure row before the slot release so a late cancel cannot drop it * fix(responses): build the close-time failure row inside its task so a bad guardrail record cannot skip the crash close * fix(proxy): always commit spend again after the shutdown wait for pending logging tasks --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 9 +- litellm/litellm_core_utils/litellm_logging.py | 14 +- litellm/llms/custom_httpx/llm_http_handler.py | 6 +- .../proxy/hooks/proxy_track_cost_callback.py | 46 +- litellm/proxy/proxy_server.py | 18 +- .../proxy/response_api_endpoints/endpoints.py | 112 +++- litellm/responses/main.py | 8 +- litellm/responses/streaming_iterator.py | 126 +++- litellm/router.py | 77 ++- litellm/router_strategy/budget_limiter.py | 3 +- .../io_token_rate_limit_check.py | 19 + .../router_code_coverage.py | 3 + ...test_responses_websocket_turn_admission.py | 629 ++++++++++++++++++ .../spend/test_failed_dispatch_tokens.py | 80 ++- .../test_litellm_logging.py | 50 ++ .../hooks/test_proxy_track_cost_callback.py | 105 ++- .../unit/proxy/proxy_server/test_lifecycle.py | 48 ++ .../response_api_endpoints/test_endpoints.py | 416 +++++++++++- tests/unit/proxy/test_proxy_server.py | 23 + .../test_responses_websocket_all_providers.py | 467 ++++++++++++- .../unit/responses/test_streaming_iterator.py | 25 + .../router_strategy/test_budget_limiter.py | 4 +- .../test_router/test_router_helper_utils.py | 223 +++++++ 23 files changed, 2396 insertions(+), 115 deletions(-) create mode 100644 tests/integration/routing/test_responses_websocket_turn_admission.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 3976612c6e1..4fcd3bde843 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -3113,14 +3113,19 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor): return response_ids[-1] if response_ids else None @staticmethod - def create_logging_object(results: Sequence[Mapping[str, object]]) -> LiteLLMRealtimeStreamLoggingObject: + def create_logging_object( + results: Sequence[Mapping[str, object]], litellm_model_name: str | None = None + ) -> LiteLLMRealtimeStreamLoggingObject: tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(results) - return LiteLLMRealtimeStreamLoggingObject( + logging_object: Final = LiteLLMRealtimeStreamLoggingObject( id=ResponsesWebSocketTokenUsageProcessor.response_id_from_responses_ws_results(results), usage=ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(results), results=cast(OpenAIRealtimeStreamList, list(results)), # cast-ok: raw WS frames, unvalidated field service_tier=next(iter(tier_partition)) if len(tier_partition) == 1 else None, ) + if litellm_model_name is not None: + logging_object.hidden_params = {"litellm_model_name": litellm_model_name} + return logging_object _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed" diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5d9652cc8a2..025741b18b2 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -989,6 +989,16 @@ class Logging(LiteLLMLoggingBaseClass): return model_id return None + def _routed_deployment_model(self) -> str | None: + litellm_params: Final = self._model_call_details_view.get("litellm_params") + metadata: Final = ( + StandardLoggingPayloadSetup.merge_litellm_metadata(_LITELLM_PARAMS.validate_python(litellm_params)) + if isinstance(litellm_params, dict) + else {} + ) + deployment: Final = metadata.get("deployment") + return deployment if isinstance(deployment, str) else None + def get_deployment_model_for_cost(self) -> str | None: """The provider-qualified model to price against. @@ -2256,6 +2266,7 @@ class Logging(LiteLLMLoggingBaseClass): and litellm_params.get(CallTypes.agenerate_content.value, False) is not True and litellm_params.get(CallTypes.agenerate_content_stream.value, False) is not True and litellm_params.get(CallTypes.arealtime.value, False) is not True + and litellm_params.get(CallTypes.aresponses_websocket.value, False) is not True ) _is_sync_litellm_request = is_sync_litellm_request @@ -2511,7 +2522,8 @@ class Logging(LiteLLMLoggingBaseClass): elif self.call_type == CallTypes.aresponses_websocket.value and isinstance(result, list): # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped logging_result = ResponsesWebSocketTokenUsageProcessor.create_logging_object( - results=result # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream + results=result, # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream + litellm_model_name=self._routed_deployment_model(), ) elif self.relays_llm_passthrough and isinstance(result, Response): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index b4dae7ba8b6..b03b8050299 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6911,6 +6911,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str | None = None, first_message: str | None = None, request_defaults: ResponsesWebSocketRequestDefaults | None = None, + turn_admission: Callable[[], Awaitable[None]] | None = None, frame_guard: "ResponsesWsFrameGuard | None" = None, **kwargs: object, ) -> Exception | None: @@ -6973,12 +6974,12 @@ class BaseLLMHTTPHandler: custom_llm_provider=custom_llm_provider, first_message=first_message, quota_callbacks=_ws_quota_callbacks, + turn_admission=turn_admission, guardrail_callbacks=_ws_guardrail_callbacks, # pyright: ignore[reportUnknownArgumentType] # duck-typed callbacks from litellm.callbacks frame_guard=frame_guard, **kwargs, ) - await handler.run() - return None + return await handler.run() import websockets from websockets.asyncio.client import ClientConnection @@ -7069,6 +7070,7 @@ class BaseLLMHTTPHandler: authorized_model=model, custom_llm_provider=custom_llm_provider, request_defaults=request_defaults, + turn_admission=turn_admission, frame_guard=frame_guard, ) return await streaming.bidirectional_forward() diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 6fbd0b8f1b3..3195394b9aa 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -2,8 +2,11 @@ import asyncio import traceback from collections.abc import Callable, Mapping, Sequence from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol, cast +from pydantic import TypeAdapter, ValidationError + import litellm from litellm._logging import verbose_proxy_logger from litellm.batches.batch_utils import batch_cost_is_final @@ -17,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost +from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( get_key_object, @@ -100,6 +104,37 @@ def _proxy_stamped_used_client_oauth_token( return stamped if isinstance(stamped, bool) else None +def _str_keyed_fields(value: object) -> Mapping[str, object]: + return value if is_str_keyed_mapping(value) else {} + + +def _is_model_group(value: object) -> bool: + return isinstance(value, str) and bool(value) + + +_TAG_LIST: Final = TypeAdapter(list[str]) + + +def _is_tag_list(value: object) -> bool: + try: + return bool(_TAG_LIST.validate_python(value, strict=True)) + except ValidationError: + return False + + +_ROUTED_METADATA_FIELD_CHECKS: Final = MappingProxyType({"model_group": _is_model_group, "tags": _is_tag_list}) + + +def _routed_metadata_fallback(request_data: Mapping[str, object]) -> Mapping[str, object]: + caller_metadata: Final = _str_keyed_fields(request_data.get("metadata")) + routed_metadata: Final = _str_keyed_fields(request_data.get("litellm_metadata")) + return { + field: routed_metadata[field] + for field, is_valid in _ROUTED_METADATA_FIELD_CHECKS.items() + if not caller_metadata.get(field) and is_valid(routed_metadata.get(field)) + } + + def _failure_snapshot_with_guardrails_recorded_after_it( failure_snapshot: object, recorded_guardrail_information: object ) -> Mapping[str, object] | None: @@ -220,10 +255,11 @@ class _ProxyDBLogger(CustomLogger): metadata=_metadata, original_exception=original_exception ) - _metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_data, request_route) + request_fields: Final = _str_keyed_fields(request_data) + _metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route) existing_metadata: Final[dict] = request_data.get("metadata", None) or {} - existing_metadata.update(_metadata) + existing_metadata.update({**_routed_metadata_fallback(request_fields), **_metadata}) litellm_metadata_bucket: Final = request_data.get("litellm_metadata") if ( @@ -471,11 +507,7 @@ class _ProxyDBLogger(CustomLogger): await _release_budget_reservation(budget_reservation=budget_reservation) # Non-model call types (health checks, afile_delete) have no model or standard_logging_object. # Use .get() for "stream" to avoid KeyError on health checks. - # WS session wrappers (_aresponses_websocket, _arealtime) also reach here with - # result=None; their per-turn costs are tracked on the inner aresponses/realtime calls. - if sl_object is None and ( - not kwargs.get("model") or kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime") - ): + if sl_object is None and (not kwargs.get("model") or kwargs.get("call_type") == "_arealtime"): verbose_proxy_logger.warning( "Cost tracking - skipping, no standard_logging_object for call_type=%s", kwargs.get("call_type", "unknown"), diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 69e92948a0e..660d7d529cb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -945,6 +945,7 @@ from litellm.proxy.video_endpoints.endpoints import router as video_router from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.credentials_repository import CredentialsRepository from litellm.repositories.prisma_protocols import TableActions +from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.router import ( AssistantsTypedDict, Deployment, @@ -1239,6 +1240,19 @@ async def _flush_spend_logs_queue_on_shutdown() -> None: verbose_proxy_logger.exception("Error flushing spend logs queue on shutdown: %s", e) +async def _flush_spend_on_shutdown() -> None: + await flush_spend_counters_on_shutdown() + await _flush_spend_logs_queue_on_shutdown() + + +async def _flush_spend_around_logging_drain_on_shutdown( + flush_spend: Callable[[], Awaitable[None]] = _flush_spend_on_shutdown, +) -> None: + await flush_spend() + await drain_logging_tasks() + await flush_spend() + + async def _drain_audit_tasks_on_shutdown() -> None: try: from litellm.proxy.management_helpers.audit_logs import drain_audit_tasks @@ -1843,9 +1857,7 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState except Exception as e: verbose_proxy_logger.error("Error stopping in-flight scheduled jobs: %s", e) - await flush_spend_counters_on_shutdown() - - await _flush_spend_logs_queue_on_shutdown() + await _flush_spend_around_logging_drain_on_shutdown() await proxy_config.stop_config_sync_subscriber() diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index f032cd79783..a0fb2b190e0 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -78,7 +78,8 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio request_tags_from_request_data, ) from litellm.proxy.route_llm_request import raise_if_required_body_param_missing -from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard +from litellm.proxy.utils import handle_exception_on_proxy +from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task from litellm.router import Router from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( @@ -1861,12 +1862,14 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec def _responses_ws_failure_frame(failure: Exception) -> str: - raw_status: Final = getattr(failure, "status_code", None) - status: Final = raw_status if isinstance(raw_status, int) and not isinstance(raw_status, bool) else 500 + proxy_exception: Final = handle_exception_on_proxy(failure) + status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500 error_type: Final = ( "rate_limit_exceeded" if status == 429 else "invalid_request_error" if 400 <= status < 500 else "server_error" ) - return json.dumps({"type": "error", "status": status, "error": {"type": error_type, "message": str(failure)}}) + return json.dumps( + {"type": "error", "status": status, "error": {"type": error_type, "message": proxy_exception.message}} + ) async def _enforce_responses_ws_first_frame_model_auth( @@ -1934,7 +1937,10 @@ class _PreCallGuardrailWalk(Protocol): def _responses_ws_frame_guard( - admitted: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth, proxy_logging_obj: _PreCallGuardrailWalk + admitted: Mapping[str, object], + user_api_key_dict: UserAPIKeyAuth, + proxy_logging_obj: _PreCallGuardrailWalk, + record_refused_turn: Callable[[tuple[Mapping[str, object], ...], Exception], None], ) -> ResponsesWsFrameGuard: """Socket admission skips guardrails (the first frame's ``input`` is not part of the admitted request), so every ``response.create`` frame gets the pre_call guardrail walk the HTTP ``/v1/responses`` path runs on its body.""" @@ -1993,10 +1999,12 @@ def _responses_ws_frame_guard( if isinstance(nested, dict) else {**kept, **guarded_fields} ) + guardrail_information: Final = _recorded_guardrail_information(guarded_data.get(metadata_key)) return GuardedResponsesWsFrame( frame=json.dumps(rewritten), - guardrail_information=_recorded_guardrail_information(guarded_data.get(metadata_key)), + guardrail_information=guardrail_information, pii_tokens=_recorded_pii_tokens(guarded_data.get("metadata")), + bill_refusal=partial(record_refused_turn, guardrail_information), ) return guard @@ -2042,6 +2050,33 @@ def _recorded_pii_tokens(metadata: object) -> Mapping[str, str]: _NO_PII_TOKENS: Final[Mapping[str, str]] = MappingProxyType({}) +@dataclass(slots=True) +class _RefusedTurns: + """Turns the deployment refused after their guardrails passed. The socket's close-time failure row bills their + guardrail records, so its failure hooks, model group and call type cover those turns too, and the first refusal + is that row's exception when the session limit cancels the relay before it returns one.""" + + first: Exception | None = None + records: tuple[Mapping[str, object], ...] = () + + def add(self, records: tuple[Mapping[str, object], ...], refusal: Exception) -> None: + self.records = (*self.records, *records) + if self.first is None: + self.first = refusal + + def billed_on(self, request_data: Mapping[str, object]) -> Mapping[str, object]: + if not self.records: + return request_data + metadata_key: Final = get_metadata_variable_name_from_kwargs(request_data) + bucket: Final = request_data.get(metadata_key) + fields: Final = _WS_FRAME.validate_python(bucket) if isinstance(bucket, dict) else {} + recorded: Final = (*_recorded_guardrail_information(fields), *self.records) + return { + **request_data, + metadata_key: {**fields, _GUARDRAIL_INFORMATION_KEY: [dict(record) for record in recorded]}, + } + + def _recorded_guardrail_information(bucket: object) -> tuple[Mapping[str, object], ...]: recorded: Final = ( _WS_FRAME.validate_python(bucket).get(_GUARDRAIL_INFORMATION_KEY) if isinstance(bucket, dict) else None @@ -2145,39 +2180,58 @@ async def _responses_websocket_session( await websocket.close(code=1008, reason="Pre-call error") return + refused_turns: Final = _RefusedTurns() routed_data: Final = dict( data, user_api_key_dict=user_api_key_dict, - responses_ws_frame_guard=_responses_ws_frame_guard(data, user_api_key_dict, proxy_logging_obj), + responses_ws_frame_guard=_responses_ws_frame_guard( + data, user_api_key_dict, proxy_logging_obj, refused_turns.add + ), **_routing_hints_from_first_ws_frame(first_message), ) - # Phase 2: route to upstream provider - try: - llm_call: Final = await route_request( - data=routed_data, - route_type="_aresponses_websocket", - llm_router=llm_router, - user_model=user_model, - ) - failure: Final = await llm_call - if isinstance(failure, Exception): - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, - original_exception=failure, - request_data=routed_data, - ) - except Exception as e: - verbose_proxy_logger.exception("Responses WebSocket error") - with contextlib.suppress(Exception): - await websocket.send_text(_responses_ws_failure_frame(e)) + + async def write_failure_row(exception: Exception) -> None: await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, - original_exception=e, - request_data=routed_data, + original_exception=exception, + request_data=dict(refused_turns.billed_on(routed_data)), ) - await websocket.close(code=1011, reason="Internal server error") + + def bill_socket_failure(exception: Exception) -> asyncio.Task[object]: + return spawn_logging_task( + asyncio.get_running_loop(), write_failure_row(exception), task_name="Responses WebSocket failure row" + ) + + async def relay_and_bill_failure() -> asyncio.Task[object] | None: + try: + llm_call: Final = await route_request( + data=routed_data, + route_type="_aresponses_websocket", + llm_router=llm_router, + user_model=user_model, + ) + outcome: Final = await llm_call + except asyncio.CancelledError: + if refused_turns.first is not None: + _ = bill_socket_failure(refused_turns.first) + raise + except Exception as e: + crash_row: Final = bill_socket_failure(e) + with contextlib.suppress(Exception): + await websocket.send_text(_responses_ws_failure_frame(e)) + with contextlib.suppress(Exception): + await websocket.close(code=1011, reason="Internal server error") + return crash_row + return bill_socket_failure(outcome) if isinstance(outcome, Exception) else None + + # Phase 2: route to upstream provider + try: + failure_row: Final = await relay_and_bill_failure() finally: await proxy_logging_obj.arelease_responses_websocket_slot(user_api_key_dict) + if failure_row is not None: + with contextlib.suppress(Exception): + await asyncio.shield(failure_row) @router.websocket("/v1/responses") diff --git a/litellm/responses/main.py b/litellm/responses/main.py index e775e155dea..727a5ccfedf 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars import json -from collections.abc import Coroutine, Generator, Iterable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Coroutine, Generator, Iterable, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass from functools import partial @@ -73,7 +73,7 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency from litellm.types.responses.main import * from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, all_litellm_params +from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params from litellm.utils import ( ProviderConfigManager, client, @@ -2609,6 +2609,7 @@ async def _aresponses_websocket( api_base: str | None = None, api_key: str | None = None, timeout: float | None = None, + turn_admission: Callable[[], Awaitable[None]] | None = None, responses_ws_frame_guard: "ResponsesWsFrameGuard | None" = None, **kwargs, ) -> Exception | None: @@ -2624,7 +2625,7 @@ async def _aresponses_websocket( litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") user: Final = kwargs.get("user", None) litellm_params: Final = GenericLiteLLMParams(**kwargs) - litellm_params_dict: Final = get_litellm_params(**kwargs) + litellm_params_dict: Final = {**get_litellm_params(**kwargs), CallTypes.aresponses_websocket.value: True} ( provider_model, @@ -2702,6 +2703,7 @@ async def _aresponses_websocket( litellm_metadata=_build_litellm_metadata_for_ws(kwargs), custom_llm_provider=_custom_llm_provider, request_defaults=_build_responses_websocket_request_defaults(deployment_kwargs), + turn_admission=turn_admission, frame_guard=responses_ws_frame_guard, **remaining_kwargs, ) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 3d07f22c81a..ffde8c66c20 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import contextlib import copy import json import time @@ -198,7 +199,7 @@ def _running_loop() -> asyncio.AbstractEventLoop | None: return None -def _spawn_logging_task( +def spawn_logging_task( running_loop: asyncio.AbstractEventLoop, coroutine: Coroutine[object, object, object], *, task_name: str ) -> asyncio.Task[object]: task: Final = running_loop.create_task(coroutine) @@ -208,6 +209,14 @@ def _spawn_logging_task( return task +async def drain_logging_tasks(timeout: float = 5.0) -> None: + if not _PENDING_LOGGING_TASKS: + return + _, pending = await asyncio.wait(tuple(_PENDING_LOGGING_TASKS), timeout=timeout) + if pending: + verbose_logger.warning("Timed out draining %d logging tasks on shutdown, dropping them", len(pending)) + + _ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType( { "server_error": 500, @@ -930,7 +939,7 @@ class BaseResponsesAPIStreamingIterator: running_loop: Final = _running_loop() if running_loop is not None: self._record_pending_logging_task( - _spawn_logging_task( + spawn_logging_task( running_loop, async_post_call_success_deployment_hook( request_data=request_payload, @@ -966,7 +975,7 @@ class BaseResponsesAPIStreamingIterator: running_loop: Final = _running_loop() if running_loop is not None: self._record_pending_logging_task( - _spawn_logging_task( + spawn_logging_task( running_loop, self._run_failure_handlers_in_order(exception, traceback_exception, end_time), task_name="Responses stream failure logging", @@ -1947,6 +1956,38 @@ async def _enforce_frame_project_quota( ) +async def _admit_frame_turn(turn_admission: Callable[[], Awaitable[None]] | None, raw_message: str) -> Exception | None: + frame: Final = _json_object_or_none(raw_message) + if turn_admission is None or frame is None or frame.get("type") != "response.create": + return None + try: + await turn_admission() + except RateLimitError as refusal: + return refusal + except Exception as e: + verbose_logger.exception("Responses WS turn admission failed: %s", e) + return e + return None + + +def _turn_refusal_frame(refusal: RateLimitError) -> str: + return json.dumps( + {"type": "error", "status": 429, "error": {"type": "rate_limit_exceeded", "message": str(refusal)}} + ) + + +def _turn_admission_error_frame(failure: Exception) -> str: + return json.dumps({"type": "error", "status": 500, "error": {"type": "server_error", "message": str(failure)}}) + + +def _turn_rejection_frame(rejection: Exception) -> str: + match rejection: + case RateLimitError(): + return _turn_refusal_frame(rejection) + case _: + return _turn_admission_error_frame(rejection) + + RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [ "response.created", "response.completed", @@ -2019,16 +2060,30 @@ def _unmask_responses_event(response_str: str, pii_tokens: Mapping[str, str], un @dataclass(frozen=True, slots=True) class GuardedResponsesWsFrame: """A ``response.create`` frame after the proxy's pre_call guardrails ran on it: the frame to forward (guardrails - may rewrite ``input`` and ``instructions``), the guardrail records that bill against this turn only, and the - Presidio ``pii_tokens`` the walk masked with, which the handler needs to unmask the provider's reply.""" + may rewrite ``input`` and ``instructions``), the guardrail records that bill against this turn only, the + Presidio ``pii_tokens`` the walk masked with, which the handler needs to unmask the provider's reply, and how + to carry those guardrail records onto the connection's failure row when the deployment then refuses the turn.""" frame: str guardrail_information: tuple[Mapping[str, object], ...] = () pii_tokens: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) + bill_refusal: Callable[[Exception], None] | None = None ResponsesWsFrameGuard: TypeAlias = Callable[[str], Awaitable[GuardedResponsesWsFrame]] + +async def _reject_guarded_turn( + websocket: ResponsesClientWebSocket, guarded: GuardedResponsesWsFrame, rejection: Exception +) -> None: + """Answers the refused turn and leaves its guardrail records to the connection's close-time failure row, which + runs every failure hook once per socket, like the first refusal it records.""" + if guarded.bill_refusal is not None: + guarded.bill_refusal(rejection) + with contextlib.suppress(Exception): + await websocket.send_text(_turn_rejection_frame(rejection)) + + _RESPONSES_WS_TERMINAL_EVENT_TYPES: Final = _HISTORY_TERMINAL_EVENT_TYPES | _RESPONSES_WS_FAILURE_EVENT_TYPES @@ -2212,6 +2267,7 @@ class ResponsesWebSocketStreaming: authorized_model: str | None = None, custom_llm_provider: str | None = None, request_defaults: ResponsesWebSocketRequestDefaults | None = None, + turn_admission: Callable[[], Awaitable[None]] | None = None, frame_guard: ResponsesWsFrameGuard | None = None, ): self.websocket = websocket @@ -2233,6 +2289,7 @@ class ResponsesWebSocketStreaming: # response.create frame to prevent deployment-substitution attacks. self.authorized_model: str | None = authorized_model self.request_defaults: ResponsesWebSocketRequestDefaults | None = request_defaults + self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission def _start_turn( self, client_frame: str | None, guardrail_information: tuple[Mapping[str, object], ...] = () @@ -2284,9 +2341,11 @@ class ResponsesWebSocketStreaming: await self._send_error_event("invalid_request_error", str(e)) return None - async def _forward_guarded_frame(self, message: str) -> None: + async def _forward_guarded_frame( + self, message: str, turn_admission: Callable[[], Awaitable[None]] | None = None + ) -> None: guarded: Final = await self._guard_frame(message) - if guarded is None: + if guarded is None or not await self._admit_or_reject_turn(guarded, message, turn_admission): return if guarded.pii_tokens: self._remember_pii_tokens(guarded.pii_tokens) @@ -2680,10 +2739,27 @@ class ResponsesWebSocketStreaming: self.quota_callbacks, self.user_api_key_dict, self.authorized_model, message ) except RateLimitError as e: - await self._send_error_event("rate_limit_exceeded", str(e)) + await self._send_turn_refusal(e) return False return True + async def _admit_or_reject_turn( + self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: Callable[[], Awaitable[None]] | None + ) -> bool: + rejection: Final = await _admit_frame_turn(turn_admission, message) + if rejection is None: + return True + if self._failure is None: + self._failure = rejection + await _reject_guarded_turn(self.websocket, guarded, rejection) + return False + + async def _send_turn_refusal(self, refusal: RateLimitError) -> None: + try: + await self.websocket.send_text(_turn_refusal_frame(refusal)) + except Exception: # noqa: BLE001, S110 # client may already be gone + pass + async def _send_error_event(self, error_type: str, message: str) -> None: try: await self.websocket.send_text( @@ -2702,7 +2778,7 @@ class ResponsesWebSocketStreaming: message = await self.websocket.receive_text() if not await self._enforce_or_reject_frame(message): continue - await self._forward_guarded_frame(message) + await self._forward_guarded_frame(message, self.turn_admission) except Exception as e: verbose_logger.debug("Responses WS client_to_backend ended: %s", e) @@ -2779,6 +2855,7 @@ class ManagedResponsesWebSocketHandler: custom_llm_provider: str | None = None, first_message: str | None = None, quota_callbacks: Sequence[ProjectQuotaCallback] | None = None, + turn_admission: Callable[[], Awaitable[None]] | None = None, frame_guard: ResponsesWsFrameGuard | None = None, guardrail_callbacks: Sequence[PresidioGuardrailCallback] | None = None, **kwargs: object, @@ -2802,6 +2879,8 @@ class ManagedResponsesWebSocketHandler: self._connection_provider = self._resolve_provider(model) or custom_llm_provider self.first_message = first_message self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else () + self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission + self._failure: Exception | None = None # Carry through safe pass-through kwargs (e.g. extra_headers) self.extra_kwargs: dict[str, object] = {k: v for k, v in kwargs.items() if k not in _MANAGED_WS_SKIP_KWARGS} # In-memory session history: response_id → full accumulated message list. @@ -2830,12 +2909,11 @@ class ManagedResponsesWebSocketHandler: return None async def _send_error(self, message: str, error_type: str = "server_error") -> None: - try: - await self.websocket.send_text( - json.dumps({"type": "error", "error": {"type": error_type, "message": message}}) - ) - except Exception: - pass + await self._send_frame(json.dumps({"type": "error", "error": {"type": error_type, "message": message}})) + + async def _send_frame(self, frame: str) -> None: + with contextlib.suppress(Exception): + await self.websocket.send_text(frame) def _get_history_messages(self, previous_response_id: str) -> list[dict[str, object]]: """ @@ -3197,7 +3275,9 @@ class ManagedResponsesWebSocketHandler: # Core request handler # ------------------------------------------------------------------ - async def _process_response_create(self, raw_message: str) -> None: + async def _process_response_create( + self, raw_message: str, turn_admission: Callable[[], Awaitable[None]] | None = None + ) -> None: """ Parse one ``response.create`` event, call ``litellm.aresponses(stream=True)``, and forward every streaming event to the client. @@ -3238,12 +3318,17 @@ class ManagedResponsesWebSocketHandler: self.quota_callbacks, self.user_api_key_dict, self.model_group or self.model, raw_message ) except RateLimitError as e: - await self._send_error(str(e), error_type="rate_limit_exceeded") + await self._send_frame(_turn_refusal_frame(e)) return - guarded: Final = await self._guard_frame(raw_message) if guarded is None: return + rejection: Final = await _admit_frame_turn(turn_admission, raw_message) + if rejection is not None: + if self._failure is None: + self._failure = rejection + await _reject_guarded_turn(self.websocket, guarded, rejection) + return if guarded.pii_tokens: self._pii_tokens = {**self._pii_tokens, **guarded.pii_tokens} @@ -3287,7 +3372,7 @@ class ManagedResponsesWebSocketHandler: # Main entry point # ------------------------------------------------------------------ - async def run(self) -> None: + async def run(self) -> Exception | None: """ Main loop: accept ``response.create`` events sequentially and handle each one before waiting for the next message. @@ -3303,8 +3388,9 @@ class ManagedResponsesWebSocketHandler: verbose_logger.debug("ManagedResponsesWS: client disconnected: %s", exc) break - await self._process_response_create(message) + await self._process_response_create(message, self.turn_admission) except Exception as exc: verbose_logger.exception("ManagedResponsesWS: unexpected error: %s", exc) await self._send_error(f"Internal server error: {exc}") + return self._failure diff --git a/litellm/router.py b/litellm/router.py index 65ba67d9ab8..6e9ffc5341e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -289,6 +289,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( deployment_has_io_token_limits, refund_stale_reservation_before_retry, set_io_token_rate_limit_request_kwargs, + without_io_token_limits, ) from litellm.router_utils.pre_call_checks.model_rate_limit_check import ( ModelRateLimitingCheck, @@ -5965,6 +5966,7 @@ class Router: ) custom_llm_provider: Final = provider_for_generic_call(data) + generic_function_name: Final[str] = getattr(original_generic_function, "__name__", "") response_kwargs: Final = { **data, @@ -5972,6 +5974,11 @@ class Router: **kwargs, "model": model_name, **_with_router_resolved_session_model(kwargs.get("session"), model_name), + **( + {"turn_admission": partial(self.async_admit_deployment_turn, model, deployment, kwargs)} + if generic_function_name == "_aresponses_websocket" + else {} + ), } # Only set custom_llm_provider if it's not None if custom_llm_provider is not None: @@ -5981,7 +5988,7 @@ class Router: self, deployment, response_kwargs, - surface_for_call(getattr(original_generic_function, "__name__", "")), + surface_for_call(generic_function_name), ) async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): response = await original_generic_function(**compacted_input) @@ -8936,8 +8943,7 @@ class Router: Track remaining tpm/rpm quota for model in model_list """ try: - # WS session wrappers fire with result=None; per-turn costs tracked by inner calls. - if kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime"): + if kwargs.get("call_type") == "_arealtime": return if is_batch_retrieve_call_type(kwargs.get("call_type")): return @@ -9605,6 +9611,71 @@ class Router: ) raise e + def _deployment_as_configured_now(self, deployment: Mapping[str, object]) -> Mapping[str, object]: + model_info: Final = deployment.get("model_info") + model_id: Final = ( + _MODEL_INFO_ADAPTER.validate_python(model_info).get("id") if isinstance(model_info, dict) else None + ) + current: Final = self.get_deployment(model_id) if isinstance(model_id, str) else None + return deployment if current is None else current.model_dump(exclude_none=True) + + async def async_admit_deployment_turn( + self, model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object] + ) -> None: + deployment: Final = self._deployment_as_configured_now(opened_on) + budget_limiter: Final = self.get_router_deployment_budget_limiter() + if budget_limiter is not None: + try: + await budget_limiter.async_filter_deployments( # pyright: ignore[reportUnknownMemberType] # legacy signature takes untyped deployment dicts + model=model_group, + healthy_deployments=[dict(deployment)], + messages=None, + request_kwargs=dict(request_kwargs), + ) + except ValueError as e: + raise litellm.RateLimitError( + message=str(e), + llm_provider="", + model=model_group, + category=litellm.RateLimitErrorCategory.LITELLM_RATE_LIMIT, + ) from e + await self._admit_turn_through_routing_strategy(model_group, deployment, request_kwargs) + await self.async_routing_strategy_pre_call_checks( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped deployment dict + deployment=dict(without_io_token_limits(deployment)), parent_otel_span=None + ) + + async def _admit_turn_through_routing_strategy( + self, model_group: str, deployment: Mapping[str, object], request_kwargs: Mapping[str, object] + ) -> None: + routing_kwargs: Final = {key: value for key, value in request_kwargs.items() if key != "litellm_logging_obj"} + strategy, selector = self._get_routing_context( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped request dict + model=model_group, request_kwargs=routing_kwargs + ) + if strategy == "simple-shuffle" or selector is None: + return + selected: Final = await self._select_deployment_async( # pyright: ignore[reportUnknownMemberType] # legacy signature takes untyped deployment dicts + strategy=strategy, + selector=selector, + model=model_group, + healthy_deployments=[dict(deployment)], + messages=None, + input=None, + request_kwargs=routing_kwargs, + ) + if selected is None: + no_deployment: Final = await async_raise_no_deployment_exception( + litellm_router_instance=self, model=model_group, parent_otel_span=None + ) + raise litellm.RateLimitError( + message=str(no_deployment), + llm_provider="", + model=model_group, + category=litellm.RateLimitErrorCategory.LITELLM_RATE_LIMIT, + ) from no_deployment + await self._async_override_selector_pre_call_check( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped deployment dict + strategy, selector, dict(deployment), None + ) + @contextlib.asynccontextmanager async def _deployment_slot( self, deployment: dict, kwargs: Mapping[str, object], parent_otel_span: Span | None diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 1622a30b95f..7573bb6716f 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -505,8 +505,7 @@ class RouterBudgetLimiting(CustomLogger): async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): """Original method now uses helper functions""" verbose_router_logger.debug("in RouterBudgetLimiting.async_log_success_event") - # WS session wrappers fire with result=None; per-turn costs tracked by inner calls. - if kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime"): + if kwargs.get("call_type") == "_arealtime": return standard_logging_payload: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None) if standard_logging_payload is None: diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 0d701411c94..086f5517c29 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -23,6 +23,7 @@ from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache from litellm.litellm_core_utils.token_counter import offload_token_count +from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET from litellm.types.router import RouterCacheEnum, RouterErrors from litellm.utils import get_utc_datetime @@ -96,6 +97,24 @@ def deployment_has_io_token_limits(deployment: dict) -> bool: return itpm is not None or otpm is not None +_IO_TOKEN_LIMIT_KEYS: Final = frozenset({"itpm", "otpm"}) +_IO_TOKEN_LIMIT_SECTIONS: Final = frozenset({"litellm_params", "model_info"}) + + +def without_io_token_limits(deployment: Mapping[str, object]) -> Mapping[str, object]: + return { + key: _without_io_token_limit_keys(value) if key in _IO_TOKEN_LIMIT_SECTIONS else value + for key, value in deployment.items() + if key not in _IO_TOKEN_LIMIT_KEYS + } + + +def _without_io_token_limit_keys(section: object) -> object: + if not is_str_keyed_mapping(section): + return section + return {key: value for key, value in section.items() if key not in _IO_TOKEN_LIMIT_KEYS} + + def _get_cache_keys(deployment: dict, current_minute: str) -> tuple[str, str] | None: model_id: Final = deployment.get("model_info", {}).get("id") deployment_name: Final = deployment.get("litellm_params", {}).get("model") diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index e669107900d..681476456c2 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -134,6 +134,9 @@ ignored_function_names = [ "_provider_model_id", # Tested through get_router_model_info in test_router_model_cost_isolation.py "_names_route_model", # Tested through get_router_model_info in test_router_model_cost_isolation.py "_declared_base_model", # Tested through get_router_model_info in test_router_model_cost_isolation.py + "async_admit_deployment_turn", # Bound as the WebSocket turn_admission partial, tested through it in test_router_helper_utils.py + "_deployment_as_configured_now", # Read by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py + "_admit_turn_through_routing_strategy", # Run by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py ] diff --git a/tests/integration/routing/test_responses_websocket_turn_admission.py b/tests/integration/routing/test_responses_websocket_turn_admission.py new file mode 100644 index 00000000000..a246c89e7b3 --- /dev/null +++ b/tests/integration/routing/test_responses_websocket_turn_admission.py @@ -0,0 +1,629 @@ +import asyncio +import hashlib +import os +import threading +import uuid +from collections.abc import AsyncIterator, Callable, Iterator, Mapping +from contextlib import asynccontextmanager, contextmanager +from dataclasses import dataclass +from functools import partial +from datetime import UTC, datetime +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import websockets +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + gateway_from_environment, + held, + list_value, + object_value, + string_value, +) +from integration._support.database import read_rows +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.responses_ws_peer import ( + PROVIDER_MODEL, + USAGE, + Peer, + Script, + completed, + create, + drain, + events, + responses_peer, + turn, + until_terminal, +) +from integration._support.tls import write_self_signed_cert +from pydantic import JsonValue +from redis import Redis + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 240) + +PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" +INPUT_RATE: Final = 0.001 +OUTPUT_RATE: Final = 0.002 +TURN_SPEND: Final = USAGE["input_tokens"] * INPUT_RATE + USAGE["output_tokens"] * OUTPUT_RATE +TURN_TOKENS: Final = USAGE["total_tokens"] +MANAGED_PROVIDER: Final = "mistral" +SPEND_ROWS_FOR_KEY: Final = ( + 'SELECT request_id, spend, status, model_group, metadata, request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s' +) +TURN_CHECK_FAULT: Final = "turn_check_fault" +FAULTING_TURN_CHECK: Final = f""" +from litellm.integrations.custom_logger import CustomLogger + + +class FaultingTurnCheck(CustomLogger): + async def async_pre_call_check(self, deployment, parent_otel_span): + fault = (deployment.get("model_info") or {{}}).get("{TURN_CHECK_FAULT}") + if fault: + raise RuntimeError(fault) + return None + + +instance = FaultingTurnCheck() +""" +FrameReader = Callable[[], int] + + +@dataclass(frozen=True, slots=True) +class Deployment: + name: str + identity: str + forwarded_turns: FrameReader + + +def _key_hash(key: str) -> str: + return hashlib.sha256(key.encode()).hexdigest() + + +def _redis() -> Redis: + return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), decode_responses=True) + + +def _config(directory: Path, router_settings: Mapping[str, JsonValue], callbacks: tuple[str, ...]) -> Path: + base: Final = JSON_OBJECT.validate_python(yaml.safe_load(PROXY_CONFIG.read_text())) + path: Final = directory / "turn-admission.yaml" + path.write_text( + yaml.safe_dump( + { + **base, + "litellm_settings": { + **object_value(base["litellm_settings"]), + **({"callbacks": list(callbacks)} if callbacks else {}), + }, + "router_settings": { + **object_value(base["router_settings"]), + "redis_host": os.environ["REDIS_HOST"], + "redis_port": int(os.environ["REDIS_PORT"]), + **router_settings, + }, + } + ) + ) + return path + + +@contextmanager +def _admitting_proxy( + tmp_path_factory: pytest.TempPathFactory, + cert: tuple[Path, Path], + router_settings: Mapping[str, JsonValue], + callback_sources: Mapping[str, str] = MappingProxyType({}), +) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("responses-ws-turn-admission") + for module, source in callback_sources.items(): + (directory / f"{module}.py").write_text(source) + config: Final = _config(directory, router_settings, tuple(f"{module}.instance" for module in callback_sources)) + with gateway_from_environment() as base: + with owned_proxy(base, directory, {"SSL_CERT_FILE": str(cert[0])}, config=config) as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def cert(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path]: + return write_self_signed_cert(tmp_path_factory.mktemp("responses-ws-turn-admission-cert")) + + +@pytest.fixture(scope="module") +def limited(tmp_path_factory: pytest.TempPathFactory, cert: tuple[Path, Path]) -> Iterator[Gateway]: + with _admitting_proxy(tmp_path_factory, cert, {"optional_pre_call_checks": ["enforce_model_rate_limits"]}) as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def faulting(tmp_path_factory: pytest.TempPathFactory, cert: tuple[Path, Path]) -> Iterator[Gateway]: + with _admitting_proxy(tmp_path_factory, cert, {}, {"faulting_turn_check": FAULTING_TURN_CHECK}) as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def usage_routed(tmp_path_factory: pytest.TempPathFactory, cert: tuple[Path, Path]) -> Iterator[Gateway]: + with _admitting_proxy(tmp_path_factory, cert, {"routing_strategy": "usage-based-routing-v2"}) as proxy: + yield proxy + + +def _native(scenario: Scenario, peer: Peer, **limits: JsonValue) -> Deployment: + identity: Final = f"ws-admission-{uuid.uuid4().hex}" + name: Final = scenario.model( + model_info={"id": identity}, + model=f"openai/{PROVIDER_MODEL}", + api_base=peer.url, + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + **limits, + ) + return Deployment( + name, identity, lambda: sum(frame.get("type") == "response.create" for frame in drain(peer.frames)) + ) + + +def _upstream_chats(gateway: Gateway, provider_model: str) -> int: + with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as upstream: + observed: Final = JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"] + assert isinstance(observed, list) + return sum(object_value(object_value(entry)["body"]).get("model") == provider_model for entry in observed) + + +def _managed(scenario: Scenario, **limits: JsonValue) -> Deployment: + identity: Final = f"ws-admission-{uuid.uuid4().hex}" + provider_model: Final = f"ws-managed-{uuid.uuid4().hex}" + name: Final = scenario.model( + model_info={"id": identity}, + model=f"{MANAGED_PROVIDER}/{provider_model}", + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + **limits, + ) + return Deployment(name, identity, lambda: _upstream_chats(scenario.gateway, provider_model)) + + +def _socket_url(gateway: Gateway, model: str) -> str: + return f"{str(gateway.client.base_url).rstrip('/').replace('http://', 'ws://')}/v1/responses?model={model}" + + +@asynccontextmanager +async def _socket( + gateway: Gateway, key: str, model: str, headers: Mapping[str, str] | None = None +) -> AsyncIterator[websockets.ClientConnection]: + async with websockets.connect( + _socket_url(gateway, model), + additional_headers={"Authorization": f"Bearer {key}", **(headers or {})}, + open_timeout=10, + ) as connection: + yield connection + + +async def _say(connection: websockets.ClientConnection, model: str, text: str = "") -> tuple[dict[str, JsonValue], ...]: + return await turn(connection, create(model, text or f"turn {uuid.uuid4().hex}")) + + +def _assert_refused(received: tuple[dict[str, JsonValue], ...]) -> None: + assert [event.get("type") for event in received] == ["error"], received + assert received[0]["status"] == 429, received + assert object_value(received[0]["error"])["type"] == "rate_limit_exceeded", received + + +def _fresh_minute() -> str: + eventually(lambda: datetime.now(UTC), lambda now: now.second < 30, seconds=65) + return datetime.now(UTC).strftime("%H-%M") + + +def _deployment_spend(deployment: Deployment) -> float: + with _redis() as redis: + return float(redis.get(f"deployment_spend:{deployment.identity}:1d") or 0) + + +def _configured(gateway: Gateway, deployment: Deployment, section: str) -> dict[str, JsonValue]: + entries: Final = list_value(gateway.get("/model/info", {"litellm_model_id": deployment.identity})["data"]) + return object_value(object_value(entries[0])[section]) + + +def _configured_tpm(gateway: Gateway, deployment: Deployment) -> JsonValue: + return _configured(gateway, deployment, "litellm_params").get("tpm") + + +def _tokens_counted(identity: str, minute: str) -> int: + with _redis() as redis: + return sum(int(redis.get(key) or 0) for key in redis.scan_iter(f"{identity}:*:tpm:{minute}")) + + +def _settled_count(read: Callable[[], int]) -> int: + reached: Final = eventually(read, lambda tokens: tokens > 0, seconds=15) + return held(read, lambda tokens: tokens == reached, holding=3, seconds=20) + + +async def _turns_against_tpm( + gateway: Gateway, key: str, deployment: Deployment +) -> tuple[tuple[dict[str, JsonValue], ...], tuple[dict[str, JsonValue], ...], tuple[dict[str, JsonValue], ...], int]: + minute: Final = await asyncio.to_thread(_fresh_minute) + tokens: Final = partial(_tokens_counted, deployment.identity, minute) + async with _socket(gateway, key, deployment.name) as connection: + first: Final = await _say(connection, deployment.name) + counted_after_first: Final = await asyncio.to_thread(_settled_count, tokens) + second: Final = await _say(connection, deployment.name) + if second[-1].get("type") == "response.completed": + await _until(tokens, lambda counted: counted > counted_after_first) + return first, second, await _say(connection, deployment.name), counted_after_first + + +async def _until(read: Callable[[], float], satisfied: Callable[[float], bool]) -> float: + return await asyncio.to_thread(eventually, read, satisfied, 15) + + +def _failure_rows(key: str) -> list[dict[str, JsonValue]]: + rows: Final = eventually( + lambda: read_rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)), + lambda found: any(row["status"] == "failure" for row in found), + seconds=70, + ) + return [row for row in rows if row["status"] == "failure"] + + +@pytest.mark.parametrize("relay", ("native", "managed")) +def test_a_turn_past_the_deployment_budget_is_refused_on_the_open_socket_and_logged_once_as_its_failure( + limited: Gateway, cert: tuple[Path, Path], relay: str +) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + budget: Final = {"max_budget": TURN_SPEND / 2, "budget_duration": "1d"} + deployment: Final = _native(scenario, peer, **budget) if relay == "native" else _managed(scenario, **budget) + key: Final = scenario.key(models=[deployment.name]) + tag: Final = f"ws-turn-{uuid.uuid4().hex}" + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(limited, key, deployment.name, {"x-litellm-tags": tag}) as connection: + first: Final = await _say(connection, deployment.name) + await _until(lambda: _deployment_spend(deployment), lambda spent: spent > TURN_SPEND / 2) + return (first, await _say(connection, deployment.name), await _say(connection, deployment.name)) + + first, second, third = asyncio.run(session()) + + completed(first) + _assert_refused(second) + _assert_refused(third) + assert deployment.forwarded_turns() == 1 + failures: Final = _failure_rows(key) + assert len(failures) == 1, failures + assert failures[0]["model_group"] == deployment.name, failures + assert tag in list_value(failures[0]["request_tags"]), failures + assert float(failures[0]["spend"]) == 0.0, failures + error: Final = object_value(object_value(failures[0]["metadata"])["error_information"]) + assert str(error["error_code"]) == "429", error + + +def _set_turn_check_fault(gateway: Gateway, deployment: Deployment, fault: str) -> None: + gateway.request( + "PATCH", + f"/model/{deployment.identity}/update", + {"model_info": {"id": deployment.identity, TURN_CHECK_FAULT: fault}}, + ).raise_for_status() + eventually( + lambda: _configured(gateway, deployment, "model_info").get(TURN_CHECK_FAULT) or "", + lambda configured: configured == fault, + seconds=15, + ) + + +def _assert_turn_check_failed(received: tuple[dict[str, JsonValue], ...], fault: str) -> None: + assert [event.get("type") for event in received] == ["error"], received + assert received[0]["status"] == 500, received + error: Final = object_value(received[0]["error"]) + assert error["type"] == "server_error", received + assert fault in string_value(error["message"]), received + + +def _first_turn_held_until(release: threading.Event) -> Script: + def script(response_id: str) -> tuple[dict[str, JsonValue], ...]: + if response_id == "resp_peer_1": + release.wait(timeout=60) + return events(f"resp_{uuid.uuid4().hex}", "seven") + + return script + + +@pytest.mark.parametrize("relay", ("native", "managed")) +def test_a_turn_whose_admission_check_fails_gets_a_500_frame_and_the_socket_serves_the_next_turn( + faulting: Gateway, cert: tuple[Path, Path], relay: str +) -> None: + with responses_peer(cert) as peer, faulting.scenario() as scenario: + deployment: Final = _native(scenario, peer) if relay == "native" else _managed(scenario) + key: Final = scenario.key(models=[deployment.name]) + fault: Final = f"turn check unavailable {uuid.uuid4().hex}" + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(faulting, key, deployment.name) as connection: + first: Final = await _say(connection, deployment.name) + await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, fault) + second: Final = await _say(connection, deployment.name) + third: Final = await _say(connection, deployment.name) + await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, "") + return first, second, third, await _say(connection, deployment.name) + + first, second, third, fourth = asyncio.run(session()) + + completed(first) + _assert_turn_check_failed(second, fault) + _assert_turn_check_failed(third, fault) + completed(fourth) + assert deployment.forwarded_turns() == 2 + failures: Final = _failure_rows(key) + assert len(failures) == 1, failures + assert failures[0]["model_group"] == deployment.name, failures + recorded: Final = object_value(object_value(failures[0]["metadata"])["error_information"]) + assert fault in string_value(recorded["error_message"]), recorded + + +def test_a_turn_still_streaming_when_the_next_turns_check_fails_completes_and_is_billed( + faulting: Gateway, cert: tuple[Path, Path] +) -> None: + release: Final = threading.Event() + with responses_peer(cert, _first_turn_held_until(release)) as peer, faulting.scenario() as scenario: + deployment: Final = _native(scenario, peer) + key: Final = scenario.key(models=[deployment.name]) + fault: Final = f"turn check unavailable {uuid.uuid4().hex}" + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(faulting, key, deployment.name) as connection: + try: + await connection.send(create(deployment.name, f"streaming {uuid.uuid4().hex}")) + await asyncio.to_thread(_set_turn_check_fault, faulting, deployment, fault) + rejected: Final = await _say(connection, deployment.name) + finally: + release.set() + return rejected, await until_terminal(connection, ()) + + rejected, streamed = asyncio.run(session()) + + _assert_turn_check_failed(rejected, fault) + billed: Final = completed(streamed) + assert deployment.forwarded_turns() == 1 + rows: Final = eventually( + lambda: read_rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)), + lambda found: any(row["request_id"] == billed["id"] for row in found), + seconds=70, + ) + row: Final = next(row for row in rows if row["request_id"] == billed["id"]) + assert row["status"] == "success", row + assert float(row["spend"]) == pytest.approx(TURN_SPEND), row + + +def test_a_socket_opened_after_websocket_turns_spent_the_budget_is_refused_and_so_is_http( + limited: Gateway, cert: tuple[Path, Path] +) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + deployment: Final = _native(scenario, peer, max_budget=TURN_SPEND / 2, budget_duration="1d") + key: Final = scenario.key(models=[deployment.name]) + + async def spend_then_reopen() -> tuple[tuple[dict[str, JsonValue], ...], int | None]: + async with _socket(limited, key, deployment.name) as spending: + completed(await _say(spending, deployment.name)) + await _until(lambda: _deployment_spend(deployment), lambda spent: spent > TURN_SPEND / 2) + async with _socket(limited, key, deployment.name) as refused: + received: Final = await _say(refused, deployment.name) + await asyncio.wait_for(refused.wait_closed(), timeout=20) + return received, refused.close_code + + received, close_code = asyncio.run(spend_then_reopen()) + + _assert_refused(received) + assert close_code == 1011, close_code + assert peer.paths.qsize() == 1 + assert deployment.forwarded_turns() == 1 + http: Final = limited.request( + "POST", "/v1/responses", {"model": deployment.name, "input": f"after {uuid.uuid4().hex}"}, key=key + ) + assert http.status_code == 429, http.text + assert peer.paths.qsize() == 1 + + +def test_a_socket_the_router_refuses_to_open_gets_the_status_and_message_http_returns( + limited: Gateway, cert: tuple[Path, Path] +) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + name: Final = scenario.model( + model=f"openai/{PROVIDER_MODEL}", + api_base=peer.url, + tags=[f"granted-{uuid.uuid4().hex}"], + model_info={"enable_tag_filtering": True}, + ) + key: Final = scenario.key(models=[name]) + foreign: Final = {"x-litellm-tags": f"foreign-{uuid.uuid4().hex}"} + http: Final = limited.request( + "POST", "/v1/responses", {"model": name, "input": "tag routing"}, key=key, headers=foreign + ) + assert http.status_code != 200, http.text + http_error: Final = object_value(JSON_OBJECT.validate_json(http.content)["error"]) + + async def refused_open() -> tuple[dict[str, JsonValue], ...]: + async with _socket(limited, key, name, foreign) as connection: + return await _say(connection, name) + + received: Final = asyncio.run(refused_open()) + + assert [event.get("type") for event in received] == ["error"], received + assert received[0]["status"] == http.status_code, (received, http.text) + assert object_value(received[0]["error"])["message"] == string_value(http_error["message"]), ( + received, + http.text, + ) + assert peer.paths.qsize() == 0 + + +def test_each_turn_counts_as_one_request_toward_the_deployment_rpm(limited: Gateway, cert: tuple[Path, Path]) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + deployment: Final = _native(scenario, peer, rpm=1) + key: Final = scenario.key(models=[deployment.name]) + _fresh_minute() + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(limited, key, deployment.name) as connection: + return (await _say(connection, deployment.name), await _say(connection, deployment.name)) + + first, second = asyncio.run(session()) + + completed(first) + _assert_refused(second) + assert deployment.forwarded_turns() == 1 + + +def test_a_deployment_tpm_refuses_a_turn_only_once_earlier_turns_used_it_up( + limited: Gateway, cert: tuple[Path, Path] +) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + deployment: Final = _native(scenario, peer, tpm=TURN_TOKENS + 2) + key: Final = scenario.key(models=[deployment.name]) + + first, second, third, counted_after_first = asyncio.run(_turns_against_tpm(limited, key, deployment)) + + completed(first) + completed(second) + _assert_refused(third) + assert counted_after_first == TURN_TOKENS + assert deployment.forwarded_turns() == 2 + + +def test_a_tpm_limit_lowered_while_the_socket_is_open_applies_to_its_next_turn( + limited: Gateway, cert: tuple[Path, Path] +) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + deployment: Final = _native(scenario, peer, tpm=100 * TURN_TOKENS) + key: Final = scenario.key(models=[deployment.name]) + minute: Final = _fresh_minute() + + def lower_tpm() -> None: + limited.request( + "PATCH", f"/model/{deployment.identity}/update", {"litellm_params": {"tpm": TURN_TOKENS - 2}} + ).raise_for_status() + eventually(lambda: _configured_tpm(limited, deployment), lambda tpm: tpm == TURN_TOKENS - 2, seconds=15) + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(limited, key, deployment.name) as connection: + first: Final = await _say(connection, deployment.name) + await _until(partial(_tokens_counted, deployment.identity, minute), lambda tokens: tokens > 0) + await asyncio.to_thread(lower_tpm) + return (first, await _say(connection, deployment.name)) + + first, second = asyncio.run(session()) + + completed(first) + _assert_refused(second) + assert deployment.forwarded_turns() == 1 + + +def test_a_socket_on_an_otpm_limited_deployment_reserves_once_and_serves_every_turn( + limited: Gateway, cert: tuple[Path, Path] +) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + deployment: Final = _native(scenario, peer, otpm=10_000) + key: Final = scenario.key(models=[deployment.name]) + _fresh_minute() + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(limited, key, deployment.name) as connection: + return tuple([await _say(connection, deployment.name) for _ in range(5)]) + + turns: Final = asyncio.run(session()) + + assert [completed(received)["status"] for received in turns] == ["completed"] * 5 + assert deployment.forwarded_turns() == 5 + + +def test_usage_based_routing_refuses_a_turn_only_once_earlier_turns_used_up_the_deployment_tpm( + usage_routed: Gateway, cert: tuple[Path, Path] +) -> None: + with responses_peer(cert) as peer, usage_routed.scenario() as scenario: + deployment: Final = _native(scenario, peer, tpm=TURN_TOKENS + 2) + key: Final = scenario.key(models=[deployment.name]) + + first, second, third, counted_after_first = asyncio.run(_turns_against_tpm(usage_routed, key, deployment)) + + completed(first) + completed(second) + _assert_refused(third) + assert counted_after_first == TURN_TOKENS + assert deployment.forwarded_turns() == 2 + + +def _assert_http_chat_counted_once(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + identity: Final = f"ws-admission-{uuid.uuid4().hex}" + name: Final = scenario.model(model_info={"id": identity}, tpm=1_000_000) + key: Final = scenario.key(models=[name]) + minute: Final = _fresh_minute() + answered: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": name, "messages": [{"role": "user", "content": f"http turn {uuid.uuid4().hex}"}]}, + key=key, + ) + assert answered.status_code == 200, answered.text + tokens: Final = object_value(JSON_OBJECT.validate_json(answered.content)["usage"])["total_tokens"] + assert isinstance(tokens, int) and tokens > 0, answered.text + + assert _settled_count(partial(_tokens_counted, identity, minute)) == tokens + + +def test_an_http_call_counts_its_tokens_once_toward_the_deployment_tpm_under_model_rate_limits( + limited: Gateway, +) -> None: + _assert_http_chat_counted_once(limited) + + +def test_an_http_call_counts_its_tokens_once_toward_the_deployment_tpm_under_usage_based_routing( + usage_routed: Gateway, +) -> None: + _assert_http_chat_counted_once(usage_routed) + + +def test_usage_based_routing_serves_every_turn_while_the_deployment_has_headroom( + usage_routed: Gateway, cert: tuple[Path, Path] +) -> None: + with responses_peer(cert) as peer, usage_routed.scenario() as scenario: + deployment: Final = _native(scenario, peer, tpm=1_000_000, rpm=1_000) + key: Final = scenario.key(models=[deployment.name]) + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(usage_routed, key, deployment.name) as connection: + return tuple([await _say(connection, deployment.name) for _ in range(3)]) + + turns: Final = asyncio.run(session()) + + assert [completed(received)["status"] for received in turns] == ["completed"] * 3 + assert deployment.forwarded_turns() == 3 + + +def test_a_frame_over_the_project_input_quota_gets_a_429_error_frame_and_never_reaches_the_provider( + limited: Gateway, +) -> None: + with limited.scenario() as scenario: + deployment: Final = _managed(scenario) + team: Final = scenario.team() + key: Final = scenario.key( + models=[deployment.name], + team_id=team, + project_id=scenario.project(team, model_itpm_limit={deployment.name: 100}), + ) + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(limited, key, deployment.name) as connection: + return ( + await _say(connection, deployment.name, "short"), + await _say(connection, deployment.name, "long " * 400), + await _say(connection, deployment.name, "long " * 400), + ) + + first, second, third = asyncio.run(session()) + + completed(first) + _assert_refused(third) + answered: Final = sum(received[-1].get("type") == "response.completed" for received in (first, second, third)) + assert deployment.forwarded_turns() == answered diff --git a/tests/integration/spend/test_failed_dispatch_tokens.py b/tests/integration/spend/test_failed_dispatch_tokens.py index 5778ff9654d..db5c49623eb 100644 --- a/tests/integration/spend/test_failed_dispatch_tokens.py +++ b/tests/integration/spend/test_failed_dispatch_tokens.py @@ -1,11 +1,13 @@ import json import uuid +from collections.abc import Callable, Mapping from typing import Final import pytest -from integration._support.client import Gateway, eventually +from integration._support.client import Gateway, eventually, list_value from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue @pytest.mark.covers("spend.failed_dispatch.failure_row_records_estimated_input_tokens") @@ -48,3 +50,79 @@ def test_provider_500_after_dispatch_records_estimated_prompt_tokens_on_failure_ assert row["status"] == "failure" and float(row["spend"]) == 0 and row["completion_tokens"] == 0, row assert row["prompt_tokens"] > 0, f"failure row lost the dispatched input tokens: {row}" assert row["total_tokens"] == row["prompt_tokens"], row + + +def _chat_body(model: str, text: str) -> Mapping[str, JsonValue]: + return {"model": model, "messages": [{"role": "user", "content": text}]} + + +def _responses_body(model: str, text: str) -> Mapping[str, JsonValue]: + return {"model": model, "input": text} + + +def _messages_body(model: str, text: str) -> Mapping[str, JsonValue]: + return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": text}]} + + +FAILURE_ROUTES: Final = pytest.mark.parametrize( + ("route", "body"), + ( + ("/v1/chat/completions", _chat_body), + ("/v1/responses", _responses_body), + ("/v1/messages", _messages_body), + ), + ids=("chat", "responses", "messages"), +) + + +def _provider_500_failure_row( + gateway: Gateway, + route: str, + body: Callable[[str, str], Mapping[str, JsonValue]], + headers: Mapping[str, str] | None = None, +) -> tuple[str, dict[str, JsonValue]]: + def provider(request: Request) -> Reply: + assert request.method == "POST" + return Reply( + status=500, + body=b'{"error":{"message":"synthetic provider outage","type":"server_error","code":"500"}}', + ) + + with wire_server(provider) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0) + key: Final = scenario.key(models=[model]) + failed: Final = gateway.request( + "POST", route, body(model, f"failed dispatch group {uuid.uuid4().hex}"), key=key, headers=headers + ) + assert failed.status_code == 500, failed.text + assert wire.drain() + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, model_group, request_tags FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (failed.headers["x-litellm-call-id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["status"] == "failure", rows + return model, rows[0] + + +@FAILURE_ROUTES +def test_a_provider_500_failure_row_names_the_model_group_the_caller_asked_for( + gateway: Gateway, route: str, body: Callable[[str, str], Mapping[str, JsonValue]] +) -> None: + model, row = _provider_500_failure_row(gateway, route, body) + + assert row["model_group"] == model, row + + +@FAILURE_ROUTES +def test_a_provider_500_failure_row_keeps_the_tags_the_caller_sent( + gateway: Gateway, route: str, body: Callable[[str, str], Mapping[str, JsonValue]] +) -> None: + tag: Final = f"failed-dispatch-{uuid.uuid4().hex}" + + _, row = _provider_500_failure_row(gateway, route, body, {"x-litellm-tags": tag}) + + assert tag in list_value(row["request_tags"]), row diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index e392ee746b3..8d546f3be59 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -2066,6 +2066,56 @@ async def test_arealtime_marks_litellm_params_async(monkeypatch): assert LitellmLogging.is_sync_litellm_request(captured["litellm_params"]) is False +@pytest.mark.asyncio +async def test_aresponses_websocket_turn_reaches_a_custom_logger_through_the_async_hook_only( + monkeypatch: pytest.MonkeyPatch, +): + from litellm.litellm_core_utils import thread_pool_executor + from litellm.responses.main import base_llm_http_handler + + class InlineExecutor: + def submit(self, fn, *args, **kwargs): + fn(*args, **kwargs) + + sessions: list[LitellmLogging] = [] + + async def open_session(**kwargs): + sessions.append(kwargs["logging_obj"]) + + async_logged = asyncio.Event() + sync_call_ids: list[str] = [] + async_call_ids: list[str] = [] + + class CaptureLogger(CustomLogger): + def log_success_event(self, kwargs, response_obj, start_time, end_time): + sync_call_ids.append(kwargs["litellm_call_id"]) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + async_call_ids.append(kwargs["litellm_call_id"]) + async_logged.set() + + logger = CaptureLogger() + monkeypatch.setattr(litellm, "callbacks", [logger]) + monkeypatch.setattr(litellm, "success_callback", ["cache", logger]) + monkeypatch.setattr(litellm, "_async_success_callback", [logger]) + monkeypatch.setattr(thread_pool_executor, "executor", InlineExecutor()) + with patch.object( # test-quality-ok: the provider socket is the seam; how a turn of the session logs is under test + base_llm_http_handler, "async_responses_websocket", AsyncMock(side_effect=open_session) + ): + await litellm._aresponses_websocket(model="openai/gpt-4o", websocket=MagicMock(), api_key="sk-test") + turn = sessions[0].copy_for_new_call("ws-turn-2", dt_object.now()) + + await turn.dispatch_success_handlers( + litellm.ResponsesAPIResponse( + id="resp_turn_2", created_at=0, output=[], parallel_tool_calls=False, tool_choice="auto", tools=[] + ), + prefer_async_handlers=True, + ) + await asyncio.wait_for(async_logged.wait(), timeout=10) + + assert (async_call_ids, sync_call_ids) == (["ws-turn-2"], []) + + @pytest.mark.asyncio async def test_aresponses_websocket_hands_back_the_provider_failure_without_a_success_log( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 2f0e8a268c2..1f0dd67d86b 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -4,7 +4,7 @@ import logging from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Final -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, create_autospec, patch import httpx import pytest @@ -136,6 +136,77 @@ async def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_ assert metadata["original_key"] == "original_value" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata_buckets", "expected_model_group"), + [ + ({"metadata": {}, "litellm_metadata": {"model_group": "ws-budget-sock"}}, "ws-budget-sock"), + ({"metadata": {"model_group": "caller-group"}, "litellm_metadata": {"model_group": "routed"}}, "caller-group"), + ({"metadata": {}, "litellm_metadata": {}}, ""), + ], +) +async def test_async_post_call_failure_hook_logs_the_routed_model_group_on_the_failure_row( + metadata_buckets: dict[str, dict[str, str]], expected_model_group: str +): + request_data: Final = { + "model": "ws-budget-sock", + "input": "Reply with exactly the word two", + "proxy_server_request": {"request_id": "test_request_id"}, + **metadata_buckets, + } + + spend_writer: Final = create_autospec(DBSpendUpdateWriter, instance=True) + await ProxyDBLogger(spend_writer=lambda: spend_writer).async_post_call_failure_hook( + request_data=request_data, + original_exception=litellm.RateLimitError(message="crossed budget", llm_provider="", model="ws-budget-sock"), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), + ) + + row: Final = get_logging_payload( + kwargs=spend_writer.update_database.call_args.kwargs["kwargs"], + response_obj={}, + start_time=datetime.now(), + end_time=datetime.now(), + ) + assert row["model_group"] == expected_model_group + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata_buckets", "expected_tags"), + [ + ({"metadata": {}, "litellm_metadata": {"tags": ["ws-tag"]}}, ["ws-tag"]), + ({"metadata": {"tags": ["caller-tag"]}, "litellm_metadata": {"tags": ["routed-tag"]}}, ["caller-tag"]), + ({"metadata": {}, "litellm_metadata": {"tags": "not-a-list"}}, []), + ({"metadata": {}, "litellm_metadata": {"tags": ["ws-tag", 1, {"a": 1}]}}, []), + ], +) +async def test_async_post_call_failure_hook_logs_the_routed_tags_on_the_failure_row( + metadata_buckets: dict[str, dict[str, object]], expected_tags: list[str] +): + request_data: Final = { + "model": "ws-budget-sock", + "input": "Reply with exactly the word two", + "proxy_server_request": {"request_id": "test_request_id"}, + **metadata_buckets, + } + + spend_writer: Final = create_autospec(DBSpendUpdateWriter, instance=True) + await ProxyDBLogger(spend_writer=lambda: spend_writer).async_post_call_failure_hook( + request_data=request_data, + original_exception=litellm.RateLimitError(message="crossed budget", llm_provider="", model="ws-budget-sock"), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), + ) + + row: Final = get_logging_payload( + kwargs=spend_writer.update_database.call_args.kwargs["kwargs"], + response_obj={}, + start_time=datetime.now(), + end_time=datetime.now(), + ) + assert json.loads(row["request_tags"]) == expected_tags + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_does_not_clobber_guardrail_info_in_metadata(): logger = ProxyDBLogger() @@ -1105,6 +1176,38 @@ async def test_track_cost_callback_skips_when_no_standard_logging_object(): mock_proxy_logging.failed_tracking_alert.assert_not_called() +class _FailedTrackingAlerts: + def __init__(self) -> None: + self.messages: tuple[str, ...] = () + + async def failed_tracking_alert(self, error_message: str, failing_model: str) -> None: + self.messages = (*self.messages, error_message) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("call_type", "alerted"), [("_aresponses_websocket", True), ("_arealtime", False)]) +async def test_track_cost_callback_alerts_on_a_websocket_turn_with_no_logging_object(call_type: str, alerted: bool): + kwargs: Final = { + "call_type": call_type, + "model": "gpt-5-mini", + "litellm_call_id": "turn-call-id", + "litellm_params": {}, + "stream": False, + } + alerts: Final = _FailedTrackingAlerts() + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", alerts): + await ProxyDBLogger()._PROXY_track_cost_callback( + kwargs=kwargs, + completion_response=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + await asyncio.sleep(0) + + assert any(f"call_type: {call_type}" in message for message in alerts.messages) is alerted + + @pytest.mark.asyncio async def test_track_cost_callback_defers_in_progress_background_interaction(): # test-quality-ok: writing no spend row and raising no alert is the whole observable contract of the deferral path """ diff --git a/tests/unit/proxy/proxy_server/test_lifecycle.py b/tests/unit/proxy/proxy_server/test_lifecycle.py index 71daa3bb873..3914c337f59 100644 --- a/tests/unit/proxy/proxy_server/test_lifecycle.py +++ b/tests/unit/proxy/proxy_server/test_lifecycle.py @@ -972,6 +972,54 @@ async def test_proxy_startup_event_honors_disable_model_info_refresh( ) +@pytest.mark.asyncio +async def test_proxy_shutdown_commits_spend_before_waiting_on_a_slow_logging_task_and_again_after_it() -> None: + """A worker killed while shutdown waits on a slow logging task keeps the spend it had already counted, + and a worker that finishes the wait also commits the spend that task adds.""" + from litellm.responses import streaming_iterator as streaming_module + + task_may_finish: Final = asyncio.Event() + task_finished: Final = asyncio.Event() + flushes: Final[list[tuple[str, bool]]] = [] # mutable-ok: records what each shutdown flush saw, which is the assertion + + async def slow_logging_task() -> None: + await task_may_finish.wait() + task_finished.set() + + def recording_flush(name: str) -> Callable[[], Awaitable[None]]: + async def flush() -> None: + flushes.append((name, task_finished.is_set())) + task_may_finish.set() + + return flush + + scheduler: Final = AsyncIOScheduler() + clean_env: Final = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} | { + "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true" + } + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.object(ps, "scheduler", scheduler), + patch.object(ps, "flush_spend_counters_on_shutdown", recording_flush("spend counters")), + patch.object(ps, "_flush_spend_logs_queue_on_shutdown", recording_flush("spend logs")), + ): + try: + async with proxy_startup_event(app=None): + streaming_module.spawn_logging_task( + asyncio.get_running_loop(), slow_logging_task(), task_name="slow spend row" + ) + finally: + if scheduler.running: + scheduler.shutdown(wait=False) + + assert flushes == [ + ("spend counters", False), + ("spend logs", False), + ("spend counters", True), + ("spend logs", True), + ] + + def test_otel_global_provider_published_after_callback_init(): """The OTel V2 global-provider publish must run after callback initialization in ``proxy_startup_event``. diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index fc329eaec5f..e3ce513dbcb 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -9,17 +9,20 @@ import json import unittest from collections.abc import Mapping from typing import Any, Final, Literal, cast -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, call, patch import httpx import pytest import respx +from fastapi import HTTPException from fastapi.testclient import TestClient from httpx import Response import litellm +from litellm.proxy._types import ProxyException from litellm.proxy.proxy_server import app from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.router import RouterErrors from tests._master_key import MASTER_KEY @@ -1648,7 +1651,7 @@ class TestManagedResponsesWSFirstMessage: processed: list = [] - async def fake_process(msg: str) -> None: + async def fake_process(msg: str, turn_admission: object = None) -> None: processed.append(msg) handler = ManagedResponsesWebSocketHandler( @@ -1693,30 +1696,32 @@ class TestResponsesWSStreamingFirstMessage: backend_ws.send.assert_awaited_once_with(first) +@pytest.mark.parametrize( + ("failure", "expected_status", "expected_type"), + [ + (ValueError("No deployments available - crossed budget: Exceeded budget for deployment"), 429, "rate_limit_exceeded"), + (litellm.BadRequestError(message="bad input", model="m", llm_provider="openai"), 400, "invalid_request_error"), + (RuntimeError("boom"), 500, "server_error"), + (ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}=['free']"), 401, "invalid_request_error"), + (ProxyException(message="Authentication Error", type="auth_error", param=None, code=401), 401, "invalid_request_error"), + (HTTPException(status_code=403, detail="key not allowed"), 403, "invalid_request_error"), + ], + ids=["crossed-budget", "bad-request", "unexpected", "tag-routing", "proxy-auth", "http-exception"], +) +def test_responses_ws_failure_frame_uses_the_http_status_for_the_same_failure( + failure: Exception, expected_status: int, expected_type: str +): + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_frame + from litellm.proxy.utils import handle_exception_on_proxy + + frame: Final = json.loads(_responses_ws_failure_frame(failure)) + + assert frame["status"] == expected_status + assert frame["error"]["type"] == expected_type + assert frame["error"]["message"] == handle_exception_on_proxy(failure).message + + class TestWSSessionCostTracking: - @pytest.mark.asyncio - async def test_router_budget_limiter_skips_aresponses_websocket_call_type(self): - """ - RouterBudgetLimiting.async_log_success_event must not raise when - call_type='_aresponses_websocket', even when standard_logging_object is None. - Per-turn costs are tracked by individual aresponses calls inside the session; - the outer session wrapper fires with result=None. - """ - from litellm.router_strategy.budget_limiter import RouterBudgetLimiting - - limiter = RouterBudgetLimiting.__new__(RouterBudgetLimiting) - kwargs = { - "call_type": "_aresponses_websocket", - "standard_logging_object": None, - "litellm_params": {"custom_llm_provider": "vertex_ai"}, - } - await limiter.async_log_success_event( - kwargs=kwargs, - response_obj=None, - start_time=None, - end_time=None, - ) - @pytest.mark.asyncio async def test_router_budget_limiter_skips_arealtime_call_type(self): """Same guard applies to _arealtime WS session wrappers.""" @@ -1996,7 +2001,7 @@ class TestResponsesWSFirstFrameModelAuth: "standard_logging_guardrail_information": [{"guardrail_name": "admission"}], }, } - guard = _responses_ws_frame_guard(admitted, MagicMock(), proxy_logging_obj) + guard = _responses_ws_frame_guard(admitted, MagicMock(), proxy_logging_obj, MagicMock()) first = await guard(json.dumps({"type": "response.create", "model": "alias", "input": "hi"})) second = await guard(json.dumps({"type": "response.create", "input": "again"})) @@ -2027,7 +2032,7 @@ class TestResponsesWSFirstFrameModelAuth: rate_limiter.async_post_call_failure_hook = AsyncMock() user_api_key_dict = MagicMock() guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, user_api_key_dict, proxy_logging_obj + {"model": "gpt-4o-mini", "litellm_metadata": {}}, user_api_key_dict, proxy_logging_obj, MagicMock() ) with ( @@ -2059,7 +2064,7 @@ class TestResponsesWSFirstFrameModelAuth: proxy_logging_obj = MagicMock() proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj + {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() ) tools = [{"type": "function", "name": "a"}, {"type": "function", "name": "b"}] @@ -2078,7 +2083,7 @@ class TestResponsesWSFirstFrameModelAuth: proxy_logging_obj = MagicMock() proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj + {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() ) metadata = { "user_api_key_metadata": {"disable_global_guardrails": True}, @@ -2109,6 +2114,7 @@ class TestResponsesWSFirstFrameModelAuth: {"model": "gpt-4o-mini", "litellm_metadata": {"guardrails": ["key-required"]}}, MagicMock(), proxy_logging_obj, + MagicMock(), ) await guard(json.dumps(frame)) @@ -2131,6 +2137,7 @@ class TestResponsesWSFirstFrameModelAuth: {"model": "gpt-4o-mini", "litellm_metadata": {"litellm_parent_otel_span": span}}, MagicMock(), proxy_logging_obj, + MagicMock(), ) guarded = await guard(json.dumps({"type": "response.create", "input": "hi"})) @@ -2156,7 +2163,7 @@ class TestResponsesWSFirstFrameModelAuth: spend_logger = MagicMock(spec=ProxyDBLogger) spend_logger.async_post_call_failure_hook = AsyncMock() guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj + {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() ) with ( @@ -2189,7 +2196,7 @@ class TestResponsesWSFirstFrameModelAuth: spend_logger = MagicMock(spec=ProxyDBLogger) spend_logger.async_post_call_failure_hook = AsyncMock() guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj + {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() ) body = { "input": "forbidden", @@ -2215,7 +2222,7 @@ class TestResponsesWSFirstFrameModelAuth: proxy_logging_obj = MagicMock() proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj + {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() ) body = {"instructions": "SECRET system prompt", "input": "hi", "store": False} frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body} @@ -2240,7 +2247,7 @@ class TestResponsesWSFirstFrameModelAuth: proxy_logging_obj = MagicMock() proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj + {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() ) with patch.object(litellm, "callbacks", []): @@ -2264,7 +2271,7 @@ class TestResponsesWSFirstFrameModelAuth: proxy_logging_obj = MagicMock() proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj + {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() ) with patch.object(litellm, "callbacks", []), pytest.raises(ValueError, match="both at the top level"): @@ -2283,7 +2290,7 @@ class TestResponsesWSFirstFrameModelAuth: spend_logger = MagicMock(spec=ProxyDBLogger) spend_logger.async_post_call_failure_hook = AsyncMock(side_effect=RuntimeError("db down")) guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj + {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() ) with patch.object(litellm, "callbacks", [spend_logger]), pytest.raises(ValueError, match="blocked"): @@ -2360,6 +2367,345 @@ class TestResponsesWSFirstFrameModelAuth: assert booked["user_api_key_dict"] is user_api_key_dict assert booked["request_data"]["model"] == "gpt-4o-mini" + @pytest.mark.asyncio + @pytest.mark.parametrize("ending", ["refusal", "crash", "session_limit"]) + async def test_close_time_failure_row_bills_the_guardrails_of_the_turns_the_deployment_refused(self, ending: str): + from litellm.exceptions import RateLimitError + from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint + + ws = MagicMock() + ws.headers = {} + ws.query_params = {} + ws.scope = {"headers": []} + ws.url = "ws://testserver/v1/responses" + ws.accept = AsyncMock() + ws.receive_text = AsyncMock( + return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []}) + ) + ws.close = AsyncMock() + processor = MagicMock() + processor.common_processing_pre_call_logic = AsyncMock( + return_value=({"model": "gpt-4o-mini", "litellm_metadata": {"model_group": "gpt-4o-mini"}}, MagicMock()) + ) + guardrail_costs = {"two": 0.2, "three": 0.3, "four": 0.4} + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") + later_refusal = RateLimitError(message="still over budget", llm_provider="", model="") + crash = RuntimeError("relay crashed") + + async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only): + recorded = [{"guardrail_cost": guardrail_costs[data["input"]]}] + return { + **data, + "litellm_metadata": {**data["litellm_metadata"], "standard_logging_guardrail_information": recorded}, + } + + def route_request(data, **kwargs): + async def relay(): + for turn in ("two", "three", "four"): + guarded = await data["responses_ws_frame_guard"]( + json.dumps({"type": "response.create", "input": turn}) + ) + if turn != "three": + guarded.bill_refusal(refusal if turn == "two" else later_refusal) + if ending == "crash": + raise crash + if ending == "session_limit": + await asyncio.Event().wait() + return refusal + + return relay() + + row_written = asyncio.Event() + proxy_logging_obj = MagicMock() + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) + proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=lambda **kwargs: row_written.set()) + proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) + + with ( + patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above + "litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth", + new_callable=AsyncMock, + ), + patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test + "litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing", + return_value=processor, + ), + patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome + "litellm.proxy.route_llm_request.route_request", + new_callable=AsyncMock, + side_effect=route_request, + ), + patch( # test-quality-ok: a short session limit stands in for the configured one so the cap fires mid-session + "litellm.proxy.response_api_endpoints.endpoints._resolve_responses_ws_session_limit_seconds", + return_value=0.05 if ending == "session_limit" else 3600.0, + ), + patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row + "litellm.proxy.proxy_server.proxy_logging_obj", + proxy_logging_obj, + ), + ): + await responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock()) + await asyncio.wait_for(row_written.wait(), timeout=1) + + proxy_logging_obj.post_call_failure_hook.assert_awaited_once() + proxy_logging_obj.arelease_responses_websocket_slot.assert_awaited_once() + booked = proxy_logging_obj.post_call_failure_hook.await_args.kwargs + assert booked["original_exception"] is (crash if ending == "crash" else refusal) + assert booked["request_data"]["litellm_metadata"] == { + "model_group": "gpt-4o-mini", + "standard_logging_guardrail_information": [{"guardrail_cost": 0.2}, {"guardrail_cost": 0.4}], + } + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "relay_outcome", + ["still_relaying", "already_returned", "returns_despite_the_cancel", "crashes_despite_the_cancel"], + ) + async def test_session_limit_frees_the_slot_at_once_and_never_cuts_the_failure_row(self, relay_outcome: str): + from litellm.exceptions import RateLimitError + from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint + + ws = MagicMock() + ws.headers = {} + ws.query_params = {} + ws.scope = {"headers": []} + ws.url = "ws://testserver/v1/responses" + ws.accept = AsyncMock() + ws.receive_text = AsyncMock( + return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []}) + ) + async def close_over_the_network(**kwargs): + await asyncio.sleep(0) + + ws.close = AsyncMock(side_effect=close_over_the_network) + processor = MagicMock() + processor.common_processing_pre_call_logic = AsyncMock( + return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock()) + ) + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") + crash = RuntimeError("relay crashed") + row_may_finish = asyncio.Event() + row_written = asyncio.Event() + + async def slow_failure_hook(**kwargs): + await row_may_finish.wait() + row_written.set() + + def route_request(data, **kwargs): + async def relay(): + guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"})) + guarded.bill_refusal(refusal) + if relay_outcome == "still_relaying": + await asyncio.Event().wait() + if relay_outcome in ("returns_despite_the_cancel", "crashes_despite_the_cancel"): + with contextlib.suppress(asyncio.CancelledError): + await asyncio.Event().wait() + if relay_outcome == "crashes_despite_the_cancel": + raise crash + return refusal + + return relay() + + proxy_logging_obj = MagicMock() + proxy_logging_obj.pre_call_hook = AsyncMock( + side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: data + ) + proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=slow_failure_hook) + async def release_over_the_network(user_api_key_dict): + await asyncio.sleep(0.01) + + proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(side_effect=release_over_the_network) + + with ( + patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above + "litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth", + new_callable=AsyncMock, + ), + patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test + "litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing", + return_value=processor, + ), + patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome + "litellm.proxy.route_llm_request.route_request", + new_callable=AsyncMock, + side_effect=route_request, + ), + patch( # test-quality-ok: a short session limit stands in for the configured one so the cap fires mid-session + "litellm.proxy.response_api_endpoints.endpoints._resolve_responses_ws_session_limit_seconds", + return_value=0.05, + ), + patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row + "litellm.proxy.proxy_server.proxy_logging_obj", + proxy_logging_obj, + ), + ): + endpoint = asyncio.ensure_future( + responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock()) + ) + finished, _ = await asyncio.wait({endpoint}, timeout=1) + assert endpoint in finished + limit_close = call(code=1000, reason="Session duration limit reached") + crash_close = call(code=1011, reason="Internal server error") + crashed = relay_outcome == "crashes_despite_the_cancel" + assert ws.close.await_args_list == ([limit_close, crash_close] if crashed else [limit_close]) + proxy_logging_obj.arelease_responses_websocket_slot.assert_awaited_once() + assert not row_written.is_set() + row_may_finish.set() + await asyncio.wait_for(row_written.wait(), timeout=1) + + assert proxy_logging_obj.post_call_failure_hook.await_args.kwargs["original_exception"] is (crash if crashed else refusal) + + @pytest.mark.asyncio + @pytest.mark.parametrize("ending", ["client_closed", "crash"]) + async def test_a_closed_socket_frees_its_slot_before_its_failure_row_and_waits_for_the_row(self, ending: str): + from litellm.exceptions import RateLimitError + from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint + + ws = MagicMock() + ws.headers = {} + ws.query_params = {} + ws.scope = {"headers": []} + ws.url = "ws://testserver/v1/responses" + ws.accept = AsyncMock() + ws.receive_text = AsyncMock( + return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []}) + ) + ws.close = AsyncMock() + processor = MagicMock() + processor.common_processing_pre_call_logic = AsyncMock( + return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock()) + ) + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") + crash = RuntimeError("relay crashed") + row_started = asyncio.Event() + row_may_finish = asyncio.Event() + row_written = asyncio.Event() + + async def slow_failure_hook(**kwargs): + row_started.set() + await row_may_finish.wait() + row_written.set() + + def route_request(data, **kwargs): + async def relay(): + guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"})) + guarded.bill_refusal(refusal) + if ending == "crash": + raise crash + return refusal + + return relay() + + proxy_logging_obj = MagicMock() + proxy_logging_obj.pre_call_hook = AsyncMock( + side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: data + ) + proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=slow_failure_hook) + proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) + + with ( + patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above + "litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth", + new_callable=AsyncMock, + ), + patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test + "litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing", + return_value=processor, + ), + patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome + "litellm.proxy.route_llm_request.route_request", + new_callable=AsyncMock, + side_effect=route_request, + ), + patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row + "litellm.proxy.proxy_server.proxy_logging_obj", + proxy_logging_obj, + ), + ): + endpoint = asyncio.ensure_future( + responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock()) + ) + await asyncio.wait_for(row_started.wait(), timeout=1) + proxy_logging_obj.arelease_responses_websocket_slot.assert_awaited_once() + finished_early, _ = await asyncio.wait({endpoint}, timeout=0.05) + assert endpoint not in finished_early + row_may_finish.set() + finished, _ = await asyncio.wait({endpoint}, timeout=1) + assert endpoint in finished + assert row_written.is_set() + + booked = proxy_logging_obj.post_call_failure_hook.await_args.kwargs + assert booked["original_exception"] is (crash if ending == "crash" else refusal) + + @pytest.mark.asyncio + async def test_a_crashed_socket_whose_failure_row_cannot_be_built_still_gets_its_error_frame_and_close(self): + from litellm.exceptions import RateLimitError + from litellm.proxy.response_api_endpoints.endpoints import responses_websocket_endpoint + + ws = MagicMock() + ws.headers = {} + ws.query_params = {} + ws.scope = {"headers": []} + ws.url = "ws://testserver/v1/responses" + ws.accept = AsyncMock() + ws.receive_text = AsyncMock( + return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []}) + ) + sent = [] + closes = [] + ws.send_text = AsyncMock(side_effect=sent.append) + ws.close = AsyncMock(side_effect=lambda code, reason: closes.append((code, reason))) + processor = MagicMock() + processor.common_processing_pre_call_logic = AsyncMock( + return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock()) + ) + refusal = RateLimitError(message="crossed budget", llm_provider="", model="") + + def route_request(data, **kwargs): + async def relay(): + guarded = await data["responses_ws_frame_guard"](json.dumps({"type": "response.create", "input": "two"})) + guarded.bill_refusal(refusal) + data["litellm_metadata"]["standard_logging_guardrail_information"] = "written by a broken callback" + raise RuntimeError("relay crashed") + + return relay() + + proxy_logging_obj = MagicMock() + proxy_logging_obj.pre_call_hook = AsyncMock( + side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: { + **data, + "litellm_metadata": {"standard_logging_guardrail_information": [{"guardrail_name": "turn-check"}]}, + } + ) + proxy_logging_obj.post_call_failure_hook = AsyncMock() + proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) + + with ( + patch( # test-quality-ok: first-frame model auth needs a live router and key table and has its own tests above + "litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth", + new_callable=AsyncMock, + ), + patch( # test-quality-ok: the pre-call processor needs a live proxy; what the endpoint does with the relay's outcome is under test + "litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing", + return_value=processor, + ), + patch( # test-quality-ok: routing is the seam that hands the relay its frame guard and returns its outcome + "litellm.proxy.route_llm_request.route_request", + new_callable=AsyncMock, + side_effect=route_request, + ), + patch( # test-quality-ok: the failure hook is the proxy's only path to a failed spend log row + "litellm.proxy.proxy_server.proxy_logging_obj", + proxy_logging_obj, + ), + ): + await asyncio.wait_for( + responses_websocket_endpoint(websocket=ws, model=None, user_api_key_dict=MagicMock()), timeout=1 + ) + + assert [json.loads(frame)["error"]["message"] for frame in sent] == ["relay crashed"] + assert closes == [(1011, "Internal server error")] + @pytest.mark.asyncio async def test_endpoint_sends_an_error_frame_when_routing_rejects_the_connection(self): from litellm.proxy.response_api_endpoints.endpoints import ( diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index 2f2563b9ee4..f8ea1e5d075 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -235,6 +235,29 @@ async def test_proxy_shutdown_closes_admin_connector_when_drain_is_cancelled( # Your bearer token +@pytest.mark.asyncio +async def test_shutdown_commits_spend_a_logging_task_queued_while_the_first_commit_was_writing(): + from litellm.responses.streaming_iterator import spawn_logging_task + + queued: list[str] = [] + committed: list[str] = [] + + async def commit_spend() -> None: + committed.extend(queued) + queued.clear() + await asyncio.sleep(0.05) + + async def failure_row() -> None: + await asyncio.sleep(0.01) + queued.append("failure row spend") + + spawn_logging_task(asyncio.get_running_loop(), failure_row(), task_name="failure row") + + await litellm.proxy.proxy_server._flush_spend_around_logging_drain_on_shutdown(flush_spend=commit_spend) + + assert committed == ["failure row spend"] + + token = MASTER_KEY headers = {"Authorization": f"Bearer {token}"} diff --git a/tests/unit/responses/test_responses_websocket_all_providers.py b/tests/unit/responses/test_responses_websocket_all_providers.py index b38808fb57c..bfb2e16e1a7 100644 --- a/tests/unit/responses/test_responses_websocket_all_providers.py +++ b/tests/unit/responses/test_responses_websocket_all_providers.py @@ -7,6 +7,7 @@ Tests that: 3. Providers without native websocket support use ManagedResponsesWebSocketHandler """ +import asyncio import json from collections.abc import Mapping from typing import Final @@ -1331,7 +1332,7 @@ class TestWebSocketProjectQuotaEnforcement: callback, not just the first one.""" @pytest.mark.asyncio - async def test_managed_handler_blocks_frame_rejected_by_quota_callback(self, monkeypatch): + async def test_managed_handler_blocks_frame_rejected_by_quota_callback(self, monkeypatch: pytest.MonkeyPatch): from unittest.mock import AsyncMock, MagicMock import litellm @@ -1377,11 +1378,11 @@ class TestWebSocketProjectQuotaEnforcement: quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once() assert aresponses_called is False mock_websocket.send_text.assert_called_once() - error_event = mock_websocket.send_text.call_args[0][0] - assert "rate_limit_exceeded" in error_event + error_event = json.loads(mock_websocket.send_text.call_args[0][0]) + assert (error_event["status"], error_event["error"]["type"]) == (429, "rate_limit_exceeded") @pytest.mark.asyncio - async def test_managed_handler_forwards_frame_allowed_by_quota_callback(self, monkeypatch): + async def test_managed_handler_forwards_frame_allowed_by_quota_callback(self, monkeypatch: pytest.MonkeyPatch): from unittest.mock import AsyncMock, MagicMock import litellm @@ -1462,7 +1463,8 @@ class TestWebSocketProjectQuotaEnforcement: assert allowed is False mock_backend_ws.send.assert_not_called() mock_websocket.send_text.assert_called_once() - assert "rate_limit_exceeded" in mock_websocket.send_text.call_args[0][0] + error_event = json.loads(mock_websocket.send_text.call_args[0][0]) + assert (error_event["status"], error_event["error"]["type"]) == (429, "rate_limit_exceeded") @pytest.mark.asyncio async def test_native_handler_runs_the_frame_guard_on_every_turn_and_bills_each_turn_its_own_guardrail_records( @@ -1580,6 +1582,461 @@ class TestWebSocketProjectQuotaEnforcement: quota_callback.enforce_project_io_token_quota_for_frame.assert_awaited_once() +def _turn_frames() -> tuple[str, str, str]: + return tuple(json.dumps({"type": "response.create", "input": word}) for word in ("one", "two", "three")) + + +def _admission_refusing_the_next_turn(): + from unittest.mock import AsyncMock + + from litellm.exceptions import RateLimitError + + return AsyncMock(side_effect=[RateLimitError(message="crossed budget", llm_provider="", model=""), None]) + + +class _ClientSocket: + """A client socket that, like Starlette's, stops receiving once the server closes it.""" + + def __init__(self, *frames: str) -> None: + self._frames: Final = iter(frames) + self.sent: Final[list[dict[str, object]]] = [] + self.close_code: int | None = None + + async def send_text(self, data: str) -> None: + self.sent.append(json.loads(data)) + + async def receive_text(self) -> str: + frame: Final = next(self._frames, None) + if self.close_code is not None or frame is None: + raise RuntimeError("WebSocket is not connected") + return frame + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + self.close_code = code + + +class _StalledClientSocket(_ClientSocket): + """A client socket whose sends never finish, like a client that stopped reading.""" + + def __init__(self, *frames: str) -> None: + super().__init__(*frames) + self.send_started: Final = asyncio.Event() + + async def send_text(self, data: str) -> None: + self.send_started.set() + await asyncio.Event().wait() + + +def _admission_crashing_the_next_turn(): + from unittest.mock import AsyncMock + + return AsyncMock(side_effect=[ValueError("custom check failed"), None]) + + +def _guard_blocking(word: str): + from litellm.responses.streaming_iterator import GuardedResponsesWsFrame + + async def guard(frame: str) -> GuardedResponsesWsFrame: + if word in json.loads(frame)["input"]: + raise ValueError("blocked by guardrail") + return GuardedResponsesWsFrame(frame=frame) + + return guard + + +def _guard_recording(bill_refusal: MagicMock): + from litellm.responses.streaming_iterator import GuardedResponsesWsFrame + + async def guard(frame: str) -> GuardedResponsesWsFrame: + return GuardedResponsesWsFrame( + frame=frame, guardrail_information=({"guardrail_cost": 0.5},), bill_refusal=bill_refusal + ) + + return guard + + +def _errors(websocket: _ClientSocket) -> list[tuple[object, object]]: + return [(frame.get("status"), frame["error"]) for frame in websocket.sent if frame.get("type") == "error"] + + +class TestWebSocketTurnAdmission: + """A turn after the connection's first is admitted by the deployment again before it reaches the provider.""" + + @pytest.mark.asyncio + async def test_native_relay_drops_a_later_turn_the_deployment_refuses(self): + from unittest.mock import AsyncMock + + first, second, third = _turn_frames() + cancel: Final = json.dumps({"type": "response.cancel"}) + turn_admission: Final = _admission_refusing_the_next_turn() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + websocket: Final = MagicMock() + websocket.send_text = AsyncMock() + websocket.receive_text = AsyncMock(side_effect=[second, cancel, third, Exception("stop")]) + handler: Final = _make_streaming( + websocket=websocket, + backend_ws=backend_ws, + request_data={}, + first_message=first, + turn_admission=turn_admission, + ) + + await handler.client_to_backend() + + assert [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] == [ + json.loads(first), + json.loads(cancel), + json.loads(third), + ] + assert turn_admission.await_count == 2 + refusal: Final = json.loads(websocket.send_text.await_args.args[0]) + assert refusal["status"] == 429 + assert refusal["error"]["type"] == "rate_limit_exceeded" + assert "crossed budget" in refusal["error"]["message"] + + @pytest.mark.asyncio + async def test_native_relay_reports_a_refused_turn_as_the_socket_failure(self): + from unittest.mock import AsyncMock + + from litellm.exceptions import RateLimitError + + first, second, _ = _turn_frames() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.close = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=Exception("stop")) + websocket: Final = MagicMock() + websocket.send_text = AsyncMock() + websocket.receive_text = AsyncMock(side_effect=[second, Exception("client gone")]) + handler: Final = _make_streaming( + websocket=websocket, + backend_ws=backend_ws, + request_data={}, + first_message=first, + turn_admission=_admission_refusing_the_next_turn(), + ) + + failure: Final = await handler.bidirectional_forward() + + assert isinstance(failure, RateLimitError) + assert "crossed budget" in str(failure) + + @pytest.mark.asyncio + async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch): + from unittest.mock import AsyncMock + + import litellm + from litellm.exceptions import RateLimitError + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler + + answered: Final[list[object]] = [] + + async def fake_aresponses(*args: object, **kwargs: object): + answered.append(kwargs.get("input")) + + async def _empty(): + return + yield + + return _empty() + + monkeypatch.setattr(litellm, "aresponses", fake_aresponses) + first, second, third = _turn_frames() + turn_admission: Final = _admission_refusing_the_next_turn() + websocket: Final = MagicMock() + websocket.send_text = AsyncMock() + websocket.receive_text = AsyncMock(side_effect=[second, third, Exception("stop")]) + handler: Final = ManagedResponsesWebSocketHandler( + websocket=websocket, + model="test-model", + logging_obj=Logging( + model="test-model", + messages=[], + stream=True, + call_type="aresponses", + start_time=0, + litellm_call_id="test-id", + function_id="test-func", + ), + first_message=first, + turn_admission=turn_admission, + ) + + failure: Final = await handler.run() + + assert len(answered) == 2 + assert "two" not in json.dumps(answered) + assert turn_admission.await_count == 2 + assert isinstance(failure, RateLimitError) + assert "crossed budget" in str(failure) + refusals: Final = [ + json.loads(call.args[0]) + for call in websocket.send_text.await_args_list + if json.loads(call.args[0]).get("type") == "error" + ] + assert [(refusal["status"], refusal["error"]["type"]) for refusal in refusals] == [(429, "rate_limit_exceeded")] + + + @pytest.mark.asyncio + async def test_native_relay_runs_the_guardrail_before_the_deployment_counts_a_turn(self): + from unittest.mock import AsyncMock + + first, second, third = _turn_frames() + turn_admission: Final = _admission_refusing_the_next_turn() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + websocket: Final = _ClientSocket(second, third) + handler: Final = _make_streaming( + websocket=websocket, + backend_ws=backend_ws, + request_data={}, + first_message=first, + frame_guard=_guard_blocking("two"), + turn_admission=turn_admission, + ) + + await handler.client_to_backend() + + assert [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] == [json.loads(first)] + assert turn_admission.await_count == 1 + assert [(status, error["type"]) for status, error in _errors(websocket)] == [ + (None, "invalid_request_error"), + (429, "rate_limit_exceeded"), + ] + + @pytest.mark.asyncio + async def test_managed_relay_runs_the_guardrail_before_the_deployment_counts_a_turn( + self, monkeypatch: pytest.MonkeyPatch + ): + import litellm + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler + + answered: Final[list[object]] = [] + + async def fake_aresponses(*args: object, **kwargs: object): + answered.append(kwargs.get("input")) + + async def _empty(): + return + yield + + return _empty() + + monkeypatch.setattr(litellm, "aresponses", fake_aresponses) + first, second, third = _turn_frames() + turn_admission: Final = _admission_refusing_the_next_turn() + websocket: Final = _ClientSocket(second, third) + handler: Final = ManagedResponsesWebSocketHandler( + websocket=websocket, + model="test-model", + logging_obj=Logging( + model="test-model", + messages=[], + stream=True, + call_type="aresponses", + start_time=0, + litellm_call_id="test-id", + function_id="test-func", + ), + first_message=first, + frame_guard=_guard_blocking("two"), + turn_admission=turn_admission, + ) + + await handler.run() + + assert answered == ["one"] + assert turn_admission.await_count == 1 + assert [(status, error["type"]) for status, error in _errors(websocket)] == [ + (None, "invalid_request_error"), + (429, "rate_limit_exceeded"), + ] + + + @pytest.mark.asyncio + async def test_native_relay_leaves_a_refused_turn_s_guardrail_records_to_the_close_time_failure_row(self): + from unittest.mock import AsyncMock + + from litellm.exceptions import RateLimitError + + first, second, third = _turn_frames() + refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="") + bill_refusal: Final = MagicMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.close = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=asyncio.Event().wait) + handler: Final = _make_streaming( + websocket=_ClientSocket(second, third), + backend_ws=backend_ws, + request_data={}, + first_message=first, + frame_guard=_guard_recording(bill_refusal), + turn_admission=AsyncMock(side_effect=[refusal, None]), + ) + + close_time_failure: Final = await handler.bidirectional_forward() + + assert [json.loads(sent.args[0])["input"] for sent in backend_ws.send.await_args_list] == ["one", "three"] + bill_refusal.assert_called_once_with(refusal) + assert close_time_failure is refusal + + @pytest.mark.asyncio + async def test_a_refused_turn_is_recorded_even_when_the_relay_is_cancelled_while_answering_it(self): + from unittest.mock import AsyncMock + + from litellm.exceptions import RateLimitError + + first, second, third = _turn_frames() + refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="") + bill_refusal: Final = MagicMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.close = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=asyncio.Event().wait) + client: Final = _StalledClientSocket(second, third) + handler: Final = _make_streaming( + websocket=client, + backend_ws=backend_ws, + request_data={}, + first_message=first, + frame_guard=_guard_recording(bill_refusal), + turn_admission=AsyncMock(side_effect=[refusal, None]), + ) + relay: Final = asyncio.ensure_future(handler.bidirectional_forward()) + await asyncio.wait_for(client.send_started.wait(), timeout=1) + + relay.cancel() + with pytest.raises(asyncio.CancelledError): + await relay + + bill_refusal.assert_called_once_with(refusal) + + @pytest.mark.asyncio + async def test_managed_relay_leaves_a_refused_turn_s_guardrail_records_to_the_close_time_failure_row( + self, monkeypatch: pytest.MonkeyPatch + ): + from unittest.mock import AsyncMock + + import litellm + from litellm.exceptions import RateLimitError + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler + + answered: Final[list[object]] = [] + + async def fake_aresponses(*args: object, **kwargs: object): + answered.append(kwargs.get("input")) + + async def _empty(): + return + yield + + return _empty() + + monkeypatch.setattr(litellm, "aresponses", fake_aresponses) + first, second, third = _turn_frames() + refusal: Final = RateLimitError(message="crossed budget", llm_provider="", model="") + bill_refusal: Final = MagicMock() + handler: Final = ManagedResponsesWebSocketHandler( + websocket=_ClientSocket(second, third), + model="test-model", + logging_obj=Logging( + model="test-model", + messages=[], + stream=True, + call_type="aresponses", + start_time=0, + litellm_call_id="test-id", + function_id="test-func", + ), + first_message=first, + frame_guard=_guard_recording(bill_refusal), + turn_admission=AsyncMock(side_effect=[refusal, None]), + ) + + close_time_failure: Final = await handler.run() + + assert answered == ["one", "three"] + bill_refusal.assert_called_once_with(refusal) + assert close_time_failure is refusal + + @pytest.mark.asyncio + async def test_native_relay_answers_a_crashed_turn_check_with_a_500_and_keeps_serving(self): + from unittest.mock import AsyncMock + + first, second, third = _turn_frames() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.close = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=asyncio.Event().wait) + websocket: Final = _ClientSocket(second, third) + handler: Final = _make_streaming( + websocket=websocket, + backend_ws=backend_ws, + request_data={}, + first_message=first, + turn_admission=_admission_crashing_the_next_turn(), + ) + + failure: Final = await handler.bidirectional_forward() + + assert isinstance(failure, ValueError) + assert [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] == [ + json.loads(first), + json.loads(third), + ] + assert _errors(websocket) == [(500, {"type": "server_error", "message": "custom check failed"})] + assert websocket.close_code is None + + @pytest.mark.asyncio + async def test_managed_relay_answers_a_crashed_turn_check_with_a_500_and_keeps_serving( + self, monkeypatch: pytest.MonkeyPatch + ): + import litellm + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler + + answered: Final[list[object]] = [] + + async def fake_aresponses(*args: object, **kwargs: object): + answered.append(kwargs.get("input")) + + async def _empty(): + return + yield + + return _empty() + + monkeypatch.setattr(litellm, "aresponses", fake_aresponses) + first, second, third = _turn_frames() + websocket: Final = _ClientSocket(second, third) + handler: Final = ManagedResponsesWebSocketHandler( + websocket=websocket, + model="test-model", + logging_obj=Logging( + model="test-model", + messages=[], + stream=True, + call_type="aresponses", + start_time=0, + litellm_call_id="test-id", + function_id="test-func", + ), + first_message=first, + turn_admission=_admission_crashing_the_next_turn(), + ) + + failure: Final = await handler.run() + + assert isinstance(failure, ValueError) + assert answered == ["one", "three"] + assert _errors(websocket) == [(500, {"type": "server_error", "message": "custom check failed"})] + assert websocket.close_code is None + def _deployment_defaults(): from types import MappingProxyType diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 7eb3115117d..e5054ed3cef 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -2594,3 +2594,28 @@ def test_faked_stream_builds_each_refusal_summary_and_custom_tool_input_once_for (1, "content", 1): "No forecasts for Mars.", (2, "input", 0): "curl wttr.in/Berlin", } + + +@pytest.mark.asyncio +async def test_shutdown_drain_waits_for_a_pending_logging_task(): + row_written = asyncio.Event() + + async def write_row(): + await asyncio.sleep(0.05) + row_written.set() + + streaming_module.spawn_logging_task(asyncio.get_running_loop(), write_row(), task_name="row") + + await streaming_module.drain_logging_tasks(timeout=1) + + assert row_written.is_set() + + +@pytest.mark.asyncio +async def test_shutdown_drain_gives_up_on_a_stuck_logging_task_at_its_timeout(): + stuck = streaming_module.spawn_logging_task(asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck") + + await asyncio.wait_for(streaming_module.drain_logging_tasks(timeout=0.05), timeout=1) + + assert not stuck.done() + stuck.cancel() diff --git a/tests/unit/router_strategy/test_budget_limiter.py b/tests/unit/router_strategy/test_budget_limiter.py index 380aa573b3c..3c825bd560f 100644 --- a/tests/unit/router_strategy/test_budget_limiter.py +++ b/tests/unit/router_strategy/test_budget_limiter.py @@ -60,7 +60,9 @@ async def _log_success(limiter: RouterBudgetLimiting, kwargs: dict[str, object]) @pytest.mark.asyncio -@pytest.mark.parametrize("call_type", ["aresponses", "anthropic_messages", "aembedding", "arerank"]) +@pytest.mark.parametrize( + "call_type", ["aresponses", "_aresponses_websocket", "anthropic_messages", "aembedding", "arerank"] +) async def test_provider_spend_tracked_when_litellm_params_omits_provider(disable_budget_sync, call_type): """Non-chat surfaces carry the provider only on the standard logging payload.""" limiter = RouterBudgetLimiting( diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 4ac917d706d..ce60cce8408 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -1,6 +1,7 @@ import json import os from datetime import datetime, timezone +from collections.abc import Mapping from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -13,6 +14,7 @@ from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_ from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo from litellm.types.utils import ( + CallTypes, ModelResponse, StandardLoggingHiddenParams, StandardLoggingMetadata, @@ -482,6 +484,227 @@ async def test_deployment_callback_on_success(sync_mode): assert tpm_key is not None +@pytest.mark.asyncio +async def test_deployment_callback_on_success_counts_a_responses_websocket_turn(): + import time + + model_list = [ + { + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-fake", "rpm": 100}, + "model_info": {"id": "ws-deployment"}, + } + ] + router = Router(model_list=model_list) + standard_logging_payload = create_standard_logging_payload() + standard_logging_payload["total_tokens"] = 17 + + def turn_kwargs(call_type: str) -> dict: + return { + "call_type": call_type, + "litellm_params": { + "metadata": {"model_group": "gpt-5-mini"}, + "model_info": {"id": "ws-deployment"}, + }, + "standard_logging_object": standard_logging_payload, + } + + tpm_key = await router.deployment_callback_on_success( + kwargs=turn_kwargs("_aresponses_websocket"), + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + assert tpm_key is not None + assert await router.cache.async_get_cache(key=tpm_key) == 17 + + realtime_key = await router.deployment_callback_on_success( + kwargs=turn_kwargs("_arealtime"), + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + assert realtime_key is None + assert await router.cache.async_get_cache(key=tpm_key) == 17 + + +async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]: + routed: Final[dict[str, object]] = {} + + async def call(**kwargs: object) -> None: + routed.update(kwargs) + + call.__name__ = function_name + await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call) + return routed + + +async def _finish_35_token_websocket_turn(routed: Mapping[str, object], call_id: str) -> None: + import time + + from litellm.litellm_core_utils.litellm_logging import Logging + + turn: Final = Logging( + model="gpt-5-mini", + messages=[], + stream=False, + call_type=CallTypes.aresponses_websocket.value, + start_time=time.time(), + litellm_call_id=call_id, + function_id=call_id, + ) + turn.update_environment_variables(litellm_params={"litellm_metadata": routed["litellm_metadata"]}, optional_params={}) + await turn.dispatch_success_handlers( + [{"type": "response.completed", "response": {"usage": {"input_tokens": 20, "output_tokens": 15, "total_tokens": 35}}}], + prefer_async_handlers=True, + ) + + +@pytest.mark.asyncio +async def test_responses_websocket_turn_is_refused_once_its_deployment_crosses_budget(): + router = Router( + model_list=[ + { + "model_name": "ws-budget", + "litellm_params": { + "model": "openai/gpt-5-mini", + "api_key": "sk-fake", + "max_budget": 0.001, + "budget_duration": "1d", + }, + "model_info": {"id": "ws-budget-dep"}, + } + ] + ) + assert "turn_admission" not in await _kwargs_routed_to(router, "ws-budget", "aresponses") + admit_turn: Final = (await _kwargs_routed_to(router, "ws-budget", "_aresponses_websocket"))["turn_admission"] + assert callable(admit_turn) + + await admit_turn() + await router.cache.async_set_cache("deployment_spend:ws-budget-dep:1d", 0.002) + + with pytest.raises(litellm.RateLimitError, match="crossed budget"): + await admit_turn() + + +@pytest.mark.asyncio +async def test_responses_websocket_turn_after_the_first_counts_toward_deployment_rpm(): + router = Router( + model_list=[ + { + "model_name": "ws-rpm", + "litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "rpm": 1}, + "model_info": {"id": "ws-rpm-dep"}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + ) + admit_turn: Final = (await _kwargs_routed_to(router, "ws-rpm", "_aresponses_websocket"))["turn_admission"] + assert callable(admit_turn) + + with pytest.raises(litellm.RateLimitError, match="RPM limit=1"): + await admit_turn() + + +@pytest.mark.asyncio +async def test_responses_websocket_turn_is_checked_against_limits_changed_after_the_socket_opened(): + router = Router( + model_list=[ + { + "model_name": "ws-rpm-lowered", + "litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "rpm": 100}, + "model_info": {"id": "ws-rpm-lowered-dep"}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + ) + admit_turn: Final = (await _kwargs_routed_to(router, "ws-rpm-lowered", "_aresponses_websocket"))["turn_admission"] + assert callable(admit_turn) + await admit_turn() + + router.upsert_deployment( + Deployment( + model_name="ws-rpm-lowered", + litellm_params=LiteLLM_Params(model="openai/gpt-5-mini", api_key="sk-fake", rpm=1), + model_info=ModelInfo(id="ws-rpm-lowered-dep"), + ) + ) + + with pytest.raises(litellm.RateLimitError, match="RPM limit=1"): + await admit_turn() + + +@pytest.mark.asyncio +async def test_responses_websocket_turn_counts_toward_deployment_tpm(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "callbacks", []) + router = Router( + model_list=[ + { + "model_name": "ws-tpm", + "litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "tpm": 30}, + "model_info": {"id": "ws-tpm-dep"}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + ) + monkeypatch.setattr(litellm, "_async_success_callback", list(litellm.callbacks)) + routed: Final = await _kwargs_routed_to(router, "ws-tpm", "_aresponses_websocket") + admit_turn: Final = routed["turn_admission"] + assert callable(admit_turn) + await admit_turn() + + await _finish_35_token_websocket_turn(routed, "ws-tpm-turn") + + with pytest.raises(litellm.RateLimitError, match="TPM limit=30, current usage=35"): + await admit_turn() + + +@pytest.mark.asyncio +async def test_responses_websocket_turn_is_refused_by_usage_based_routing_once_deployment_tpm_is_used( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "callbacks", []) + router = Router( + model_list=[ + { + "model_name": "ws-tpm-v2", + "litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake", "tpm": 30}, + "model_info": {"id": "ws-tpm-v2-dep"}, + } + ], + routing_strategy="usage-based-routing-v2", + ) + monkeypatch.setattr(litellm, "_async_success_callback", list(litellm.callbacks)) + routed: Final = await _kwargs_routed_to(router, "ws-tpm-v2", "_aresponses_websocket") + admit_turn: Final = routed["turn_admission"] + assert callable(admit_turn) + await admit_turn() + + await _finish_35_token_websocket_turn(routed, "ws-tpm-v2-turn") + + with pytest.raises(litellm.RateLimitError): + await admit_turn() + + +@pytest.mark.asyncio +async def test_responses_websocket_turns_do_not_reserve_deployment_otpm(): + router = Router( + model_list=[ + { + "model_name": "ws-otpm", + "litellm_params": {"model": "openai/ws-otpm-unpriced", "api_key": "sk-fake", "otpm": 5000}, + "model_info": {"id": "ws-otpm-dep"}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + ) + admit_turn: Final = (await _kwargs_routed_to(router, "ws-otpm", "_aresponses_websocket"))["turn_admission"] + assert callable(admit_turn) + + for _ in range(3): + await admit_turn() + + @pytest.mark.asyncio async def test_deployment_callback_on_success_tracks_tpm_for_io_deployment(): """ From 63b740532b2c609707b7960357d5b9bd752d26e0 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 19 Oct 2026 09:03:42 -0700 Subject: [PATCH 20/50] refactor(rust): inline transcription provider resolution (#47214) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../inference-transcription/src/handler.rs | 60 ++++++++++++----- .../crates/inference-transcription/src/lib.rs | 43 +++++++++--- .../inference-transcription/src/prepare.rs | 65 ------------------- .../inference-transcription/src/types.rs | 38 ----------- 4 files changed, 79 insertions(+), 127 deletions(-) delete mode 100644 litellm-rust/crates/inference-transcription/src/prepare.rs diff --git a/litellm-rust/crates/inference-transcription/src/handler.rs b/litellm-rust/crates/inference-transcription/src/handler.rs index a6a18154bfd..5a7238c5e36 100644 --- a/litellm-rust/crates/inference-transcription/src/handler.rs +++ b/litellm-rust/crates/inference-transcription/src/handler.rs @@ -1,28 +1,57 @@ use std::time::Duration; -use litellm_http::{Client, request::truncate_error_body}; -use litellm_llms::base_llm::auth::resolve_auth; +use litellm_http::{ + Client, + request::{string_headers, truncate_error_body, with_default_headers}, +}; +use litellm_llms::base_llm::{ + audio_transcription::transformation::BaseAudioTranscriptionConfig, + auth::{ValidatedEnvironment, resolve_auth}, +}; +use litellm_secrets::source::Secrets; use serde_json::Value; use super::Error; -use crate::{ - constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::ProviderAudioTranscriptionRequest, -}; +use crate::{constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::AudioTranscriptionRequest}; +use litellm_inference::provider::ResolvedProvider; -pub async fn execute_audio_transcription_provider_call( +pub(super) async fn execute( http: &Client, auth: &litellm_auth::AuthServices, - request: ProviderAudioTranscriptionRequest, + config: &'static dyn BaseAudioTranscriptionConfig, + provider: ResolvedProvider<'_>, + call: AudioTranscriptionRequest<'_>, + secrets: Secrets, ) -> Result { - let env_lookup = |key: &str| request.secrets.get(key); - let authenticated = resolve_auth(auth, request.environment.clone(), &env_lookup).await?; + let env_lookup = |key: &str| secrets.get(key); + let forwarded = string_headers("audio transcription", call.extra_headers)?; + let validated = config.validate_environment( + forwarded, + provider.model, + &call.optional_params, + &env_lookup, + )?; + let environment = ValidatedEnvironment { + headers: with_default_headers(validated.headers, config.default_headers()), + auth: validated.auth, + }; + let url = config.get_complete_url( + call.api_base, + provider.model, + &call.optional_params, + &env_lookup, + )?; + let filtered_params = config.map_transcription_params(&call.optional_params); + let body = config + .transform_audio_transcription_request(provider.model, call.audio, filtered_params)? + .body; + let authenticated = resolve_auth(auth, environment, &env_lookup).await?; let outbound = litellm_inference::outbound::outbound_request( authenticated, - request.url.clone(), - &request.body, + url, + &body, Some( - request - .timeout + call.timeout .unwrap_or(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)), ), )?; @@ -47,8 +76,7 @@ pub async fn execute_audio_transcription_provider_call( error, )) })?; - Ok(request - .config - .transform_audio_transcription_response(&request.model, response_json)? + Ok(config + .transform_audio_transcription_response(provider.model, response_json)? .into_json()) } diff --git a/litellm-rust/crates/inference-transcription/src/lib.rs b/litellm-rust/crates/inference-transcription/src/lib.rs index d9daea29df9..47e7bc70b31 100644 --- a/litellm-rust/crates/inference-transcription/src/lib.rs +++ b/litellm-rust/crates/inference-transcription/src/lib.rs @@ -2,16 +2,29 @@ pub mod types; pub use litellm_inference::RouteError as Error; mod constants; mod handler; -mod prepare; -pub use handler::execute_audio_transcription_provider_call; + use litellm_auth::AuthServices; +use litellm_core_utils::get_llm_provider_logic::LlmProviders; +use litellm_inference::provider::resolve_llm_provider; +use litellm_llms::{ + base_llm::audio_transcription::transformation::BaseAudioTranscriptionConfig, + bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG, +}; use litellm_secrets::source::SecretSource; -pub use prepare::prepare_audio_transcription_provider_call; use serde_json::Value; use std::sync::Arc; use crate::types::AudioTranscriptionRequest; +fn audio_transcription_provider( + provider: LlmProviders, +) -> Option<&'static dyn BaseAudioTranscriptionConfig> { + match provider { + LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG), + _ => None, + } +} + #[derive(Clone)] pub struct AudioTranscriptionRoute { http: litellm_http::Client, @@ -32,6 +45,10 @@ impl AudioTranscriptionRoute { } } + pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result { + self.run(request).await + } + #[tracing::instrument(name = "litellm.route", skip_all, fields( route = "audio_transcription", model = request.model, @@ -40,13 +57,23 @@ impl AudioTranscriptionRoute { stream = false, outcome ))] - pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result { + async fn run(&self, request: AudioTranscriptionRequest<'_>) -> Result { litellm_inference::diagnostic::unary(async { - let request = - prepare_audio_transcription_provider_call(request, self.secrets.as_ref()).await?; - litellm_inference::diagnostic::provider(&request.model, &request.custom_llm_provider); + let provider = resolve_llm_provider( + request.model, + request.custom_llm_provider, + "audio transcription", + )?; + let config = audio_transcription_provider(provider.provider).ok_or_else(|| { + Error::InvalidProvider(<&str>::from(provider.provider).to_string()) + })?; + litellm_inference::diagnostic::provider( + provider.model, + <&'static str>::from(provider.provider), + ); + let secrets = self.secrets.resolve(&config.secret_names()).await?; let execute: futures_util::future::BoxFuture<'_, Result> = Box::pin( - execute_audio_transcription_provider_call(&self.http, &self.auth, request), + handler::execute(&self.http, &self.auth, config, provider, request, secrets), ); execute.await }) diff --git a/litellm-rust/crates/inference-transcription/src/prepare.rs b/litellm-rust/crates/inference-transcription/src/prepare.rs deleted file mode 100644 index 15ca7627339..00000000000 --- a/litellm-rust/crates/inference-transcription/src/prepare.rs +++ /dev/null @@ -1,65 +0,0 @@ -use litellm_core_utils::get_llm_provider_logic::LlmProviders; -use litellm_http::request::string_headers; -use litellm_http::request::with_default_headers; -use litellm_llms::{ - base_llm::{ - audio_transcription::transformation::BaseAudioTranscriptionConfig, - auth::ValidatedEnvironment, - }, - bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG, -}; -use litellm_secrets::source::SecretSource; - -use super::Error; -use crate::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; -use litellm_inference::provider::resolve_llm_provider; - -fn provider_config(provider: LlmProviders) -> Option<&'static dyn BaseAudioTranscriptionConfig> { - match provider { - LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG), - _ => None, - } -} - -#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)] -pub async fn prepare_audio_transcription_provider_call( - request: AudioTranscriptionRequest<'_>, - secrets: &dyn SecretSource, -) -> Result { - let provider_info = resolve_llm_provider( - request.model, - request.custom_llm_provider, - "audio transcription", - )?; - let model = provider_info.model.to_string(); - let config = provider_config(provider_info.provider) - .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?; - let snapshot = secrets.resolve(&config.secret_names()).await?; - let env_lookup = |key: &str| snapshot.get(key); - let forwarded = string_headers("audio transcription", request.extra_headers)?; - let validated = - config.validate_environment(forwarded, &model, &request.optional_params, &env_lookup)?; - let environment = ValidatedEnvironment { - headers: with_default_headers(validated.headers, config.default_headers()), - auth: validated.auth, - }; - let url = config.get_complete_url( - request.api_base, - &model, - &request.optional_params, - &env_lookup, - )?; - let filtered_params = config.map_transcription_params(&request.optional_params); - let transformed = - config.transform_audio_transcription_request(&model, request.audio, filtered_params)?; - Ok(ProviderAudioTranscriptionRequest { - model, - custom_llm_provider: <&str>::from(provider_info.provider).to_string(), - config, - url, - body: transformed.body, - environment, - secrets: snapshot, - timeout: request.timeout, - }) -} diff --git a/litellm-rust/crates/inference-transcription/src/types.rs b/litellm-rust/crates/inference-transcription/src/types.rs index 9b42ef9b3a4..7359d448262 100644 --- a/litellm-rust/crates/inference-transcription/src/types.rs +++ b/litellm-rust/crates/inference-transcription/src/types.rs @@ -1,9 +1,5 @@ -use litellm_secrets::source::Secrets; use std::time::Duration; -use litellm_llms::base_llm::{ - audio_transcription::transformation::BaseAudioTranscriptionConfig, auth::ValidatedEnvironment, -}; use serde_json::{Map, Value}; pub struct AudioTranscriptionRequest<'a> { @@ -16,37 +12,3 @@ pub struct AudioTranscriptionRequest<'a> { pub optional_params: Map, pub timeout: Option, } - -#[derive(Clone)] -pub struct ProviderAudioTranscriptionRequest { - pub model: String, - pub custom_llm_provider: String, - pub config: &'static dyn BaseAudioTranscriptionConfig, - pub url: String, - pub body: Value, - pub environment: ValidatedEnvironment, - pub secrets: Secrets, - pub timeout: Option, -} - -impl ProviderAudioTranscriptionRequest { - pub fn model(&self) -> &str { - &self.model - } - - pub fn custom_llm_provider(&self) -> &str { - &self.custom_llm_provider - } - - pub fn url(&self) -> &str { - &self.url - } - - pub fn body(&self) -> &Value { - &self.body - } - - pub fn with_body(self, body: Value) -> Self { - Self { body, ..self } - } -} From ede6d97785efa809fefc0af24dfa5f995128d248 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 19 Oct 2026 09:03:43 -0700 Subject: [PATCH 21/50] refactor(rust): separate responses provider resolution (#47215) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/inference-responses/src/handler.rs | 61 ++++--- .../crates/inference-responses/src/lib.rs | 108 ++++++++---- .../crates/inference-responses/src/prepare.rs | 162 ------------------ .../crates/inference-responses/src/types.rs | 19 +- .../inference-responses/tests/responses.rs | 105 ++++++++++++ 5 files changed, 232 insertions(+), 223 deletions(-) delete mode 100644 litellm-rust/crates/inference-responses/src/prepare.rs diff --git a/litellm-rust/crates/inference-responses/src/handler.rs b/litellm-rust/crates/inference-responses/src/handler.rs index eadedca7d3f..02182d23ae3 100644 --- a/litellm-rust/crates/inference-responses/src/handler.rs +++ b/litellm-rust/crates/inference-responses/src/handler.rs @@ -3,39 +3,61 @@ use litellm_host::observation::ObservationSender; use std::time::Duration; use futures_util::StreamExt; -use litellm_host::interceptors::{Interceptors, RawResponse, WireRequest}; -use litellm_llms::base_llm::auth::{Authenticated, resolve_auth}; +use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest}; +use litellm_llms::base_llm::auth::{AuthScheme, Authenticated, resolve_auth}; +use litellm_secrets::source::Secrets; use super::{ - Error, - types::{ProviderResponsesRequest, ResponsesOutput, ResponsesStreamHead}, + Error, ResponsesRoute, + types::{ResponsesCall, ResponsesOutput, ResponsesProvider, ResponsesStreamHead}, }; pub(super) async fn execute( - http: &litellm_http::Client, - auth: &litellm_auth::AuthServices, - request: ProviderResponsesRequest, - cache: Option, + route: &ResponsesRoute, + ResponsesProvider { identity, config }: ResponsesProvider, + call: ResponsesCall, + secrets: Secrets, cache_options: Option, interceptors: &impl Interceptors, observers: Option<&ObservationSender>, ) -> Result { - let authenticated = resolve_auth(auth, request.environment, &|_| None).await?; - let identity = litellm_host::interceptors::ProviderIdentity { - model: request.context.model.clone(), - provider: request.context.custom_llm_provider.clone(), + let lookup = |name: &str| secrets.get(name); + let environment = config.validate_environment( + litellm_http::request::string_headers("responses", call.extra_headers)?, + call.api_key.as_deref(), + &lookup, + )?; + let url = config.get_complete_url(call.api_base.as_deref(), &lookup); + let context = RequestContext { + model: identity.model.clone(), + custom_llm_provider: identity.provider.clone(), + optional_params: serde_json::Value::Object(call.optional_params.clone()), + secret_fields: Vec::new(), + api_key: match &environment.auth { + AuthScheme::Credential { secret, .. } => Some(secret.clone()), + _ => None, + }, }; + let body = config.transform_responses_api_request( + &identity.model, + call.input, + call.optional_params, + )?; + let authenticated = resolve_auth(&route.auth, environment, &|_| None).await?; let wire = interceptors .before_provider_request( WireRequest { - url: request.url, + url, headers: authenticated.headers, - body: request.body, + body, }, - request.context, + context, ) .await?; - let cache = cache.filter(|_| authenticated.signer.is_none()); + let cache = route + .cache + .clone() + .filter(|_| authenticated.signer.is_none()); let cache_request = litellm_inference::caching::CacheRequest::from_wire( identity, cache.as_ref().map(|_| &wire), @@ -59,9 +81,9 @@ pub(super) async fn execute( }, wire.url, &wire.body, - Some(request.timeout.unwrap_or(Duration::from_secs(600))), + Some(call.timeout.unwrap_or(Duration::from_secs(600))), )?; - let response = litellm_inference::outbound::send(outbound, http) + let response = litellm_inference::outbound::send(outbound, &route.http) .await .map_err(network)?; let status = response.status().as_u16(); @@ -103,8 +125,7 @@ pub(super) async fn execute( .map_err(Error::post_call)?; let value = serde_json::from_str(&body) .map_err(|error| Error::InvalidResponse(error.to_string().into()))?; - request - .config + config .transform_response_api_response(value) .map(ResponsesOutput::Complete) .map_err(Error::from) diff --git a/litellm-rust/crates/inference-responses/src/lib.rs b/litellm-rust/crates/inference-responses/src/lib.rs index 11939823b05..6a9d5372311 100644 --- a/litellm-rust/crates/inference-responses/src/lib.rs +++ b/litellm-rust/crates/inference-responses/src/lib.rs @@ -7,14 +7,30 @@ pub mod types; pub mod websocket; mod handler; -mod prepare; use std::sync::Arc; use litellm_auth::AuthServices; +use litellm_core_utils::get_llm_provider_logic::{ + CustomLlmProvider, LlmProviders, get_custom_llm_provider, +}; use litellm_host::interceptors::Interceptors; +use litellm_host::interceptors::ProviderIdentity; +use litellm_llms::{ + base_llm::responses::transformation::BaseResponsesApiConfig, + openai::responses::transformation::OpenAiResponsesApiConfig, + openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG, +}; use litellm_secrets::source::SecretSource; -use types::{ResponsesCall, ResponsesOutput}; +use types::{ResponsesCall, ResponsesOutput, ResponsesProvider}; + +fn responses_provider(provider: LlmProviders) -> Option<&'static dyn BaseResponsesApiConfig> { + match provider { + LlmProviders::Openai => Some(&OpenAiResponsesApiConfig), + LlmProviders::Openrouter => Some(&OPENROUTER_RESPONSES_CONFIG), + _ => None, + } +} #[derive(Clone)] pub struct ResponsesRoute { @@ -78,34 +94,68 @@ impl ResponsesRoute { observers: Option<&ObservationSender>, ) -> Result { litellm_inference::diagnostic::call(async { - self.run_provider(call, cache_options, interceptors, observers) - .await + let explicit = call + .custom_llm_provider + .as_deref() + .filter(|provider| !provider.is_empty()); + let (provider_name, resolved_model) = + match get_custom_llm_provider(&call.model, explicit) { + Some(CustomLlmProvider { + model, + custom_llm_provider, + }) => (custom_llm_provider, model), + None => ("openai", call.model.as_str()), + }; + if resolved_model.is_empty() { + return Err(Error::InvalidProvider(call.model)); + } + let provider = provider_name + .parse::() + .map_err(|_| Error::Unsupported("native HTTP responses provider"))?; + let config = responses_provider(provider) + .ok_or(Error::Unsupported("native HTTP responses provider"))?; + let wire_model = match provider { + LlmProviders::Openai if resolved_model.contains('/') => { + return Err(Error::InvalidProvider(call.model)); + } + LlmProviders::Openai => resolved_model.into(), + LlmProviders::Openrouter if explicit.is_some() && !resolved_model.contains('/') => { + format!("openrouter/{resolved_model}") + } + LlmProviders::Openrouter => resolved_model.into(), + _ => return Err(Error::Unsupported("native HTTP responses provider")), + }; + let provider = ResponsesProvider { + identity: ProviderIdentity { + model: wire_model, + provider: provider_name.into(), + }, + config, + }; + litellm_inference::diagnostic::provider( + &provider.identity.model, + &provider.identity.provider, + ); + let secrets = self + .secrets + .resolve( + provider + .config + .secret_names(call.api_key.as_deref(), call.api_base.as_deref()), + ) + .await?; + let execute: futures_util::future::BoxFuture<'_, Result> = + Box::pin(handler::execute( + self, + provider, + call, + secrets, + cache_options, + interceptors, + observers, + )); + execute.await }) .await } - - async fn run_provider( - &self, - call: ResponsesCall, - cache_options: Option, - interceptors: &impl Interceptors, - observers: Option<&ObservationSender>, - ) -> Result { - let request = prepare::prepare(call, self.secrets.as_ref()).await?; - litellm_inference::diagnostic::provider( - &request.context.model, - &request.context.custom_llm_provider, - ); - let execute: futures_util::future::BoxFuture<'_, Result> = - Box::pin(handler::execute( - &self.http, - &self.auth, - request, - self.cache.clone(), - cache_options, - interceptors, - observers, - )); - execute.await - } } diff --git a/litellm-rust/crates/inference-responses/src/prepare.rs b/litellm-rust/crates/inference-responses/src/prepare.rs deleted file mode 100644 index 05a9772abd4..00000000000 --- a/litellm-rust/crates/inference-responses/src/prepare.rs +++ /dev/null @@ -1,162 +0,0 @@ -use litellm_core_utils::get_llm_provider_logic::{ - CustomLlmProvider, LlmProviders, get_custom_llm_provider, -}; -use litellm_host::interceptors::{ProviderIdentity, RequestContext}; -use litellm_llms::{ - base_llm::responses::transformation::BaseResponsesApiConfig, - openai::responses::transformation::OpenAiResponsesApiConfig, - openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG, -}; -use litellm_secrets::source::SecretSource; -use serde_json::Value; - -use super::{ - Error, - types::{ProviderResponsesRequest, ResponsesCall}, -}; - -#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)] -pub(super) async fn prepare( - call: ResponsesCall, - secrets: &dyn SecretSource, -) -> Result { - let ResponsesProvider { identity, config } = - resolve_provider(&call.model, call.custom_llm_provider.as_deref())?; - let provider = identity.provider.as_str(); - let model = identity.model.as_str(); - let snapshot = secrets - .resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref())) - .await?; - let lookup = |name: &str| snapshot.get(name); - let environment = config.validate_environment( - litellm_http::request::string_headers("responses", call.extra_headers)?, - call.api_key.as_deref(), - &lookup, - )?; - let context = RequestContext { - model: model.into(), - custom_llm_provider: provider.into(), - optional_params: Value::Object(call.optional_params.clone()), - secret_fields: Vec::new(), - api_key: match &environment.auth { - litellm_llms::base_llm::auth::AuthScheme::Credential { secret, .. } => { - Some(secret.clone()) - } - _ => None, - }, - }; - let body = config.transform_responses_api_request(model, call.input, call.optional_params)?; - Ok(ProviderResponsesRequest { - url: config.get_complete_url(call.api_base.as_deref(), &lookup), - config, - environment, - body, - context, - timeout: call.timeout, - }) -} - -pub(super) struct ResponsesProvider { - pub identity: ProviderIdentity, - pub config: &'static dyn BaseResponsesApiConfig, -} - -/// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter -/// provider a model without a vendor segment keeps the `openrouter/` prefix as its wire id. -pub(super) fn resolve_provider( - model: &str, - custom_llm_provider: Option<&str>, -) -> Result { - let explicit = custom_llm_provider.filter(|provider| !provider.is_empty()); - let (provider, resolved) = match get_custom_llm_provider(model, explicit) { - Some(CustomLlmProvider { - model, - custom_llm_provider, - }) => (custom_llm_provider, model), - None => ("openai", model), - }; - if resolved.is_empty() { - return Err(Error::InvalidProvider(model.into())); - } - let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) = - match provider.parse::() { - Ok(LlmProviders::Openai) if resolved.contains('/') => { - return Err(Error::InvalidProvider(model.into())); - } - Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()), - Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => ( - &OPENROUTER_RESPONSES_CONFIG, - format!("openrouter/{resolved}"), - ), - Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()), - _ => return Err(Error::Unsupported("native HTTP responses provider")), - }; - Ok(ResponsesProvider { - identity: ProviderIdentity { - model: wire_model, - provider: provider.into(), - }, - config, - }) -} - -#[cfg(test)] -mod tests { - use rstest::rstest; - - use super::*; - - /// Pinned against Python's `get_llm_provider` on 2026-10-11. - #[rstest] - #[case::bare("gpt-5", None, "openai", "gpt-5")] - #[case::openai_prefix("openai/gpt-5", None, "openai", "gpt-5")] - #[case::openai_explicit("gpt-5", Some("openai"), "openai", "gpt-5")] - #[case::openrouter_prefix( - "openrouter/anthropic/claude", - None, - "openrouter", - "anthropic/claude" - )] - #[case::openrouter_explicit( - "openrouter/anthropic/claude", - Some("openrouter"), - "openrouter", - "anthropic/claude" - )] - #[case::openrouter_explicit_unprefixed( - "anthropic/claude", - Some("openrouter"), - "openrouter", - "anthropic/claude" - )] - #[case::openrouter_native_id( - "openrouter/auto", - Some("openrouter"), - "openrouter", - "openrouter/auto" - )] - #[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")] - fn providers_and_wire_models_match_python( - #[case] model: &str, - #[case] custom_llm_provider: Option<&str>, - #[case] provider: &str, - #[case] wire_model: &str, - ) { - let resolved = resolve_provider(model, custom_llm_provider).unwrap(); - assert_eq!(resolved.identity.provider, provider); - assert_eq!(resolved.identity.model, wire_model); - } - - #[rstest] - #[case::other_prefix("other/model", None)] - #[case::other_explicit("model", Some("other"))] - #[case::openai_with_vendor_segment("openai/other/model", None)] - #[case::empty_model("openai/", None)] - #[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))] - fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) { - assert!(matches!( - resolve_provider(model, custom_llm_provider), - Err(Error::InvalidProvider(_) | Error::Unsupported(_)) - )); - } -} diff --git a/litellm-rust/crates/inference-responses/src/types.rs b/litellm-rust/crates/inference-responses/src/types.rs index 18c64dc8178..e24cec801ba 100644 --- a/litellm-rust/crates/inference-responses/src/types.rs +++ b/litellm-rust/crates/inference-responses/src/types.rs @@ -2,9 +2,8 @@ use std::time::Duration; use bytes::Bytes; use litellm_host::call::CallOutput; -use litellm_llms::base_llm::{ - auth::ValidatedEnvironment, responses::transformation::BaseResponsesApiConfig, -}; +use litellm_host::interceptors::ProviderIdentity; +use litellm_llms::base_llm::responses::transformation::BaseResponsesApiConfig; use litellm_llms_types::formats::responses::ResponsesApiResponse; use serde_json::{Map, Value}; @@ -21,17 +20,13 @@ pub struct ResponsesCall { pub timeout: Option, } +pub(crate) struct ResponsesProvider { + pub identity: ProviderIdentity, + pub config: &'static dyn BaseResponsesApiConfig, +} + pub struct ResponsesStreamHead { pub headers: Vec<(String, String)>, } pub type ResponsesOutput = CallOutput; - -pub(super) struct ProviderResponsesRequest { - pub config: &'static dyn BaseResponsesApiConfig, - pub environment: ValidatedEnvironment, - pub url: String, - pub body: Value, - pub context: litellm_host::interceptors::RequestContext, - pub timeout: Option, -} diff --git a/litellm-rust/crates/inference-responses/tests/responses.rs b/litellm-rust/crates/inference-responses/tests/responses.rs index f5181aca2b1..ae07a71dabd 100644 --- a/litellm-rust/crates/inference-responses/tests/responses.rs +++ b/litellm-rust/crates/inference-responses/tests/responses.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use futures_util::TryStreamExt; use litellm_host::{call::HostedCompletion, lifecycle::CallEvent}; use litellm_inference_responses::{ + Error, route::Responses, types::{ResponsesCall, ResponsesOutput}, }; @@ -32,6 +33,110 @@ fn call() -> ResponsesCall { } } +#[rstest] +#[case::bare_openai("gpt-5", None, "gpt-5")] +#[case::openai_prefix("openai/gpt-5", None, "gpt-5")] +#[case::empty_explicit_provider("openai/gpt-5", Some(""), "gpt-5")] +#[case::explicit_openai("gpt-5", Some("openai"), "gpt-5")] +#[case::openrouter_prefix("openrouter/anthropic/claude", None, "anthropic/claude")] +#[case::explicit_openrouter_prefix( + "openrouter/anthropic/claude", + Some("openrouter"), + "anthropic/claude" +)] +#[case::explicit_openrouter_unprefixed("anthropic/claude", Some("openrouter"), "anthropic/claude")] +#[case::explicit_openrouter_native_id("auto", Some("openrouter"), "openrouter/auto")] +#[case::implicit_openrouter_native_id("openrouter/auto", None, "auto")] +#[tokio::test] +async fn provider_resolution_preserves_the_wire_model( + #[case] model: &str, + #[case] custom_llm_provider: Option<&str>, + #[case] expected_wire_model: &str, +) { + let response = json!({ + "id": "response-1", + "model": expected_wire_model, + "output": [{"type": "message", "content": []}], + "usage": {"total_tokens": 7} + }); + let upstream = upstream([json_response(response)]).await; + responses_route(no_secrets()) + .execute( + ResponsesCall { + model: model.into(), + api_key: Some("test-key".into()), + api_base: Some(upstream.uri()), + custom_llm_provider: custom_llm_provider.map(str::to_owned), + ..call() + }, + &(), + None, + ) + .await + .unwrap(); + + assert_eq!( + only_request(&upstream).await.json()["model"], + expected_wire_model + ); +} + +#[rstest] +#[case::explicit_unknown_provider("test-model", Some("unknown-provider"))] +#[case::unsupported_responses_provider("test-model", Some("anthropic"))] +#[tokio::test] +async fn unsupported_providers_fail_before_transport( + #[case] model: &str, + #[case] custom_llm_provider: Option<&str>, +) { + let upstream = upstream([]).await; + let result = responses_route(no_secrets()) + .execute( + ResponsesCall { + model: model.into(), + api_base: Some(upstream.uri()), + custom_llm_provider: custom_llm_provider.map(str::to_owned), + ..call() + }, + &(), + None, + ) + .await; + + assert!(matches!( + result, + Err(Error::Unsupported("native HTTP responses provider")) + )); + assert!(received(&upstream).await.is_empty()); +} + +#[rstest] +#[case::openai_with_vendor_segment("openai/other/model", None)] +#[case::empty_model("openai/", None)] +#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))] +#[tokio::test] +async fn invalid_provider_models_fail_before_transport( + #[case] model: &str, + #[case] custom_llm_provider: Option<&str>, +) { + let upstream = upstream([]).await; + let result = responses_route(no_secrets()) + .execute( + ResponsesCall { + model: model.into(), + api_base: Some(upstream.uri()), + custom_llm_provider: custom_llm_provider.map(str::to_owned), + ..call() + }, + &(), + None, + ) + .await; + + assert!(matches!(result, Err(Error::InvalidProvider(_)))); + assert!(received(&upstream).await.is_empty()); +} + #[rstest] #[case::direct(false)] #[case::hosted(true)] From bfcb9a028f7d29aae869d0b3446ce6b3f0f251b7 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 19 Oct 2026 09:03:44 -0700 Subject: [PATCH 22/50] refactor(rust): update OCR route orchestration (#47216) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/inference-ocr/src/arguments.rs | 4 +- .../crates/inference-ocr/src/client.rs | 81 ----- .../crates/inference-ocr/src/document.rs | 19 +- .../crates/inference-ocr/src/handler.rs | 296 ++++++++++++++++- litellm-rust/crates/inference-ocr/src/lib.rs | 78 ++++- .../crates/inference-ocr/src/prepare.rs | 309 ------------------ .../inference-ocr/src/provider_config.rs | 106 ++++-- .../crates/inference-ocr/src/types.rs | 13 +- .../inference-ocr/tests/ocr/lifecycle.rs | 13 + 9 files changed, 481 insertions(+), 438 deletions(-) delete mode 100644 litellm-rust/crates/inference-ocr/src/client.rs delete mode 100644 litellm-rust/crates/inference-ocr/src/prepare.rs diff --git a/litellm-rust/crates/inference-ocr/src/arguments.rs b/litellm-rust/crates/inference-ocr/src/arguments.rs index 05aa345f01d..1ee61189619 100644 --- a/litellm-rust/crates/inference-ocr/src/arguments.rs +++ b/litellm-rust/crates/inference-ocr/src/arguments.rs @@ -44,8 +44,8 @@ pub fn consumed_optional_param_names( model: &str, custom_llm_provider: Option<&str>, ) -> Result, Error> { - let (model, config) = resolve_provider_config(model, custom_llm_provider)?; - let provider_fields = config.get_supported_ocr_params(&model); + let (provider, config) = resolve_provider_config(model, custom_llm_provider)?; + let provider_fields = config.get_supported_ocr_params(provider.model); let auth_fields: &[&str] = match config { OcrConfigKind::AwsTextract | OcrConfigKind::AwsTextractAnalyze => AWS_AUTH_OPTION_FIELDS, OcrConfigKind::AzureAi diff --git a/litellm-rust/crates/inference-ocr/src/client.rs b/litellm-rust/crates/inference-ocr/src/client.rs deleted file mode 100644 index 21a4490048d..00000000000 --- a/litellm-rust/crates/inference-ocr/src/client.rs +++ /dev/null @@ -1,81 +0,0 @@ -use litellm_host::observation::ObservationSender; -use std::sync::Arc; - -use litellm_host::interceptors::Interceptors; -use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient}; -use litellm_llms_types::formats::ocr::LiteLLMOcrResponse; - -use super::{ - handler::perform_ocr_request, - types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest}, -}; - -#[derive(Clone)] -pub struct OcrRoute { - client: OcrClient, -} - -impl OcrRoute { - pub fn new(client: OcrClient) -> Self { - Self { client } - } - - pub async fn execute( - &self, - request: LiteLLMOcrRequest, - interceptors: &impl Interceptors, - observers: Option, - ) -> Result { - litellm_host::lifecycle::observe_unary( - observers.clone(), - self.run(request, interceptors, observers.as_ref()), - ) - .await - } - - #[tracing::instrument(name = "litellm.route", skip_all, fields( - route = "ocr", - model = %request.model, - resolved_model = %request.model, - provider = <&str>::from(request.config.provider()), - stream = false, - outcome - ))] - pub(super) async fn run( - &self, - request: LiteLLMOcrRequest, - interceptors: &impl Interceptors, - observers: Option<&ObservationSender>, - ) -> Result { - litellm_inference::diagnostic::unary(async { - let caller_document = matches!(&request.document, OcrDocumentInput::Document(_)); - let prepared = prepare_request_document(request).await?; - let execute: futures_util::future::BoxFuture<'_, Result> = - Box::pin(perform_ocr_request( - &self.client, - prepared, - interceptors, - caller_document, - observers, - )); - execute.await - }) - .await - } -} - -#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)] -async fn prepare_request_document( - request: LiteLLMOcrRequest, -) -> Result { - if let OcrDocumentInput::Document(_) = &request.document { - return request.map_document(super::document::prepare_document); - } - let logger = litellm_tracing::Logger::current(); - let span = tracing::Span::current(); - tokio::task::spawn_blocking(move || { - logger.scope(|| span.in_scope(|| request.map_document(super::document::prepare_document))) - }) - .await - .map_err(|error| Error::DocumentTask(Arc::new(error)))? -} diff --git a/litellm-rust/crates/inference-ocr/src/document.rs b/litellm-rust/crates/inference-ocr/src/document.rs index ede0ef2a510..0d9a296d289 100644 --- a/litellm-rust/crates/inference-ocr/src/document.rs +++ b/litellm-rust/crates/inference-ocr/src/document.rs @@ -1,10 +1,10 @@ -use std::{collections::BTreeMap as Map, io::Read, path::Path}; +use std::{collections::BTreeMap as Map, io::Read, path::Path, sync::Arc}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_llms::base_llm::ocr::{error::Error, transformation::OCR_INLINE_MAX_BYTES}; use litellm_llms_types::formats::ocr::OcrDocument; -use crate::types::OcrDocumentInput; +use crate::types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest}; pub fn prepare_document(input: OcrDocumentInput) -> Result { match input { @@ -24,6 +24,21 @@ pub fn prepare_document(input: OcrDocumentInput) -> Result { } } +pub(crate) async fn resolve_document( + request: LiteLLMOcrRequest, +) -> Result { + if let OcrDocumentInput::Document(_) = &request.document { + return request.map_document(prepare_document); + } + let logger = litellm_tracing::Logger::current(); + let span = tracing::Span::current(); + tokio::task::spawn_blocking(move || { + logger.scope(|| span.in_scope(|| request.map_document(prepare_document))) + }) + .await + .map_err(|error| Error::DocumentTask(Arc::new(error)))? +} + pub fn read_path_document(path: &Path, mime_type: Option<&str>) -> Result { let mut bytes = Vec::new(); std::fs::File::open(path) diff --git a/litellm-rust/crates/inference-ocr/src/handler.rs b/litellm-rust/crates/inference-ocr/src/handler.rs index 51b739f47ba..e9cfc9a8989 100644 --- a/litellm-rust/crates/inference-ocr/src/handler.rs +++ b/litellm-rust/crates/inference-ocr/src/handler.rs @@ -4,29 +4,45 @@ use litellm_host::{lifecycle::ExecutionEvent, observation::ObservationSender}; use litellm_llms::base_llm::ocr::{ error::Error, handler::{CallHooks, OcrClient}, - transformation::PreparedOcrRequest, + transformation::{OcrConnection, PreparedOcrRequest}, }; use litellm_llms_types::formats::ocr::LiteLLMOcrResponse; +use litellm_secrets::source::Secrets; use serde_json::Value; -use super::{arguments::is_secret_param, prepare::prepare_request, provider_config::OcrConfigKind}; -use crate::types::ResolvedOcrRequest; +use super::{arguments::is_secret_param, provider_config::OcrConfigKind}; +use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest}; -pub(crate) async fn perform_ocr_request( +pub(crate) async fn execute( client: &OcrClient, - request: ResolvedOcrRequest, + config: OcrConfigKind, + call: ResolvedOcrRequest, + secrets: Secrets, host: &impl Interceptors, caller_document: bool, observers: Option<&ObservationSender>, ) -> Result { - request.response_format()?; - let config = request.config; - let secrets = client - .secret_source() - .resolve(&config.secret_names()) - .await - .map_err(|error| Error::Secret(std::sync::Arc::new(error)))?; - let request = prepare_request(request, caller_document, client, secrets); + call.response_format()?; + let LiteLLMOcrRequest { + model, + document, + credentials, + transport, + optional_params, + input_sources, + azure_ad_token_provider, + .. + } = call; + let resolved = config.resolve_credentials(credentials, &secrets); + let request = PreparedOcrRequest { + model, + document, + connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets), + caller_document, + optional_params, + input_sources, + azure_ad_token_provider, + }; let interceptors = OcrCallHooks::new(host, &request, config, observers); config.ocr(client, &request, &interceptors).await } @@ -86,3 +102,257 @@ impl> CallHooks for OcrCallHooks<'_, H> { Box::pin(self.interceptors.after_provider_response(raw)) } } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options}; + use litellm_llms_types::formats::ocr::OcrResponseFormat; + use rstest::rstest; + use serde_json::{Value, json}; + use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any}; + + use super::*; + use crate::{ + OcrRoute, + types::LiteLLMOcrRequest, + wire::{OcrWireRequest, decode_request}, + }; + + fn client() -> OcrClient { + OcrClient::for_test( + litellm_http::Client::plain_for_test(), + litellm_http::Client::no_redirect_for_test(), + ) + .with_secrets(litellm_inference_testing::no_secrets()) + } + + fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest { + decode_request(OcrWireRequest { + model: model.into(), + document, + api_key: Some(litellm_auth::SecretValue::new("test-key")), + api_base: Some(base.into()), + custom_llm_provider: None, + extra_headers: None, + optional_params: options.as_object().unwrap().clone(), + input_sources: Default::default(), + timeout_seconds: Some(2.0), + }) + .unwrap() + } + + async fn run_request(request: LiteLLMOcrRequest) -> Result { + OcrRoute::new(client()).execute(request, &(), None).await + } + + async fn upstream(request_count: u64, response: Value) -> MockServer { + let server = MockServer::start().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(200).set_body_json(response)) + .up_to_n_times(request_count) + .mount(&server) + .await; + server + } + + fn image(url: &str) -> Value { + json!({"type": "image_url", "image_url": url}) + } + + fn body(request: &wiremock::Request) -> Value { + serde_json::from_slice(&request.body).unwrap() + } + + fn assert_mistral_request(request: &wiremock::Request) { + assert_eq!( + request + .headers + .get("authorization") + .unwrap() + .to_str() + .unwrap(), + "Bearer test-key" + ); + assert_eq!( + request + .headers + .get("content-type") + .unwrap() + .to_str() + .unwrap(), + "application/json" + ); + assert_eq!( + body(request), + json!({ + "model": "mistral-ocr-maas", + "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, + "pages": [0, 2], + "include_image_base64": true, + "unknown": "preserved" + }) + ); + } + + #[rstest] + #[tokio::test] + async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() { + let upstream = upstream( + 1, + json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}), + ) + .await; + let request = request( + "cohere/parse", + &upstream.uri(), + image("https://example.com/original.png"), + json!({ + "output_format": "markdown", "timeout": 30, + "extra_body": { + "output_format": {"future": true}, + "document": {"type": "image_url", "image_url": "https://example.com/a.png", + "provider_options": {"nested": [false, 0, null]}} + } + }), + ); + + run_request(request).await.unwrap(); + + let [request] = upstream + .received_requests() + .await + .unwrap() + .try_into() + .unwrap(); + assert_eq!( + body(&request), + json!({ + "model": "parse", "output_format": {"future": true}, + "document": {"type": "image_url", "image_url": "https://example.com/a.png", + "provider_options": {"nested": [false, 0, null]}} + }) + ); + } + + #[rstest] + #[tokio::test] + async fn explicit_null_options_use_defaults_before_http() { + let upstream = upstream( + 1, + json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}), + ) + .await; + let request = request( + "cohere/parse", + &upstream.uri(), + image("https://example.com/a.png"), + json!({"output_format": null, "req_format": null}), + ); + assert_eq!( + request.response_format().unwrap(), + OcrResponseFormat::Litellm + ); + + run_request(request).await.unwrap(); + + let [request] = upstream + .received_requests() + .await + .unwrap() + .try_into() + .unwrap(); + let body = body(&request); + assert_eq!(body["output_format"], "markdown"); + assert!(body.get("req_format").is_none()); + } + + #[rstest] + #[tokio::test] + async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() { + let upstream = upstream( + 2, + json!({ + "pages": [{"index": 0, "markdown": "hello"}], + "extra": "preserved" + }), + ) + .await; + let options = json!({ + "pages": [0, 2], + "include_image_base64": true, + "vertex_project": "project-1", + "vertex_location": "us-central1", + "unknown": "preserved" + }); + let document = + json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}); + let direct_call = request( + "mistral/mistral-ocr-maas", + &upstream.uri(), + document.clone(), + options.clone(), + ); + let vertex_call = request( + "vertex_ai/mistral-ocr-maas", + &upstream.uri(), + document, + options, + ); + assert_eq!(direct_call.transport.timeout, Some(Duration::from_secs(2))); + assert_eq!(vertex_call.transport.timeout, Some(Duration::from_secs(2))); + let direct_response = run_request(direct_call).await.unwrap().into_json(); + let vertex_response = run_request(vertex_call).await.unwrap().into_json(); + + let [direct_http, vertex_http] = upstream + .received_requests() + .await + .unwrap() + .try_into() + .unwrap(); + assert_eq!(direct_http.url.path(), "/v1/ocr"); + assert_eq!( + vertex_http.url.path(), + "/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict" + ); + assert_mistral_request(&direct_http); + assert_mistral_request(&vertex_http); + assert_eq!(direct_response, vertex_response); + assert_eq!(direct_response["model"], "mistral-ocr-maas"); + assert_eq!(direct_response["object"], "ocr"); + assert_eq!(direct_response["extra"], "preserved"); + } + + #[derive(serde::Deserialize)] + struct KnownParams { + pages: Option>, + } + + #[rstest] + fn parsed_provider_params_separates_known_and_extra_params() { + let arguments: CallArguments = serde_json::from_value(json!({ + "pages": [0, 2], + "future_ocr_option": true, + "extra_body": {"provider_option": "value"} + })) + .unwrap(); + let known: KnownParams = parse_options(&arguments).unwrap(); + assert_eq!(known.pages, Some(vec![0, 2])); + assert_eq!(arguments["future_ocr_option"], true); + assert_eq!(arguments["extra_body"], json!({"provider_option": "value"})); + assert_eq!( + arguments + .iter() + .filter(|(name, _)| name.as_str() != "pages") + .count(), + 2 + ); + assert_eq!( + compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(), + json!({ + "pages": [0, 2], "future_ocr_option": true, "provider_option": "value" + }) + ); + } +} diff --git a/litellm-rust/crates/inference-ocr/src/lib.rs b/litellm-rust/crates/inference-ocr/src/lib.rs index 45b664961b2..5320aa12583 100644 --- a/litellm-rust/crates/inference-ocr/src/lib.rs +++ b/litellm-rust/crates/inference-ocr/src/lib.rs @@ -1,10 +1,82 @@ pub mod arguments; -mod client; -pub use client::OcrRoute; pub mod document; pub(crate) mod handler; -pub(crate) mod prepare; pub mod provider_config; pub mod route; pub mod types; pub mod wire; + +use std::sync::Arc; + +use litellm_host::interceptors::Interceptors; +use litellm_host::observation::ObservationSender; +use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient}; +use litellm_llms_types::formats::ocr::LiteLLMOcrResponse; + +use crate::{ + handler::execute, + types::{LiteLLMOcrRequest, OcrDocumentInput}, +}; + +#[derive(Clone)] +pub struct OcrRoute { + client: OcrClient, +} + +impl OcrRoute { + pub fn new(client: OcrClient) -> Self { + Self { client } + } + + pub async fn execute( + &self, + request: LiteLLMOcrRequest, + interceptors: &impl Interceptors, + observers: Option, + ) -> Result { + litellm_host::lifecycle::observe_unary( + observers.clone(), + self.run(request, interceptors, observers.as_ref()), + ) + .await + } + + #[tracing::instrument(name = "litellm.route", skip_all, fields( + route = "ocr", + model = %call.model, + resolved_model = %call.model, + provider = <&str>::from(call.config.provider()), + stream = false, + outcome + ))] + async fn run( + &self, + call: LiteLLMOcrRequest, + interceptors: &impl Interceptors, + observers: Option<&ObservationSender>, + ) -> Result { + litellm_inference::diagnostic::unary(async { + let caller_document = matches!(&call.document, OcrDocumentInput::Document(_)); + let call = document::resolve_document(call).await?; + let config = call.config; + let secrets = self + .client + .secret_source() + .resolve(&config.secret_names()) + .await + .map_err(|error| Error::Secret(Arc::new(error)))?; + let execute: futures_util::future::BoxFuture<'_, Result> = + Box::pin(execute( + &self.client, + config, + call, + secrets, + interceptors, + caller_document, + observers, + )); + execute.await + }) + .await + } +} diff --git a/litellm-rust/crates/inference-ocr/src/prepare.rs b/litellm-rust/crates/inference-ocr/src/prepare.rs deleted file mode 100644 index 14120606864..00000000000 --- a/litellm-rust/crates/inference-ocr/src/prepare.rs +++ /dev/null @@ -1,309 +0,0 @@ -use litellm_auth::{InputSource, SecretValue, Sourced}; -use litellm_core_utils::get_llm_provider_logic::LlmProviders; -use litellm_llms::base_llm::ocr::{ - handler::OcrClient, - transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest}, -}; -use litellm_secrets::source::Secrets; - -use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest}; - -pub(crate) fn prepare_request( - request: ResolvedOcrRequest, - caller_document: bool, - client: &OcrClient, - secrets: Secrets, -) -> PreparedOcrRequest { - let credentials = request.credentials.clone(); - let (preferred_api_key_env, api_base_env) = match request.config.provider() { - LlmProviders::Mistral => ( - Some("MISTRAL_AZURE_API_KEY"), - Some("MISTRAL_AZURE_API_BASE"), - ), - LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")), - _ => (None, None), - }; - let secret = |name: &str| secrets.truthy(name); - let dynamic_api_key = credentials.dynamic_api_key.or_else(|| { - credentials.api_key.clone().or_else(|| { - preferred_api_key_env - .into_iter() - .chain(request.config.get_api_key_env_var()) - .find_map(secret) - .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment)) - }) - }); - let dynamic_api_base = credentials.dynamic_api_base.or_else(|| { - credentials.api_base.clone().or_else(|| { - api_base_env - .and_then(secret) - .map(|value| Sourced::new(value, InputSource::Environment)) - }) - }); - let resolved = request - .config - .resolve_connection_params(OcrCredentialInputs { - dynamic_api_key, - dynamic_api_base, - ..credentials - }); - let LiteLLMOcrRequest { - model, - document, - transport, - optional_params, - input_sources, - azure_ad_token_provider, - .. - } = request; - PreparedOcrRequest { - model, - document, - connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets), - caller_document, - optional_params, - input_sources, - azure_ad_token_provider, - } -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use futures_util::future::BoxFuture; - use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options}; - use litellm_host::interceptors::WireRequest; - use litellm_llms::{ - base_llm::ocr::{ - error::Error, - handler::{CallHooks, OcrClient}, - transformation::BaseOcrConfig, - }, - cohere::ocr::transformation::CohereParseConfig, - mistral::ocr::transformation::MistralOcrConfig, - vertex_ai::ocr::transformation::VertexAiOcrConfig, - }; - use litellm_llms_types::formats::ocr::OcrResponseFormat; - use serde_json::{Value, json}; - - use super::*; - use crate::{ - document::prepare_document, - types::LiteLLMOcrRequest, - wire::{OcrWireRequest, decode_request}, - }; - - /// Stands in for a host with no interceptors registered. - struct NoHooks; - - impl CallHooks for NoHooks { - fn before_provider_request( - &self, - wire: WireRequest, - ) -> BoxFuture<'_, Result> { - Box::pin(async move { Ok(wire) }) - } - - fn response_received<'a>(&'a self, _body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> { - Box::pin(async { Ok(()) }) - } - } - - fn client() -> OcrClient { - OcrClient::for_test( - litellm_http::Client::plain_for_test(), - litellm_http::Client::no_redirect_for_test(), - ) - } - - fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest { - decode_request(OcrWireRequest { - model: model.into(), - document, - api_key: Some(litellm_auth::SecretValue::new("test-key")), - api_base: Some(base.into()), - custom_llm_provider: None, - extra_headers: None, - optional_params: options.as_object().unwrap().clone(), - input_sources: Default::default(), - timeout_seconds: Some(2.0), - }) - .unwrap() - } - - fn prepared(request: LiteLLMOcrRequest) -> PreparedOcrRequest { - prepare_request( - request.map_document(prepare_document).unwrap(), - true, - &client(), - std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment), - ) - } - - fn image(url: &str) -> Value { - json!({"type": "image_url", "image_url": url}) - } - - #[rstest::rstest] - #[tokio::test] - async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() { - let request = request( - "cohere/parse", - "https://example.com", - image("https://example.com/original.png"), - json!({ - "output_format": "markdown", "timeout": 30, - "extra_body": { - "output_format": {"future": true}, - "document": {"type": "image_url", "image_url": "https://example.com/a.png", - "provider_options": {"nested": [false, 0, null]}} - } - }), - ); - - let http = CohereParseConfig - .prepare_request(&prepared(request), &client(), &NoHooks) - .await - .unwrap(); - - let body: Value = serde_json::from_slice(http.body()).unwrap(); - assert_eq!( - body, - json!({ - "model": "parse", "output_format": {"future": true}, - "document": {"type": "image_url", "image_url": "https://example.com/a.png", - "provider_options": {"nested": [false, 0, null]}} - }) - ); - } - - #[rstest::rstest] - #[tokio::test] - async fn explicit_null_options_use_defaults_before_http() { - let request = request( - "cohere/parse", - "https://example.com", - image("https://example.com/a.png"), - json!({"output_format": null, "req_format": null}), - ); - assert_eq!( - request.response_format().unwrap(), - OcrResponseFormat::Litellm - ); - - let http = CohereParseConfig - .prepare_request(&prepared(request), &client(), &NoHooks) - .await - .unwrap(); - - let body: Value = serde_json::from_slice(http.body()).unwrap(); - assert_eq!(body["output_format"], "markdown"); - assert!(body.get("req_format").is_none()); - } - - #[rstest::rstest] - #[tokio::test] - async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() { - let options = json!({ - "pages": [0, 2], - "include_image_base64": true, - "vertex_project": "project-1", - "vertex_location": "us-central1", - "unknown": "preserved" - }); - let document = - json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}); - let direct = prepared(request( - "mistral/mistral-ocr-maas", - "https://mistral.test", - document.clone(), - options.clone(), - )); - let vertex = prepared(request( - "vertex_ai/mistral-ocr-maas", - "https://vertex.test", - document, - options, - )); - - let direct_http = MistralOcrConfig - .prepare_request(&direct, &client(), &NoHooks) - .await - .unwrap(); - let vertex_http = VertexAiOcrConfig - .prepare_request(&vertex, &client(), &NoHooks) - .await - .unwrap(); - - assert_eq!(direct_http.url(), "https://mistral.test/v1/ocr"); - assert_eq!( - vertex_http.url(), - "https://vertex.test/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict" - ); - for http in [&direct_http, &vertex_http] { - assert_eq!(http.header("authorization").unwrap(), "Bearer test-key"); - assert_eq!(http.header("content-type").unwrap(), "application/json"); - assert_eq!(http.timeout(), Some(Duration::from_secs(2))); - let body: Value = serde_json::from_slice(http.body()).unwrap(); - assert_eq!( - body, - json!({ - "model": "mistral-ocr-maas", - "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, - "pages": [0, 2], - "include_image_base64": true, - "unknown": "preserved" - }) - ); - } - let payload = serde_json::to_vec( - &json!({"pages": [{"index": 0, "markdown": "hello"}], "extra": "preserved"}), - ) - .unwrap(); - let direct_response = MistralOcrConfig - .transform_ocr_response(&direct.model, &payload, OcrResponseFormat::Litellm) - .unwrap() - .into_json(); - let vertex_response = VertexAiOcrConfig - .transform_ocr_response(&vertex.model, &payload, OcrResponseFormat::Litellm) - .unwrap() - .into_json(); - assert_eq!(direct_response, vertex_response); - assert_eq!(direct_response["model"], "mistral-ocr-maas"); - assert_eq!(direct_response["object"], "ocr"); - assert_eq!(direct_response["extra"], "preserved"); - } - - #[derive(serde::Deserialize)] - struct KnownParams { - pages: Option>, - } - - #[rstest::rstest] - fn parsed_provider_params_separates_known_and_extra_params() { - let arguments: CallArguments = serde_json::from_value(json!({ - "pages": [0, 2], - "future_ocr_option": true, - "extra_body": {"provider_option": "value"} - })) - .unwrap(); - let known: KnownParams = parse_options(&arguments).unwrap(); - assert_eq!(known.pages, Some(vec![0, 2])); - assert_eq!(arguments["future_ocr_option"], true); - assert_eq!(arguments["extra_body"], json!({"provider_option": "value"})); - assert_eq!( - arguments - .iter() - .filter(|(name, _)| name.as_str() != "pages") - .count(), - 2 - ); - assert_eq!( - compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(), - json!({ - "pages": [0, 2], "future_ocr_option": true, "provider_option": "value" - }) - ); - } -} diff --git a/litellm-rust/crates/inference-ocr/src/provider_config.rs b/litellm-rust/crates/inference-ocr/src/provider_config.rs index da708d17b86..2f0346f6e47 100644 --- a/litellm-rust/crates/inference-ocr/src/provider_config.rs +++ b/litellm-rust/crates/inference-ocr/src/provider_config.rs @@ -1,5 +1,6 @@ -use litellm_core_utils::get_llm_provider_logic::LlmProviders; -use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; +use litellm_auth::{InputSource, SecretValue, Sourced}; +use litellm_core_utils::get_llm_provider_logic::{LlmProviders, get_custom_llm_provider}; +use litellm_inference::provider::{ResolvedProvider, resolve_llm_provider}; use litellm_llms::{ aws_textract::ocr::{ analyze_transformation::TextractAnalyzeDocumentConfig, common_utils::TextractOperation, @@ -25,6 +26,7 @@ use litellm_llms::{ }, }; use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrResponseFormat}; +use litellm_secrets::source::Secrets; macro_rules! with_config { ($kind:expr, $config:ident => $body:expr) => { @@ -115,7 +117,18 @@ impl OcrConfigKind { } pub(crate) fn secret_names(self) -> Vec<&'static str> { - with_config!(self, config => config.secret_names()) + let names = with_config!(self, config => config.secret_names()); + let fallback_names: &[&str] = match self.provider() { + LlmProviders::Mistral => &["MISTRAL_AZURE_API_KEY", "MISTRAL_AZURE_API_BASE"], + LlmProviders::AzureAi => &["AZURE_AI_API_BASE"], + _ => &[], + }; + let additional_names = fallback_names + .iter() + .copied() + .filter(|name| !names.contains(name)) + .collect::>(); + names.into_iter().chain(additional_names).collect() } pub(crate) fn get_health_check_document(self) -> OcrDocument { @@ -129,6 +142,43 @@ impl OcrConfigKind { with_config!(self, config => config.resolve_connection_params(inputs)) } + pub(crate) fn resolve_credentials( + self, + credentials: OcrCredentialInputs, + secrets: &Secrets, + ) -> ResolvedOcrCredentials { + let (preferred_api_key_env, api_base_env) = match self.provider() { + LlmProviders::Mistral => ( + Some("MISTRAL_AZURE_API_KEY"), + Some("MISTRAL_AZURE_API_BASE"), + ), + LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")), + _ => (None, None), + }; + let secret = |name: &str| secrets.truthy(name); + let dynamic_api_key = credentials.dynamic_api_key.or_else(|| { + credentials.api_key.clone().or_else(|| { + preferred_api_key_env + .into_iter() + .chain(self.get_api_key_env_var()) + .find_map(secret) + .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment)) + }) + }); + let dynamic_api_base = credentials.dynamic_api_base.or_else(|| { + credentials.api_base.clone().or_else(|| { + api_base_env + .and_then(secret) + .map(|value| Sourced::new(value, InputSource::Environment)) + }) + }); + self.resolve_connection_params(OcrCredentialInputs { + dynamic_api_key, + dynamic_api_base, + ..credentials + }) + } + pub(crate) async fn ocr( self, client: &OcrClient, @@ -164,7 +214,7 @@ pub fn passthrough_response( endpoint: &str, body: &[u8], ) -> Result, Error> { - let (model, config) = resolve_provider_config(model, Some("azure_ai"))?; + let (provider, config) = resolve_provider_config(model, Some("azure_ai"))?; let segments: Vec<&str> = endpoint .split('/') .filter(|segment| !segment.is_empty()) @@ -173,7 +223,7 @@ pub fn passthrough_response( OcrConfigKind::AzureAi => segments == AZURE_AI_OCR_PATH, OcrConfigKind::AzureCohere => segments == AZURE_COHERE_PARSE_PATH, OcrConfigKind::AzureDocumentIntelligence => { - segments == AzureDocumentIntelligenceOcrConfig::analyze_path(&model)? + segments == AzureDocumentIntelligenceOcrConfig::analyze_path(provider.model)? } other => { let provider: &'static str = other.provider().into(); @@ -183,24 +233,28 @@ pub fn passthrough_response( if !is_ocr_endpoint { return Ok(None); } - with_config!(config, config => config.transform_ocr_response(&model, body, OcrResponseFormat::Litellm)) + with_config!(config, config => config.transform_ocr_response(provider.model, body, OcrResponseFormat::Litellm)) .map(Some) } -pub(crate) fn resolve_provider_config( - model: &str, - custom_llm_provider: Option<&str>, -) -> Result<(String, OcrConfigKind), Error> { - let provider = - get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider { - model, - custom_llm_provider: LlmProviders::Mistral.into(), - }); - let llm_provider = provider - .custom_llm_provider - .parse::() - .map_err(|_| Error::InvalidProvider(provider.custom_llm_provider.to_string()))?; - let config = match llm_provider { +pub(crate) fn resolve_provider_config<'a>( + model: &'a str, + custom_llm_provider: Option<&'a str>, +) -> Result<(ResolvedProvider<'a>, OcrConfigKind), Error> { + let provider = match resolve_llm_provider(model, custom_llm_provider, "ocr") { + Ok(provider) => provider, + Err(_) if get_custom_llm_provider(model, custom_llm_provider).is_none() => { + ResolvedProvider { + model, + provider: LlmProviders::Mistral, + } + } + Err(litellm_inference::RouteError::InvalidProvider(provider)) => { + return Err(Error::InvalidProvider(provider)); + } + Err(error) => return Err(Error::InvalidProvider(error.to_string())), + }; + let config = match provider.provider { LlmProviders::AwsTextract => match TextractOperation::from_model(provider.model)? { TextractOperation::DetectDocumentText => OcrConfigKind::AwsTextract, TextractOperation::AnalyzeDocument => OcrConfigKind::AwsTextractAnalyze, @@ -227,11 +281,11 @@ pub(crate) fn resolve_provider_config( LlmProviders::VertexAi => OcrConfigKind::VertexAi, _ => { return Err(Error::InvalidProvider( - provider.custom_llm_provider.to_string(), + <&'static str>::from(provider.provider).to_string(), )); } }; - Ok((provider.model.to_string(), config)) + Ok((provider, config)) } fn is_document_intelligence_model(model: &str) -> bool { @@ -243,7 +297,7 @@ fn is_document_intelligence_model(model: &str) -> bool { mod tests { use std::collections::HashSet; - use litellm_auth::{InputSource, Sourced}; + use litellm_auth::Sourced; use litellm_llms::{ base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document, }; @@ -484,8 +538,8 @@ mod tests { #[case] expected_config: OcrConfigKind, ) { let expected_model = qualified_model.split_once('/').unwrap().1; - let (model, config) = resolve_provider_config(qualified_model, None).unwrap(); - assert_eq!(model, expected_model); + let (provider, config) = resolve_provider_config(qualified_model, None).unwrap(); + assert_eq!(provider.model, expected_model); assert_eq!(config, expected_config); } @@ -528,7 +582,7 @@ mod tests { expected_config ); assert_eq!( - resolve_provider_config(model, None).unwrap().0, + resolve_provider_config(model, None).unwrap().0.model, model.split_once('/').unwrap().1 ); } diff --git a/litellm-rust/crates/inference-ocr/src/types.rs b/litellm-rust/crates/inference-ocr/src/types.rs index fda65e3284e..1bd70c06116 100644 --- a/litellm-rust/crates/inference-ocr/src/types.rs +++ b/litellm-rust/crates/inference-ocr/src/types.rs @@ -75,6 +75,8 @@ impl OcrConnectionInputs { pub struct LiteLLMOcrRequest { pub model: String, + pub(crate) requested_model: String, + pub(crate) custom_llm_provider: Option, pub document: D, pub credentials: OcrCredentialInputs, pub transport: OcrTransportConfig, @@ -91,7 +93,8 @@ impl LiteLLMOcrRequest { custom_llm_provider: Option<&str>, optional_params: CallArguments, ) -> Result { - let (model, config) = resolve_provider_config(&model, custom_llm_provider)?; + let (provider, config) = resolve_provider_config(&model, custom_llm_provider)?; + let resolved_model = provider.model.to_owned(); let default_transport = OcrTransportConfig::default(); let max_response_bytes = optional_params .get("max_response_bytes") @@ -116,7 +119,9 @@ impl LiteLLMOcrRequest { .collect(); Ok(Self { - model, + model: resolved_model, + requested_model: model, + custom_llm_provider: custom_llm_provider.map(str::to_owned), document: document.into(), credentials: OcrCredentialInputs::default(), transport, @@ -135,6 +140,8 @@ impl LiteLLMOcrRequest { ) -> Result, E> { Ok(LiteLLMOcrRequest { model: self.model, + requested_model: self.requested_model, + custom_llm_provider: self.custom_llm_provider, document: map(self.document)?, credentials: self.credentials, transport: self.transport, @@ -148,6 +155,8 @@ impl LiteLLMOcrRequest { pub fn with_document(self, document: T) -> LiteLLMOcrRequest { LiteLLMOcrRequest { model: self.model, + requested_model: self.requested_model, + custom_llm_provider: self.custom_llm_provider, document, credentials: self.credentials, transport: self.transport, diff --git a/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs b/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs index d10a2e23c9a..e413781f1a3 100644 --- a/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs +++ b/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs @@ -66,6 +66,19 @@ async fn interception_runs_once_and_observers_receive_ordered_success_events() { assert_eq!(received(&upstream).await.len(), 1); } +#[rstest::rstest] +#[case::mutated_model("model-b")] +#[tokio::test] +async fn provider_request_uses_the_request_model_after_it_is_mutated(#[case] model: &str) { + let upstream = upstream([pages_response()]).await; + let mut request = ocr_request("mistral/model-a", &upstream.uri(), json!({})); + request.model = model.into(); + + perform(request).await.unwrap(); + + assert_eq!(only_request(&upstream).await.json()["model"], json!(model)); +} + #[rstest::rstest] #[tokio::test] async fn a_blocking_before_send_prevents_the_call_and_emits_one_failure() { From 97e93005777887ea06b61c5553f154f7c5f172ae Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 19 Oct 2026 09:03:44 -0700 Subject: [PATCH 23/50] refactor(rust): inline Messages provider resolution (#47217) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/inference-messages/AGENTS.md | 4 +- .../inference-messages/src/common_utils.rs | 22 +- .../crates/inference-messages/src/handler.rs | 821 +++++++++++++++--- .../crates/inference-messages/src/lib.rs | 55 +- .../crates/inference-messages/src/prepare.rs | 554 ------------ .../src/anthropic/messages/transformation.rs | 3 + .../src/azure_ai/messages/transformation.rs | 5 +- .../src/base_llm/messages/transformation.rs | 6 +- .../messages/transformation.rs | 25 +- .../anthropic_claude3_transformation.rs | 5 +- .../bedrock_mantle/messages/transformation.rs | 22 +- .../src/deepseek/messages/transformation.rs | 2 + .../src/edenai/messages/transformation.rs | 1 + .../src/minimax/messages/transformation.rs | 10 +- .../src/openrouter/messages/transformation.rs | 1 + .../src/tencent/messages/transformation.rs | 10 +- .../src/vertex_ai/messages/transformation.rs | 2 + 17 files changed, 818 insertions(+), 730 deletions(-) delete mode 100644 litellm-rust/crates/inference-messages/src/prepare.rs diff --git a/litellm-rust/crates/inference-messages/AGENTS.md b/litellm-rust/crates/inference-messages/AGENTS.md index 860eb83c1f3..6c4fad481ae 100644 --- a/litellm-rust/crates/inference-messages/AGENTS.md +++ b/litellm-rust/crates/inference-messages/AGENTS.md @@ -7,8 +7,8 @@ - delegate authentication policy, beta selection, payload rewriting, and response interpretation to those adapters - keep provider policy out of request preparation and transport handlers - calling a concrete provider helper for every provider is still a policy dependency -- Route types such as `MessagesCall`, prepared requests, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here +- Route types such as `MessagesCall`, provider calls, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here - The route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks -- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `src/lib.rs` explicitly sequences cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage +- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `handler::execute` explicitly sequences the provider call build in Python's handler order, cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage - Preserve the order of validation, normalization, caller-requested parameter removal, and provider transformation when that order affects observable behavior - Test provider dispatch, auth precedence, header handling, transformations, and responses through behavior, not source structure diff --git a/litellm-rust/crates/inference-messages/src/common_utils.rs b/litellm-rust/crates/inference-messages/src/common_utils.rs index 66e7e1cfe25..7394d7ae21e 100644 --- a/litellm-rust/crates/inference-messages/src/common_utils.rs +++ b/litellm-rust/crates/inference-messages/src/common_utils.rs @@ -82,17 +82,17 @@ mod tests { use litellm_core_utils::get_llm_provider_logic::LlmProviders; #[rstest] - #[case::tencent("tencent")] - #[case::anthropic("anthropic")] - #[case::azure_ai("azure_ai")] - #[case::bedrock("bedrock")] - #[case::deepseek("deepseek")] - #[case::edenai("edenai")] - #[case::minimax("minimax")] - #[case::vertex_ai("vertex_ai")] - fn provider_keeps_its_python_name(#[case] name: &str) { - let provider = - messages_provider(name.parse::().unwrap(), "claude-sonnet-4-5").unwrap(); + #[case::tencent(LlmProviders::Tencent, "tencent")] + #[case::anthropic(LlmProviders::Anthropic, "anthropic")] + #[case::azure_ai(LlmProviders::AzureAi, "azure_ai")] + #[case::bedrock(LlmProviders::Bedrock, "bedrock")] + #[case::deepseek(LlmProviders::Deepseek, "deepseek")] + #[case::edenai(LlmProviders::Edenai, "edenai")] + #[case::minimax(LlmProviders::Minimax, "minimax")] + #[case::vertex_ai(LlmProviders::VertexAi, "vertex_ai")] + #[case::openrouter(LlmProviders::Openrouter, "openrouter")] + fn provider_keeps_its_python_name(#[case] provider: LlmProviders, #[case] name: &str) { + let provider = messages_provider(provider, "claude-sonnet-4-5").unwrap(); assert_eq!(provider.as_str(), name); } diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index c4032baa98f..55c80d9bddf 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -1,148 +1,271 @@ use std::time::Duration; use bytes::Bytes; -use futures_util::{StreamExt, TryStreamExt, stream::BoxStream}; -use litellm_host::interceptors::{Interceptors, ProviderIdentity, RequestContext, WireRequest}; -use litellm_http::transport::Error as TransportError; +use futures_util::{FutureExt, StreamExt, TryStreamExt, stream::BoxStream}; +use litellm_auth::{AuthServices, SecretValue}; +use litellm_core_utils::{ + dot_notation_indexing::delete_nested_value, + get_provider_specific_headers::get_provider_specific_headers, settings::Lookup, +}; +use litellm_host::interceptors::{ + Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest, +}; +use litellm_http::{request::with_default_headers, transport::Error as TransportError}; use litellm_llms::base_llm::{ - auth::{Authenticated, resolve_auth}, + auth::{Authenticated, ValidatedEnvironment, resolve_auth}, messages::{ + context::MessagesTransformContext, streaming::{self, ByteStream, StreamDecoder}, transformation::BaseMessagesConfig, }, }; -use litellm_llms_types::formats::messages::MessagesResponse; +use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse}; +use litellm_secrets::source::Secrets; use litellm_tracing::ByteChunk; use serde_json::Value; use super::{ - Error, MessagesCallResponse, MessagesRoute, common_utils::truncate_error_body, - prepare::ProviderMessagesRequest, + Error, MessagesCall, MessagesCallResponse, MessagesRoute, + common_utils::{MessagesProvider, string_headers, truncate_error_body}, + constants::MESSAGES_TIMEOUT_SECS, + route::Messages, + types::invalid_request, }; -use crate::constants::MESSAGES_TIMEOUT_SECS; -use litellm_inference::{context::CallContext, outbound::outbound_request}; +use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request}; -pub(super) struct ProviderCall { - pub identity: ProviderIdentity, - pub wire: WireRequest, - pub(super) provider: super::common_utils::MessagesProvider, +struct ProviderCall { + identity: ProviderIdentity, + wire: WireRequest, + provider: MessagesProvider, signer: Option, timeout: Option, stream: bool, } impl ProviderCall { - pub fn cacheable(&self) -> bool { + fn cacheable(&self) -> bool { self.signer.is_none() } } -impl MessagesRoute { - pub(super) async fn prepare_outbound( - &self, - request: ProviderMessagesRequest, - context: &CallContext<'_, impl Interceptors>, - ) -> Result { - let ProviderMessagesRequest { - provider, - url, - body, - environment, - timeout, - api_key, - } = request; - let request_context = RequestContext { - model: body.model.clone(), - custom_llm_provider: provider.as_str().to_string(), - optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?, - secret_fields: Vec::new(), - api_key, - }; - let authenticated = - resolve_auth(&self.auth, environment, &|key| std::env::var(key).ok()).await?; - let identity = ProviderIdentity { - model: request_context.model.clone(), - provider: request_context.custom_llm_provider.clone(), - }; - let wire = context - .interceptors - .before_provider_request( - WireRequest { - url, - headers: authenticated.headers, - body: provider - .config() - .wire_body(serde_json::to_value(&body).map_err(serialize_failure)?), - }, - request_context, - ) - .await?; - let stream = match wire.body.get("stream") { - None | Some(Value::Null) => false, - Some(Value::Bool(stream)) => *stream, - Some(value) => { - return Err(Error::InvalidRequest( - litellm_llms::ErrorDetail::InvalidValue { - field: "stream", - expected: "a boolean", - actual: value.clone(), - }, - )); - } - }; - Ok(ProviderCall { - identity, - wire, - provider, - signer: authenticated.signer, - timeout, - stream, +pub(super) async fn execute( + route: &MessagesRoute, + provider: MessagesProvider, + model: String, + call: MessagesCall, + secrets: Secrets, + context: &CallContext<'_, impl Interceptors>, +) -> Result { + let request = provider_call( + &route.auth, + provider, + model, + call, + secrets.as_ref(), + context, + ) + .boxed() + .await?; + let cache = CallCache::::from_wire( + route.cache.as_ref().filter(|_| request.cacheable()), + context.cache, + &request.identity, + &request.wire, + ); + let identity = request.identity.clone(); + let (output, source) = match cache.lookup().await { + Some(hit) => hit, + None => ( + call_provider(&route.http, request, context).await?, + ResultSource::Provider, + ), + }; + let reported = match &output { + MessagesCallResponse::Complete(response) => provider.config().reported_cost(response), + MessagesCallResponse::Stream { .. } => None, + }; + context + .result_ready(ExecutionFacts { + provider: identity, + cost: Cost::settle(&source, reported), + source: source.clone(), }) - } + .await?; + Ok(cache.finish(output, &source).await) +} - pub(super) async fn call_provider( - &self, - request: ProviderCall, - context: &CallContext<'_, impl Interceptors>, - ) -> Result { - let ProviderCall { - identity, - wire, - provider, - signer, - timeout, - stream, - } = request; - let provider_name = provider.as_str(); - log_request_body(provider_name, stream, &wire.body); - let response = send( - &self.http, - Authenticated { - headers: wire.headers, - signer, +async fn provider_call( + auth: &AuthServices, + provider: MessagesProvider, + model: String, + call: MessagesCall, + secrets: &(dyn Lookup + Send + Sync), + context: &CallContext<'_, impl Interceptors>, +) -> Result { + let MessagesCall { + body, + api_key, + api_base, + litellm_params, + extra_headers, + provider_specific_header, + timeout, + shaping, + .. + } = call; + let connection_params = litellm_router_types::LitellmParams { + api_base: api_base.clone().or(litellm_params.api_base.clone()), + ..litellm_params + }; + let config = provider.config(); + let env_lookup = |key: &str| secrets.get(key); + + let sanitized = config.shape_request( + MessagesRequest { model, ..body }, + shaping.settings.reasoning_auto_summary, + )?; + let trimmed = + without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?; + + let scoped = + get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str()); + let forwarded = string_headers(Some( + extra_headers.into_iter().flatten().chain(scoped).collect(), + ))?; + let validated = config.validate_environment( + forwarded, + api_key.as_deref(), + &trimmed.model, + &trimmed.params, + &connection_params, + &env_lookup, + )?; + + let body = config.transform_anthropic_messages_request( + trimmed, + &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params), + )?; + let environment = ValidatedEnvironment { + headers: config.request_headers( + with_default_headers(validated.headers, config.default_headers()), + &body, + ), + auth: validated.auth, + }; + + let url = config.get_complete_url( + api_base.as_deref(), + &body.model, + &connection_params, + body.params.stream == Some(true), + &env_lookup, + )?; + + let request_context = RequestContext { + model: body.model.clone(), + custom_llm_provider: provider.as_str().to_string(), + optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?, + secret_fields: Vec::new(), + api_key: api_key.map(SecretValue::new), + }; + let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?; + let identity = ProviderIdentity { + model: request_context.model.clone(), + provider: request_context.custom_llm_provider.clone(), + }; + let wire = context + .interceptors + .before_provider_request( + WireRequest { + url, + headers: authenticated.headers, + body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?), }, - &wire.url, - &wire.body, - timeout, + request_context, ) .await?; - if !response.status().is_success() { - return Err(provider_error(response).await); - } - let config = provider.config(); - if stream { - return Ok(streaming_response( - response, - config.stream_decoder(), - provider_name, + let stream = match wire.body.get("stream") { + None | Some(Value::Null) => false, + Some(Value::Bool(stream)) => *stream, + Some(value) => { + return Err(Error::InvalidRequest( + litellm_llms::ErrorDetail::InvalidValue { + field: "stream", + expected: "a boolean", + actual: value.clone(), + }, )); } - let text = response.text().await.map_err(network)?; - log_response_body(&text); - context.response_received(&text).await?; - decode_response(config, &identity.model, &text) - .map(|message| MessagesCallResponse::Complete(Box::new(message))) + }; + Ok(ProviderCall { + identity, + wire, + provider, + signer: authenticated.signer, + timeout: timeout.or_else(|| shaping.settings.request_timeout()), + stream, + }) +} + +fn without_additional_drop_params( + request: MessagesRequest, + paths: &[String], +) -> Result { + if paths.is_empty() { + return Ok(request); } + let params = serde_json::to_value(request.params).map_err(invalid_request)?; + let trimmed = paths + .iter() + .fold(params, |params, path| delete_nested_value(params, path)); + Ok(MessagesRequest { + params: serde_json::from_value(trimmed).map_err(invalid_request)?, + ..request + }) +} + +async fn call_provider( + http: &litellm_http::Client, + request: ProviderCall, + context: &CallContext<'_, impl Interceptors>, +) -> Result { + let ProviderCall { + identity, + wire, + provider, + signer, + timeout, + stream, + } = request; + let provider_name = provider.as_str(); + log_request_body(provider_name, stream, &wire.body); + let response = send( + http, + Authenticated { + headers: wire.headers, + signer, + }, + &wire.url, + &wire.body, + timeout, + ) + .await?; + if !response.status().is_success() { + return Err(provider_error(response).await); + } + let config = provider.config(); + if stream { + return Ok(streaming_response( + response, + config.stream_decoder(), + provider_name, + )); + } + let text = response.text().await.map_err(network)?; + log_response_body(&text); + context.response_received(&text).await?; + decode_response(config, &identity.model, &text) + .map(|message| MessagesCallResponse::Complete(Box::new(message))) } fn serialize_failure(err: serde_json::Error) -> Error { @@ -271,10 +394,482 @@ fn log_chunk(provider: &str, stage: &str, data: &bytes::Bytes) { #[cfg(test)] mod tests { - use rstest::rstest; + use litellm_llms_types::headers::ProviderSpecificHeaders; + use rstest::{fixture, rstest}; + use serde_json::{Map, Value, json}; use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any}; use super::*; + use crate::{MessagesSettings, MessagesShaping, common_utils::messages_provider}; + use litellm_inference::provider::resolve_llm_provider; + + #[fixture] + fn shaping() -> MessagesShaping { + MessagesShaping::default() + } + + fn body(value: Value) -> MessagesRequest { + serde_json::from_value(value).unwrap() + } + + async fn provider_call_for(call: MessagesCall) -> Result { + provider_call_with_secrets(call, &|_: &str| None).await + } + + async fn provider_call_with_secrets( + call: MessagesCall, + secrets: &(dyn Lookup + Send + Sync), + ) -> Result { + let provider = resolve_llm_provider( + &call.body.model, + call.custom_llm_provider.as_deref(), + "messages", + )?; + let config = messages_provider(provider.provider, provider.model) + .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?; + let model = provider.model.to_owned(); + let interceptors = (); + let context = CallContext::new(&interceptors, litellm_inference::CallOptions::default()); + provider_call( + &AuthServices::default(), + config, + model, + call, + secrets, + &context, + ) + .await + } + + async fn wire_body(fields: Value, shaping: MessagesShaping) -> Result { + provider_call_for(MessagesCall { + body: body(fields), + api_key: Some("sk-test".into()), + api_base: Some("https://anthropic.test".into()), + custom_llm_provider: Some("anthropic".into()), + litellm_params: Default::default(), + extra_headers: None, + provider_specific_header: None, + timeout: None, + shaping, + }) + .await + .map(|call| call.wire.body) + } + + #[rstest] + #[case::api_key( + &[("ANTHROPIC_API_KEY", "sk-secret")], + &[("x-api-key", "sk-secret")], + "https://api.anthropic.com/v1/messages" + )] + #[case::auth_token( + &[("ANTHROPIC_AUTH_TOKEN", "token")], + &[("authorization", "Bearer token")], + "https://api.anthropic.com/v1/messages" + )] + #[case::api_base( + &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")], + &[("x-api-key", "sk-secret")], + "https://gateway.test/v1/messages" + )] + #[case::sdk_base_url( + &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")], + &[("x-api-key", "sk-secret")], + "https://sdk.test/v1/messages" + )] + #[tokio::test] + async fn credentials_and_base_come_from_the_resolved_secrets( + shaping: MessagesShaping, + #[case] secrets: &[(&str, &str)], + #[case] expected_auth: &[(&str, &str)], + #[case] expected_url: &str, + ) { + let lookup = |name: &str| { + secrets + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + }; + let call = provider_call_with_secrets( + MessagesCall { + body: body( + json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}), + ), + api_key: None, + api_base: None, + custom_llm_provider: Some("anthropic".into()), + litellm_params: Default::default(), + extra_headers: None, + provider_specific_header: None, + timeout: None, + shaping, + }, + &lookup, + ) + .await + .unwrap(); + let auth: Vec<(&str, &str)> = call + .wire + .headers + .iter() + .filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization")) + .map(|(name, value)| (name.as_str(), value.as_str())) + .collect(); + assert_eq!( + (auth.as_slice(), call.wire.url.as_str()), + (expected_auth, expected_url) + ); + } + + #[rstest] + #[tokio::test] + async fn missing_credentials_are_reported_before_the_request_is_transformed( + shaping: MessagesShaping, + ) { + let result = provider_call_for(MessagesCall { + body: body( + json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}]}), + ), + api_key: None, + api_base: None, + custom_llm_provider: Some("anthropic".into()), + litellm_params: Default::default(), + extra_headers: None, + provider_specific_header: None, + timeout: None, + shaping, + }) + .await; + assert!( + matches!( + result, + Err(Error::Auth(litellm_auth::Error::MissingApiKey { .. })) + ), + "{:?}", + result.err() + ); + } + + #[rstest] + #[case::nothing_configured(None, None, None)] + #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))] + #[case::call_timeout_wins( + Some(Duration::from_secs(30)), + Some(1800.0), + Some(Duration::from_secs(30)) + )] + #[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))] + #[case::unusable_setting_ignored(None, Some(-1.0), None)] + #[case::zero_setting_ignored(None, Some(0.0), None)] + #[tokio::test] + async fn timeout_falls_back_to_the_configured_request_timeout( + shaping: MessagesShaping, + #[case] call_timeout: Option, + #[case] request_timeout: Option, + #[case] expected: Option, + ) { + let call = provider_call_for(MessagesCall { + body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})), + api_key: Some("sk-test".into()), + api_base: Some("https://anthropic.test".into()), + custom_llm_provider: Some("anthropic".into()), + litellm_params: Default::default(), + extra_headers: None, + provider_specific_header: None, + timeout: call_timeout, + shaping: MessagesShaping { + settings: MessagesSettings { request_timeout, ..shaping.settings }, + ..shaping + }, + }) + .await + .unwrap(); + assert_eq!(call.timeout, expected); + } + + #[rstest] + #[case::top_level_and_nested_paths( + json!({ + "max_tokens": 1024, + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "context_management": {"edits": [{"type": "clear_thinking_20251015"}]}, + "metadata": {"user_id": "u1"}, + "tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}] + }), + &["thinking", "context_management", "tools[*].input_examples"], + json!({ + "max_tokens": 1024, + "metadata": {"user_id": "u1"}, + "tools": [{"name": "lookup", "input_schema": {"type": "object"}}] + }), + )] + #[case::no_paths( + json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}), + &[], + json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}), + )] + #[case::model_and_messages_are_never_dropped( + json!({"max_tokens": 16}), + &["model", "messages", "messages[0].content"], + json!({"max_tokens": 16}), + )] + #[tokio::test] + async fn wire_body_drops_configured_paths( + shaping: MessagesShaping, + #[case] fields: Value, + #[case] additional_drop_params: &[&str], + #[case] expected_fields: Value, + ) { + let with_messages = |fields: Value| -> Value { + let Value::Object(fields) = fields else { + unreachable!() + }; + Value::Object( + [ + ("model".to_string(), json!("claude-test")), + ( + "messages".to_string(), + json!([{"role": "user", "content": "hi"}]), + ), + ] + .into_iter() + .chain(fields) + .collect(), + ) + }; + let shaping = MessagesShaping { + settings: MessagesSettings { + additional_drop_params: additional_drop_params + .iter() + .map(ToString::to_string) + .collect(), + ..shaping.settings + }, + ..shaping + }; + assert_eq!( + wire_body(with_messages(fields), shaping).await, + Ok(with_messages(expected_fields)) + ); + } + + #[rstest] + #[case::model_prefix_picks_the_provider( + "azure_ai/claude-test", + None, + &[("x-priority", "extra"), ("x-scoped", "azure_ai")] + )] + #[case::explicit_provider( + "claude-test", + Some("anthropic"), + &[("x-priority", "scoped"), ("x-scoped", "anthropic")] + )] + #[case::provider_prefix_on_an_anthropic_model( + "anthropic/claude-test", + None, + &[("x-priority", "scoped"), ("x-scoped", "anthropic")] + )] + #[tokio::test] + async fn provider_specific_headers_follow_the_resolved_provider( + shaping: MessagesShaping, + #[case] model: &str, + #[case] custom_llm_provider: Option<&str>, + #[case] expected: &[(&str, &str)], + ) { + let configured: ProviderSpecificHeaders = serde_json::from_value(json!([ + {"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}}, + {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}} + ])) + .unwrap(); + let call = provider_call_for(MessagesCall { + body: body( + json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}), + ), + api_key: Some("sk-test".into()), + api_base: Some("https://resource.services.ai.azure.com".into()), + custom_llm_provider: custom_llm_provider.map(Into::into), + litellm_params: Default::default(), + extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])), + provider_specific_header: Some(configured), + timeout: None, + shaping, + }) + .await + .unwrap(); + let caller_headers: Vec<(&str, &str)> = call + .wire + .headers + .iter() + .filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped")) + .map(|(name, value)| (name.as_str(), value.as_str())) + .collect(); + assert_eq!(caller_headers, expected); + } + + #[rstest] + #[tokio::test] + async fn wire_body_carries_the_provider_stripped_model(shaping: MessagesShaping) { + assert_eq!( + wire_body( + json!({ + "model": "anthropic/claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 16 + }), + shaping, + ) + .await, + Ok(json!({ + "model": "claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 16 + })) + ); + } + + #[rstest] + #[tokio::test] + async fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) { + let shaping = MessagesShaping { + settings: MessagesSettings { + reasoning_auto_summary: true, + additional_drop_params: vec!["thinking.display".to_string()], + ..shaping.settings + }, + ..shaping + }; + assert_eq!( + wire_body( + json!({ + "model": "claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 4096, + "thinking": {"type": "enabled", "budget_tokens": 2048} + }), + shaping, + ) + .await, + Ok(json!({ + "model": "claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 4096, + "thinking": {"type": "enabled", "budget_tokens": 2048} + })) + ); + } + + #[rstest] + #[tokio::test] + async fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation( + shaping: MessagesShaping, + ) { + let shaping = MessagesShaping { + settings: MessagesSettings { + additional_drop_params: vec!["metadata.user_id".to_string()], + ..shaping.settings + }, + ..shaping + }; + assert!(matches!( + wire_body( + json!({ + "model": "claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 16, + "metadata": {"user_id": 123} + }), + shaping, + ) + .await, + Err(Error::InvalidRequest(_)) + )); + } + + #[rstest] + #[tokio::test] + async fn wire_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) { + assert_eq!( + wire_body( + json!({ + "model": "claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 16, + "metadata": {"user_id": 123} + }), + shaping, + ) + .await, + Err(Error::InvalidRequest( + litellm_llms::ErrorDetail::InvalidValue { + field: "metadata.user_id", + expected: "a string", + actual: json!(123), + } + )) + ); + } + + #[rstest] + #[case::mantle( + "bedrock_mantle/claude-test", + "https://bedrock-mantle.us-west-2.api.aws", + "/anthropic/v1/messages", + "bedrock" + )] + #[case::platform( + "bedrock/claude_platform/claude-test", + "https://aws-external-anthropic.us-west-2.api.aws", + "/v1/messages", + "aws-external-anthropic" + )] + #[tokio::test] + async fn explicit_endpoint_region_is_used_by_the_signer( + shaping: MessagesShaping, + #[case] model: &str, + #[case] base: &str, + #[case] path: &str, + #[case] service: &str, + ) { + let call = provider_call_with_secrets(MessagesCall { + body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})), + api_key: None, + api_base: Some(base.into()), + custom_llm_provider: None, + litellm_params: crate::litellm_params(Map::from_iter([ + ("model".into(), json!(model)), + ("aws_access_key_id".into(), json!("AKIDEXAMPLE")), + ("aws_secret_access_key".into(), json!("secret")), + ])) + .unwrap(), + extra_headers: None, + provider_specific_header: None, + timeout: None, + shaping, + }, &|name: &str| match name { + "AWS_REGION_NAME" => Some("us-east-1".into()), + "ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()), + _ => None, + }) + .await + .unwrap(); + assert_eq!(call.wire.url, format!("{base}{path}")); + let signed = outbound_request( + Authenticated { + headers: call.wire.headers, + signer: call.signer, + }, + call.wire.url, + &call.wire.body, + None, + ) + .unwrap(); + let authorization = signed.header("authorization").unwrap_or_default(); + assert!( + authorization.contains(&format!("/us-west-2/{service}/aws4_request")), + "{authorization}" + ); + } #[rstest] #[case::event( diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs index c690c7a2ef4..f91d70445f3 100644 --- a/litellm-rust/crates/inference-messages/src/lib.rs +++ b/litellm-rust/crates/inference-messages/src/lib.rs @@ -1,18 +1,19 @@ mod common_utils; mod constants; mod handler; -mod prepare; pub mod route; mod types; use futures_util::FutureExt; use litellm_auth::AuthServices; -use litellm_host::interceptors::{Cost, ExecutionFacts, Interceptors, ResultSource}; - -use litellm_inference::{caching::CallCache, context::CallContext}; +use litellm_host::interceptors::Interceptors; +use litellm_inference::context::CallContext; +use litellm_inference::provider::resolve_llm_provider; use litellm_secrets::source::SecretSource; use std::sync::Arc; +use crate::common_utils::messages_provider; + pub use litellm_inference::RouteError as Error; pub use types::{ MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params, @@ -74,39 +75,23 @@ impl MessagesRoute { context: CallContext<'_, impl Interceptors>, ) -> Result { litellm_inference::diagnostic::call(async { - let prepared = prepare::prepare(call, self.secrets.as_ref()).await?; + let provider = resolve_llm_provider( + &call.body.model, + call.custom_llm_provider.as_deref(), + "messages", + )?; + let config = messages_provider(provider.provider, provider.model).ok_or_else(|| { + Error::InvalidProvider(<&str>::from(provider.provider).to_string()) + })?; litellm_inference::diagnostic::provider( - &prepared.body.model, - prepared.provider.as_str(), + provider.model, + <&'static str>::from(provider.provider), ); - let request = self.prepare_outbound(prepared, &context).boxed().await?; - let cache = CallCache::::from_wire( - self.cache.as_ref().filter(|_| request.cacheable()), - context.cache, - &request.identity, - &request.wire, - ); - let identity = request.identity.clone(); - let config = request.provider.config(); - let (output, source) = match cache.lookup().await { - Some(hit) => hit, - None => ( - self.call_provider(request, &context).await?, - ResultSource::Provider, - ), - }; - let reported = match &output { - MessagesCallResponse::Complete(response) => config.reported_cost(response), - MessagesCallResponse::Stream { .. } => None, - }; - context - .result_ready(ExecutionFacts { - provider: identity, - cost: Cost::settle(&source, reported), - source: source.clone(), - }) - .await?; - Ok(cache.finish(output, &source).await) + let model = provider.model.to_owned(); + let secrets = self.secrets.resolve(config.config().secret_names()).await?; + handler::execute(self, config, model, call, secrets, &context) + .boxed() + .await }) .await } diff --git a/litellm-rust/crates/inference-messages/src/prepare.rs b/litellm-rust/crates/inference-messages/src/prepare.rs deleted file mode 100644 index 4b7873b4808..00000000000 --- a/litellm-rust/crates/inference-messages/src/prepare.rs +++ /dev/null @@ -1,554 +0,0 @@ -use std::time::Duration; - -use litellm_auth::SecretValue; -use litellm_core_utils::{ - dot_notation_indexing::delete_nested_value, - get_provider_specific_headers::get_provider_specific_headers, settings::Lookup, -}; -use litellm_http::request::with_default_headers; -use litellm_llms::base_llm::{ - auth::ValidatedEnvironment, messages::context::MessagesTransformContext, -}; -use litellm_llms_types::formats::messages::MessagesRequest; -use litellm_secrets::source::SecretSource; - -use super::{ - Error, MessagesCall, - common_utils::{MessagesProvider, messages_provider, string_headers}, - types::invalid_request, -}; -use litellm_inference::provider::resolve_llm_provider; - -struct ResolvedProvider { - model: String, - provider: MessagesProvider, -} - -pub(super) struct ProviderMessagesRequest { - pub(super) provider: MessagesProvider, - pub(super) url: String, - pub(super) body: MessagesRequest, - pub(super) environment: ValidatedEnvironment, - pub(super) timeout: Option, - /// The caller's own credential, reported to the host beside the wire request. - pub(super) api_key: Option, -} - -#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)] -pub(super) async fn prepare( - call: MessagesCall, - secrets: &dyn SecretSource, -) -> Result { - let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?; - let secrets = secrets - .resolve(resolved.provider.config().secret_names()) - .await?; - prepare_provider_request(call, resolved, secrets.as_ref()) -} - -fn resolve_provider( - model: &str, - custom_llm_provider: Option<&str>, -) -> Result { - let resolved = resolve_llm_provider(model, custom_llm_provider, "messages")?; - let provider = messages_provider(resolved.provider, resolved.model) - .ok_or_else(|| Error::InvalidProvider(<&str>::from(resolved.provider).to_string()))?; - Ok(ResolvedProvider { - model: resolved.model.to_string(), - provider, - }) -} - -fn prepare_provider_request( - call: MessagesCall, - resolved: ResolvedProvider, - secrets: &dyn Lookup, -) -> Result { - let ResolvedProvider { model, provider } = resolved; - let MessagesCall { - body, - api_key, - api_base, - litellm_params, - extra_headers, - provider_specific_header, - timeout, - shaping, - .. - } = call; - let connection_params = litellm_router_types::LitellmParams { - api_base: api_base.clone().or(litellm_params.api_base.clone()), - ..litellm_params - }; - let config = provider.config(); - let env_lookup = |key: &str| secrets.get(key); - - let sanitized = config.shape_request( - MessagesRequest { model, ..body }, - shaping.settings.reasoning_auto_summary, - )?; - let trimmed = - without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?; - let transformed = config.transform_anthropic_messages_request( - trimmed, - &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params), - )?; - - let scoped = - get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str()); - let forwarded = string_headers(Some( - extra_headers.into_iter().flatten().chain(scoped).collect(), - ))?; - let forwarded = config.request_headers( - with_default_headers(forwarded, config.default_headers()), - &transformed, - ); - let environment = config.validate_environment( - forwarded, - api_key.as_deref(), - &transformed.model, - &connection_params, - &env_lookup, - )?; - - let url = config.get_complete_url( - api_base.as_deref(), - &transformed.model, - &connection_params, - transformed.params.stream == Some(true), - &env_lookup, - )?; - - Ok(ProviderMessagesRequest { - provider, - url, - body: transformed, - environment, - timeout: timeout.or_else(|| shaping.settings.request_timeout()), - api_key: api_key.map(SecretValue::new), - }) -} - -fn without_additional_drop_params( - request: MessagesRequest, - paths: &[String], -) -> Result { - if paths.is_empty() { - return Ok(request); - } - let params = serde_json::to_value(request.params).map_err(invalid_request)?; - let trimmed = paths - .iter() - .fold(params, |params, path| delete_nested_value(params, path)); - Ok(MessagesRequest { - params: serde_json::from_value(trimmed).map_err(invalid_request)?, - ..request - }) -} - -#[cfg(test)] -mod tests { - use litellm_llms::base_llm::auth::resolve_auth; - use litellm_llms_types::headers::ProviderSpecificHeaders; - use rstest::{fixture, rstest}; - use serde_json::{Map, Value, json}; - - use super::*; - use crate::{MessagesSettings, MessagesShaping}; - - #[fixture] - fn shaping() -> MessagesShaping { - MessagesShaping::default() - } - - fn body(value: Value) -> MessagesRequest { - serde_json::from_value(value).unwrap() - } - - fn prepare(call: MessagesCall) -> Result { - prepare_with_secrets(call, &|_: &str| None) - } - - fn prepare_with_secrets( - call: MessagesCall, - secrets: &dyn Lookup, - ) -> Result { - let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?; - prepare_provider_request(call, resolved, secrets) - } - - /// The headers as they go on the wire, credential applied. - fn wire_headers(prepared: &ProviderMessagesRequest) -> Vec<(String, String)> { - tokio::runtime::Builder::new_current_thread() - .build() - .unwrap() - .block_on(resolve_auth( - &litellm_auth::AuthServices::default(), - prepared.environment.clone(), - &|_| None, - )) - .unwrap() - .headers - } - - #[rstest] - #[case::api_key( - &[("ANTHROPIC_API_KEY", "sk-secret")], - &[("x-api-key", "sk-secret")], - "https://api.anthropic.com/v1/messages" - )] - #[case::auth_token( - &[("ANTHROPIC_AUTH_TOKEN", "token")], - &[("authorization", "Bearer token")], - "https://api.anthropic.com/v1/messages" - )] - #[case::api_base( - &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")], - &[("x-api-key", "sk-secret")], - "https://gateway.test/v1/messages" - )] - #[case::sdk_base_url( - &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")], - &[("x-api-key", "sk-secret")], - "https://sdk.test/v1/messages" - )] - fn credentials_and_base_come_from_the_resolved_secrets( - shaping: MessagesShaping, - #[case] secrets: &[(&str, &str)], - #[case] expected_auth: &[(&str, &str)], - #[case] expected_url: &str, - ) { - let lookup = |name: &str| { - secrets - .iter() - .find(|(key, _)| *key == name) - .map(|(_, value)| value.to_string()) - }; - let prepared = prepare_with_secrets( - MessagesCall { - body: body( - json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}), - ), - api_key: None, - api_base: None, - custom_llm_provider: Some("anthropic".into()), - litellm_params: Default::default(), - extra_headers: None, - provider_specific_header: None, - timeout: None, - shaping, - }, - &lookup, - ) - .unwrap(); - let headers = wire_headers(&prepared); - let auth: Vec<(&str, &str)> = headers - .iter() - .filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization")) - .map(|(name, value)| (name.as_str(), value.as_str())) - .collect(); - assert_eq!( - (auth.as_slice(), prepared.url.as_str()), - (expected_auth, expected_url) - ); - } - - #[rstest] - #[case::nothing_configured(None, None, None)] - #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))] - #[case::call_timeout_wins( - Some(Duration::from_secs(30)), - Some(1800.0), - Some(Duration::from_secs(30)) - )] - #[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))] - #[case::unusable_setting_ignored(None, Some(-1.0), None)] - #[case::zero_setting_ignored(None, Some(0.0), None)] - fn timeout_falls_back_to_the_configured_request_timeout( - shaping: MessagesShaping, - #[case] call_timeout: Option, - #[case] request_timeout: Option, - #[case] expected: Option, - ) { - let prepared = prepare(MessagesCall { - body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})), - api_key: Some("sk-test".into()), - api_base: Some("https://anthropic.test".into()), - custom_llm_provider: Some("anthropic".into()), - litellm_params: Default::default(), - extra_headers: None, - provider_specific_header: None, - timeout: call_timeout, - shaping: MessagesShaping { - settings: MessagesSettings { request_timeout, ..shaping.settings }, - ..shaping - }, - }) - .unwrap(); - assert_eq!(prepared.timeout, expected); - } - - fn prepared_body(fields: Value, shaping: MessagesShaping) -> Result { - prepare(MessagesCall { - body: body(fields), - api_key: Some("sk-test".into()), - api_base: Some("https://anthropic.test".into()), - custom_llm_provider: Some("anthropic".into()), - litellm_params: Default::default(), - extra_headers: None, - provider_specific_header: None, - timeout: None, - shaping, - }) - .map(|prepared| serde_json::to_value(prepared.body).unwrap()) - } - - #[rstest] - #[case::top_level_and_nested_paths( - json!({ - "max_tokens": 1024, - "thinking": {"type": "enabled", "budget_tokens": 2048}, - "context_management": {"edits": [{"type": "clear_thinking_20251015"}]}, - "metadata": {"user_id": "u1"}, - "tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}] - }), - &["thinking", "context_management", "tools[*].input_examples"], - json!({ - "max_tokens": 1024, - "metadata": {"user_id": "u1"}, - "tools": [{"name": "lookup", "input_schema": {"type": "object"}}] - }), - )] - #[case::no_paths( - json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}), - &[], - json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}), - )] - #[case::model_and_messages_are_never_dropped( - json!({"max_tokens": 16}), - &["model", "messages", "messages[0].content"], - json!({"max_tokens": 16}), - )] - fn prepared_body_drops_configured_paths( - shaping: MessagesShaping, - #[case] fields: Value, - #[case] additional_drop_params: &[&str], - #[case] expected_fields: Value, - ) { - let with_messages = |fields: Value| -> Value { - let Value::Object(fields) = fields else { - unreachable!() - }; - Value::Object( - [ - ("model".to_string(), json!("claude-test")), - ( - "messages".to_string(), - json!([{"role": "user", "content": "hi"}]), - ), - ] - .into_iter() - .chain(fields) - .collect(), - ) - }; - let shaping = MessagesShaping { - settings: MessagesSettings { - additional_drop_params: additional_drop_params - .iter() - .map(ToString::to_string) - .collect(), - ..shaping.settings - }, - ..shaping - }; - assert_eq!( - prepared_body(with_messages(fields), shaping), - Ok(with_messages(expected_fields)) - ); - } - - #[rstest] - #[case::model_prefix_picks_the_provider( - "azure_ai/claude-test", - None, - &[("x-priority", "extra"), ("x-scoped", "azure_ai")] - )] - #[case::explicit_provider( - "claude-test", - Some("anthropic"), - &[("x-priority", "scoped"), ("x-scoped", "anthropic")] - )] - #[case::provider_prefix_on_an_anthropic_model( - "anthropic/claude-test", - None, - &[("x-priority", "scoped"), ("x-scoped", "anthropic")] - )] - fn provider_specific_headers_follow_the_resolved_provider( - shaping: MessagesShaping, - #[case] model: &str, - #[case] custom_llm_provider: Option<&str>, - #[case] expected: &[(&str, &str)], - ) { - let configured: ProviderSpecificHeaders = serde_json::from_value(json!([ - {"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}}, - {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}} - ])) - .unwrap(); - let prepared = prepare(MessagesCall { - body: body( - json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}), - ), - api_key: Some("sk-test".into()), - api_base: Some("https://resource.services.ai.azure.com".into()), - custom_llm_provider: custom_llm_provider.map(Into::into), - litellm_params: Default::default(), - extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])), - provider_specific_header: Some(configured), - timeout: None, - shaping, - }) - .unwrap(); - let caller_headers: Vec<(&str, &str)> = prepared - .environment - .headers - .iter() - .filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped")) - .map(|(name, value)| (name.as_str(), value.as_str())) - .collect(); - assert_eq!(caller_headers, expected); - } - - #[rstest] - fn prepared_body_carries_the_provider_stripped_model(shaping: MessagesShaping) { - assert_eq!( - prepared_body( - json!({ - "model": "anthropic/claude-test", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 16 - }), - shaping, - ), - Ok(json!({ - "model": "claude-test", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 16 - })) - ); - } - - #[rstest] - fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) { - let shaping = MessagesShaping { - settings: MessagesSettings { - reasoning_auto_summary: true, - additional_drop_params: vec!["thinking.display".to_string()], - ..shaping.settings - }, - ..shaping - }; - assert_eq!( - prepared_body( - json!({ - "model": "claude-test", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 4096, - "thinking": {"type": "enabled", "budget_tokens": 2048} - }), - shaping, - ), - Ok(json!({ - "model": "claude-test", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 4096, - "thinking": {"type": "enabled", "budget_tokens": 2048} - })) - ); - } - - #[rstest] - fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(shaping: MessagesShaping) { - let shaping = MessagesShaping { - settings: MessagesSettings { - additional_drop_params: vec!["metadata.user_id".to_string()], - ..shaping.settings - }, - ..shaping - }; - assert!(matches!( - prepared_body( - json!({ - "model": "claude-test", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 16, - "metadata": {"user_id": 123} - }), - shaping, - ), - Err(Error::InvalidRequest(_)) - )); - } - - #[rstest] - fn prepared_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) { - assert_eq!( - prepared_body( - json!({ - "model": "claude-test", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 16, - "metadata": {"user_id": 123} - }), - shaping, - ), - Err(Error::InvalidRequest( - litellm_llms::ErrorDetail::InvalidValue { - field: "metadata.user_id", - expected: "a string", - actual: json!(123), - } - )) - ); - } - #[rstest] - #[case::mantle( - "bedrock_mantle/claude-test", - "https://bedrock-mantle.us-west-2.api.aws", - "/anthropic/v1/messages", - "bedrock" - )] - #[case::platform( - "bedrock/claude_platform/claude-test", - "https://aws-external-anthropic.us-west-2.api.aws", - "/v1/messages", - "aws-external-anthropic" - )] - fn explicit_endpoint_region_is_used_by_the_signer( - shaping: MessagesShaping, - #[case] model: &str, - #[case] base: &str, - #[case] path: &str, - #[case] service: &str, - ) { - let prepared = prepare_with_secrets(MessagesCall { - body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})), - api_key: None, - api_base: Some(base.into()), - custom_llm_provider: None, - litellm_params: Default::default(), - extra_headers: None, - provider_specific_header: None, - timeout: None, - shaping, - }, &|name: &str| match name { - "AWS_REGION_NAME" => Some("us-east-1".into()), - "ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()), - _ => None, - }).unwrap(); - assert_eq!(prepared.url, format!("{base}{path}")); - assert!( - matches!(prepared.environment.auth, litellm_llms::base_llm::auth::AuthScheme::AwsSigV4 { region, service: actual, .. } if region == "us-west-2" && actual == service) - ); - } -} diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index bcd1e08ad86..6657cbf262f 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -74,6 +74,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { headers: Headers, api_key: Option<&str>, _model: &str, + _optional_params: &MessagesOptionalParams, _litellm_params: &LitellmParams, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { @@ -887,6 +888,7 @@ mod tests { headers(forwarded), api_key, "claude", + &MessagesOptionalParams::default(), &LitellmParams::default(), &env(vars), ) @@ -1206,6 +1208,7 @@ mod tests { Vec::new(), None, "claude", + &MessagesOptionalParams::default(), &LitellmParams::default(), &record, ); diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index 3bec142ecf8..bf934fb4d4c 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -1,7 +1,7 @@ use litellm_auth::SecretValue; use litellm_http::request::{has_bearer_auth, has_header}; use litellm_llms_types::{ - formats::messages::MessagesRequest, + formats::messages::{MessagesOptionalParams, MessagesRequest}, providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH}, }; use litellm_router_types::LitellmParams; @@ -83,6 +83,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { headers: Headers, api_key: Option<&str>, _model: &str, + _optional_params: &MessagesOptionalParams, _litellm_params: &LitellmParams, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { @@ -237,6 +238,7 @@ mod tests { .collect(), api_key, "claude", + &MessagesOptionalParams::default(), &LitellmParams::default(), &|_| None, ) @@ -622,6 +624,7 @@ mod tests { Vec::new(), None, "claude", + &MessagesOptionalParams::default(), &LitellmParams::default(), &record, ); diff --git a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs index 6f0ea5c8f2a..8a88af40d6b 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs @@ -1,4 +1,6 @@ -use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse}; +use litellm_llms_types::formats::messages::{ + MessagesOptionalParams, MessagesRequest, MessagesResponse, +}; use litellm_router_types::LitellmParams; use serde_json::Value; @@ -55,6 +57,7 @@ pub trait BaseMessagesConfig: Sync { headers: Headers, api_key: Option<&str>, model: &str, + optional_params: &MessagesOptionalParams, litellm_params: &LitellmParams, env_lookup: &dyn Fn(&str) -> Option, ) -> Result; @@ -110,6 +113,7 @@ mod tests { headers: Headers, _api_key: Option<&str>, _model: &str, + _optional_params: &MessagesOptionalParams, _litellm_params: &LitellmParams, _env_lookup: &dyn Fn(&str) -> Option, ) -> Result { diff --git a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs index e2e64e54664..72958fd6cb9 100644 --- a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs @@ -5,7 +5,7 @@ use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region}; use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{header_value, with_header, without_headers}; use litellm_llms_types::{ - formats::messages::MessagesRequest, + formats::messages::{MessagesOptionalParams, MessagesRequest}, providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH}, }; use litellm_router_types::LitellmParams; @@ -60,6 +60,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig { headers: Headers, api_key: Option<&str>, _model: &str, + optional_params: &MessagesOptionalParams, params: &LitellmParams, env: &dyn Fn(&str) -> Option, ) -> Result { @@ -67,11 +68,8 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig { .filter(|value| !value.trim().is_empty()) .or_else(|| { WORKSPACE_FIELDS.iter().find_map(|field| { - params - .extra - .get(*field)? - .as_str() - .filter(|value| !value.trim().is_empty()) + workspace_value(optional_params.extra.get(*field)) + .or_else(|| workspace_value(params.extra.get(*field))) }) }); let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS) @@ -182,6 +180,10 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig { } } +fn workspace_value(value: Option<&Value>) -> Option<&str> { + value?.as_str().filter(|value| !value.trim().is_empty()) +} + fn platform_region( base: Option<&str>, params: &LitellmParams, @@ -260,6 +262,7 @@ mod tests { vec![], Some("key"), "claude-test", + &MessagesOptionalParams::default(), &LitellmParams::default(), &|_| None ) @@ -280,6 +283,7 @@ mod tests { ], Some("explicit"), "claude-test", + &MessagesOptionalParams::default(), &LitellmParams::default(), &|name| (name == "ANTHROPIC_AWS_WORKSPACE_ID").then(|| "workspace".into()), ) @@ -345,7 +349,14 @@ mod tests { .get_complete_url(None, "claude-test", ¶ms, false, &env) .unwrap(); let environment = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG - .validate_environment(vec![], None, "claude-test", ¶ms, &env) + .validate_environment( + vec![], + None, + "claude-test", + &MessagesOptionalParams::default(), + ¶ms, + &env, + ) .unwrap(); assert_eq!( url, diff --git a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs index b467ec723a5..1f6b6edf28d 100644 --- a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs @@ -16,7 +16,7 @@ use litellm_auth_aws::{ resolve_bedrock_region, }; use litellm_llms_types::formats::messages::{ - MessagesRequest, + MessagesOptionalParams, MessagesRequest, streaming::{MessagesStreamEvent, MessagesStreamUsage}, }; use litellm_llms_types::providers::bedrock::{ @@ -146,6 +146,7 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { headers: Headers, api_key: Option<&str>, model: &str, + _optional_params: &MessagesOptionalParams, litellm_params: &LitellmParams, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { @@ -618,6 +619,7 @@ mod tests { Vec::new(), None, "anthropic.claude-3", + &MessagesOptionalParams::default(), &litellm_params, &|_| None, ) @@ -658,6 +660,7 @@ mod tests { vec![("authorization".into(), "Bearer forwarded".into())], api_key, "anthropic.claude-3", + &MessagesOptionalParams::default(), &LitellmParams::default(), &env, ) diff --git a/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs b/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs index 798f6648250..ae13704f4a0 100644 --- a/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs @@ -6,7 +6,7 @@ use litellm_auth_aws::bedrock_model_id_and_region; use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{with_header, without_headers}; use litellm_llms_types::{ - formats::messages::MessagesRequest, + formats::messages::{MessagesOptionalParams, MessagesRequest}, providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS}, }; use litellm_router_types::LitellmParams; @@ -59,6 +59,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig { headers: Headers, api_key: Option<&str>, model: &str, + _optional_params: &MessagesOptionalParams, params: &LitellmParams, env: &dyn Fn(&str) -> Option, ) -> Result { @@ -317,7 +318,14 @@ mod tests { "https://private.test/anthropic/v1/messages" ); let environment = BEDROCK_MANTLE_MESSAGES_CONFIG - .validate_environment(vec![], None, model, ¶ms, &|_| None) + .validate_environment( + vec![], + None, + model, + &MessagesOptionalParams::default(), + ¶ms, + &|_| None, + ) .unwrap(); assert!( matches!(environment.auth, AuthScheme::AwsSigV4 { region: actual, service: "bedrock", .. } if actual == region) @@ -347,7 +355,14 @@ mod tests { .get_complete_url(None, "claude-test", ¶ms, false, &|_| None) .unwrap(); let environment = BEDROCK_MANTLE_MESSAGES_CONFIG - .validate_environment(vec![], None, "claude-test", ¶ms, &|_| None) + .validate_environment( + vec![], + None, + "claude-test", + &MessagesOptionalParams::default(), + ¶ms, + &|_| None, + ) .unwrap(); assert_eq!( url, @@ -401,6 +416,7 @@ mod tests { ], key, "claude-test", + &MessagesOptionalParams::default(), &LitellmParams::default(), &|name| match name { "BEDROCK_MANTLE_API_KEY" => Some("mantle".into()), diff --git a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs index fefedbbbf5e..168c3e63920 100644 --- a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs @@ -104,6 +104,7 @@ impl BaseMessagesConfig for DeepSeekAnthropicMessagesConfig { headers: Headers, api_key: Option<&str>, _model: &str, + _optional_params: &MessagesOptionalParams, _litellm_params: &LitellmParams, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { @@ -298,6 +299,7 @@ mod tests { .collect(), api_key, "deepseek-v4-pro", + &MessagesOptionalParams::default(), &LitellmParams::default(), env, ) diff --git a/litellm-rust/crates/llms/src/edenai/messages/transformation.rs b/litellm-rust/crates/llms/src/edenai/messages/transformation.rs index 7fd001ddbbc..2c3a7eaef1d 100644 --- a/litellm-rust/crates/llms/src/edenai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/edenai/messages/transformation.rs @@ -36,6 +36,7 @@ impl BaseMessagesConfig for EdenAIAnthropicMessagesConfig { headers: Headers, api_key: Option<&str>, _model: &str, + _optional_params: &MessagesOptionalParams, _params: &LitellmParams, env: &dyn Fn(&str) -> Option, ) -> Result { diff --git a/litellm-rust/crates/llms/src/minimax/messages/transformation.rs b/litellm-rust/crates/llms/src/minimax/messages/transformation.rs index e472ccf8ec9..012a657fceb 100644 --- a/litellm-rust/crates/llms/src/minimax/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/minimax/messages/transformation.rs @@ -42,6 +42,7 @@ impl BaseMessagesConfig for MinimaxMessagesConfig { headers: Headers, api_key: Option<&str>, _model: &str, + _optional_params: &MessagesOptionalParams, _params: &LitellmParams, env: &dyn Fn(&str) -> Option, ) -> Result { @@ -183,7 +184,14 @@ mod tests { .flatten() }; let environment = MINIMAX_MESSAGES_CONFIG - .validate_environment(vec![], key, "test-model", &Default::default(), &env) + .validate_environment( + vec![], + key, + "test-model", + &MessagesOptionalParams::default(), + &Default::default(), + &env, + ) .unwrap(); assert!(matches!( environment.auth, diff --git a/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs b/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs index 20bc0f99ed8..5b4e4f77f2a 100644 --- a/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs @@ -34,6 +34,7 @@ impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig { headers: Headers, api_key: Option<&str>, _model: &str, + _optional_params: &MessagesOptionalParams, _params: &LitellmParams, env: &dyn Fn(&str) -> Option, ) -> Result { diff --git a/litellm-rust/crates/llms/src/tencent/messages/transformation.rs b/litellm-rust/crates/llms/src/tencent/messages/transformation.rs index e96025952dd..a70336748ad 100644 --- a/litellm-rust/crates/llms/src/tencent/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/tencent/messages/transformation.rs @@ -43,6 +43,7 @@ impl BaseMessagesConfig for TencentAnthropicMessagesConfig { headers: Headers, api_key: Option<&str>, _model: &str, + _optional_params: &MessagesOptionalParams, _params: &LitellmParams, env: &dyn Fn(&str) -> Option, ) -> Result { @@ -229,7 +230,14 @@ mod tests { .flatten() }; let environment = TENCENT_MESSAGES_CONFIG - .validate_environment(vec![], key, "test-model", &Default::default(), &env) + .validate_environment( + vec![], + key, + "test-model", + &MessagesOptionalParams::default(), + &Default::default(), + &env, + ) .unwrap(); assert!(matches!( environment.auth, diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index 0e665304661..2e77dbc1ca8 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -114,6 +114,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { headers: Headers, _api_key: Option<&str>, _model: &str, + _optional_params: &MessagesOptionalParams, litellm_params: &LitellmParams, _env_lookup: &dyn Fn(&str) -> Option, ) -> Result { @@ -439,6 +440,7 @@ mod tests { .collect(), Some("ignored-api-key"), "claude-sonnet-4-5", + &MessagesOptionalParams::default(), ¶ms(Some("proj"), None), &no_env, ) From 0675c22d56a2c8592c6c716cd60177406a9751b9 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 19 Oct 2026 09:03:45 -0700 Subject: [PATCH 24/50] refactor(rust): inline chat provider resolution (#47218) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/inference-chat/src/common_utils.rs | 57 ++ .../crates/inference-chat/src/handler.rs | 116 +++- litellm-rust/crates/inference-chat/src/lib.rs | 53 +- .../crates/inference-chat/src/prepare.rs | 638 ------------------ .../crates/inference-chat/src/route.rs | 40 +- .../crates/inference-chat/src/types.rs | 29 +- .../crates/inference-chat/tests/request.rs | 439 ++++++++++++ 7 files changed, 617 insertions(+), 755 deletions(-) delete mode 100644 litellm-rust/crates/inference-chat/src/prepare.rs create mode 100644 litellm-rust/crates/inference-chat/tests/request.rs diff --git a/litellm-rust/crates/inference-chat/src/common_utils.rs b/litellm-rust/crates/inference-chat/src/common_utils.rs index 90b75dcddaa..4063f33432d 100644 --- a/litellm-rust/crates/inference-chat/src/common_utils.rs +++ b/litellm-rust/crates/inference-chat/src/common_utils.rs @@ -6,9 +6,11 @@ use litellm_llms::{ bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG, }; +use litellm_llms_types::formats::chat_completions::ChatMessage; use serde_json::{Map, Value}; use super::Error; +use crate::types::{ChatCompletionsCall, ResolvedChatCompletionsRequest}; const HEADER_CONTEXT: &str = "chat completions"; @@ -42,3 +44,58 @@ pub(super) fn string_headers( ) -> Result, Error> { shared_string_headers(HEADER_CONTEXT, extra_headers).map_err(Error::from) } + +pub(super) fn resolve_request( + call: ChatCompletionsCall, + config: &'static dyn BaseConfig, +) -> Result { + let messages = serde_json::from_value::>(call.messages).map_err(|err| { + Error::InvalidRequest(litellm_llms::ErrorDetail::invalid( + "chat completions messages", + err, + )) + })?; + if messages.is_empty() { + return Err(Error::InvalidRequest( + "chat completions requires at least one message".into(), + )); + } + if let Some(reason) = config.unsupported_reason(&messages, &call.optional_params) { + return Err(Error::Unsupported(reason.0)); + } + Ok(ResolvedChatCompletionsRequest { + messages, + optional_params: call.optional_params, + api_key: call.api_key, + api_base: call.api_base, + extra_headers: call.extra_headers, + timeout: call.timeout, + }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::chat_completions_provider; + use litellm_core_utils::get_llm_provider_logic::LlmProviders; + + #[rstest] + #[case::anthropic(LlmProviders::Anthropic, ("anthropic-version", "2023-06-01"))] + #[case::bedrock(LlmProviders::Bedrock, ("Content-Type", "application/json"))] + fn resolves_supported_provider_configs( + #[case] provider: LlmProviders, + #[case] expected_header: (&str, &str), + ) { + let config = chat_completions_provider(provider) + .expect("provider has a chat completions config") + .config(); + assert!(config.default_headers().contains(&expected_header)); + } + + #[rstest] + #[case::openai(LlmProviders::Openai)] + fn rejects_providers_without_a_chat_config(#[case] provider: LlmProviders) { + assert!(chat_completions_provider(provider).is_none()); + } +} diff --git a/litellm-rust/crates/inference-chat/src/handler.rs b/litellm-rust/crates/inference-chat/src/handler.rs index 95dd05e90d6..595e851c6fd 100644 --- a/litellm-rust/crates/inference-chat/src/handler.rs +++ b/litellm-rust/crates/inference-chat/src/handler.rs @@ -3,44 +3,66 @@ use std::time::Duration; use litellm_auth::AuthServices; use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest}; -use litellm_http::{Client, outbound::OutboundRequest, request::truncate_error_body}; +use litellm_http::{ + Client, + outbound::OutboundRequest, + request::{truncate_error_body, with_default_headers}, +}; use litellm_llms::base_llm::{ - auth::{Authenticated, resolve_auth}, + auth::{Authenticated, ValidatedEnvironment, resolve_auth}, + chat::transformation::BaseConfig, chat::transformation::ProviderChatResponseData, }; use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse; +use litellm_secrets::source::Secrets; use serde_json::Value; use super::Error; -use crate::{constants::CHAT_COMPLETIONS_TIMEOUT_SECS, types::ProviderChatCompletionsRequest}; +use crate::{ + common_utils::string_headers, constants::CHAT_COMPLETIONS_TIMEOUT_SECS, + types::ResolvedChatCompletionsRequest, +}; +#[expect(clippy::too_many_arguments)] // mirrors BaseLLMHTTPHandler.completion, which takes each stage input separately pub(super) async fn execute( http: &Client, auth: &AuthServices, - request: ProviderChatCompletionsRequest, + config: &'static dyn BaseConfig, + provider: litellm_inference::provider::ResolvedProvider<'_>, + call: ResolvedChatCompletionsRequest, + secrets: Secrets, cache: Option, cache_options: Option, interceptors: &impl Interceptors, observers: Option<&ObservationSender>, ) -> Result { - let ProviderChatCompletionsRequest { - model, - custom_llm_provider, - config, - url, - body, - optional_params, - environment, - secrets, - timeout, - api_key, - } = request; + let validated = config.validate_environment( + string_headers(call.extra_headers)?, + call.api_key.as_deref(), + provider.model, + &call.optional_params, + &|key| secrets.get(key), + )?; + let url = config.get_complete_url( + call.api_base.as_deref(), + provider.model, + &call.optional_params, + &|key| secrets.get(key), + )?; + let body = config + .transform_request(provider.model, call.messages, call.optional_params.clone())? + .body; + let environment = ValidatedEnvironment { + headers: with_default_headers(validated.headers, config.default_headers()), + auth: validated.auth, + }; + let provider_name = <&'static str>::from(provider.provider); let context = RequestContext { - model: model.clone(), - custom_llm_provider, - optional_params: Value::Object(optional_params), + model: provider.model.to_owned(), + custom_llm_provider: provider_name.to_owned(), + optional_params: Value::Object(call.optional_params), secret_fields: Vec::new(), - api_key, + api_key: call.api_key.map(litellm_auth::SecretValue::new), }; let authenticated = resolve_auth(auth, environment, &|key| secrets.get(key)).await?; let identity = litellm_host::interceptors::ProviderIdentity { @@ -76,7 +98,7 @@ pub(super) async fn execute( }, wire.url, &wire.body, - timeout, + call.timeout, )?; let response = litellm_inference::outbound::send(outbound, http) @@ -121,7 +143,7 @@ pub(super) async fn execute( )) })?; config - .transform_response(&model, ProviderChatResponseData { body }) + .transform_response(provider.model, ProviderChatResponseData { body }) .map_err(Error::from) .map_err(as_response_error) }, @@ -134,7 +156,7 @@ pub(super) async fn execute( /// /// A config reports the same variants on either side of the call: a missing /// field or an unsupported block can mean "this request cannot be translated" -/// during prepare and "this response cannot be normalized" here. Only the +/// before the request is sent and "this response cannot be normalized" here. Only the /// second kind has already been billed, and a host that keeps a reference /// implementation must not retry those, so collapse them to one variant that /// can only mean the provider was already called. @@ -178,9 +200,10 @@ mod tests { use super::*; use crate::{ - prepare::{prepare_provider_request, resolve_request}, - types::ChatCompletionsRequest, + common_utils::{chat_completions_provider, resolve_request}, + types::{ChatCompletionsCall, ResolvedChatCompletionsRequest}, }; + use litellm_inference::provider::resolve_llm_provider; const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#; @@ -217,22 +240,33 @@ mod tests { } } - fn prepared(api_base: &str) -> ProviderChatCompletionsRequest { - prepare_provider_request( - resolve_request(ChatCompletionsRequest { - model: "anthropic/claude-sonnet-4-5", + fn prepared( + api_base: &str, + ) -> ( + litellm_inference::provider::ResolvedProvider<'static>, + &'static dyn BaseConfig, + ResolvedChatCompletionsRequest, + ) { + let provider = + resolve_llm_provider("anthropic/claude-sonnet-4-5", None, "chat completions").unwrap(); + let config = chat_completions_provider(provider.provider) + .expect("provider has a chat completions config") + .config(); + let request = resolve_request( + ChatCompletionsCall { + model: "anthropic/claude-sonnet-4-5".into(), messages: json!([{"role": "user", "content": "hi"}]), optional_params: json!({"max_tokens": 16}).as_object().unwrap().clone(), - api_key: Some("sk-test"), - api_base: Some(api_base), + api_key: Some("sk-test".into()), + api_base: Some(api_base.into()), custom_llm_provider: None, extra_headers: None, timeout: None, - }) - .unwrap(), - std::sync::Arc::new(|_: &str| None), + }, + config, ) - .unwrap() + .unwrap(); + (provider, config, request) } #[rstest] @@ -247,10 +281,14 @@ mod tests { .await; let interceptors = RecordingHooks::default(); + let (provider, config, request) = prepared(&upstream.uri()); execute( &Client::plain_for_test(), &AuthServices::default(), - prepared(&upstream.uri()), + config, + provider, + request, + std::sync::Arc::new(|_: &str| None), None, None, &interceptors, @@ -288,10 +326,14 @@ mod tests { .await; let interceptors = RecordingHooks::default(); + let (provider, config, request) = prepared(&upstream.uri()); let error = execute( &Client::plain_for_test(), &AuthServices::default(), - prepared(&upstream.uri()), + config, + provider, + request, + std::sync::Arc::new(|_: &str| None), None, None, &interceptors, diff --git a/litellm-rust/crates/inference-chat/src/lib.rs b/litellm-rust/crates/inference-chat/src/lib.rs index 65e82af6a8d..f6b391e0ba1 100644 --- a/litellm-rust/crates/inference-chat/src/lib.rs +++ b/litellm-rust/crates/inference-chat/src/lib.rs @@ -5,14 +5,13 @@ pub use litellm_inference::RouteError as Error; mod common_utils; pub mod constants; pub(crate) mod handler; -mod prepare; use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse; -use prepare::{prepare_provider_request, resolve_request}; use litellm_auth::AuthServices; +use litellm_inference::provider::resolve_llm_provider; use litellm_secrets::source::SecretSource; use std::sync::Arc; -use types::ChatCompletionsRequest; +use types::{ChatCompletionsCall, ChatCompletionsRequest}; #[derive(Clone)] pub struct ChatCompletionsRoute { @@ -55,7 +54,7 @@ impl ChatCompletionsRoute { } = options.into(); litellm_host::lifecycle::observe_unary( observers.clone(), - self.run_call( + self.run( request.into(), cache_options, interceptors, @@ -65,30 +64,52 @@ impl ChatCompletionsRoute { .await } + #[tracing::instrument(name = "litellm.route", skip_all, fields( + route = "chat_completions", + model = %call.model, + provider, + resolved_model, + stream = false, + outcome + ))] async fn run( &self, - request: ChatCompletionsRequest<'_>, + call: ChatCompletionsCall, cache_options: Option, interceptors: &impl litellm_host::interceptors::Interceptors, observers: Option<&ObservationSender>, ) -> Result { - let resolved = resolve_request(request)?; - let snapshot = self - .secrets - .resolve(&resolved.config.secret_names()) - .await?; - let prepared = prepare_provider_request(resolved, snapshot)?; - litellm_inference::diagnostic::provider(&prepared.model, &prepared.custom_llm_provider); - let execute: futures_util::future::BoxFuture<'_, Result> = - Box::pin(handler::execute( + litellm_inference::diagnostic::unary(async { + let model = call.model.clone(); + let custom_llm_provider = call.custom_llm_provider.clone(); + let provider = + resolve_llm_provider(&model, custom_llm_provider.as_deref(), "chat completions")?; + let config = common_utils::chat_completions_provider(provider.provider) + .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))? + .config(); + let call = common_utils::resolve_request(call, config)?; + litellm_inference::diagnostic::provider( + provider.model, + <&'static str>::from(provider.provider), + ); + let secrets = self.secrets.resolve(&config.secret_names()).await?; + let execute: futures_util::future::BoxFuture< + '_, + Result, + > = Box::pin(handler::execute( &self.http, &self.auth, - prepared, + config, + provider, + call, + secrets, self.cache.clone(), cache_options, interceptors, observers, )); - execute.await + execute.await + }) + .await } } diff --git a/litellm-rust/crates/inference-chat/src/prepare.rs b/litellm-rust/crates/inference-chat/src/prepare.rs deleted file mode 100644 index 7e1cf43094d..00000000000 --- a/litellm-rust/crates/inference-chat/src/prepare.rs +++ /dev/null @@ -1,638 +0,0 @@ -use litellm_auth::SecretValue; -use litellm_core_utils::settings::Lookup; -use litellm_http::request::with_default_headers; -use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig}; -use litellm_llms_types::formats::chat_completions::ChatMessage; -use litellm_secrets::source::Secrets; -use serde_json::Value; - -use super::{ - Error, - common_utils::{chat_completions_provider, string_headers}, -}; -use crate::types::{ - ChatCompletionsRequest, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest, -}; -use litellm_inference::provider::resolve_llm_provider; - -pub(super) struct ResolvedProvider { - pub(super) model: String, - pub(super) custom_llm_provider: String, - pub(super) config: &'static dyn BaseConfig, -} - -pub(super) fn resolve_provider_config<'a>( - model: &'a str, - custom_llm_provider: Option<&'a str>, -) -> Result { - let provider_info = resolve_llm_provider(model, custom_llm_provider, "chat completions")?; - let config = chat_completions_provider(provider_info.provider) - .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))? - .config(); - Ok(ResolvedProvider { - model: provider_info.model.to_string(), - custom_llm_provider: <&str>::from(provider_info.provider).to_string(), - config, - }) -} - -pub(super) fn parse_messages(messages: Value) -> Result, Error> { - serde_json::from_value(messages).map_err(|err| { - Error::InvalidRequest(litellm_llms::ErrorDetail::invalid( - "chat completions messages", - err, - )) - }) -} - -pub(super) fn resolve_request( - request: ChatCompletionsRequest<'_>, -) -> Result, Error> { - let ResolvedProvider { - model, - custom_llm_provider, - config, - } = resolve_provider_config(request.model, request.custom_llm_provider)?; - let messages = parse_messages(request.messages)?; - if messages.is_empty() { - return Err(Error::InvalidRequest( - "chat completions requires at least one message".into(), - )); - } - if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) { - return Err(Error::Unsupported(reason.0)); - } - Ok(ResolvedChatCompletionsRequest { - model, - custom_llm_provider, - config, - messages, - optional_params: request.optional_params, - api_key: request.api_key, - api_base: request.api_base, - extra_headers: request.extra_headers, - timeout: request.timeout, - }) -} - -fn validate_environment( - request: &ResolvedChatCompletionsRequest<'_>, - model: &str, - config: &dyn BaseConfig, - secrets: &dyn Lookup, -) -> Result { - let env_lookup = |key: &str| secrets.get(key); - let forwarded = string_headers(request.extra_headers.clone())?; - let validated = config.validate_environment( - forwarded, - request.api_key, - model, - &request.optional_params, - &env_lookup, - )?; - Ok(ValidatedEnvironment { - headers: with_default_headers(validated.headers, config.default_headers()), - auth: validated.auth, - }) -} - -#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)] -pub(super) fn prepare_provider_request( - request: ResolvedChatCompletionsRequest<'_>, - secrets: Secrets, -) -> Result { - let environment = - validate_environment(&request, &request.model, request.config, secrets.as_ref())?; - let model = request.model; - let config = request.config; - let env_lookup = |key: &str| secrets.get(key); - let url = config.get_complete_url( - request.api_base, - &model, - &request.optional_params, - &env_lookup, - )?; - let transformed = - config.transform_request(&model, request.messages, request.optional_params.clone())?; - - Ok(ProviderChatCompletionsRequest { - model, - custom_llm_provider: request.custom_llm_provider, - config, - url, - body: transformed.body, - optional_params: request.optional_params, - environment, - secrets, - timeout: request.timeout, - api_key: request.api_key.map(|key| SecretValue::new(key.to_string())), - }) -} - -#[cfg(test)] -mod tests { - use litellm_auth::CredentialPlacement; - use litellm_llms::base_llm::auth::{AuthScheme, resolve_auth}; - use serde_json::{Map, Value, json}; - - use super::{prepare_provider_request, resolve_request}; - use crate::{ - Error, - types::{ChatCompletionsRequest, ProviderChatCompletionsRequest}, - }; - - fn prepare_chat_completions_call( - request: ChatCompletionsRequest<'_>, - ) -> Result { - prepare_provider_request( - resolve_request(request)?, - std::sync::Arc::new(|_: &str| None), - ) - } - - /// The headers as they go on the wire, credential applied. - fn wire_headers(prepared: &ProviderChatCompletionsRequest) -> Vec<(String, String)> { - tokio::runtime::Builder::new_current_thread() - .build() - .unwrap() - .block_on(resolve_auth( - &litellm_auth::AuthServices::default(), - prepared.environment.clone(), - &|_| None, - )) - .unwrap() - .headers - } - - fn request<'a>( - model: &'a str, - provider: Option<&'a str>, - messages: Value, - optional_params: Value, - ) -> ChatCompletionsRequest<'a> { - ChatCompletionsRequest { - model, - messages, - optional_params: match optional_params { - Value::Object(map) => map, - other => panic!("params must be an object, got {other}"), - }, - api_key: Some("sk-test"), - api_base: None, - custom_llm_provider: provider, - extra_headers: None, - timeout: None, - } - } - - fn preparation_error(request: ChatCompletionsRequest<'_>) -> Error { - prepare_chat_completions_call(request) - .err() - .expect("request preparation should fail") - } - - #[test] - fn resolves_the_provider_from_the_model_prefix() { - let prepared = prepare_chat_completions_call(request( - "anthropic/claude-sonnet-4-5", - None, - json!([{"role": "user", "content": "hi"}]), - json!({"max_tokens": 16}), - )) - .expect("prepares"); - assert_eq!(prepared.model, "claude-sonnet-4-5"); - assert_eq!(prepared.url, "https://api.anthropic.com/v1/messages"); - assert_eq!(prepared.body["model"], json!("claude-sonnet-4-5")); - } - - #[test] - fn strips_an_explicit_provider_prefix_from_the_model() { - let prepared = prepare_chat_completions_call(request( - "anthropic/claude-sonnet-4-5", - Some("anthropic"), - json!([{"role": "user", "content": "hi"}]), - json!({}), - )) - .expect("prepares"); - assert_eq!(prepared.model, "claude-sonnet-4-5"); - } - - #[test] - fn adds_the_auth_and_default_headers() { - let prepared = prepare_chat_completions_call(request( - "claude-sonnet-4-5", - Some("anthropic"), - json!([{"role": "user", "content": "hi"}]), - json!({}), - )) - .expect("prepares"); - assert!( - wire_headers(&prepared).contains(&("x-api-key".to_string(), "sk-test".to_string())) - ); - assert!( - wire_headers(&prepared) - .contains(&("anthropic-version".to_string(), "2023-06-01".to_string())) - ); - assert!(matches!( - prepared.environment.auth, - AuthScheme::Credential { - placement: CredentialPlacement::Header("x-api-key"), - .. - } - )); - } - - #[test] - fn the_deployment_credential_replaces_a_caller_supplied_auth_header() { - // Python builds `{**headers, **anthropic_headers}`, so the deployment's key - // overwrites a forwarded one. Honouring the caller's would let whoever sends - // the request choose the Anthropic principal it bills to. - let mut call = request( - "claude-sonnet-4-5", - Some("anthropic"), - json!([{"role": "user", "content": "hi"}]), - json!({}), - ); - call.extra_headers = Some(Map::from_iter([( - "X-Api-Key".to_string(), - json!("sk-caller"), - )])); - let prepared = prepare_chat_completions_call(call).expect("prepares"); - let headers = wire_headers(&prepared); - let keys: Vec<_> = headers - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key")) - .collect(); - assert_eq!(keys.len(), 1, "got {:?}", headers); - assert_eq!(keys[0].1, "sk-test"); - } - - #[test] - fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() { - // Anthropic's `validate_environment` pops `x-api-key` and sets `authorization` - // for an OAuth token, so re-adding the key here would put the credential into - // a header the host removed on purpose. - let mut call = request( - "claude-sonnet-4-5", - Some("anthropic"), - json!([{"role": "user", "content": "hi"}]), - json!({}), - ); - call.extra_headers = Some(Map::from_iter([ - ( - "Authorization".to_string(), - json!("Bearer sk-ant-oat01-token"), - ), - ("X-Api-Key".to_string(), json!("sk-caller")), - ])); - let prepared = prepare_chat_completions_call(call).expect("prepares"); - assert!( - !wire_headers(&prepared) - .iter() - .any(|(name, value)| name.eq_ignore_ascii_case("x-api-key") && value == "sk-test"), - "the resolved key must not be applied over an OAuth bearer, got {:?}", - wire_headers(&prepared) - ); - assert!( - wire_headers(&prepared) - .iter() - .any(|(name, value)| name.eq_ignore_ascii_case("authorization") - && value == "Bearer sk-ant-oat01-token") - ); - } - - #[test] - fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() { - // Only an OAuth bearer replaces the credential. Python sends the deployment's - // `x-api-key` alongside any other forwarded `authorization`, so deferring on - // the mere presence of that header would drop the deployment's auth. - let mut call = request( - "claude-sonnet-4-5", - Some("anthropic"), - json!([{"role": "user", "content": "hi"}]), - json!({}), - ); - call.extra_headers = Some(Map::from_iter([ - ("Authorization".to_string(), json!("Bearer unrelated")), - ("X-Api-Key".to_string(), json!("sk-caller")), - ])); - let prepared = prepare_chat_completions_call(call).expect("prepares"); - let headers = wire_headers(&prepared); - let keys: Vec<_> = headers - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key")) - .collect(); - assert_eq!(keys.len(), 1, "got {:?}", headers); - assert_eq!(keys[0].1, "sk-test"); - assert!( - wire_headers(&prepared) - .iter() - .any(|(name, value)| name.eq_ignore_ascii_case("authorization") - && value == "Bearer unrelated"), - "the unrelated authorization must survive, got {:?}", - wire_headers(&prepared) - ); - } - - #[rstest::rstest] - fn rejects_empty_messages_before_resolving_credentials() { - let call = ChatCompletionsRequest { - api_key: None, - ..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({})) - }; - assert!(matches!(preparation_error(call), Error::InvalidRequest(_))); - } - - #[test] - fn rejects_an_unknown_provider() { - assert_eq!( - preparation_error(request( - "openai/gpt-4o", - None, - json!([{"role": "user", "content": "hi"}]), - json!({}), - )), - Error::InvalidProvider("openai".to_string()) - ); - } - - #[test] - fn rejects_a_model_with_no_resolvable_provider() { - assert!(matches!( - preparation_error(request( - "claude-sonnet-4-5", - None, - json!([{"role": "user", "content": "hi"}]), - json!({}), - )), - Error::InvalidProvider(_) - )); - } - - #[test] - fn rejects_an_empty_or_malformed_message_list() { - assert_eq!( - preparation_error(request( - "anthropic/claude-sonnet-4-5", - None, - json!([]), - json!({}), - )), - Error::InvalidRequest( - "chat completions requires at least one message" - .to_string() - .into() - ) - ); - assert!(matches!( - preparation_error(request( - "anthropic/claude-sonnet-4-5", - None, - json!("not a list"), - json!({}), - )), - Error::InvalidRequest(_) - )); - } - - #[test] - fn rejects_non_string_extra_headers() { - let mut call = request( - "anthropic/claude-sonnet-4-5", - None, - json!([{"role": "user", "content": "hi"}]), - json!({}), - ); - call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))])); - assert_eq!( - preparation_error(call), - Error::Headers(litellm_http::request::HeaderError { - context: "chat completions", - name: "x-trace".to_string(), - actual: "number", - }) - ); - } - - #[test] - fn prepares_a_bedrock_call_without_resolving_credentials() { - let mut call = request( - "bedrock/us-east-1/anthropic.claude-v2", - None, - json!([{"role": "user", "content": "hi"}]), - json!({"maxTokens": 16}), - ); - call.api_key = None; - let prepared = prepare_chat_completions_call(call).expect("prepares"); - assert_eq!( - prepared.url, - "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse" - ); - assert!(matches!( - &prepared.environment.auth, - AuthScheme::AwsSigV4 { region, service: "bedrock", .. } if region == "us-east-1" - )); - // SigV4 signs the serialized body, so prepare must not have added an - // Authorization header; the signer does it over the bytes sent. - assert!( - !prepared - .environment - .headers - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case("authorization")) - ); - assert_eq!(prepared.body["inferenceConfig"], json!({"maxTokens": 16})); - } - - #[tokio::test] - async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() { - // Python signs only the AWS header set and reattaches the rest, so a header - // the caller forwarded rides along without joining the canonical request. - // Signing it makes Converse 403 on a deployment that works on Python. - let mut call = request( - "bedrock/us-east-1/anthropic.claude-v2", - None, - json!([{"role": "user", "content": "hi"}]), - json!({ - "maxTokens": 16, - "aws_access_key_id": "AKIDEXAMPLE", - "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" - }), - ); - // A key would resolve to a bearer token and never reach the signer. - call.api_key = None; - call.extra_headers = Some(Map::from_iter([( - "x-request-id".to_string(), - json!("abc-123"), - )])); - let prepared = prepare_chat_completions_call(call).expect("prepares"); - let authenticated = resolve_auth( - &litellm_auth::AuthServices::default(), - prepared.environment, - &|_| None, - ) - .await - .expect("resolves"); - let signed = crate::handler::outbound_request( - authenticated, - prepared.url, - &prepared.body, - prepared.timeout, - ) - .expect("signs"); - - let authorization = signed - .header("authorization") - .expect("carries an authorization header") - .to_string(); - assert!( - authorization.starts_with("AWS4-HMAC-SHA256"), - "expected a SigV4 signature, got {authorization}" - ); - assert!( - !authorization.contains("x-request-id"), - "forwarded header reached SignedHeaders: {authorization}" - ); - // It still goes on the wire, it is just not part of the signature. - assert!( - signed - .headers() - .iter() - .any(|(name, value)| name == "x-request-id" && value == "abc-123"), - "forwarded header was dropped instead of reattached" - ); - } - - #[rstest::rstest] - #[case::authorization("Authorization")] - #[case::amz_date("x-amz-date")] - #[case::security_token("x-amz-security-token")] - #[case::date("Date")] - #[tokio::test] - async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) { - let call = ChatCompletionsRequest { - api_key: None, - extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])), - ..request( - "bedrock/us-east-1/anthropic.claude-v2", - None, - json!([{"role": "user", "content": "hi"}]), - json!({ - "maxTokens": 16, - "aws_access_key_id": "AKIDEXAMPLE", - "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" - }), - ) - }; - let prepared = prepare_chat_completions_call(call).expect("prepares"); - let authenticated = resolve_auth( - &litellm_auth::AuthServices::default(), - prepared.environment, - &|_| None, - ) - .await - .expect("resolves"); - let error = crate::handler::outbound_request( - authenticated, - prepared.url, - &prepared.body, - prepared.timeout, - ) - .expect_err("conflicting signing headers must fail"); - assert!( - matches!(error, Error::Unsupported(_)), - "{forwarded} returned {error:?}" - ); - } - - #[test] - fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() { - // `get_request_headers` assigns `headers["Authorization"]` unconditionally - // once a bearer token resolves, so the deployment's identity wins on - // Python. Keeping the caller's would authorize and bill the call as a - // different principal, and only when the deployment carries `rust: true`. - let mut call = request( - "bedrock/us-east-1/anthropic.claude-v2", - None, - json!([{"role": "user", "content": "hi"}]), - json!({"maxTokens": 16}), - ); - call.extra_headers = Some(Map::from_iter([( - "Authorization".to_string(), - json!("Bearer caller-supplied"), - )])); - let prepared = prepare_chat_completions_call(call).expect("prepares"); - let headers = wire_headers(&prepared); - let authorizations: Vec<_> = headers - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case("authorization")) - .map(|(_, value)| value.as_str()) - .collect(); - assert_eq!( - authorizations, - vec!["Bearer sk-test"], - "the deployment token must be the only authorization on the wire" - ); - } - - #[test] - fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() { - // The opposite precedence, and deliberate: Anthropic's own transform - // honours a forwarded OAuth bearer, so the Bedrock fix above must not be - // generalized into a rule that the configured key always wins. - // - // An OAuth bearer is the whole of that exception. This forwarded a plain - // `x-api-key` until round 17, which read as the same claim and was not: - // Python overwrites a forwarded `x-api-key` with the deployment's. - let mut call = request( - "claude-sonnet-4-5", - Some("anthropic"), - json!([{"role": "user", "content": "hi"}]), - json!({}), - ); - call.extra_headers = Some(Map::from_iter([( - "authorization".to_string(), - json!("Bearer sk-ant-oat01-forwarded"), - )])); - let prepared = prepare_chat_completions_call(call).expect("prepares"); - let headers = wire_headers(&prepared); - let keys: Vec<_> = headers - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key")) - .map(|(_, value)| value.as_str()) - .collect(); - assert!(keys.is_empty(), "got {:?}", headers); - assert!( - wire_headers(&prepared) - .iter() - .any(|(name, value)| name.eq_ignore_ascii_case("authorization") - && value == "Bearer sk-ant-oat01-forwarded") - ); - } - - #[test] - fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() { - // The configured bearer identity has its own account and quota boundary, - // so a request carrying one must not be signed as whatever principal the - // host's AWS credentials resolve to. - let prepared = prepare_chat_completions_call(request( - "bedrock/us-east-1/anthropic.claude-v2", - None, - json!([{"role": "user", "content": "hi"}]), - json!({"maxTokens": 16}), - )) - .expect("prepares"); - assert!(matches!( - &prepared.environment.auth, - AuthScheme::Credential { placement: CredentialPlacement::Bearer, secret } - if secret.expose() == "sk-test" - )); - assert!( - wire_headers(&prepared) - .iter() - .any(|(name, value)| name.eq_ignore_ascii_case("authorization") - && value == "Bearer sk-test"), - "prepare did not carry the bearer token" - ); - } -} diff --git a/litellm-rust/crates/inference-chat/src/route.rs b/litellm-rust/crates/inference-chat/src/route.rs index 50fc412a705..4a7bc0e169f 100644 --- a/litellm-rust/crates/inference-chat/src/route.rs +++ b/litellm-rust/crates/inference-chat/src/route.rs @@ -1,4 +1,3 @@ -use litellm_host::observation::ObservationSender; use std::convert::Infallible; use litellm_host::{ @@ -7,10 +6,7 @@ use litellm_host::{ }; use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse; -use super::{ - ChatCompletionsRoute, Error, - types::{ChatCompletionsCall, ChatCompletionsRequest}, -}; +use super::{ChatCompletionsRoute, Error, types::ChatCompletionsCall}; pub struct ChatCompletions; @@ -37,44 +33,12 @@ impl ChatCompletionsRoute { call, observers, move |call, _, interceptors, observers| async move { - self.run_call(call, cache_options, &interceptors, observers.as_ref()) + self.run(call, cache_options, &interceptors, observers.as_ref()) .await .map(CallOutput::Complete) }, ) } - - #[tracing::instrument(name = "litellm.route", skip_all, fields( - route = "chat_completions", - model = %call.model, - provider, - resolved_model, - stream = false, - outcome - ))] - pub(super) async fn run_call( - &self, - call: ChatCompletionsCall, - cache_options: Option, - interceptors: &impl litellm_host::interceptors::Interceptors, - observers: Option<&ObservationSender>, - ) -> Result { - litellm_inference::diagnostic::unary(async { - let request = ChatCompletionsRequest { - model: &call.model, - messages: call.messages, - optional_params: call.optional_params, - api_key: call.api_key.as_deref(), - api_base: call.api_base.as_deref(), - custom_llm_provider: call.custom_llm_provider.as_deref(), - extra_headers: call.extra_headers, - timeout: call.timeout, - }; - self.run(request, cache_options, interceptors, observers) - .await - }) - .await - } } impl litellm_inference::caching::Cachable for ChatCompletions { diff --git a/litellm-rust/crates/inference-chat/src/types.rs b/litellm-rust/crates/inference-chat/src/types.rs index 5d7d04804f7..23b1d26de54 100644 --- a/litellm-rust/crates/inference-chat/src/types.rs +++ b/litellm-rust/crates/inference-chat/src/types.rs @@ -1,8 +1,5 @@ -use litellm_secrets::source::Secrets; use std::time::Duration; -use litellm_auth::SecretValue; -use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig}; use litellm_llms_types::formats::chat_completions::ChatMessage; use serde_json::{Map, Value}; @@ -49,31 +46,11 @@ impl From> for ChatCompletionsCall { } } -pub struct ResolvedChatCompletionsRequest<'a> { - pub model: String, - pub custom_llm_provider: String, - pub config: &'static dyn BaseConfig, +pub struct ResolvedChatCompletionsRequest { pub messages: Vec, pub optional_params: Map, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, + pub api_key: Option, + pub api_base: Option, pub extra_headers: Option>, pub timeout: Option, } - -pub struct ProviderChatCompletionsRequest { - pub model: String, - pub custom_llm_provider: String, - pub config: &'static dyn BaseConfig, - pub url: String, - pub body: Value, - /// The route's parameters before the provider transformation, reported to the host - /// beside the wire request. - pub optional_params: Map, - /// The forwarded and default headers plus how the call authenticates; the credential - /// itself is applied when the request is sent. - pub environment: ValidatedEnvironment, - pub secrets: Secrets, - pub timeout: Option, - pub api_key: Option, -} diff --git a/litellm-rust/crates/inference-chat/tests/request.rs b/litellm-rust/crates/inference-chat/tests/request.rs new file mode 100644 index 00000000000..5988a7baee5 --- /dev/null +++ b/litellm-rust/crates/inference-chat/tests/request.rs @@ -0,0 +1,439 @@ +use std::time::Duration; + +use litellm_inference_chat::{Error, types::ChatCompletionsRequest}; +use rstest::rstest; +use serde_json::{Map, Value, json}; +use wiremock::ResponseTemplate; + +mod support; +use support::*; + +const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#; + +fn request<'a>( + model: &'a str, + provider: Option<&'a str>, + messages: Value, + optional_params: Value, +) -> ChatCompletionsRequest<'a> { + let Value::Object(optional_params) = optional_params else { + panic!("params must be an object"); + }; + ChatCompletionsRequest { + model, + messages, + optional_params, + api_key: Some("sk-test"), + api_base: None, + custom_llm_provider: provider, + extra_headers: None, + timeout: Some(Duration::from_secs(10)), + } +} + +async fn complete( + request: ChatCompletionsRequest<'_>, +) -> Result { + chat_completions_route().execute(request, &(), None).await +} + +fn object(value: Value) -> Map { + let Value::Object(map) = value else { + panic!("expected an object"); + }; + map +} + +fn anthropic_response() -> ResponseTemplate { + ResponseTemplate::new(200).set_body_raw(ANTHROPIC_MESSAGE, "application/json") +} + +fn bedrock_response() -> ResponseTemplate { + json_response(json!({ + "output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15} + })) +} + +#[rstest] +#[tokio::test] +async fn resolves_the_provider_from_the_model_prefix() { + let upstream = upstream([anthropic_response()]).await; + let base = upstream.uri(); + complete(ChatCompletionsRequest { + api_base: Some(&base), + ..request( + "anthropic/claude-sonnet-4-5", + None, + json!([{"role": "user", "content": "hi"}]), + json!({"max_tokens": 16}), + ) + }) + .await + .expect("call succeeds"); + + let sent = only_request(&upstream).await; + assert_eq!(sent.url.path(), "/v1/messages"); + assert_eq!(sent.json()["model"], "claude-sonnet-4-5"); +} + +#[rstest] +#[tokio::test] +async fn strips_an_explicit_provider_prefix_from_the_model() { + let upstream = upstream([anthropic_response()]).await; + let base = upstream.uri(); + complete(ChatCompletionsRequest { + api_base: Some(&base), + ..request( + "anthropic/claude-sonnet-4-5", + Some("anthropic"), + json!([{"role": "user", "content": "hi"}]), + json!({}), + ) + }) + .await + .expect("call succeeds"); + + assert_eq!( + only_request(&upstream).await.json()["model"], + "claude-sonnet-4-5" + ); +} + +#[rstest] +#[tokio::test] +async fn adds_the_auth_and_default_headers() { + let upstream = upstream([anthropic_response()]).await; + let base = upstream.uri(); + complete(ChatCompletionsRequest { + api_base: Some(&base), + ..request( + "claude-sonnet-4-5", + Some("anthropic"), + json!([{"role": "user", "content": "hi"}]), + json!({}), + ) + }) + .await + .expect("call succeeds"); + + let sent = only_request(&upstream).await; + assert_eq!(sent.header_values("x-api-key"), ["sk-test"]); + assert_eq!(sent.header("anthropic-version"), Some("2023-06-01")); +} + +#[rstest] +#[tokio::test] +async fn the_deployment_credential_replaces_a_caller_supplied_auth_header() { + let upstream = upstream([anthropic_response()]).await; + let base = upstream.uri(); + complete(ChatCompletionsRequest { + api_base: Some(&base), + extra_headers: Some(object(json!({"X-Api-Key": "sk-caller"}))), + ..request( + "claude-sonnet-4-5", + Some("anthropic"), + json!([{"role": "user", "content": "hi"}]), + json!({}), + ) + }) + .await + .expect("call succeeds"); + + assert_eq!( + only_request(&upstream).await.header_values("x-api-key"), + ["sk-test"] + ); +} + +#[rstest] +#[tokio::test] +async fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() { + let upstream = upstream([anthropic_response()]).await; + let base = upstream.uri(); + complete(ChatCompletionsRequest { + api_base: Some(&base), + extra_headers: Some(object(json!({ + "Authorization": "Bearer sk-ant-oat01-token", + "X-Api-Key": "sk-caller" + }))), + ..request( + "claude-sonnet-4-5", + Some("anthropic"), + json!([{"role": "user", "content": "hi"}]), + json!({}), + ) + }) + .await + .expect("call succeeds"); + + let sent = only_request(&upstream).await; + assert_eq!(sent.header_values("x-api-key"), ["sk-caller"]); + assert_eq!( + sent.header_values("authorization"), + ["Bearer sk-ant-oat01-token"] + ); +} + +#[rstest] +#[tokio::test] +async fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() { + let upstream = upstream([anthropic_response()]).await; + let base = upstream.uri(); + complete(ChatCompletionsRequest { + api_base: Some(&base), + extra_headers: Some(object(json!({ + "Authorization": "Bearer unrelated", + "X-Api-Key": "sk-caller" + }))), + ..request( + "claude-sonnet-4-5", + Some("anthropic"), + json!([{"role": "user", "content": "hi"}]), + json!({}), + ) + }) + .await + .expect("call succeeds"); + + let sent = only_request(&upstream).await; + assert_eq!(sent.header_values("x-api-key"), ["sk-test"]); + assert_eq!(sent.header_values("authorization"), ["Bearer unrelated"]); +} + +#[rstest] +#[tokio::test] +async fn rejects_empty_messages_before_resolving_credentials() { + let error = complete(ChatCompletionsRequest { + api_key: None, + ..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({})) + }) + .await + .expect_err("empty messages are rejected"); + assert!(matches!(error, Error::InvalidRequest(_))); +} + +#[rstest] +#[tokio::test] +async fn rejects_an_unknown_provider() { + assert_eq!( + complete(request( + "openai/gpt-4o", + None, + json!([{"role": "user", "content": "hi"}]), + json!({}), + )) + .await + .unwrap_err(), + Error::InvalidProvider("openai".into()) + ); +} + +#[rstest] +#[tokio::test] +async fn rejects_a_model_with_no_resolvable_provider() { + assert!(matches!( + complete(request( + "claude-sonnet-4-5", + None, + json!([{"role": "user", "content": "hi"}]), + json!({}), + )) + .await, + Err(Error::InvalidProvider(_)) + )); +} + +#[rstest] +#[case::empty(json!([]), Some("chat completions requires at least one message"))] +#[case::malformed(json!("not a list"), None)] +#[tokio::test] +async fn rejects_an_empty_or_malformed_message_list( + #[case] messages: Value, + #[case] expected: Option<&str>, +) { + let error = complete(request( + "anthropic/claude-sonnet-4-5", + None, + messages, + json!({}), + )) + .await + .unwrap_err(); + if let Some(expected) = expected { + assert_eq!(error, Error::InvalidRequest(expected.to_string().into())); + } else { + assert!(matches!(error, Error::InvalidRequest(_))); + } +} + +#[rstest] +#[tokio::test] +async fn rejects_non_string_extra_headers() { + assert_eq!( + complete(ChatCompletionsRequest { + extra_headers: Some(Map::from_iter([("x-trace".to_string(), json!(7))])), + ..request( + "anthropic/claude-sonnet-4-5", + None, + json!([{"role": "user", "content": "hi"}]), + json!({}), + ) + }) + .await + .unwrap_err(), + Error::Headers(litellm_http::request::HeaderError { + context: "chat completions", + name: "x-trace".to_string(), + actual: "number", + }) + ); +} + +#[rstest] +#[tokio::test] +async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() { + let upstream = upstream([bedrock_response()]).await; + let base = upstream.uri(); + let mut call = request( + "bedrock/us-east-1/anthropic.claude-v2", + None, + json!([{"role": "user", "content": "hi"}]), + json!({ + "maxTokens": 16, + "aws_access_key_id": "AKIDEXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }), + ); + call.api_key = None; + call.api_base = Some(&base); + call.extra_headers = Some(Map::from_iter([( + "x-request-id".to_string(), + json!("abc-123"), + )])); + complete(call).await.expect("call succeeds"); + + let sent = only_request(&upstream).await; + let authorization = sent.header("authorization").expect("signed request"); + assert!(authorization.starts_with("AWS4-HMAC-SHA256")); + assert!(!authorization.contains("x-request-id")); + assert_eq!(sent.header("x-request-id"), Some("abc-123")); + assert_eq!(sent.url.path(), "/model/anthropic.claude-v2/converse"); + let body: Value = serde_json::from_slice(&sent.body).unwrap(); + assert_eq!(body["inferenceConfig"], json!({"maxTokens": 16})); + assert_eq!( + body["messages"], + json!([{"role": "user", "content": [{"text": "hi"}]}]) + ); +} + +#[rstest] +#[case::authorization("Authorization")] +#[case::amz_date("x-amz-date")] +#[case::security_token("x-amz-security-token")] +#[case::date("Date")] +#[tokio::test] +async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) { + let upstream = wiremock::MockServer::start().await; + let base = upstream.uri(); + let call = ChatCompletionsRequest { + api_key: None, + api_base: Some(&base), + extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])), + ..request( + "bedrock/us-east-1/anthropic.claude-v2", + None, + json!([{"role": "user", "content": "hi"}]), + json!({ + "maxTokens": 16, + "aws_access_key_id": "AKIDEXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + }), + ) + }; + let error = complete(call) + .await + .expect_err("conflicting signing headers fail"); + assert!(matches!(error, Error::Unsupported(_)), "{error:?}"); + assert!(received(&upstream).await.is_empty()); +} + +#[rstest] +#[tokio::test] +async fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() { + let upstream = upstream([bedrock_response()]).await; + let base = upstream.uri(); + complete(ChatCompletionsRequest { + api_base: Some(&base), + extra_headers: Some(Map::from_iter([( + "Authorization".to_string(), + json!("Bearer caller-supplied"), + )])), + ..request( + "bedrock/us-east-1/anthropic.claude-v2", + None, + json!([{"role": "user", "content": "hi"}]), + json!({"maxTokens": 16}), + ) + }) + .await + .expect("call succeeds"); + + assert_eq!( + only_request(&upstream).await.header_values("authorization"), + ["Bearer sk-test"] + ); +} + +#[rstest] +#[tokio::test] +async fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() { + let upstream = upstream([anthropic_response()]).await; + let base = upstream.uri(); + complete(ChatCompletionsRequest { + api_base: Some(&base), + extra_headers: Some(Map::from_iter([( + "authorization".to_string(), + json!("Bearer sk-ant-oat01-forwarded"), + )])), + ..request( + "claude-sonnet-4-5", + Some("anthropic"), + json!([{"role": "user", "content": "hi"}]), + json!({}), + ) + }) + .await + .expect("call succeeds"); + + let sent = only_request(&upstream).await; + assert!(sent.header_values("x-api-key").is_empty()); + assert_eq!( + sent.header_values("authorization"), + ["Bearer sk-ant-oat01-forwarded"] + ); +} + +#[rstest] +#[tokio::test] +async fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() { + let upstream = upstream([bedrock_response()]).await; + let base = upstream.uri(); + complete(ChatCompletionsRequest { + api_base: Some(&base), + ..request( + "bedrock/us-east-1/anthropic.claude-v2", + None, + json!([{"role": "user", "content": "hi"}]), + json!({"maxTokens": 16}), + ) + }) + .await + .expect("call succeeds"); + + assert_eq!( + only_request(&upstream).await.header_values("authorization"), + ["Bearer sk-test"] + ); +} From ce0dbe60dfe9bea323d94c34d170a49ab737680e Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:36:32 -0700 Subject: [PATCH 25/50] chore(cost-map): add fireworks minimax m3.1 flash priority prices from the prices api (#47346) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 16 ++++++++++++++++ model_prices_and_context_window.json | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index bb0cdd51d9c..e32eb4ba747 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76100,5 +76100,21 @@ "supported_output_modalities": [ "image" ] + }, + "fireworks_ai/accounts/fireworks/models/minimax-m3p1-flash": { + "cache_read_input_token_cost_priority": 2.25e-08, + "input_cost_per_token_priority": 2.25e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token_priority": 9e-07, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bb0cdd51d9c..e32eb4ba747 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76100,5 +76100,21 @@ "supported_output_modalities": [ "image" ] + }, + "fireworks_ai/accounts/fireworks/models/minimax-m3p1-flash": { + "cache_read_input_token_cost_priority": 2.25e-08, + "input_cost_per_token_priority": 2.25e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token_priority": 9e-07, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } From 3eeee3e76118783671bd7c9dd44a192eedf917ad Mon Sep 17 00:00:00 2001 From: Alex Ker Date: Mon, 19 Oct 2026 12:40:13 -0400 Subject: [PATCH 26/50] fix(baseten): list current Model APIs instead of retired 2023 deployment IDs (#46945) * fix(baseten): list current Model APIs instead of retired 2023 deployment IDs `baseten/*` wildcard expansion and `/v1/models` read a hardcoded set of three 2023 deployment IDs (Falcon 7B / WizardLM / MosaicML) that no longer exist, while the model cost map already carries current Baseten entries. - Populate `baseten_models` from the model cost map, like fireworks_ai. - Add `BasetenConfig.get_models()` against the Model API `/v1/models` endpoint and register it in `ProviderConfigManager.get_provider_model_info`, so `check_provider_endpoint: true` returns the live catalog. - Remove the retired deployment IDs from `constants.py`. Co-Authored-By: Claude Opus 5.5 * fix(baseten): mark retired Model APIs deprecated and skip them in listings 15 Baseten entries in the model cost map are no longer servable: the Model API returns 410 "model version ... has been deprecated" (DeepSeek-V3.1 returns 400 deactivated), and none appear in `inference.baseten.co/v1/models`. - Set `deprecation_date` on those entries (kept for spend tracking). - Skip entries past their `deprecation_date` when building `baseten_models`, so `baseten/*` expansion only lists models that still serve. Co-Authored-By: Claude Opus 5.5 * test(baseten): assert the custom api_base model listing on the returned models The custom api_base test only inspected the patched client's call args, which trips the TQ002 mock-echo gate. Serve the custom URL with respx and assert on the models that come back instead, so a wrong host or a broken parse both fail. * fix(baseten): drop newly retired models on cost map reload and fake discovery at the HTTP boundary A cost map refresh only ever added to the Baseten set, so a model that gained a past deprecation_date after boot stayed listed under baseten/* until restart. Discard it on reload. Tests now serve the catalog with respx instead of patching our own HTTP client, the isinstance registration test is gone since the get_valid_models test already proves discovery end to end, and new test locals carry Final. --------- Co-authored-by: AlexKer Co-authored-by: Claude Opus 5.5 Co-authored-by: ryan-crabbe-berri --- litellm/__init__.py | 18 ++- litellm/constants.py | 8 -- litellm/llms/baseten/chat.py | 36 +++++- ...odel_prices_and_context_window_backup.json | 15 +++ litellm/utils.py | 2 + model_prices_and_context_window.json | 15 +++ .../baseten/chat/test_baseten_completions.py | 109 +++++++++++++++++- 7 files changed, 191 insertions(+), 12 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index cc5fbc1e72e..b3e80bfece5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -7,6 +7,7 @@ _apply_process_defaults() ### INIT VARIABLES ######################### import threading import os +import datetime from collections.abc import Mapping, Sequence from typing import ( @@ -62,7 +63,6 @@ from litellm.constants import ( modelscope_models, empower_models, together_ai_models, - baseten_models, WANDB_MODELS, REPEATED_STREAMING_CHUNK_LIMIT, request_timeout, @@ -624,6 +624,7 @@ bedrock_models: Set = set() bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS) fal_ai_models: Set = set() fireworks_ai_models: Set = set() +baseten_models: Set[str] = set() fireworks_ai_embedding_models: Set = set() deepinfra_models: Set = set() perplexity_models: Set = set() @@ -742,6 +743,16 @@ def is_openai_finetune_model(key: str) -> bool: return key.startswith("ft:") and not key.count(":") > 1 +def _is_past_deprecation_date(value: Mapping[str, object]) -> bool: + deprecation_date: Final = value.get("deprecation_date") + if not isinstance(deprecation_date, str): + return False + try: + return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today() + except ValueError: + return False + + def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None: for key, value in tuple(model_cost_map.items()): if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key): @@ -842,6 +853,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. if "-to-" not in key and "fireworks-ai-default" not in key: fireworks_ai_models.add(key) + elif value.get("litellm_provider") == "baseten": + if _is_past_deprecation_date(value): + baseten_models.discard(key) + else: + baseten_models.add(key) elif value.get("litellm_provider") == "fireworks_ai-embedding-models": # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. if "-to-" not in key: diff --git a/litellm/constants.py b/litellm/constants.py index da95b97036a..5473a3c6390 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1256,14 +1256,6 @@ together_ai_models: Final[set] = set( # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...) -baseten_models: Final[set] = set( - [ - "qvv0xeq", - "q841o8w", - "31dxrj3", - ] -) # FALCON 7B # WizardLM # Mosaic ML - featherless_ai_models: Final[set] = set( [ "featherless-ai/Qwerky-72B", diff --git a/litellm/llms/baseten/chat.py b/litellm/llms/baseten/chat.py index e92f47c41c0..381d487c36d 100644 --- a/litellm/llms/baseten/chat.py +++ b/litellm/llms/baseten/chat.py @@ -1,6 +1,10 @@ from typing import Final +import litellm from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.secret_managers.main import get_secret_str + +BASETEN_MODEL_API_BASE: Final = "https://inference.baseten.co/v1" class BasetenConfig(OpenAIGPTConfig): @@ -89,11 +93,39 @@ class BasetenConfig(OpenAIGPTConfig): Get the OpenAI compatible provider info for Baseten """ # Default to Model API - default_api_base: Final = "https://inference.baseten.co/v1" + default_api_base: Final = BASETEN_MODEL_API_BASE default_api_key: Final = api_key or "BASETEN_API_KEY" return default_api_base, default_api_key + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or get_secret_str("BASETEN_API_KEY") + + @staticmethod + def get_api_base(api_base: str | None = None) -> str | None: + return api_base or get_secret_str("BASETEN_API_BASE") or BASETEN_MODEL_API_BASE + + def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + """ + Calls Baseten's Model API `/v1/models` endpoint and returns the live catalog. + """ + resolved_api_key: Final = self.get_api_key(api_key) + if resolved_api_key is None: + raise ValueError( + "BASETEN_API_KEY is not set. Please set the environment variable, to query Baseten's `/models` endpoint." + ) + resolved_api_base: Final = (self.get_api_base(api_base) or BASETEN_MODEL_API_BASE).rstrip("/") + response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # untyped params + url=f"{resolved_api_base.removesuffix('/v1')}/v1/models", + headers={"Authorization": f"Bearer {resolved_api_key}"}, + ) + if response.status_code != 200: + raise ValueError( + f"Failed to fetch models from Baseten. Status code: {response.status_code}, Response: {response.text}" + ) + return ["baseten/" + model["id"] for model in response.json()["data"]] + @staticmethod def is_dedicated_deployment(model: str) -> bool: """ @@ -118,4 +150,4 @@ class BasetenConfig(OpenAIGPTConfig): return f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" else: # Use Model API - return "https://inference.baseten.co/v1" + return BASETEN_MODEL_API_BASE diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e32eb4ba747..e0ace9c4f39 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -33684,24 +33684,28 @@ "output_cost_per_token": 1.2e-06 }, "baseten/MiniMaxAI/MiniMax-M2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.2e-06 }, "baseten/nvidia/Nemotron-120B-A12B": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 7.5e-07 }, "baseten/zai-org/GLM-5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3.15e-06 }, "baseten/zai-org/GLM-4.7": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "baseten", @@ -33719,24 +33723,28 @@ "supports_vision": false }, "baseten/zai-org/GLM-4.6": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.2e-06 }, "baseten/moonshotai/Kimi-K2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3e-06 }, "baseten/moonshotai/Kimi-K2-Thinking": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.5e-06 }, "baseten/moonshotai/Kimi-K2-Instruct-0905": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", @@ -33760,12 +33768,14 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V3.1": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.5e-06 }, "baseten/deepseek-ai/DeepSeek-V3-0324": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 7.7e-07, "litellm_provider": "baseten", "mode": "chat", @@ -72125,6 +72135,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.6": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72142,6 +72153,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.7-Code": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72193,6 +72205,7 @@ "supports_vision": false }, "baseten/thinkingmachines/inkling": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, "litellm_provider": "baseten", @@ -72210,6 +72223,7 @@ "supports_vision": true }, "baseten/thinkingmachines/inkling-small": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "baseten", @@ -72278,6 +72292,7 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V4-Pro": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.45e-07, "input_cost_per_token": 1.74e-06, "litellm_provider": "baseten", diff --git a/litellm/utils.py b/litellm/utils.py index 66d78a03075..ff2c9012ec8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9704,6 +9704,8 @@ class ProviderConfigManager: ) -> BaseLLMModelInfo | None: if LlmProviders.FIREWORKS_AI == provider: return litellm.FireworksAIConfig() + elif LlmProviders.BASETEN == provider: + return litellm.BasetenConfig() elif LlmProviders.OPENAI == provider: return litellm.OpenAIGPTConfig() elif LlmProviders.GEMINI == provider: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e32eb4ba747..e0ace9c4f39 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -33684,24 +33684,28 @@ "output_cost_per_token": 1.2e-06 }, "baseten/MiniMaxAI/MiniMax-M2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.2e-06 }, "baseten/nvidia/Nemotron-120B-A12B": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 7.5e-07 }, "baseten/zai-org/GLM-5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3.15e-06 }, "baseten/zai-org/GLM-4.7": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "baseten", @@ -33719,24 +33723,28 @@ "supports_vision": false }, "baseten/zai-org/GLM-4.6": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.2e-06 }, "baseten/moonshotai/Kimi-K2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3e-06 }, "baseten/moonshotai/Kimi-K2-Thinking": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.5e-06 }, "baseten/moonshotai/Kimi-K2-Instruct-0905": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", @@ -33760,12 +33768,14 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V3.1": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.5e-06 }, "baseten/deepseek-ai/DeepSeek-V3-0324": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 7.7e-07, "litellm_provider": "baseten", "mode": "chat", @@ -72125,6 +72135,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.6": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72142,6 +72153,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.7-Code": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72193,6 +72205,7 @@ "supports_vision": false }, "baseten/thinkingmachines/inkling": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, "litellm_provider": "baseten", @@ -72210,6 +72223,7 @@ "supports_vision": true }, "baseten/thinkingmachines/inkling-small": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "baseten", @@ -72278,6 +72292,7 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V4-Pro": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.45e-07, "input_cost_per_token": 1.74e-06, "litellm_provider": "baseten", diff --git a/tests/unit/llms/baseten/chat/test_baseten_completions.py b/tests/unit/llms/baseten/chat/test_baseten_completions.py index 7cac16cca1c..4f135a1a106 100644 --- a/tests/unit/llms/baseten/chat/test_baseten_completions.py +++ b/tests/unit/llms/baseten/chat/test_baseten_completions.py @@ -1,8 +1,14 @@ import os from typing import Final -import pytest from unittest.mock import patch + +import httpx +import pytest +import respx + +import litellm from litellm.llms.baseten.chat import BasetenConfig +from litellm.utils import get_valid_models class TestBasetenRouting: @@ -50,6 +56,107 @@ class TestBasetenModelAPI: assert api_key == "test-key" +class TestBasetenModelDiscovery: + """Test Baseten model listing for `baseten/*` wildcard expansion and /v1/models""" + + def test_models_by_provider_comes_from_registry(self): + """Static listing uses the model cost map, not the retired 2023 deployment IDs.""" + baseten_models: Final = litellm.models_by_provider["baseten"] + servable_registry_models: Final = { + key + for key, value in litellm.model_cost.items() + if value.get("litellm_provider") == "baseten" and not litellm._is_past_deprecation_date(value) + } + + assert servable_registry_models + assert servable_registry_models <= set(baseten_models) + for retired_id in ("qvv0xeq", "q841o8w", "31dxrj3"): + assert retired_id not in baseten_models + + def test_models_by_provider_skips_deprecated_models(self): + """Retired Model APIs stay in the cost map for spend tracking but are not listed.""" + assert litellm.model_cost["baseten/zai-org/GLM-5"]["deprecation_date"] + assert "baseten/zai-org/GLM-5" not in litellm.models_by_provider["baseten"] + + def test_cost_map_reload_drops_a_newly_retired_model(self): + """A refreshed cost map that marks a listed model retired removes it without a restart.""" + model: Final = "baseten/acme/reload-probe" + try: + litellm.add_known_models({model: {"litellm_provider": "baseten"}}) + assert model in litellm.models_by_provider["baseten"] + + litellm.add_known_models({model: {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"}}) + + assert model not in litellm.models_by_provider["baseten"] + finally: + litellm.baseten_models.discard(model) + litellm.add_known_models({}) + + @pytest.mark.parametrize( + "value, expected", + [ + ({}, False), + ({"deprecation_date": "2000-01-01"}, True), + ({"deprecation_date": "2999-01-01"}, False), + ({"deprecation_date": "not-a-date"}, False), + ], + ) + def test_is_past_deprecation_date(self, value, expected): + assert litellm._is_past_deprecation_date(value) is expected + + def test_get_models_calls_model_api(self, respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.get("https://inference.baseten.co/v1/models").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [ + {"id": "zai-org/GLM-5", "object": "model"}, + {"id": "moonshotai/Kimi-K2.5", "object": "model"}, + ], + }, + ) + ) + + models: Final = BasetenConfig().get_models(api_key="test-key") + + assert models == ["baseten/zai-org/GLM-5", "baseten/moonshotai/Kimi-K2.5"] + assert route.calls.last.request.headers["Authorization"] == "Bearer test-key" + + def test_get_models_respects_custom_api_base(self, respx_mock: respx.MockRouter) -> None: + respx_mock.get("https://custom.baseten.example/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "zai-org/GLM-5"}]}) + ) + + models: Final = BasetenConfig().get_models(api_key="test-key", api_base="https://custom.baseten.example/v1/") + + assert models == ["baseten/zai-org/GLM-5"] + + @patch.dict(os.environ, {}, clear=True) + def test_get_models_requires_api_key(self): + with pytest.raises(ValueError, match="BASETEN_API_KEY"): + BasetenConfig().get_models() + + def test_get_models_raises_on_error_status(self, respx_mock: respx.MockRouter) -> None: + respx_mock.get("https://inference.baseten.co/v1/models").mock( + return_value=httpx.Response(403, text="forbidden") + ) + + with pytest.raises(ValueError, match="403"): + BasetenConfig().get_models(api_key="test-key") + + @patch.dict(os.environ, {"BASETEN_API_KEY": "test-key"}) + def test_get_valid_models_check_provider_endpoint(self, respx_mock: respx.MockRouter) -> None: + """check_provider_endpoint=True returns the live catalog instead of the static list.""" + respx_mock.get("https://inference.baseten.co/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "brand-new/model"}]}) + ) + + models: Final = get_valid_models(check_provider_endpoint=True, custom_llm_provider="baseten") + + assert models == ["baseten/brand-new/model"] + + class TestBasetenReasoningEffort: def test_reasoning_effort_passthrough_with_drop_params(self): config: Final = BasetenConfig() From 6fc60bc00d7e22402eee404719c52f0d72fce95f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:58:19 -0700 Subject: [PATCH 27/50] feat(router): discover kubernetes pods behind a headless service api_base (#44342) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 7 + litellm/router.py | 51 +- .../router_utils/kubernetes_pod_discovery.py | 462 ++++++ litellm/types/litellm_params.py | 1 + litellm/types/router.py | 1 + .../router_code_coverage.py | 6 +- .../test_kubernetes_pod_discovery.py | 1266 +++++++++++++++++ tests/unit/types/test_litellm_params.py | 1 + .../panels/AddModelPanel.integration.test.tsx | 19 + .../add_model/advanced_settings.test.tsx | 31 +- .../add_model/advanced_settings.tsx | 14 + 11 files changed, 1851 insertions(+), 8 deletions(-) create mode 100644 litellm/router_utils/kubernetes_pod_discovery.py create mode 100644 tests/unit/router_utils/test_kubernetes_pod_discovery.py diff --git a/litellm/constants.py b/litellm/constants.py index 5473a3c6390..db49a40a37b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -40,6 +40,13 @@ class ServerStreamingClassification(str, Enum): SERVER_STREAMING_CLASSIFICATION_MARKER: Final = ServerStreamingClassification.MARKER DEFER_PYDANTIC_BUILD: Final = os.getenv("DEFER_PYDANTIC_BUILD", "true") in ("true", "1", "on") DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")) +KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS: Final = get_env_float( + "KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS", 5.0 +) +KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS: Final = get_env_float( + "KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS", 300.0 +) +KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS: Final = 2.0 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"}) ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5) diff --git a/litellm/router.py b/litellm/router.py index 6e9ffc5341e..23b516ea573 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -266,6 +266,13 @@ from litellm.router_utils.handle_error import ( send_llm_exception_alert, ) from litellm.router_utils.health_state_cache import DeploymentHealthCache +from litellm.router_utils.kubernetes_pod_discovery import ( + KubernetesPodDiscovery, + async_resolve_pods_after, + async_resolve_pods_after_bound, + resolve_pods_after, + resolve_pods_after_bound, +) from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( MID_STREAM_CONTINUATION_MARKER, ContinuationOutput, @@ -1289,6 +1296,7 @@ class Router: cache_config: Final[dict[str, Any]] = {} self.client_ttl = client_ttl + self.kubernetes_pod_discovery: Final = KubernetesPodDiscovery() if redis_url is not None or (redis_host is not None and redis_port is not None): cache_type = "redis" @@ -13512,6 +13520,13 @@ class Router: client = self.cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) return client elif client_type == "async": + if ( + deployment["litellm_params"].get( # pyright: ignore[reportUnknownMemberType] # legacy mapping + "kubernetes_pod_discovery" + ) + is True + ): + return None if kwargs.get("stream") is True: cache_key = f"{model_id}_stream_async_client" client = self.cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) @@ -13521,6 +13536,13 @@ class Router: client = self.cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) return client else: + if ( + deployment["litellm_params"].get( # pyright: ignore[reportUnknownMemberType] # legacy mapping + "kubernetes_pod_discovery" + ) + is True + ): + return None if kwargs.get("stream") is True: cache_key = f"{model_id}_stream_client" client = self.cache.get_cache(key=cache_key, parent_otel_span=parent_otel_span) @@ -14550,7 +14572,7 @@ class Router: Router._pop_effort_from_nested_carrier(request_kwargs, "output_config") Router._pop_effort_from_nested_carrier(request_kwargs, "reasoning") - async def async_get_available_deployment( + async def _async_get_available_deployment_unresolved( # noqa: ANN202 # legacy selector result remains inferred self, model: str, request_kwargs: dict, @@ -14587,7 +14609,7 @@ class Router: and self.routing_strategy != "latency-based-routing" and self.routing_strategy != "least-busy" ): # prevent regressions for other routing strategies, that don't have async get available deployments implemented. - return self.get_available_deployment( + return self._get_available_deployment_unresolved( model=model, messages=messages, input=input, @@ -14726,6 +14748,12 @@ class Router: ) raise e + async_get_available_deployment = ( # pyright: ignore[reportUnknownVariableType] # legacy selector return + async_resolve_pods_after( + _async_get_available_deployment_unresolved # pyright: ignore[reportUnknownArgumentType] # legacy type + ) + ) + async def async_get_available_deployment_for_pass_through( self, model: str, @@ -15492,7 +15520,7 @@ class Router: } return cast(StandardLoggingRoutingDecision, kept) # cast-ok: dropping optional keys preserves the type - def get_available_deployment( + def _get_available_deployment_unresolved( # noqa: ANN202 # legacy selector result remains inferred self, model: str, messages: list[dict[str, str]] | None = None, @@ -15665,6 +15693,12 @@ class Router: phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return deployment + get_available_deployment = ( # pyright: ignore[reportUnknownVariableType] # legacy selector return + resolve_pods_after( + _get_available_deployment_unresolved # pyright: ignore[reportUnknownArgumentType] # legacy annotations + ) + ) + def get_available_deployment_for_pass_through( self, model: str, @@ -16086,15 +16120,22 @@ class Router: CustomRoutingStrategy: litellm.router.CustomRoutingStrategyBase """ + strategy: Final = CustomRoutingStrategy setattr( self, "get_available_deployment", - CustomRoutingStrategy.get_available_deployment, + resolve_pods_after_bound( + strategy.get_available_deployment, # pyright: ignore[reportUnknownArgumentType] # legacy stub + self.kubernetes_pod_discovery, + ), ) setattr( self, "async_get_available_deployment", - CustomRoutingStrategy.async_get_available_deployment, + async_resolve_pods_after_bound( + strategy.async_get_available_deployment, # pyright: ignore[reportUnknownArgumentType] # legacy stub + self.kubernetes_pod_discovery, + ), ) def _reset_custom_routing_strategy(self) -> None: diff --git a/litellm/router_utils/kubernetes_pod_discovery.py b/litellm/router_utils/kubernetes_pod_discovery.py new file mode 100644 index 00000000000..7b98a52a4a0 --- /dev/null +++ b/litellm/router_utils/kubernetes_pod_discovery.py @@ -0,0 +1,462 @@ +import asyncio +import concurrent.futures +import ipaddress +import socket +import threading +import time +import urllib.request +from collections.abc import Awaitable, Callable, Coroutine, Mapping, Sequence +from concurrent.futures import Future +from dataclasses import dataclass, replace +from functools import wraps +from types import MappingProxyType +from typing import ( + Concatenate, + Final, + ParamSpec, + Protocol, + TypeAlias, + TypeVar, + cast, # noqa: TID251 # typeshed lacks proxy_bypass_environment +) + +import httpx + +from litellm._logging import verbose_router_logger +from litellm.constants import ( + KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS, + KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS, + KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS, +) + +_SocketAddress: TypeAlias = tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes] +_AddressInfo: TypeAlias = tuple[socket.AddressFamily, socket.SocketKind, int, str, _SocketAddress] +_SyncLookup: TypeAlias = Future[list[_AddressInfo]] +_AsyncLookup: TypeAlias = asyncio.Future[list[_AddressInfo]] +_CacheKey: TypeAlias = tuple[str, int | None] +_DeploymentT: Final = TypeVar("_DeploymentT") + + +@dataclass(frozen=True, slots=True) +class _PodSet: + ips: tuple[str, ...] + resolved_at: float + last_used_at: float + cursor: int = 0 + + +def _lookup_outcome( + done: _SyncLookup | _AsyncLookup, +) -> Sequence[_AddressInfo] | OSError: + try: + return done.result() + except OSError as error: + return error + except BaseException as error: # noqa: BLE001 # a cancelled or crashed lookup is a failed refresh, never a crash + return OSError(str(error)) + + +class KubernetesPodDiscovery: + def __init__( + self, + refresh_interval_seconds: float = KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS, + idle_eviction_seconds: float = KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS, + proxy_environment: Callable[[], Mapping[str, str]] = urllib.request.getproxies_environment, + clock: Callable[[], float] = time.monotonic, + dns_timeout_seconds: float = KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS, + dns_executor: concurrent.futures.Executor | None = None, + ) -> None: + self.refresh_interval_seconds = refresh_interval_seconds + self.idle_eviction_seconds = idle_eviction_seconds + self.dns_timeout_seconds = dns_timeout_seconds + self._dns_executor = dns_executor or concurrent.futures.ThreadPoolExecutor( + thread_name_prefix="litellm-k8s-pod-dns" + ) + self.proxy_environment = proxy_environment + self.clock = clock + self._cache: Mapping[_CacheKey, _PodSet] = MappingProxyType({}) + self._refreshing: frozenset[_CacheKey] = frozenset() + self._timed_out: frozenset[_CacheKey] = frozenset() + self._https_warning_hosts: frozenset[str] = frozenset() + self._proxy_warning_hosts: frozenset[str] = frozenset() + self._lock = threading.Lock() + + def _begin_refresh(self, key: _CacheKey, now: float) -> tuple[_PodSet | None, bool]: + with self._lock: + self._cache = MappingProxyType( + { + cache_key: pod_set + for cache_key, pod_set in self._cache.items() + if cache_key in self._refreshing or now - pod_set.last_used_at <= self.idle_eviction_seconds + } + ) + cached: Final = self._cache.get(key) + refreshing: Final = key in self._refreshing + if cached is not None and (now - cached.resolved_at < self.refresh_interval_seconds or refreshing): + return cached, False + if refreshing: + return None, False + self._refreshing = self._refreshing | {key} + return None, True + + def resolve_deployment(self, deployment: _DeploymentT) -> _DeploymentT: + eligible: Final = self._eligible(deployment) + if eligible is None: + return deployment + url, key, deployment_mapping = eligible + host, port = key + now: Final = self.clock() + cached, should_refresh = self._begin_refresh(key, now) + if cached is not None: + return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + if not should_refresh: + return deployment + + lookup: Final = self._dns_executor.submit( + socket.getaddrinfo, host, port, type=socket.SOCK_STREAM, flags=socket.AI_ADDRCONFIG + ) + lookup.add_done_callback(lambda done: self._finish_refresh(key, done)) + try: + return self._apply_lookup( + deployment, deployment_mapping, key, url, self._await_lookup(key, host, lookup), now + ) + finally: + self._end_refresh(key) + + async def async_resolve_deployment(self, deployment: _DeploymentT) -> _DeploymentT: + eligible: Final = self._eligible(deployment) + if eligible is None: + return deployment + url, key, deployment_mapping = eligible + host, port = key + now: Final = self.clock() + cached, should_refresh = self._begin_refresh(key, now) + if cached is not None: + return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + if not should_refresh: + return deployment + + lookup: Final = asyncio.ensure_future( + asyncio.get_running_loop().getaddrinfo(host, port, type=socket.SOCK_STREAM, flags=socket.AI_ADDRCONFIG) + ) + lookup.add_done_callback(lambda done: self._finish_refresh(key, done)) + try: + result: Final = await self._await_async_lookup(key, host, lookup) + return self._apply_lookup(deployment, deployment_mapping, key, url, result, now) + finally: + self._end_refresh(key) + + def _eligible(self, deployment: object) -> tuple[httpx.URL, _CacheKey, Mapping[str, object]] | None: + if not isinstance(deployment, Mapping): + return None + typed_deployment: Final = cast( # cast-ok: runtime Mapping check precedes typed lookup + Mapping[str, object], deployment + ) + target: Final = self._target(typed_deployment) + if target is None: + return None + url, host, port, target_deployment = target + if url.scheme == "https": + self._warn_https_once(host) + return None + if url.scheme != "http" or self._is_ip_literal(host): + return None + return (url, (host, port), target_deployment) + + @staticmethod + def _target( + deployment_mapping: Mapping[str, object], + ) -> tuple[httpx.URL, str, int | None, Mapping[str, object]] | None: + raw_params: Final = deployment_mapping.get("litellm_params") + if not isinstance(raw_params, Mapping): + return None + params: Final = cast( # cast-ok: runtime Mapping validation precedes the string field checks + Mapping[str, object], raw_params + ) + if params.get("kubernetes_pod_discovery") is not True: + return None + raw_api_base: Final = params.get("api_base") + if not isinstance(raw_api_base, str): + return None + try: + url: Final = httpx.URL(raw_api_base) + except httpx.InvalidURL: + return None + return (url, url.host, url.port, deployment_mapping) + + def _await_lookup(self, key: _CacheKey, host: str, lookup: _SyncLookup) -> Sequence[_AddressInfo] | OSError: + try: + return lookup.result(timeout=self.dns_timeout_seconds) + except (TimeoutError, concurrent.futures.TimeoutError): + return self._abandon_lookup(key, host, lookup) + except OSError as error: + return error + + async def _await_async_lookup( + self, key: _CacheKey, host: str, lookup: _AsyncLookup + ) -> Sequence[_AddressInfo] | OSError: + try: + return await asyncio.wait_for(asyncio.shield(lookup), timeout=self.dns_timeout_seconds) + except (TimeoutError, asyncio.TimeoutError): + return self._abandon_lookup(key, host, lookup) + except asyncio.CancelledError: + self._abandon_lookup(key, host, lookup) + raise + except OSError as error: + return error + + def _abandon_lookup( + self, + key: _CacheKey, + host: str, + lookup: _SyncLookup | _AsyncLookup, + ) -> Sequence[_AddressInfo] | OSError: + with self._lock: + if lookup.done(): + return _lookup_outcome(lookup) + self._timed_out = self._timed_out | {key} + return socket.gaierror(socket.EAI_AGAIN, f"DNS lookup for {host} exceeded {self.dns_timeout_seconds}s") + + def _end_refresh(self, key: _CacheKey) -> None: + with self._lock: + if key not in self._timed_out: + self._refreshing = self._refreshing - {key} + + def _finish_refresh( + self, + key: _CacheKey, + done: _SyncLookup | _AsyncLookup, + ) -> None: + with self._lock: + timed_out: Final = key in self._timed_out + self._timed_out = self._timed_out - {key} + self._refreshing = self._refreshing - {key} if timed_out else self._refreshing + late_result: Final = _lookup_outcome(done) if timed_out else None + if late_result is None: + return + if not isinstance(late_result, OSError): + self._store(key, self._pod_ips(late_result), self.clock()) + elif isinstance(late_result, socket.gaierror) and self._is_authoritative_no_pods(late_result): + self._store(key, (), self.clock()) + + def _apply_lookup( + self, + deployment: _DeploymentT, + deployment_mapping: Mapping[str, object], + key: _CacheKey, + url: httpx.URL, + result: Sequence[_AddressInfo] | OSError, + now: float, + ) -> _DeploymentT: + if isinstance(result, OSError): + if isinstance(result, socket.gaierror) and self._is_authoritative_no_pods(result): + self._store(key, (), self.clock()) + return deployment + self._stamp_failed_refresh(key, self.clock()) + verbose_router_logger.debug("Kubernetes pod discovery DNS refresh failed for %s: %s", key[0], result) + return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + ips: Final = self._pod_ips(result) + self._store(key, ips, self.clock()) + if not ips: + return deployment + return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + + @staticmethod + def _is_ip_literal(host: str) -> bool: + try: + ipaddress.ip_address(host) + except ValueError: + return False + return True + + @staticmethod + def _is_authoritative_no_pods(error: socket.gaierror) -> bool: + no_pod_errors: Final = (socket.EAI_NONAME, socket.EAI_NODATA) + return error.errno in no_pod_errors + + @staticmethod + def _pod_ip(record: _AddressInfo) -> str | None: + address: Final = record[4][0] + return address if isinstance(address, str) else None + + @classmethod + def _pod_ips(cls, records: Sequence[_AddressInfo]) -> tuple[str, ...]: + addresses: Final = tuple(cls._pod_ip(record) for record in records) + return tuple(sorted({address for address in addresses if address is not None})) + + def _store(self, key: _CacheKey, ips: tuple[str, ...], resolved_at: float) -> None: + with self._lock: + cached: Final = self._cache.get(key) + cursor: Final = cached.cursor % len(ips) if cached is not None and cached.ips and ips else 0 + self._cache = MappingProxyType( + { + **self._cache, + key: _PodSet(ips=ips, resolved_at=resolved_at, last_used_at=resolved_at, cursor=cursor), + } + ) + + def _stamp_failed_refresh(self, key: _CacheKey, resolved_at: float) -> None: + with self._lock: + cached: Final = self._cache.get(key) + stamped: Final = ( + replace(cached, resolved_at=resolved_at) + if cached is not None + else _PodSet(ips=(), resolved_at=resolved_at, last_used_at=resolved_at) + ) + self._cache = MappingProxyType({**self._cache, key: stamped}) + + def _next_ip(self, key: _CacheKey, now: float) -> str | None: + with self._lock: + cached: Final = self._cache.get(key) + if cached is None or not cached.ips: + return None + ip: Final = cached.ips[cached.cursor] + next_cursor: Final = (cached.cursor + 1) % len(cached.ips) + self._cache = MappingProxyType( + { + **self._cache, + key: replace(cached, cursor=next_cursor, last_used_at=max(cached.last_used_at, now)), + } + ) + return ip + + def _deployment_with_cached_ip( + self, + deployment: _DeploymentT, + deployment_mapping: Mapping[str, object], + key: _CacheKey, + url: httpx.URL, + now: float, + ) -> _DeploymentT: + ip: Final = self._next_ip(key, now) + if ip is None: + return deployment + if self._proxy_bypasses_only_service_host(key, ip): + self._warn_proxy_once(key[0]) + return deployment + params: Final = deployment_mapping.get("litellm_params") + if not isinstance(params, Mapping): + return deployment + typed_params: Final = cast( # cast-ok: runtime Mapping validation precedes the copied provider parameters + Mapping[str, object], params + ) + return cast( # cast-ok: resolving produces a copied dict for the selector's Mapping type + _DeploymentT, + { + **deployment_mapping, + "litellm_params": {**typed_params, "api_base": str(url.copy_with(host=ip))}, + }, + ) + + def _proxy_bypasses_only_service_host(self, key: _CacheKey, ip: str) -> bool: + proxies: Final = self.proxy_environment() + if not (proxies.get("http") or proxies.get("all")): + return False + proxy_bypass_environment: Final = cast( # cast-ok: stdlib helper accepts the supplied proxy mapping + Callable[[str, dict[str, str]], bool], + urllib.request.proxy_bypass_environment, # pyright: ignore[reportAttributeAccessIssue] # typeshed gap + ) + no_proxy: Final = proxies.get("no", "") + entries: Final = tuple(entry.strip() for entry in no_proxy.split(",")) + host: Final = key[0] + port: Final = key[1] if key[1] is not None else 80 + bracketed_ip: Final = f"[{ip}]" if ipaddress.ip_address(ip).version == 6 else ip + service_bypassed: Final = "*" in entries or any( + proxy_bypass_environment(candidate, dict(proxies)) for candidate in (host, f"{host}:{port}") + ) + pod_bypassed: Final = any( + proxy_bypass_environment(candidate, dict(proxies)) for candidate in (ip, f"{bracketed_ip}:{port}") + ) + return service_bypassed and not pod_bypassed + + def _warn_https_once(self, host: str) -> None: + with self._lock: + should_warn: Final = host not in self._https_warning_hosts + if should_warn: + self._https_warning_hosts = self._https_warning_hosts | {host} + if should_warn: + verbose_router_logger.warning( + "Kubernetes pod discovery is disabled for HTTPS host %s because substituting an IP " + "breaks TLS hostname verification", + host, + ) + + def _warn_proxy_once(self, host: str) -> None: + with self._lock: + should_warn: Final = host not in self._proxy_warning_hosts + if should_warn: + self._proxy_warning_hosts = self._proxy_warning_hosts | {host} + if should_warn: + verbose_router_logger.warning( + "Kubernetes pod discovery keeps the service hostname %s: NO_PROXY exempts it but not its pod IPs", + host, + ) + + +class _HasPodDiscovery(Protocol): + kubernetes_pod_discovery: KubernetesPodDiscovery + + +_P: Final = ParamSpec("_P") +_R: Final = TypeVar("_R") +_S: Final = TypeVar("_S", bound=_HasPodDiscovery) + + +def resolve_pods_after( + fn: Callable[Concatenate[_S, _P], _R], +) -> Callable[Concatenate[_S, _P], _R]: + @wraps(fn) + def wrapped( + self: _S, + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature + ) -> _R: + deployment: Final = fn(self, *args, **kwargs) + return self.kubernetes_pod_discovery.resolve_deployment(deployment) + + return wrapped + + +def async_resolve_pods_after( + fn: Callable[Concatenate[_S, _P], Awaitable[_R]], +) -> Callable[Concatenate[_S, _P], Coroutine[object, object, _R]]: + @wraps(fn) + async def wrapped( + self: _S, + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature + ) -> _R: + deployment: Final = await fn(self, *args, **kwargs) + return await self.kubernetes_pod_discovery.async_resolve_deployment(deployment) + + return wrapped + + +def resolve_pods_after_bound( + fn: Callable[_P, _R], + discovery: KubernetesPodDiscovery, +) -> Callable[_P, _R]: + @wraps(fn) + def wrapped( + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature + ) -> _R: + deployment: Final = fn(*args, **kwargs) + return discovery.resolve_deployment(deployment) + + return wrapped + + +def async_resolve_pods_after_bound( + fn: Callable[_P, Awaitable[_R]], + discovery: KubernetesPodDiscovery, +) -> Callable[_P, Coroutine[object, object, _R]]: + @wraps(fn) + async def wrapped( + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature + ) -> _R: + deployment: Final = await fn(*args, **kwargs) + return await discovery.async_resolve_deployment(deployment) + + return wrapped diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 174d07a2cea..b05d25462aa 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -183,6 +183,7 @@ class DeploymentOptions: tpm: int | None = None itpm: int | None = None otpm: int | None = None + kubernetes_pod_discovery: bool | None = None default_api_key_rpm_limit: int | None = None default_api_key_tpm_limit: int | None = None max_parallel_requests: int | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index 1bd1c2e5502..68dbc701c9e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -720,6 +720,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): use_chat_completions_api: bool | None ## PASS-THROUGH ENDPOINTS ## use_in_pass_through: bool | None + kubernetes_pod_discovery: ReadOnly[bool | None] litellm_credential_name: str | None ## UNIFIED PROJECT/REGION ## region_name: str | None diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 681476456c2..4ea242258f6 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -42,9 +42,11 @@ def get_all_functions_called_in_tests(base_dir): print("dir_path: ", dir_path) for root, _, files in os.walk(dir_path): for file in files: - if file.endswith(".py") and ("router" in file.lower() or test_dir == "unit"): + file_path = os.path.join(root, file) + if file.endswith(".py") and ( + "router" in os.path.relpath(file_path, dir_path).lower() or test_dir == "unit" + ): print("file: ", file) - file_path = os.path.join(root, file) with open(file_path, "r", encoding="utf-8") as f: try: tree = ast.parse(f.read()) diff --git a/tests/unit/router_utils/test_kubernetes_pod_discovery.py b/tests/unit/router_utils/test_kubernetes_pod_discovery.py new file mode 100644 index 00000000000..b3fc6eede62 --- /dev/null +++ b/tests/unit/router_utils/test_kubernetes_pod_discovery.py @@ -0,0 +1,1266 @@ +import asyncio +import copy +import json +import logging +import re +import socket +import threading +from collections.abc import Callable, Mapping +from concurrent.futures import Future, ThreadPoolExecutor +from itertools import chain, count, repeat +from typing import Final, TypeAlias, cast + +import httpx +import pytest +import respx +from openai import AsyncOpenAI, OpenAI + +import litellm +from litellm.router import CustomRoutingStrategyBase, Router +from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery + +_SERVICE_URL: Final = "http://vllm-headless.ns.svc.cluster.local:8000/v1" +_SocketAddress: TypeAlias = tuple[str, int] | tuple[str, int, int, int] +_AddrInfo: TypeAlias = tuple[socket.AddressFamily, socket.SocketKind, int, str, _SocketAddress] +_NO_POD_ERRNOS: Final = tuple(sorted((socket.EAI_NONAME, socket.EAI_NODATA))) +_CHAT_RESPONSE: Final = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "my-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +def _deployment(api_base: str = _SERVICE_URL, kubernetes_pod_discovery: bool | None = True) -> dict[str, object]: + params: Final = { + "model": "openai/my-model", + "api_base": api_base, + "api_key": "fake", + **({"kubernetes_pod_discovery": kubernetes_pod_discovery} if kubernetes_pod_discovery is not None else {}), + } + return { + "model_name": "gpu-model", + "litellm_params": params, + "model_info": {"id": "registered-id"}, + } + + +class _ModelListRoutingStrategy(CustomRoutingStrategyBase): + def __init__(self, router: Router) -> None: + self._router: Final = router + + def get_available_deployment(self, *args: object, **kwargs: object) -> Mapping[str, object]: + return self._router.model_list[0] + + async def async_get_available_deployment(self, *args: object, **kwargs: object) -> Mapping[str, object]: + return self._router.model_list[0] + + +def _record(ip: str, port: int) -> _AddrInfo: + address: Final = (ip, port, 0, 0) if ":" in ip else (ip, port) + family: Final = socket.AF_INET6 if ":" in ip else socket.AF_INET + return (family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", address) + + +def _records(*ips: str, port: int = 8000) -> list[_AddrInfo]: + return [_record(ip, port) for ip in ips] + + +def _clock(ticks: tuple[float, ...]) -> Callable[[], float]: + timestamps: Final = iter(ticks) + + def monotonic() -> float: + return next(timestamps) + + return monotonic + + +def _proxy_environment(no_proxy: str) -> Callable[[], Mapping[str, str]]: + return lambda: {"http": "http://proxy.example:8080", "no": no_proxy} + + +def _empty_proxy_environment() -> Mapping[str, str]: + return {} + + +def _stub_sync_dns(monkeypatch: pytest.MonkeyPatch, *ips: str) -> None: + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records(*ips, port=int(port or 8000)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + + +def _api_base(deployment: Mapping[str, object]) -> str: + params: Final = deployment["litellm_params"] + assert isinstance(params, Mapping) + api_base: Final = params["api_base"] + assert isinstance(api_base, str) + return api_base + + +def _request_body(content: bytes) -> Mapping[str, object]: + body: Final = json.loads(content) + assert isinstance(body, dict) + return cast(Mapping[str, object], body) + + +def test_sync_resolution_round_robins_sorted_ips_without_mutating_deployment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + original: Final = _deployment() + before: Final = copy.deepcopy(original) + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + assert type == socket.SOCK_STREAM + return _records("10.0.0.3", "10.0.0.1", "10.0.0.2", "10.0.0.1") + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=30, + proxy_environment=_empty_proxy_environment, + ) + + results: Final = tuple(discovery.resolve_deployment(original) for _ in range(3)) + + assert tuple(_api_base(result) for result in results) == ( + "http://10.0.0.1:8000/v1", + "http://10.0.0.2:8000/v1", + "http://10.0.0.3:8000/v1", + ) + assert original == before + assert results[0] is not original + assert results[0]["model_info"] is original["model_info"] + + +@pytest.mark.asyncio +async def test_async_resolution_round_robins_sorted_ips_and_preserves_url_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + original: Final = _deployment(api_base=f"{_SERVICE_URL}/custom?version=1") + before: Final = copy.deepcopy(original) + loop: Final = asyncio.get_running_loop() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.3", "10.0.0.1", "10.0.0.2", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=30, + proxy_environment=_empty_proxy_environment, + ) + + results: Final = ( + await discovery.async_resolve_deployment(original), + await discovery.async_resolve_deployment(original), + await discovery.async_resolve_deployment(original), + ) + + assert tuple(_api_base(result) for result in results) == ( + "http://10.0.0.1:8000/v1/custom?version=1", + "http://10.0.0.2:8000/v1/custom?version=1", + "http://10.0.0.3:8000/v1/custom?version=1", + ) + assert original == before + assert results[0] is not original + assert results[0]["model_info"] is original["model_info"] + + +def test_refresh_reuses_cached_set_until_interval_then_uses_new_pods( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock: Final = _clock((10.0, 10.0, 14.99, 15.0, 15.0)) + responses: Final = iter((_records("10.0.0.1", "10.0.0.2"), _records("10.0.0.3"))) + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return list(next(responses)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + first: Final = _api_base(discovery.resolve_deployment(_deployment())) + cached: Final = _api_base(discovery.resolve_deployment(_deployment())) + refreshed: Final = _api_base(discovery.resolve_deployment(_deployment())) + + assert (first, cached, refreshed) == ( + "http://10.0.0.1:8000/v1", + "http://10.0.0.2:8000/v1", + "http://10.0.0.3:8000/v1", + ) + assert next(lookups) == 2 + + +def test_refresh_carries_round_robin_cursor_into_new_ip_set(monkeypatch: pytest.MonkeyPatch) -> None: + clock: Final = _clock((0.0, 0.0, 5.0, 5.0)) + responses: Final = iter((_records("10.0.0.1", "10.0.0.2", "10.0.0.3"), _records("10.0.0.4", "10.0.0.5"))) + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return list(next(responses)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + first: Final = _api_base(discovery.resolve_deployment(_deployment())) + refreshed: Final = _api_base(discovery.resolve_deployment(_deployment())) + + assert (first, refreshed) == ("http://10.0.0.1:8000/v1", "http://10.0.0.5:8000/v1") + + +def test_idle_cache_eviction_restarts_round_robin_cursor(monkeypatch: pytest.MonkeyPatch) -> None: + clock: Final = _clock((0.0, 0.0, 0.0, 11.0, 11.0)) + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + idle_eviction_seconds=10, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + first: Final = _api_base(discovery.resolve_deployment(_deployment())) + second: Final = _api_base(discovery.resolve_deployment(_deployment())) + after_eviction: Final = _api_base(discovery.resolve_deployment(_deployment())) + + assert (first, second, after_eviction) == ( + "http://10.0.0.1:8000/v1", + "http://10.0.0.2:8000/v1", + "http://10.0.0.1:8000/v1", + ) + assert next(lookups) == 2 + + +def test_active_pod_set_survives_idle_eviction_when_refresh_interval_is_longer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock_values: Final = iter(chain((0.0, 0.0, 200.0, 400.0, 550.0), repeat(550.0))) + + def clock() -> float: + return next(clock_values) + + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=600, + idle_eviction_seconds=300, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + api_bases: Final = tuple(_api_base(discovery.resolve_deployment(_deployment())) for _ in range(4)) + + assert api_bases == ( + "http://10.0.0.1:8000/v1", + "http://10.0.0.2:8000/v1", + "http://10.0.0.3:8000/v1", + "http://10.0.0.1:8000/v1", + ) + assert next(lookups) == 1 + + +def test_slow_dns_refresh_does_not_move_last_use_backward(monkeypatch: pytest.MonkeyPatch) -> None: + clock_values: Final = iter(chain((0.0, 250.0), repeat(500.0))) + + def clock() -> float: + return next(clock_values) + + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=600, + idle_eviction_seconds=300, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + api_bases: Final = tuple(_api_base(discovery.resolve_deployment(_deployment())) for _ in range(2)) + + assert api_bases == ("http://10.0.0.1:8000/v1", "http://10.0.0.2:8000/v1") + assert next(lookups) == 1 + + +def test_proxy_bypassing_only_service_hostname_keeps_api_base_and_warns_once( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_proxy_environment(".svc.cluster.local")) + original: Final = _deployment() + + with caplog.at_level(logging.WARNING): + first: Final = discovery.resolve_deployment(original) + second: Final = discovery.resolve_deployment(original) + + assert first is original + assert second is original + assert _api_base(first) == _SERVICE_URL + assert sum("NO_PROXY exempts it but not its pod IPs" in record.getMessage() for record in caplog.records) == 1 + + +def test_port_qualified_no_proxy_service_keeps_api_base_and_warns_once( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery( + proxy_environment=_proxy_environment("vllm-headless.ns.svc.cluster.local:8000") + ) + original: Final = _deployment() + + with caplog.at_level(logging.WARNING): + result: Final = discovery.resolve_deployment(original) + + assert result is original + assert _api_base(result) == _SERVICE_URL + assert sum("NO_PROXY exempts it but not its pod IPs" in record.getMessage() for record in caplog.records) == 1 + + +def test_port_qualified_no_proxy_service_and_pod_substitutes_pod_ip(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery( + proxy_environment=_proxy_environment("vllm-headless.ns.svc.cluster.local:8000,10.244.1.3:8000") + ) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://10.244.1.3:8000/v1" + + +def test_port_qualified_no_proxy_other_service_port_substitutes_pod_ip( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery( + proxy_environment=_proxy_environment("vllm-headless.ns.svc.cluster.local:9000") + ) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://10.244.1.3:8000/v1" + + +def test_proxy_with_pod_cidr_in_no_proxy_keeps_service_hostname(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_proxy_environment(".svc.cluster.local,10.244.0.0/16")) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == _SERVICE_URL + + +def test_proxy_with_exact_pod_ip_in_no_proxy_keeps_pod_ip_substitution(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_proxy_environment(".svc.cluster.local,10.244.1.3")) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://10.244.1.3:8000/v1" + + +def test_proxy_with_empty_no_proxy_keeps_pod_ip_substitution(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_proxy_environment("")) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://10.244.1.3:8000/v1" + + +def test_empty_proxy_environment_keeps_pod_ip_substitution(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://10.244.1.3:8000/v1" + + +@pytest.mark.parametrize("no_pods_errno", _NO_POD_ERRNOS) +def test_authoritative_no_pods_uses_service_url_and_transient_failure_keeps_cached_pods( + monkeypatch: pytest.MonkeyPatch, no_pods_errno: int +) -> None: + clock: Final = _clock((0.0, 0.0, 0.0, 0.0, 5.0, 5.0)) + missing: Final = _deployment() + no_pod_lookups: Final = count() + + def missing_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(no_pod_lookups) + raise socket.gaierror(no_pods_errno, "missing") + + monkeypatch.setattr(socket, "getaddrinfo", missing_getaddrinfo) + discovery_without_pods: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + assert discovery_without_pods.resolve_deployment(missing) is missing + assert discovery_without_pods.resolve_deployment(missing) is missing + assert _api_base(missing) == _SERVICE_URL + assert next(no_pod_lookups) == 1 + + responses: Final = iter((_records("10.0.0.4"), socket.gaierror(socket.EAI_AGAIN, "temporary"))) + + def transient_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + response: Final = next(responses) + if isinstance(response, socket.gaierror): + raise response + return list(response) + + monkeypatch.setattr(socket, "getaddrinfo", transient_getaddrinfo) + discovery_with_pods: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + first: Final = _api_base(discovery_with_pods.resolve_deployment(_deployment())) + after_failure: Final = _api_base(discovery_with_pods.resolve_deployment(_deployment())) + + assert (first, after_failure) == ("http://10.0.0.4:8000/v1", "http://10.0.0.4:8000/v1") + + +@pytest.mark.parametrize( + ("api_base", "flag"), + ( + pytest.param(_SERVICE_URL, None, id="flag-absent"), + pytest.param(_SERVICE_URL, False, id="flag-false"), + pytest.param("https://vllm-headless.ns.svc.cluster.local:8000/v1", True, id="https"), + pytest.param("http://10.0.0.9:8000/v1", True, id="ip-literal"), + pytest.param("http://[zzzz]", True, id="malformed-url"), + ), +) +def test_unsupported_deployments_are_returned_unchanged_without_dns( + monkeypatch: pytest.MonkeyPatch, api_base: str, flag: bool | None +) -> None: + deployment: Final = _deployment(api_base=api_base, kubernetes_pod_discovery=flag) + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return [] + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + assert discovery.resolve_deployment(deployment) is deployment + assert next(lookups) == 0 + + +@pytest.mark.asyncio +async def test_non_mapping_deployments_are_returned_unchanged() -> None: + deployment: Final = object() + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + assert discovery.resolve_deployment(deployment) is deployment + assert await discovery.async_resolve_deployment(deployment) is deployment + + +def test_https_discovery_warning_is_emitted_once_per_host(caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.WARNING) + deployment: Final = _deployment(api_base="https://vllm-headless.ns.svc.cluster.local:8000/v1") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + results: Final = ( + discovery.resolve_deployment(deployment), + discovery.resolve_deployment(deployment), + ) + warnings: Final = tuple( + record + for record in caplog.records + if "Kubernetes pod discovery is disabled for HTTPS host" in record.getMessage() + ) + + assert results == (deployment, deployment) + assert all(result is deployment for result in results) + assert len(warnings) == 1 + + +def test_ipv6_pod_address_is_bracketed_in_url(monkeypatch: pytest.MonkeyPatch) -> None: + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("2001:db8::2") + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://[2001:db8::2]:8000/v1" + + +def test_transient_dns_failure_on_cold_cache_waits_for_the_refresh_interval(monkeypatch: pytest.MonkeyPatch) -> None: + clock: Final = _clock((0.0, 0.0, 1.0, 6.0, 6.0, 6.0, 6.0, 6.0)) + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + raise socket.gaierror(socket.EAI_AGAIN, "temporary") + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + deployment: Final = _deployment() + + within_interval: Final = tuple(discovery.resolve_deployment(deployment) for _ in range(2)) + lookups_within_interval: Final = next(lookups) + after_interval: Final = discovery.resolve_deployment(deployment) + + assert all(result is deployment for result in within_interval) + assert after_interval is deployment + assert (lookups_within_interval, next(lookups)) == (1, 3) + + +def test_slow_sync_dns_lookup_falls_back_to_the_service_url_within_the_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + release: Final = threading.Event() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + release.wait() + return _records("10.0.0.1", port=int(port or 8000)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment, dns_timeout_seconds=0.01) + deployment: Final = _deployment() + + during_slow_lookup: Final = discovery.resolve_deployment(deployment) + release.set() + + assert during_slow_lookup is deployment + assert _api_base(during_slow_lookup) == _SERVICE_URL + + +@pytest.mark.asyncio +async def test_slow_async_dns_lookup_falls_back_to_the_service_url_within_the_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + release: Final = asyncio.Event() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + await release.wait() + return _records("10.0.0.1", port=int(port or 8000)) + + monkeypatch.setattr(asyncio.get_running_loop(), "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment, dns_timeout_seconds=0.01) + deployment: Final = _deployment() + + during_slow_lookup: Final = await discovery.async_resolve_deployment(deployment) + release.set() + + assert during_slow_lookup is deployment + assert _api_base(during_slow_lookup) == _SERVICE_URL + + +def _stuck_then_ten_clock() -> Callable[[], float]: + ticks: Final = chain((0.0, 0.0), repeat(10.0)) + + def monotonic() -> float: + return next(ticks) + + return monotonic + + +def test_timed_out_sync_lookup_is_not_resubmitted_and_its_late_pods_are_used(monkeypatch: pytest.MonkeyPatch) -> None: + release: Final = threading.Event() + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + release.wait() + return _records("10.0.0.1", port=int(port or 8000)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + dns_executor: Final = ThreadPoolExecutor(max_workers=1) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=_stuck_then_ten_clock(), + proxy_environment=_empty_proxy_environment, + dns_timeout_seconds=0.01, + dns_executor=dns_executor, + ) + deployment: Final = _deployment() + + timed_out: Final = discovery.resolve_deployment(deployment) + after_interval: Final = discovery.resolve_deployment(deployment) + release.set() + dns_executor.shutdown(wait=True) + late: Final = _api_base(discovery.resolve_deployment(deployment)) + + assert (timed_out is deployment, after_interval is deployment) == (True, True) + assert late == "http://10.0.0.1:8000/v1" + assert next(lookups) == 1 + + +@pytest.mark.asyncio +async def test_timed_out_async_lookup_is_not_resubmitted_and_its_late_pods_are_used( + monkeypatch: pytest.MonkeyPatch, +) -> None: + release: Final = asyncio.Event() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + await release.wait() + return _records("10.0.0.1", port=int(port or 8000)) + + monkeypatch.setattr(asyncio.get_running_loop(), "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=_stuck_then_ten_clock(), + proxy_environment=_empty_proxy_environment, + dns_timeout_seconds=0.01, + ) + deployment: Final = _deployment() + + timed_out: Final = await discovery.async_resolve_deployment(deployment) + after_interval: Final = await discovery.async_resolve_deployment(deployment) + release.set() + for _ in range(5): + await asyncio.sleep(0) + late: Final = _api_base(await discovery.async_resolve_deployment(deployment)) + + assert (timed_out is deployment, after_interval is deployment) == (True, True) + assert late == "http://10.0.0.1:8000/v1" + assert next(lookups) == 1 + + +def test_late_authoritative_no_pods_after_a_timed_out_refresh_clears_the_cached_pods( + monkeypatch: pytest.MonkeyPatch, +) -> None: + release: Final = threading.Event() + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + if next(lookups) == 0: + return _records("10.0.0.1", port=int(port or 8000)) + release.wait() + raise socket.gaierror(socket.EAI_NONAME, "gone") + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + dns_executor: Final = ThreadPoolExecutor(max_workers=1) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=_stuck_then_ten_clock(), + proxy_environment=_empty_proxy_environment, + dns_timeout_seconds=0.01, + dns_executor=dns_executor, + ) + deployment: Final = _deployment() + + first: Final = _api_base(discovery.resolve_deployment(deployment)) + during_timed_out_refresh: Final = _api_base(discovery.resolve_deployment(deployment)) + release.set() + dns_executor.shutdown(wait=True) + after_late_no_pods: Final = discovery.resolve_deployment(deployment) + + assert (first, during_timed_out_refresh) == ("http://10.0.0.1:8000/v1", "http://10.0.0.1:8000/v1") + assert after_late_no_pods is deployment + assert next(lookups) == 2 + + +@pytest.mark.asyncio +async def test_cancelled_request_does_not_resubmit_the_lookup_it_abandoned(monkeypatch: pytest.MonkeyPatch) -> None: + release: Final = asyncio.Event() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + await release.wait() + return _records("10.0.0.1", port=int(port or 8000)) + + monkeypatch.setattr(asyncio.get_running_loop(), "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=_stuck_then_ten_clock(), + proxy_environment=_empty_proxy_environment, + dns_timeout_seconds=60, + ) + deployment: Final = _deployment() + + disconnected: Final = asyncio.create_task(discovery.async_resolve_deployment(deployment)) + for _ in range(3): + await asyncio.sleep(0) + disconnected.cancel() + cancelled: Final = await asyncio.gather(disconnected, return_exceptions=True) + after_interval: Final = await discovery.async_resolve_deployment(deployment) + release.set() + for _ in range(5): + await asyncio.sleep(0) + late: Final = _api_base(await discovery.async_resolve_deployment(deployment)) + + assert isinstance(cancelled[0], asyncio.CancelledError) + assert after_interval is deployment + assert late == "http://10.0.0.1:8000/v1" + assert next(lookups) == 1 + + +def _ipv4_only_host_records(port: int, flags: int) -> list[_AddrInfo]: + """A resolver on a host with no IPv6 address: dual-stack records only when AI_ADDRCONFIG is absent.""" + if flags & socket.AI_ADDRCONFIG: + return _records("10.0.0.1", "10.0.0.2", port=port) + return _records("10.0.0.1", "fd00::1", "10.0.0.2", "fd00::2", port=port) + + +def test_dual_stack_service_on_ipv4_only_host_rotates_only_ipv4_pods(monkeypatch: pytest.MonkeyPatch) -> None: + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _ipv4_only_host_records(int(port or 8000), flags) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + hosts: Final = tuple(httpx.URL(_api_base(discovery.resolve_deployment(_deployment()))).host for _ in range(4)) + + assert hosts == ("10.0.0.1", "10.0.0.2", "10.0.0.1", "10.0.0.2") + + +@pytest.mark.asyncio +async def test_dual_stack_service_on_ipv4_only_host_rotates_only_ipv4_pods_async( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _ipv4_only_host_records(int(port or 8000), flags) + + monkeypatch.setattr(asyncio.get_running_loop(), "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + hosts: Final = tuple( + [httpx.URL(_api_base(await discovery.async_resolve_deployment(_deployment()))).host for _ in range(4)] + ) + + assert hosts == ("10.0.0.1", "10.0.0.2", "10.0.0.1", "10.0.0.2") + + +@pytest.mark.asyncio +async def test_concurrent_async_refresh_uses_cached_ips_while_refresh_is_in_flight( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock: Final = _clock((0.0, 0.0, 5.0, 5.0, 5.0, 5.0, 5.0)) + loop: Final = asyncio.get_running_loop() + refresh_started: Final = asyncio.Event() + finish_refresh: Final = asyncio.Event() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + lookup_number: Final = next(lookups) + if lookup_number == 0: + return _records("10.0.0.1", port=int(port or 0)) + if lookup_number > 1: + return _records("10.0.0.3", port=int(port or 0)) + refresh_started.set() + await finish_refresh.wait() + return _records("10.0.0.2", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + first: Final = _api_base(await discovery.async_resolve_deployment(_deployment())) + refresh_task: Final = asyncio.create_task(discovery.async_resolve_deployment(_deployment())) + await refresh_started.wait() + concurrent: Final = _api_base(await discovery.async_resolve_deployment(_deployment())) + finish_refresh.set() + refreshed: Final = _api_base(await refresh_task) + + assert (first, concurrent, refreshed) == ( + "http://10.0.0.1:8000/v1", + "http://10.0.0.1:8000/v1", + "http://10.0.0.2:8000/v1", + ) + assert next(lookups) == 2 + + +@pytest.mark.parametrize("cache_existing", (False, True), ids=("cold-cache", "stale-cache")) +def test_concurrent_sync_refresh_uses_cached_ips_or_service_url_while_in_flight( + monkeypatch: pytest.MonkeyPatch, + cache_existing: bool, +) -> None: + ticks: Final = (0.0, 0.0, 5.0, 5.0, 5.0) if cache_existing else (0.0, 0.0, 0.0) + clock: Final = _clock(ticks) + refresh_started: Final = threading.Event() + finish_refresh: Final = threading.Event() + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + lookup_number: Final = next(lookups) + if cache_existing and lookup_number == 0: + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + refresh_started.set() + assert finish_refresh.wait(timeout=3) + return _records("10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + if cache_existing: + assert _api_base(discovery.resolve_deployment(_deployment())) == "http://10.0.0.1:8000/v1" + + refresh_result: Final = Future[dict[str, object]]() + + def refresh() -> None: + refresh_result.set_result(discovery.resolve_deployment(_deployment())) + + thread: Final = threading.Thread(target=refresh, daemon=True) + thread.start() + try: + assert refresh_started.wait(timeout=2) + concurrent: Final = discovery.resolve_deployment(_deployment()) + assert _api_base(concurrent) == ("http://10.0.0.2:8000/v1" if cache_existing else _SERVICE_URL) + assert next(lookups) == (2 if cache_existing else 1) + finally: + finish_refresh.set() + thread.join(timeout=2) + assert not thread.is_alive() + assert _api_base(refresh_result.result(timeout=2)) == "http://10.0.0.3:8000/v1" + + +@pytest.mark.asyncio +async def test_router_sends_pod_hosts_without_forwarding_discovery_flag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock: Final = _clock((0.0, 0.0, 0.0, 0.0, 0.0, 5.0, 5.0)) + loop: Final = asyncio.get_running_loop() + async_lookups: Final = count() + sync_lookups: Final = count() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(async_lookups) + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(sync_lookups) + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post( + url__regex=re.compile( + r"http://(?:10\.0\.0\.1|10\.0\.0\.2|vllm-headless\.ns\.svc\.cluster\.local):8000/v1/chat/completions" + ) + ).mock(return_value=httpx.Response(200, json=_CHAT_RESPONSE)) + discovery_enabled_router: Final = Router(model_list=[_deployment()]) + control_router: Final = Router(model_list=[_deployment(kubernetes_pod_discovery=None)]) + discovery_enabled_router.kubernetes_pod_discovery = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + cached_async_client: Final = AsyncOpenAI(api_key="fake", base_url=_SERVICE_URL) + cached_sync_client: Final = OpenAI(api_key="fake", base_url=_SERVICE_URL) + discovery_enabled_router.cache.set_cache( + key="registered-id_async_client", + value=cached_async_client, + local_only=True, + ) + discovery_enabled_router.cache.set_cache( + key="registered-id_client", + value=cached_sync_client, + local_only=True, + ) + for _ in range(4): + await discovery_enabled_router.acompletion( + model="gpu-model", messages=[{"role": "user", "content": "hello"}] + ) + discovery_enabled_router.completion(model="gpu-model", messages=[{"role": "user", "content": "hello"}]) + await control_router.acompletion(model="gpu-model", messages=[{"role": "user", "content": "hello"}]) + + hosts: Final = tuple(call.request.url.host for call in route.calls) + bodies: Final = tuple(_request_body(call.request.content) for call in route.calls) + await cached_async_client.close() + cached_sync_client.close() + + assert hosts == ( + "10.0.0.1", + "10.0.0.2", + "10.0.0.1", + "10.0.0.2", + "10.0.0.1", + "vllm-headless.ns.svc.cluster.local", + ) + assert all("kubernetes_pod_discovery" not in body for body in bodies) + assert next(async_lookups) == 1 + assert next(sync_lookups) == 1 + + +@pytest.mark.asyncio +async def test_custom_routing_strategy_resolves_pod_hosts_for_sync_and_async_requests( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock: Final = _clock((0.0,) * 40) + loop: Final = asyncio.get_running_loop() + async_lookups: Final = count() + sync_lookups: Final = count() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(async_lookups) + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(sync_lookups) + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post( + url__regex=re.compile( + r"http://(?:10\.0\.0\.1|10\.0\.0\.2|vllm-headless\.ns\.svc\.cluster\.local):8000/v1/chat/completions" + ) + ).mock(return_value=httpx.Response(200, json=_CHAT_RESPONSE)) + async_router: Final = Router(model_list=[_deployment()]) + sync_router: Final = Router(model_list=[_deployment()]) + async_router.kubernetes_pod_discovery = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + sync_router.kubernetes_pod_discovery = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + async_router.set_custom_routing_strategy(_ModelListRoutingStrategy(async_router)) + sync_router.set_custom_routing_strategy(_ModelListRoutingStrategy(sync_router)) + for _ in range(4): + await async_router.acompletion(model="gpu-model", messages=[{"role": "user", "content": "hello"}]) + for _ in range(4): + sync_router.completion(model="gpu-model", messages=[{"role": "user", "content": "hello"}]) + hosts: Final = tuple(call.request.url.host for call in route.calls) + + assert hosts == ("10.0.0.1", "10.0.0.2") * 4 + assert next(async_lookups) == 1 + assert next(sync_lookups) == 1 + + +@pytest.mark.asyncio +async def test_unresolved_router_selectors_keep_service_hostname() -> None: + router: Final = Router(model_list=[_deployment()]) + + sync_selected: Final = router._get_available_deployment_unresolved(model="gpu-model", request_kwargs={}) + async_selected: Final = await router._async_get_available_deployment_unresolved( + model="gpu-model", + request_kwargs={}, + ) + + assert _api_base(sync_selected) == _SERVICE_URL + assert _api_base(async_selected) == _SERVICE_URL + + +@pytest.mark.asyncio +async def test_router_async_retry_uses_next_discovered_pod(monkeypatch: pytest.MonkeyPatch) -> None: + clock: Final = _clock((0.0,) * 8) + loop: Final = asyncio.get_running_loop() + attempts: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + def response_for(request: httpx.Request) -> httpx.Response: + attempt: Final = next(attempts) + status_code: Final = 500 if attempt == 0 else 200 + body: Final = {"error": {"message": "retryable", "type": "server_error"}} if attempt == 0 else _CHAT_RESPONSE + return httpx.Response(status_code, json=body, request=request) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post( + url__regex=re.compile(r"http://(?:10\.0\.0\.1|10\.0\.0\.2):8000/v1/chat/completions") + ).mock(side_effect=response_for) + router: Final = Router(model_list=[_deployment()], num_retries=1) + router.kubernetes_pod_discovery = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + response: Final = await router.acompletion( + model="gpu-model", + messages=[{"role": "user", "content": "hello"}], + ) + hosts: Final = tuple(call.request.url.host for call in route.calls) + + assert response.choices[0].message.content == "ok" + assert hosts == ("10.0.0.1", "10.0.0.2") + + +def test_router_sync_retry_uses_next_discovered_pod(monkeypatch: pytest.MonkeyPatch) -> None: + clock: Final = _clock((0.0,) * 8) + attempts: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + def response_for(request: httpx.Request) -> httpx.Response: + attempt: Final = next(attempts) + status_code: Final = 500 if attempt == 0 else 200 + body: Final = {"error": {"message": "retryable", "type": "server_error"}} if attempt == 0 else _CHAT_RESPONSE + return httpx.Response(status_code, json=body, request=request) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post( + url__regex=re.compile(r"http://(?:10\.0\.0\.1|10\.0\.0\.2):8000/v1/chat/completions") + ).mock(side_effect=response_for) + router: Final = Router(model_list=[_deployment()], num_retries=1) + router.kubernetes_pod_discovery = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + response: Final = router.completion( + model="gpu-model", + messages=[{"role": "user", "content": "hello"}], + ) + hosts: Final = tuple(call.request.url.host for call in route.calls) + + assert response.choices[0].message.content == "ok" + assert hosts == ("10.0.0.1", "10.0.0.2") diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index aef43a15a72..ce763378656 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -131,6 +131,7 @@ OPTION_NAMES: Final = ( "use_litellm_proxy", "use_chat_completions_api", "use_in_pass_through", + "kubernetes_pod_discovery", "allowed_openai_params", "fallbacks", "context_window_fallback_dict", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index 99656df3b0e..43b9a5541db 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -132,6 +132,7 @@ const alwaysMounted = { const advancedOpenExtras = { guardrails: undefined, + kubernetes_pod_discovery: undefined, tags: undefined, use_in_pass_through: undefined, vector_store_ids: undefined, @@ -304,6 +305,24 @@ describe("AddModelPanel submit payload contract", () => { }); }); + it("submits Kubernetes pod discovery in litellm_params", async () => { + const { user, openAdvanced, fillRequired, submit } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.click(screen.getByRole("switch", { name: "Kubernetes pod discovery" })); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { + ...alwaysMounted, + ...advancedOpenExtras, + kubernetes_pod_discovery: true, + }, + model_info: { ...baseModelInfo }, + }); + }); + it("drops a collapsed section's keys and the value typed into it", async () => { const { user, openAdvanced, closeAdvanced, fillRequired, submit } = await setup(); await fillRequired(); diff --git a/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx b/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx index 15a13219d18..e3d98492cb9 100644 --- a/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx @@ -1,6 +1,8 @@ import { act, fireEvent, render, waitFor, screen } from "@testing-library/react"; +import { useFormContext } from "react-hook-form"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { MountedFormHost } from "../../../tests/mounted-form-host"; +import type { MountedFormValues } from "../common_components/MountedFormField"; import AdvancedSettings from "./advanced_settings"; const mockUsePtuCostAttributionEnabled = vi.fn(); @@ -11,7 +13,16 @@ vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () => const PTU_LABELS = ["PTU Count", "Calculated Cost per PTU / Hour (USD)", "PTU Effective From (UTC)"]; -const renderAdvancedSettings = () => +const KubernetesPodDiscoveryValue = () => { + const form = useFormContext(); + return ( + + {String(form.watch("kubernetes_pod_discovery") === true)} + + ); +}; + +const renderAdvancedSettings = (includeDiscoveryValue = false) => render( tagsList={{}} accessToken="test-token" /> + {includeDiscoveryValue && } , ); @@ -34,6 +46,23 @@ describe("AdvancedSettings", () => { renderAdvancedSettings(); }); + it("updates the Kubernetes pod discovery form value when enabled", async () => { + renderAdvancedSettings(true); + act(() => { + fireEvent.click(screen.getByText("Advanced Settings")); + }); + + const toggle = await screen.findByRole("switch", { name: "Kubernetes pod discovery" }); + const formValue = screen.getByRole("status", { name: "Kubernetes pod discovery form value" }); + expect(toggle).not.toBeChecked(); + expect(formValue).toHaveTextContent("false"); + + fireEvent.click(toggle); + + expect(toggle).toBeChecked(); + expect(formValue).toHaveTextContent("true"); + }); + it("should render tags list", async () => { renderAdvancedSettings(); fireEvent.click(screen.getByText("Advanced Settings")); diff --git a/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx b/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx index 6025db8e2e2..b17d38c5586 100644 --- a/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx +++ b/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx @@ -457,6 +457,20 @@ const AdvancedSettings: React.FC = ({ )} + + {(control) => ( + + )} + + Date: Mon, 19 Oct 2026 09:58:19 -0700 Subject: [PATCH 28/50] feat(router): keep a session on one kubernetes pod (#44393) Requests with an explicit session id stick to one discovered pod via rendezvous hashing Requests without a session id keep round robin Pod routing records are included in logs, spend logs, and traces Resolves LIT-9168 Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/integrations/opentelemetry.py | 12 +- litellm/litellm_core_utils/core_helpers.py | 51 +- litellm/litellm_core_utils/litellm_logging.py | 13 +- litellm/proxy/_types.py | 2 + litellm/proxy/litellm_pre_call_utils.py | 10 +- .../spend_tracking/spend_tracking_utils.py | 22 +- litellm/router.py | 10 + .../router_utils/kubernetes_pod_discovery.py | 220 ++++- litellm/types/utils.py | 8 + .../gcs_pubsub/spend_logs_payload.json | 2 +- tests/unit/integrations/test_opentelemetry.py | 29 +- .../test_litellm_logging.py | 98 +- .../test_spend_management_endpoints.py | 13 +- .../test_spend_tracking_utils.py | 85 +- .../unit/proxy/test_litellm_pre_call_utils.py | 7 +- .../test_kubernetes_pod_discovery.py | 876 +++++++++++++++++- tests/unit/test_router/test_router.py | 88 ++ .../src/components/logs/constants.ts | 6 + .../LogDetailContent.integration.test.tsx | 31 + .../logs/detail/LogDetailContent.tsx | 12 +- .../src/components/logs/types.ts | 11 +- 22 files changed, 1541 insertions(+), 66 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index db49a40a37b..c7c1576bc20 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -47,6 +47,7 @@ KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS: Final = get_env_float( "KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS", 300.0 ) KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS: Final = 2.0 +KUBERNETES_POD_ROUTING_KEY: Final = "kubernetes_pod_routing" AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"}) ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index fb0723289ee..90392d9a45d 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -13,6 +13,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger +from litellm.constants import KUBERNETES_POD_ROUTING_KEY from litellm.integrations._types.open_inference import ( OpenInferenceSpanKindValues, ) @@ -50,6 +51,7 @@ from litellm.types.utils import ( Function, LLMResponseTypes, StandardCallbackDynamicParams, + StandardLoggingMetadata, StandardLoggingPayload, ) @@ -2420,9 +2422,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ############################################# ############ LLM CALL METADATA ############## ############################################# - metadata: Final = standard_logging_payload["metadata"] + metadata: Final = cast( # cast-ok: payload metadata follows the standard logging contract + StandardLoggingMetadata, standard_logging_payload["metadata"] + ) for key, value in metadata.items(): - self.safe_set_attribute(span=span, key=f"metadata.{key}", value=value) + self.safe_set_attribute( + span=span, + key=f"metadata.{key}", + value=safe_dumps(value) if key == KUBERNETES_POD_ROUTING_KEY and value is not None else value, + ) decision: Final = metadata.get("routing_decision") span.set_attributes(routing_decision_attributes(decision)) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index e706bcadef8..65d0089304b 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -11,13 +11,14 @@ import httpx from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.constants import KUBERNETES_POD_ROUTING_KEY from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - from litellm.types.utils import ModelResponseStream + from litellm.types.utils import ModelResponseStream, StandardLoggingKubernetesPodRouting Span = _Span | Any else: @@ -385,6 +386,54 @@ def proxy_stamped_used_client_oauth_token(metadata: object, litellm_params: Mapp return metadata.get("used_client_oauth_token") if isinstance(metadata, Mapping) else None +def proxy_stamped_kubernetes_pod_routing( + metadata: object, + litellm_params: Mapping[str, object] | None, +) -> "StandardLoggingKubernetesPodRouting | None": + litellm_metadata: Final = litellm_params.get("litellm_metadata") if litellm_params is not None else None + litellm_routing: Final = _standard_logging_kubernetes_pod_routing(litellm_metadata) + if litellm_routing is not None: + return litellm_routing + return _standard_logging_kubernetes_pod_routing(metadata) + + +def _standard_logging_kubernetes_pod_routing( + metadata: object, +) -> "StandardLoggingKubernetesPodRouting | None": + if not isinstance(metadata, Mapping): + return None + metadata_mapping: Final = cast( # cast-ok: routing metadata is a string-keyed mapping + Mapping[str, object], metadata + ) + routing: Final = metadata_mapping.get(KUBERNETES_POD_ROUTING_KEY) + if not isinstance(routing, Mapping): + return None + routing_mapping: Final = cast( # cast-ok: pod routing records use string keys + Mapping[str, object], routing + ) + service_host: Final = routing_mapping.get("service_host") + pod_ip: Final = routing_mapping.get("pod_ip") + pod_count: Final = routing_mapping.get("pod_count") + selection: Final = routing_mapping.get("selection") + if ( + not isinstance(service_host, str) + or not isinstance(pod_ip, str) + or not isinstance(pod_count, int) + or isinstance(pod_count, bool) + or pod_count < 1 + ): + return None + if selection in ("round_robin", "session_affinity", "session_affinity_retry"): + routing_record: Final[StandardLoggingKubernetesPodRouting] = { + "service_host": service_host, + "pod_ip": pod_ip, + "pod_count": pod_count, + "selection": selection, + } + return routing_record + return None + + def get_litellm_metadata_from_kwargs(kwargs: dict): """ Helper to get litellm metadata from all litellm request kwargs diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 025741b18b2..0ac216c2029 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -43,6 +43,7 @@ from litellm.constants import ( DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, EMPTY_MAPPING, INTERNAL_CALLBACK_NAMES, + KUBERNETES_POD_ROUTING_KEY, PROVIDER_REQUEST_ID_HEADERS, REDACTED_BY_LITELLM, ) @@ -75,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import ( from litellm.litellm_core_utils.core_helpers import ( get_provider_response_headers_from_hidden_params, is_expected_client_error, + proxy_stamped_kubernetes_pod_routing, proxy_stamped_used_client_oauth_token, reconstruct_model_name, set_response_cost_in_hidden_params, @@ -298,7 +300,9 @@ else: _GENERIC_API_LOGGER_CLS: Final = GenericAPILogger _in_memory_loggers: Final[list[CustomLogger]] = [] -_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(("used_client_oauth_token", "usage_object")) +_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( + ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY) +) _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS ) @@ -6275,7 +6279,7 @@ class StandardLoggingPayloadSetup: @staticmethod def get_standard_logging_metadata( metadata: Mapping[str, object] | None, - litellm_params: dict | None = None, + litellm_params: dict[str, object] | None = None, prompt_integration: str | None = None, applied_guardrails: list[str] | None = None, mcp_tool_call_metadata: StandardLoggingMCPToolCall | None = None, @@ -6315,6 +6319,8 @@ class StandardLoggingPayloadSetup: prompt_integration=prompt_integration, ) + kubernetes_pod_routing: Final = proxy_stamped_kubernetes_pod_routing(metadata, litellm_params) + # Initialize with default values clean_metadata = StandardLoggingMetadata( user_api_key_hash=None, @@ -6351,6 +6357,7 @@ class StandardLoggingPayloadSetup: user_api_key_auth_metadata=None, team_alias=None, team_id=None, + kubernetes_pod_routing=kubernetes_pod_routing, used_client_oauth_token=resolve_used_client_oauth_token( proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, @@ -7418,6 +7425,7 @@ def get_standard_logging_metadata( user_api_key_auth_metadata=None, team_alias=None, team_id=None, + kubernetes_pod_routing=None, used_client_oauth_token=None, ) if isinstance(metadata, dict): @@ -7491,6 +7499,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: requester_ip_address="127.0.0.1", requester_metadata=None, user_api_key_end_user_id="test_end_user", + kubernetes_pod_routing=None, ) hidden_params: Final = StandardLoggingHiddenParams( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8ff5d1a1002..c2cdd47304a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -105,6 +105,7 @@ from litellm.types.utils import ( ProviderField, StandardCallbackDynamicParams, StandardLoggingGuardrailInformation, + StandardLoggingKubernetesPodRouting, StandardLoggingMCPToolCall, StandardLoggingModelInformation, StandardLoggingPayloadErrorInformation, @@ -3682,6 +3683,7 @@ class SpendLogsMetadata(TypedDict): mcp_tool_call_metadata: StandardLoggingMCPToolCall | None vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None routing_decision: StandardLoggingRoutingDecision | None + kubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None] internal_call_origin: InternalCallOrigin | None litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index ce2e3cd30c4..06033f092dd 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1672,11 +1672,9 @@ class LiteLLMProxyRequestSetup: metadata_from_headers["agent_id"] = agent_id_from_header verbose_proxy_logger.debug("Extracted agent_id from header: %s", agent_id_from_header) - spend_log_session_id: Final = ( + caller_session_beside_trace: Final = ( _caller_session_beside_trace_header(normalized_headers, request_body, _metadata_variable_name) - if litellm.request_correlation_in_logs - and chain_id is not None - and chain_id == normalized_headers.get("x-litellm-trace-id") + if chain_id is not None and chain_id == normalized_headers.get("x-litellm-trace-id") else None ) if chain_id: @@ -1738,8 +1736,8 @@ class LiteLLMProxyRequestSetup: if isinstance(data[_metadata_variable_name], dict): data[_metadata_variable_name].update( - {**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: spend_log_session_id} - if spend_log_session_id + {**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: caller_session_beside_trace} + if caller_session_beside_trace else metadata_from_headers ) return data diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index d1303b012c8..78818cde266 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -19,6 +19,7 @@ from litellm.constants import ( CLI_SESSION_KEY_PREFIX, EMPTY_MAPPING, INTERNAL_CALL_ORIGIN_METADATA_KEY, + KUBERNETES_POD_ROUTING_KEY, LITELLM_PROXY_MASTER_KEY_ALIAS, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, @@ -43,6 +44,7 @@ from litellm.litellm_core_utils.classifier_logging import ( ) from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, + proxy_stamped_kubernetes_pod_routing, proxy_stamped_used_client_oauth_token, reconstruct_model_name, ) @@ -76,6 +78,7 @@ from litellm.types.utils import ( CostBreakdown, LlmProviders, StandardLoggingGuardrailInformation, + StandardLoggingKubernetesPodRouting, StandardLoggingMCPToolCall, StandardLoggingModelInformation, StandardLoggingPayload, @@ -178,6 +181,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "autorouter_savings_estimate", "autorouter_baseline_observation", "used_client_oauth_token", + KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", "user_api_key_user_email", ) @@ -210,6 +214,7 @@ def _get_spend_logs_metadata( router_metadata: SpendLogsRouterMetadata | None = None, azure_spillover: AzureSpillover | None = None, used_client_oauth_token: bool | None = None, + kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -257,6 +262,7 @@ def _get_spend_logs_metadata( router_metadata=router_metadata, azure_spillover=azure_spillover, used_client_oauth_token=used_client_oauth_token, + kubernetes_pod_routing=None, ) verbose_proxy_logger.debug("getting payload for SpendLogs, available keys in metadata: %s", [*metadata]) @@ -271,6 +277,7 @@ def _get_spend_logs_metadata( router_metadata=router_metadata, azure_spillover=azure_spillover, used_client_oauth_token=used_client_oauth_token, + kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, user_api_key_user_email=_stored_user_email(metadata), ) @@ -557,6 +564,16 @@ def get_logging_payload( # standardize this function to be used across, s3, dynamoDB, langfuse logging litellm_params: Final = kwargs.get("litellm_params", {}) metadata: Final = get_litellm_metadata_from_kwargs(kwargs) + routing_litellm_params: Final[Mapping[str, object] | None] = ( + litellm_params if isinstance(litellm_params, Mapping) else None + ) + routing_metadata: Final[object] = ( + routing_litellm_params.get("metadata") if routing_litellm_params is not None else None + ) + kubernetes_pod_routing: Final = proxy_stamped_kubernetes_pod_routing( + routing_metadata, + routing_litellm_params, + ) completion_start_time: Final = kwargs.get("completion_start_time", end_time) call_type: Final = kwargs.get("call_type") cache_hit: Final = kwargs.get("cache_hit", False) @@ -591,7 +608,9 @@ def get_logging_payload( id = get_spend_logs_id(call_type or "acompletion", response_obj_dict, kwargs, metadata) standard_logging_payload: Final = cast(StandardLoggingPayload | None, kwargs.get("standard_logging_object", None)) - end_user_id = get_end_user_id_for_cost_tracking(litellm_params) + end_user_id = get_end_user_id_for_cost_tracking( + dict(litellm_params) if isinstance(litellm_params, Mapping) else litellm_params + ) api_key = metadata.get("user_api_key", "") @@ -769,6 +788,7 @@ def get_logging_payload( used_client_oauth_token=resolve_used_client_oauth_token( proxy_stamped_used_client_oauth_token(litellm_params.get("metadata"), litellm_params), custom_llm_provider ), + kubernetes_pod_routing=kubernetes_pod_routing, azure_spillover=azure_spillover( response_headers=kwargs.get("response_headers") if isinstance(kwargs.get("response_headers"), Mapping) diff --git a/litellm/router.py b/litellm/router.py index 23b516ea573..113565fecaa 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -68,6 +68,7 @@ from litellm.constants import ( DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER, DEFAULT_MAX_LRU_CACHE_SIZE, INTERNAL_CALL_ORIGIN_METADATA_KEY, + KUBERNETES_POD_ROUTING_KEY, MID_STREAM_FAILURE_LOG_WAIT_SECONDS, OUTPUT_TOKEN_CEILING_PARAMS, PROMPT_CACHE_PIN_CACHE_MAX_SIZE, @@ -4606,6 +4607,7 @@ class Router: deployment_that_just_failed: Final = _deployment_that_just_failed( _FALLBACK_HOP_ADAPTER.validate_python(kwargs).get(metadata_variable_name) ) + routing: Final = deployment.get(KUBERNETES_POD_ROUTING_KEY) kwargs.setdefault(metadata_variable_name, {}).update( { @@ -4613,8 +4615,16 @@ class Router: "model_info": model_info, "api_base": deployment_api_base, "deployment_model_name": deployment_model_name, + KUBERNETES_POD_ROUTING_KEY: routing, } ) + other_bucket: Final = "metadata" if metadata_variable_name == "litellm_metadata" else "litellm_metadata" + other_bucket_metadata: Final = kwargs.get(other_bucket) + if isinstance(other_bucket_metadata, Mapping) and KUBERNETES_POD_ROUTING_KEY in other_bucket_metadata: + kwargs[other_bucket] = { # rebind-ok: this method updates the router's request kwargs in place + **other_bucket_metadata, + KUBERNETES_POD_ROUTING_KEY: routing, + } # A retry/fallback reuses this same kwargs dict for the next deployment. # Refund and clear any reservation the previous deployment attempt left diff --git a/litellm/router_utils/kubernetes_pod_discovery.py b/litellm/router_utils/kubernetes_pod_discovery.py index 7b98a52a4a0..05ba7be0344 100644 --- a/litellm/router_utils/kubernetes_pod_discovery.py +++ b/litellm/router_utils/kubernetes_pod_discovery.py @@ -1,5 +1,7 @@ import asyncio import concurrent.futures +import hashlib +import inspect import ipaddress import socket import threading @@ -13,6 +15,7 @@ from types import MappingProxyType from typing import ( Concatenate, Final, + Literal, ParamSpec, Protocol, TypeAlias, @@ -24,9 +27,12 @@ import httpx from litellm._logging import verbose_router_logger from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS, KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS, KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS, + KUBERNETES_POD_ROUTING_KEY, + SESSION_ID_GENERATED_METADATA_KEY, ) _SocketAddress: TypeAlias = tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes] @@ -34,9 +40,22 @@ _AddressInfo: TypeAlias = tuple[socket.AddressFamily, socket.SocketKind, int, st _SyncLookup: TypeAlias = Future[list[_AddressInfo]] _AsyncLookup: TypeAlias = asyncio.Future[list[_AddressInfo]] _CacheKey: TypeAlias = tuple[str, int | None] +_PodSelection: TypeAlias = tuple[str, int, Literal["round_robin", "session_affinity", "session_affinity_retry"]] _DeploymentT: Final = TypeVar("_DeploymentT") +def _retry_count_from_metadata(metadata: object) -> int | None: + if not isinstance(metadata, Mapping): + return None + metadata_mapping: Final = cast( # cast-ok: request metadata is a string-keyed mapping + Mapping[str, object], metadata + ) + retry_count: Final = metadata_mapping.get("request_retry_count") + if type(retry_count) is int and retry_count >= 0: + return retry_count + return None + + @dataclass(frozen=True, slots=True) class _PodSet: ips: tuple[str, ...] @@ -99,16 +118,24 @@ class KubernetesPodDiscovery: self._refreshing = self._refreshing | {key} return None, True - def resolve_deployment(self, deployment: _DeploymentT) -> _DeploymentT: + def resolve_deployment( + self, + deployment: _DeploymentT, + request_kwargs: Mapping[str, object] | None = None, + ) -> _DeploymentT: eligible: Final = self._eligible(deployment) if eligible is None: return deployment url, key, deployment_mapping = eligible host, port = key now: Final = self.clock() + session_id: Final = self._session_id(request_kwargs) + retry_count: Final = self._request_retry_count(request_kwargs) cached, should_refresh = self._begin_refresh(key, now) if cached is not None: - return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + return self._deployment_with_cached_ip( + deployment, deployment_mapping, key, url, now, session_id, retry_count + ) if not should_refresh: return deployment @@ -118,21 +145,36 @@ class KubernetesPodDiscovery: lookup.add_done_callback(lambda done: self._finish_refresh(key, done)) try: return self._apply_lookup( - deployment, deployment_mapping, key, url, self._await_lookup(key, host, lookup), now + deployment, + deployment_mapping, + key, + url, + self._await_lookup(key, host, lookup), + now, + session_id, + retry_count, ) finally: self._end_refresh(key) - async def async_resolve_deployment(self, deployment: _DeploymentT) -> _DeploymentT: + async def async_resolve_deployment( + self, + deployment: _DeploymentT, + request_kwargs: Mapping[str, object] | None = None, + ) -> _DeploymentT: eligible: Final = self._eligible(deployment) if eligible is None: return deployment url, key, deployment_mapping = eligible host, port = key now: Final = self.clock() + session_id: Final = self._session_id(request_kwargs) + retry_count: Final = self._request_retry_count(request_kwargs) cached, should_refresh = self._begin_refresh(key, now) if cached is not None: - return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + return self._deployment_with_cached_ip( + deployment, deployment_mapping, key, url, now, session_id, retry_count + ) if not should_refresh: return deployment @@ -142,10 +184,56 @@ class KubernetesPodDiscovery: lookup.add_done_callback(lambda done: self._finish_refresh(key, done)) try: result: Final = await self._await_async_lookup(key, host, lookup) - return self._apply_lookup(deployment, deployment_mapping, key, url, result, now) + return self._apply_lookup(deployment, deployment_mapping, key, url, result, now, session_id, retry_count) finally: self._end_refresh(key) + @staticmethod + def _session_id(request_kwargs: Mapping[str, object] | None) -> str | None: + if request_kwargs is None: + return None + metadata_dicts: Final = tuple( + cast(Mapping[str, object], metadata) # cast-ok: runtime dict check precedes metadata lookup + for metadata in (request_kwargs.get("litellm_metadata"), request_kwargs.get("metadata")) + if isinstance(metadata, dict) + ) + if any(metadata.get(SESSION_ID_GENERATED_METADATA_KEY) for metadata in metadata_dicts): + return None + caller_session_id: Final = next( + ( + caller_session + for metadata in metadata_dicts + if (caller_session := metadata.get(CALLER_SESSION_ID_METADATA_KEY)) is not None and caller_session != "" + ), + None, + ) + if caller_session_id is not None: + return str(caller_session_id) + top_level_session_id: Final = request_kwargs.get("litellm_session_id") + if top_level_session_id is not None and top_level_session_id != "": + return str(top_level_session_id) + metadata_session_id: Final = next( + ( + session_id + for metadata in metadata_dicts + if (session_id := metadata.get("session_id")) is not None and session_id != "" + ), + None, + ) + if metadata_session_id is not None: + return str(metadata_session_id) + return None + + @staticmethod + def _request_retry_count(request_kwargs: Mapping[str, object] | None) -> int: + if request_kwargs is None: + return 0 + retry_counts: Final = tuple( + _retry_count_from_metadata(request_kwargs.get(metadata_key)) + for metadata_key in ("litellm_metadata", "metadata") + ) + return next((retry_count for retry_count in retry_counts if retry_count is not None), 0) + def _eligible(self, deployment: object) -> tuple[httpx.URL, _CacheKey, Mapping[str, object]] | None: if not isinstance(deployment, Mapping): return None @@ -247,6 +335,8 @@ class KubernetesPodDiscovery: url: httpx.URL, result: Sequence[_AddressInfo] | OSError, now: float, + session_id: str | None, + retry_count: int, ) -> _DeploymentT: if isinstance(result, OSError): if isinstance(result, socket.gaierror) and self._is_authoritative_no_pods(result): @@ -254,12 +344,14 @@ class KubernetesPodDiscovery: return deployment self._stamp_failed_refresh(key, self.clock()) verbose_router_logger.debug("Kubernetes pod discovery DNS refresh failed for %s: %s", key[0], result) - return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + return self._deployment_with_cached_ip( + deployment, deployment_mapping, key, url, now, session_id, retry_count + ) ips: Final = self._pod_ips(result) self._store(key, ips, self.clock()) if not ips: return deployment - return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now, session_id, retry_count) @staticmethod def _is_ip_literal(host: str) -> bool: @@ -305,20 +397,55 @@ class KubernetesPodDiscovery: ) self._cache = MappingProxyType({**self._cache, key: stamped}) - def _next_ip(self, key: _CacheKey, now: float) -> str | None: + def _next_ip( + self, + key: _CacheKey, + now: float, + session_id: str | None, + retry_count: int, + ) -> _PodSelection | None: + session_hash_prefix: Final = ( + hashlib.blake2b(f"{session_id}\x00".encode(), digest_size=8) if session_id is not None else None + ) + + def session_rank(address: str) -> int: + if session_hash_prefix is None: + return 0 + h: Final = session_hash_prefix.copy() + h.update(address.encode()) + return int.from_bytes(h.digest(), "big") + with self._lock: cached: Final = self._cache.get(key) if cached is None or not cached.ips: return None - ip: Final = cached.ips[cached.cursor] - next_cursor: Final = (cached.cursor + 1) % len(cached.ips) + pod_count: Final = len(cached.ips) + ranked_ips: Final = ( + tuple( + sorted( + cached.ips, + key=lambda address: (-session_rank(address), address), + ) + ) + if session_id is not None + else () + ) + ip: Final = cached.ips[cached.cursor] if session_id is None else ranked_ips[retry_count % pod_count] + next_cursor: Final = (cached.cursor + 1) % len(cached.ips) if session_id is None else cached.cursor + selection: Final = ( + "round_robin" + if session_id is None + else "session_affinity_retry" + if retry_count > 0 + else "session_affinity" + ) self._cache = MappingProxyType( { **self._cache, key: replace(cached, cursor=next_cursor, last_used_at=max(cached.last_used_at, now)), } ) - return ip + return ip, pod_count, selection def _deployment_with_cached_ip( self, @@ -327,10 +454,13 @@ class KubernetesPodDiscovery: key: _CacheKey, url: httpx.URL, now: float, + session_id: str | None, + retry_count: int, ) -> _DeploymentT: - ip: Final = self._next_ip(key, now) - if ip is None: + selection: Final = self._next_ip(key, now, session_id, retry_count) + if selection is None: return deployment + ip, pod_count, selection_kind = selection if self._proxy_bypasses_only_service_host(key, ip): self._warn_proxy_once(key[0]) return deployment @@ -345,6 +475,12 @@ class KubernetesPodDiscovery: { **deployment_mapping, "litellm_params": {**typed_params, "api_base": str(url.copy_with(host=ip))}, + KUBERNETES_POD_ROUTING_KEY: { + "service_host": key[0], + "pod_ip": ip, + "pod_count": pod_count, + "selection": selection_kind, + }, }, ) @@ -402,9 +538,39 @@ _R: Final = TypeVar("_R") _S: Final = TypeVar("_S", bound=_HasPodDiscovery) +def _request_kwargs_from_call( + signature: inspect.Signature, + *args: object, + **kwargs: object, # kwargs-ok: forwards the call to Signature.bind_partial +) -> Mapping[str, object] | None: + try: + bound_arguments: Final = cast( # cast-ok: Signature arguments use string parameter names + Mapping[str, object], + signature.bind_partial(*args, **kwargs).arguments, + ) + request_kwargs: Final = bound_arguments.get( + "request_kwargs", + kwargs.get("request_kwargs"), + ) + except TypeError: + return _mapping_request_kwargs(kwargs.get("request_kwargs")) + return _mapping_request_kwargs(request_kwargs) + + +def _mapping_request_kwargs(request_kwargs: object) -> Mapping[str, object] | None: + if not isinstance(request_kwargs, Mapping): + return None + return cast( # cast-ok: router selector request kwargs use string keys + Mapping[str, object], + request_kwargs, + ) + + def resolve_pods_after( fn: Callable[Concatenate[_S, _P], _R], ) -> Callable[Concatenate[_S, _P], _R]: + signature: Final = inspect.signature(fn) + @wraps(fn) def wrapped( self: _S, @@ -412,7 +578,10 @@ def resolve_pods_after( **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature ) -> _R: deployment: Final = fn(self, *args, **kwargs) - return self.kubernetes_pod_discovery.resolve_deployment(deployment) + return self.kubernetes_pod_discovery.resolve_deployment( + deployment, + _request_kwargs_from_call(signature, self, *args, **kwargs), + ) return wrapped @@ -420,6 +589,8 @@ def resolve_pods_after( def async_resolve_pods_after( fn: Callable[Concatenate[_S, _P], Awaitable[_R]], ) -> Callable[Concatenate[_S, _P], Coroutine[object, object, _R]]: + signature: Final = inspect.signature(fn) + @wraps(fn) async def wrapped( self: _S, @@ -427,7 +598,10 @@ def async_resolve_pods_after( **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature ) -> _R: deployment: Final = await fn(self, *args, **kwargs) - return await self.kubernetes_pod_discovery.async_resolve_deployment(deployment) + return await self.kubernetes_pod_discovery.async_resolve_deployment( + deployment, + _request_kwargs_from_call(signature, self, *args, **kwargs), + ) return wrapped @@ -436,13 +610,18 @@ def resolve_pods_after_bound( fn: Callable[_P, _R], discovery: KubernetesPodDiscovery, ) -> Callable[_P, _R]: + signature: Final = inspect.signature(fn) + @wraps(fn) def wrapped( *args: _P.args, **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature ) -> _R: deployment: Final = fn(*args, **kwargs) - return discovery.resolve_deployment(deployment) + return discovery.resolve_deployment( + deployment, + _request_kwargs_from_call(signature, *args, **kwargs), + ) return wrapped @@ -451,12 +630,17 @@ def async_resolve_pods_after_bound( fn: Callable[_P, Awaitable[_R]], discovery: KubernetesPodDiscovery, ) -> Callable[_P, Coroutine[object, object, _R]]: + signature: Final = inspect.signature(fn) + @wraps(fn) async def wrapped( *args: _P.args, **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature ) -> _R: deployment: Final = await fn(*args, **kwargs) - return await discovery.async_resolve_deployment(deployment) + return await discovery.async_resolve_deployment( + deployment, + _request_kwargs_from_call(signature, *args, **kwargs), + ) return wrapped diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 419c2624111..fd1d58adafb 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3242,6 +3242,13 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): tier_litellm_params: Mapping[str, object] # writable-ok: Pydantic warns on ReadOnly TypedDict fields +class StandardLoggingKubernetesPodRouting(TypedDict): + service_host: ReadOnly[str] + pod_ip: ReadOnly[str] + pod_count: ReadOnly[int] + selection: ReadOnly[Literal["round_robin", "session_affinity", "session_affinity_retry"]] + + # Fields whose values quote the caller's prompt. Dropped when an operator turns message # logging off. Every other field aggregates the prompt without reproducing it and is kept, # so a redacted row stays explainable. `test_every_routing_decision_field_is_classified` @@ -3308,6 +3315,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): mcp_tool_call_metadata: StandardLoggingMCPToolCall | None vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None routing_decision: StandardLoggingRoutingDecision | None + kubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None] applied_guardrails: list[str] | None usage_object: dict | None cold_storage_object_key: str | None # S3/GCS object key for cold storage retrieval diff --git a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json index c6cf8ee7ee4..cfdf707c977 100644 --- a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json +++ b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}", + "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index b40c1a96500..9cc06ccaafc 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -51,7 +51,7 @@ from litellm.proxy import proxy_server from litellm.proxy._types import SpanAttributes from litellm.types.services import ServiceLoggerPayload, ServiceTypes from collections.abc import AsyncIterator -from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE +from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LOGGING_WORKER_MAX_TIME_PER_COROUTINE from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.utils import ChatCompletionMessageToolCall, Function from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -363,6 +363,33 @@ def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metada assert finished.attributes["metadata.routing_decision"] == str(decision) +def test_v1_pod_routing_none_matches_other_none_metadata_attributes() -> None: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel: Final = OpenTelemetry(tracer_provider=provider) + with provider.get_tracer(__name__).start_as_current_span("inference") as span: + otel.set_attributes( + span, + { + "model": "answer", + "standard_logging_object": { + "id": "routing-call", + "call_type": "acompletion", + "metadata": { + KUBERNETES_POD_ROUTING_KEY: None, + "other_none": None, + }, + }, + }, + {"model": "answer", "choices": [], "usage": {}}, + ) + (finished,) = exporter.get_finished_spans() + routing_attribute: Final = finished.attributes["metadata.kubernetes_pod_routing"] + assert routing_attribute == finished.attributes["metadata.other_none"] + assert routing_attribute != "null" + + @pytest.mark.parametrize("decision", ["custom", 42, ["custom"]]) def test_v1_malformed_routing_metadata_preserves_inference_attributes(decision: object) -> None: exporter: Final = InMemorySpanExporter() diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 8d546f3be59..d1258e43154 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -30,7 +30,12 @@ from litellm._internal_context import in_post_response_phase from litellm._logging import session_id_var, trace_id_var, verbose_logger from litellm._service_logger import ServiceLogging from litellm.caching.caching import DualCache -from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE, REDACTED_BY_LITELLM, SENTRY_PII_DENYLIST +from litellm.constants import ( + KUBERNETES_POD_ROUTING_KEY, + LOGGING_WORKER_MAX_TIME_PER_COROUTINE, + REDACTED_BY_LITELLM, + SENTRY_PII_DENYLIST, +) from litellm.cost_calculator import ocr_batch_cost from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils import litellm_logging @@ -5713,6 +5718,97 @@ def test_get_standard_logging_object_payload_takes_used_client_oauth_token_from_ assert payload["metadata"]["used_client_oauth_token"] is expected +@pytest.mark.parametrize( + ("metadata", "litellm_metadata", "expected"), + [ + ( + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + } + }, + {}, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + }, + ), + ( + {}, + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.2", + "pod_count": 3, + "selection": "session_affinity", + } + }, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.2", + "pod_count": 3, + "selection": "session_affinity", + }, + ), + ( + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + } + }, + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.3", + "pod_count": 3, + "selection": "session_affinity_retry", + } + }, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.3", + "pod_count": 3, + "selection": "session_affinity_retry", + }, + ), + ], +) +def test_standard_logging_payload_resolves_kubernetes_pod_routing_from_metadata_buckets( + logging_obj: LitellmLogging, + metadata: dict[str, object], + litellm_metadata: dict[str, object], + expected: dict[str, object], +) -> None: + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + now: Final = datetime(2026, 1, 1) + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": "gpt-4o", + "messages": [], + "litellm_params": {"metadata": metadata, "litellm_metadata": litellm_metadata}, + }, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["metadata"][KUBERNETES_POD_ROUTING_KEY] == expected + + def test_get_standard_logging_object_payload_carries_matched_access_groups(logging_obj): """Access groups stamped at auth time reach the logging payload, so integrations see what a request billed.""" from datetime import datetime diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 47cfa2df117..ed8ae2db8b1 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3748,7 +3748,18 @@ class TestSpendLogsPayload: } ) - differences = _compare_nested_dicts(payload, expected_payload, ignore_keys=ignored_keys) + expected_payload_with_pod_routing: Final[SpendLogsPayload] = { + **expected_payload, + "metadata": expected_payload["metadata"].replace( + '"routing_decision": null, ', + '"routing_decision": null, "kubernetes_pod_routing": null, ', + ), + } + differences = _compare_nested_dicts( + payload, + expected_payload_with_pod_routing, + ignore_keys=ignored_keys, + ) if differences: pytest.fail(f"Dictionary mismatch: {differences}") diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 262357fe1bd..34e17b3cda8 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -14,7 +14,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm import litellm.constants as litellm_constants import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils -from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL +from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo @@ -76,6 +76,89 @@ def _get_additional_usage_values_for_usage(usage: litellm.Usage) -> dict: return metadata["additional_usage_values"] +@pytest.mark.parametrize( + ("metadata", "litellm_metadata", "expected"), + [ + ( + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + } + }, + {}, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + }, + ), + ( + {}, + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.2", + "pod_count": 3, + "selection": "session_affinity", + } + }, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.2", + "pod_count": 3, + "selection": "session_affinity", + }, + ), + ( + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + } + }, + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.3", + "pod_count": 3, + "selection": "session_affinity_retry", + } + }, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.3", + "pod_count": 3, + "selection": "session_affinity_retry", + }, + ), + ], +) +def test_get_logging_payload_includes_kubernetes_pod_routing( + metadata: dict[str, object], + litellm_metadata: dict[str, object], + expected: dict[str, object], +) -> None: + now: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc) + payload: Final = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": metadata, "litellm_metadata": litellm_metadata}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=now, + end_time=now, + ) + + assert json.loads(payload["metadata"])[KUBERNETES_POD_ROUTING_KEY] == expected + assert _get_spend_logs_metadata(None)["kubernetes_pod_routing"] is None + + @pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)]) def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch, store_prompts, redact): from litellm.proxy import proxy_server diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 93202d93566..2943fe79979 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -3309,12 +3309,11 @@ def test_add_litellm_metadata_from_request_headers_x_litellm_trace_id_sets_chain assert data["litellm_trace_id"] == "foo" -@pytest.mark.parametrize(("correlation_in_logs", "expected_spend_log_session"), [(True, "convo-1"), (False, None)]) +@pytest.mark.parametrize("correlation_in_logs", [True, False], ids=["correlation-on", "correlation-off"]) @pytest.mark.parametrize("session_in_metadata", [False, True], ids=["top-level", "metadata"]) -def test_x_litellm_trace_id_keeps_the_body_session_for_spend_logs_only( +def test_x_litellm_trace_id_keeps_the_body_session_beside_it_whatever_the_log_setting( monkeypatch: pytest.MonkeyPatch, correlation_in_logs: bool, - expected_spend_log_session: str | None, session_in_metadata: bool, ): monkeypatch.setattr(litellm, "request_correlation_in_logs", correlation_in_logs) @@ -3325,7 +3324,7 @@ def test_x_litellm_trace_id_keeps_the_body_session_for_spend_logs_only( LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( headers={"x-litellm-trace-id": "req-trace-1"}, data=data, _metadata_variable_name="metadata" ) - assert metadata.get(CALLER_SESSION_ID_METADATA_KEY) == expected_spend_log_session + assert metadata.get(CALLER_SESSION_ID_METADATA_KEY) == "convo-1" assert data["litellm_session_id"] == "req-trace-1" assert metadata["session_id"] == "req-trace-1" assert data["litellm_trace_id"] == "req-trace-1" diff --git a/tests/unit/router_utils/test_kubernetes_pod_discovery.py b/tests/unit/router_utils/test_kubernetes_pod_discovery.py index b3fc6eede62..001b8863b17 100644 --- a/tests/unit/router_utils/test_kubernetes_pod_discovery.py +++ b/tests/unit/router_utils/test_kubernetes_pod_discovery.py @@ -1,5 +1,6 @@ import asyncio import copy +import hashlib import json import logging import re @@ -16,10 +17,22 @@ import respx from openai import AsyncOpenAI, OpenAI import litellm +from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, + KUBERNETES_POD_ROUTING_KEY, + SESSION_ID_GENERATED_METADATA_KEY, +) from litellm.router import CustomRoutingStrategyBase, Router -from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery +from litellm.router_utils.kubernetes_pod_discovery import ( + KubernetesPodDiscovery, + async_resolve_pods_after, + async_resolve_pods_after_bound, + resolve_pods_after, + resolve_pods_after_bound, +) _SERVICE_URL: Final = "http://vllm-headless.ns.svc.cluster.local:8000/v1" +_THREE_POD_CHAT_PATTERN: Final = re.compile(r"http://(?:10\.0\.0\.1|10\.0\.0\.2|10\.0\.0\.3):8000/v1/chat/completions") _SocketAddress: TypeAlias = tuple[str, int] | tuple[str, int, int, int] _AddrInfo: TypeAlias = tuple[socket.AddressFamily, socket.SocketKind, int, str, _SocketAddress] _NO_POD_ERRNOS: Final = tuple(sorted((socket.EAI_NONAME, socket.EAI_NODATA))) @@ -91,6 +104,14 @@ def _empty_proxy_environment() -> Mapping[str, str]: return {} +def _patch_router_discovery( + monkeypatch: pytest.MonkeyPatch, + *discoveries: KubernetesPodDiscovery, +) -> None: + discovery_iter: Final = iter(discoveries) + monkeypatch.setattr(litellm.router, "KubernetesPodDiscovery", lambda: next(discovery_iter)) + + def _stub_sync_dns(monkeypatch: pytest.MonkeyPatch, *ips: str) -> None: def getaddrinfo( host: str | None, @@ -113,12 +134,88 @@ def _api_base(deployment: Mapping[str, object]) -> str: return api_base +def _ranked_session_ips(session_id: str) -> tuple[str, ...]: + ips: Final = ("10.0.0.1", "10.0.0.2", "10.0.0.3") + return tuple( + sorted( + ips, + key=lambda ip: ( + -int.from_bytes(hashlib.blake2b(f"{session_id}\x00{ip}".encode(), digest_size=8).digest(), "big"), + ip, + ), + ) + ) + + +def test_session_hash_ranking_matches_full_hash_for_large_session_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ips: Final = ("10.0.0.1", "10.0.0.2", "10.0.0.3") + _stub_sync_dns(monkeypatch, *ips) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + for session_id in ("s1", "session-2", "session-3"): + expected: Final = _ranked_session_ips(session_id) + actual: Final = tuple( + httpx.URL( + _api_base( + discovery.resolve_deployment( + _deployment(), + {"metadata": {"session_id": session_id, "request_retry_count": retry_count}}, + ) + ) + ).host + for retry_count in range(len(ips)) + ) + assert actual == expected + + large_ips: Final = tuple(f"10.1.0.{index}" for index in range(1, 101)) + _stub_sync_dns(monkeypatch, *large_ips) + large_discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + large_session_id: Final = "x" * (1024 * 1024) + + result: Final = large_discovery.resolve_deployment( + _deployment(), + {"metadata": {"session_id": large_session_id}}, + ) + + assert httpx.URL(_api_base(result)).host in large_ips + + def _request_body(content: bytes) -> Mapping[str, object]: body: Final = json.loads(content) assert isinstance(body, dict) return cast(Mapping[str, object], body) +def _cache_async_client(router: Router) -> AsyncOpenAI: + client: Final = AsyncOpenAI(api_key="fake", base_url=_SERVICE_URL) + router.cache.set_cache(key="registered-id_async_client", value=client, local_only=True) + return client + + +def _cache_sync_client(router: Router) -> OpenAI: + client: Final = OpenAI(api_key="fake", base_url=_SERVICE_URL) + router.cache.set_cache(key="registered-id_client", value=client, local_only=True) + return client + + +async def _router_acompletion(router: Router, **request_kwargs: object) -> None: + await router.acompletion( + model="gpu-model", + messages=[{"role": "user", "content": "hello"}], + **request_kwargs, + ) + + +def _router_completion(router: Router, **request_kwargs: object) -> None: + router.completion( + model="gpu-model", + messages=[{"role": "user", "content": "hello"}], + **request_kwargs, + ) + + def test_sync_resolution_round_robins_sorted_ips_without_mutating_deployment( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1050,19 +1147,17 @@ async def test_router_sends_pod_hosts_without_forwarding_discovery_flag( monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - with respx.mock(assert_all_called=True) as respx_mock: route: Final = respx_mock.post( url__regex=re.compile( r"http://(?:10\.0\.0\.1|10\.0\.0\.2|vllm-headless\.ns\.svc\.cluster\.local):8000/v1/chat/completions" ) ).mock(return_value=httpx.Response(200, json=_CHAT_RESPONSE)) + discovery_enabled: Final = KubernetesPodDiscovery(clock=clock, proxy_environment=_empty_proxy_environment) + control: Final = KubernetesPodDiscovery(clock=clock, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery_enabled, control) discovery_enabled_router: Final = Router(model_list=[_deployment()]) control_router: Final = Router(model_list=[_deployment(kubernetes_pod_discovery=None)]) - discovery_enabled_router.kubernetes_pod_discovery = KubernetesPodDiscovery( - clock=clock, - proxy_environment=_empty_proxy_environment, - ) cached_async_client: Final = AsyncOpenAI(api_key="fake", base_url=_SERVICE_URL) cached_sync_client: Final = OpenAI(api_key="fake", base_url=_SERVICE_URL) discovery_enabled_router.cache.set_cache( @@ -1100,6 +1195,506 @@ async def test_router_sends_pod_hosts_without_forwarding_discovery_flag( assert next(sync_lookups) == 1 +@pytest.mark.asyncio +async def test_router_session_requests_stick_to_one_pod_for_sync_and_async( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + async_lookups: Final = count() + sync_lookups: Final = count() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(async_lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(sync_lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + async_discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + sync_discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, async_discovery, sync_discovery) + async_router: Final = Router(model_list=[_deployment()]) + sync_router: Final = Router(model_list=[_deployment()]) + async_client: Final = _cache_async_client(async_router) + sync_client: Final = _cache_sync_client(sync_router) + for _ in range(6): + await _router_acompletion(async_router, metadata={"session_id": "s1"}) + for _ in range(6): + _router_completion(sync_router, metadata={"session_id": "s1"}) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await async_client.close() + sync_client.close() + + assert len(set(hosts[:6])) == 1 + assert len(set(hosts[6:])) == 1 + assert next(async_lookups) == 1 + assert next(sync_lookups) == 1 + + +@pytest.mark.asyncio +async def test_router_session_mapping_repeats_across_pods( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + session_ids: Final = tuple(f"session-{index}" for index in range(30)) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for session_id in session_ids: + await _router_acompletion(router, metadata={"session_id": session_id}) + await _router_acompletion(router) + for session_id in session_ids: + await _router_acompletion(router, metadata={"session_id": session_id}) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + first_pass: Final = hosts[: len(session_ids)] + second_pass: Final = hosts[len(session_ids) + 1 :] + assert len(set(first_pass)) > 1 + assert hosts[len(session_ids)] == "10.0.0.1" + assert second_pass == first_pass + assert next(lookups) == 1 + + +@pytest.mark.asyncio +async def test_generated_session_id_uses_round_robin_pod_selection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for _ in range(6): + await _router_acompletion( + router, + metadata={ + "session_id": "generated-session", + SESSION_ID_GENERATED_METADATA_KEY: True, + }, + ) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert hosts == ("10.0.0.1", "10.0.0.2", "10.0.0.3") * 2 + + +@pytest.mark.asyncio +async def test_empty_session_id_uses_round_robin_pod_selection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for _ in range(6): + await _router_acompletion(router, metadata={"session_id": ""}) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert hosts == ("10.0.0.1", "10.0.0.2", "10.0.0.3") * 2 + + +@pytest.mark.asyncio +async def test_session_requests_do_not_advance_round_robin_cursor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + await _router_acompletion(router) + await _router_acompletion(router, metadata={"session_id": "s1"}) + await _router_acompletion(router, metadata={"session_id": "s1"}) + await _router_acompletion(router) + await _router_acompletion(router, metadata={"session_id": "s1"}) + await _router_acompletion(router) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert tuple(hosts[index] for index in (0, 3, 5)) == ( + "10.0.0.1", + "10.0.0.2", + "10.0.0.3", + ) + + +@pytest.mark.asyncio +async def test_session_pod_membership_change_remaps_only_affected_sessions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + lookup_number: Final = next(lookups) + ips: Final = ("10.0.0.1", "10.0.0.2", "10.0.0.3") if lookup_number == 0 else ("10.0.0.2", "10.0.0.3") + return _records(*ips, port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + session_ids: Final = tuple(f"session-{index}" for index in range(60)) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=10, + clock=_clock((0.0,) * 62 + (10.0,) * 10), + proxy_environment=_empty_proxy_environment, + ) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for session_id in session_ids: + await _router_acompletion(router, metadata={"session_id": session_id}) + initial_hosts: Final = tuple(call.request.url.host for call in route.calls) + removed_session_ids: Final = tuple( + session_id for session_id, host in zip(session_ids, initial_hosts) if host == "10.0.0.1" + ) + surviving_session_ids: Final = tuple( + session_id for session_id, host in zip(session_ids, initial_hosts) if host == "10.0.0.2" + ) + assert removed_session_ids + assert len(surviving_session_ids) >= 2 + await _router_acompletion(router) + await _router_acompletion(router, metadata={"session_id": surviving_session_ids[0]}) + await _router_acompletion(router, metadata={"session_id": removed_session_ids[0]}) + await _router_acompletion(router, metadata={"session_id": surviving_session_ids[1]}) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert hosts[60] == "10.0.0.1" + assert hosts[61] == "10.0.0.2" + assert hosts[62] in ("10.0.0.2", "10.0.0.3") + assert hosts[63] == "10.0.0.2" + assert next(lookups) == 2 + + +@pytest.mark.asyncio +async def test_custom_routing_strategy_keeps_sync_and_async_sessions_on_one_pod( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + async_lookups: Final = count() + sync_lookups: Final = count() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(async_lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(sync_lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + async_discovery: Final = KubernetesPodDiscovery( + clock=lambda: 0.0, + proxy_environment=_empty_proxy_environment, + ) + sync_discovery: Final = KubernetesPodDiscovery( + clock=lambda: 0.0, + proxy_environment=_empty_proxy_environment, + ) + _patch_router_discovery(monkeypatch, async_discovery, sync_discovery) + async_router: Final = Router(model_list=[_deployment()]) + sync_router: Final = Router(model_list=[_deployment()]) + async_router.set_custom_routing_strategy(_ModelListRoutingStrategy(async_router)) + sync_router.set_custom_routing_strategy(_ModelListRoutingStrategy(sync_router)) + async_client: Final = _cache_async_client(async_router) + sync_client: Final = _cache_sync_client(sync_router) + for _ in range(6): + await _router_acompletion(async_router, metadata={"session_id": "custom-session"}) + for _ in range(6): + _router_completion(sync_router, metadata={"session_id": "custom-session"}) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await async_client.close() + sync_client.close() + + assert len(set(hosts[:6])) == 1 + assert len(set(hosts[6:])) == 1 + assert next(async_lookups) == 1 + assert next(sync_lookups) == 1 + + +@pytest.mark.asyncio +async def test_top_level_litellm_session_id_keeps_requests_on_one_pod( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for _ in range(6): + await _router_acompletion(router, litellm_session_id="s1") + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert len(set(hosts)) == 1 + + +def test_caller_session_beside_trace_header_keeps_the_pod_across_trace_ids(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + caller_session_id: Final = "chat-session-a" + caller_pod: Final = _ranked_session_ips(caller_session_id)[0] + trace_ids: Final = tuple( + trace_id + for trace_id in (f"trace-{turn}" for turn in range(20)) + if _ranked_session_ips(trace_id)[0] != caller_pod + )[:3] + assert len(trace_ids) == 3 + + pods: Final = tuple( + httpx.URL( + _api_base( + discovery.resolve_deployment( + _deployment(), + { + "litellm_session_id": trace_id, + "metadata": {CALLER_SESSION_ID_METADATA_KEY: caller_session_id}, + }, + ) + ) + ).host + for trace_id in trace_ids + ) + + assert pods == (caller_pod, caller_pod, caller_pod) + + +def test_proxy_headers_with_trace_and_session_id_pin_the_pod_without_log_correlation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + monkeypatch.setattr(litellm, "request_correlation_in_logs", False) + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + caller_session_id: Final = "chat-session-a" + caller_pod: Final = _ranked_session_ips(caller_session_id)[0] + trace_ids: Final = tuple( + trace_id + for trace_id in (f"trace-{turn}" for turn in range(20)) + if _ranked_session_ips(trace_id)[0] != caller_pod + )[:3] + assert len(trace_ids) == 3 + + def resolve_turn(trace_id: str) -> str: + data: Final = LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + {"x-litellm-trace-id": trace_id, "x-litellm-session-id": caller_session_id}, + {"model": "gpu-model", "metadata": {}}, + "metadata", + ) + return httpx.URL(_api_base(discovery.resolve_deployment(_deployment(), data))).host + + assert tuple(resolve_turn(trace_id) for trace_id in trace_ids) == (caller_pod, caller_pod, caller_pod) + + +def test_top_level_litellm_session_id_precedes_metadata_session_id(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + top_level_session_id: Final = "s1" + metadata_session_id: Final = "metadata-session" + top_level_kwargs: Final = {"litellm_session_id": top_level_session_id} + metadata_kwargs: Final = {"metadata": {"session_id": metadata_session_id}} + combined_kwargs: Final = { + "litellm_session_id": top_level_session_id, + "metadata": {"session_id": metadata_session_id}, + } + + assert _ranked_session_ips(top_level_session_id)[0] != _ranked_session_ips(metadata_session_id)[0] + + top_level_pod: Final = httpx.URL(_api_base(discovery.resolve_deployment(_deployment(), top_level_kwargs))).host + metadata_pod: Final = httpx.URL(_api_base(discovery.resolve_deployment(_deployment(), metadata_kwargs))).host + combined_pod: Final = httpx.URL(_api_base(discovery.resolve_deployment(_deployment(), combined_kwargs))).host + + assert top_level_pod != metadata_pod + assert combined_pod == top_level_pod + + +@pytest.mark.asyncio +async def test_generated_metadata_ignores_top_level_litellm_session_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for _ in range(6): + await _router_acompletion( + router, + litellm_session_id="s1", + metadata={ + "session_id": "s1", + SESSION_ID_GENERATED_METADATA_KEY: True, + }, + ) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert hosts == ("10.0.0.1", "10.0.0.2", "10.0.0.3") * 2 + + @pytest.mark.asyncio async def test_custom_routing_strategy_resolves_pod_hosts_for_sync_and_async_requests( monkeypatch: pytest.MonkeyPatch, @@ -1134,23 +1729,23 @@ async def test_custom_routing_strategy_resolves_pod_hosts_for_sync_and_async_req monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - with respx.mock(assert_all_called=True) as respx_mock: route: Final = respx_mock.post( url__regex=re.compile( r"http://(?:10\.0\.0\.1|10\.0\.0\.2|vllm-headless\.ns\.svc\.cluster\.local):8000/v1/chat/completions" ) ).mock(return_value=httpx.Response(200, json=_CHAT_RESPONSE)) + async_discovery: Final = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + sync_discovery: Final = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + _patch_router_discovery(monkeypatch, async_discovery, sync_discovery) async_router: Final = Router(model_list=[_deployment()]) sync_router: Final = Router(model_list=[_deployment()]) - async_router.kubernetes_pod_discovery = KubernetesPodDiscovery( - clock=clock, - proxy_environment=_empty_proxy_environment, - ) - sync_router.kubernetes_pod_discovery = KubernetesPodDiscovery( - clock=clock, - proxy_environment=_empty_proxy_environment, - ) async_router.set_custom_routing_strategy(_ModelListRoutingStrategy(async_router)) sync_router.set_custom_routing_strategy(_ModelListRoutingStrategy(sync_router)) for _ in range(4): @@ -1202,16 +1797,13 @@ async def test_router_async_retry_uses_next_discovered_pod(monkeypatch: pytest.M monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - with respx.mock(assert_all_called=True) as respx_mock: route: Final = respx_mock.post( url__regex=re.compile(r"http://(?:10\.0\.0\.1|10\.0\.0\.2):8000/v1/chat/completions") ).mock(side_effect=response_for) + discovery: Final = KubernetesPodDiscovery(clock=clock, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) router: Final = Router(model_list=[_deployment()], num_retries=1) - router.kubernetes_pod_discovery = KubernetesPodDiscovery( - clock=clock, - proxy_environment=_empty_proxy_environment, - ) response: Final = await router.acompletion( model="gpu-model", @@ -1245,16 +1837,13 @@ def test_router_sync_retry_uses_next_discovered_pod(monkeypatch: pytest.MonkeyPa monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - with respx.mock(assert_all_called=True) as respx_mock: route: Final = respx_mock.post( url__regex=re.compile(r"http://(?:10\.0\.0\.1|10\.0\.0\.2):8000/v1/chat/completions") ).mock(side_effect=response_for) + discovery: Final = KubernetesPodDiscovery(clock=clock, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) router: Final = Router(model_list=[_deployment()], num_retries=1) - router.kubernetes_pod_discovery = KubernetesPodDiscovery( - clock=clock, - proxy_environment=_empty_proxy_environment, - ) response: Final = router.completion( model="gpu-model", @@ -1264,3 +1853,240 @@ def test_router_sync_retry_uses_next_discovered_pod(monkeypatch: pytest.MonkeyPa assert response.choices[0].message.content == "ok" assert hosts == ("10.0.0.1", "10.0.0.2") + + +@pytest.mark.parametrize( + ("litellm_retry_count", "metadata_retry_count", "expected_rank"), + [ + (0, None, 0), + (None, None, 0), + (True, None, 0), + ("1", None, 0), + (-1, None, 0), + (1, None, 1), + (True, 1, 1), + (1, 0, 1), + ], +) +def test_session_retry_count_selects_ranked_pod( + monkeypatch: pytest.MonkeyPatch, + litellm_retry_count: object, + metadata_retry_count: object, + expected_rank: int, +) -> None: + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + request_kwargs: Final = { + "litellm_metadata": { + "session_id": "s1", + **({"request_retry_count": litellm_retry_count} if litellm_retry_count is not None else {}), + }, + "metadata": ({"request_retry_count": metadata_retry_count} if metadata_retry_count is not None else {}), + } + + resolved: Final = discovery.resolve_deployment(_deployment(), request_kwargs) + + assert httpx.URL(_api_base(resolved)).host == _ranked_session_ips("s1")[expected_rank] + + +@pytest.mark.asyncio +async def test_router_session_retry_uses_next_rendezvous_pod(monkeypatch: pytest.MonkeyPatch) -> None: + loop: Final = asyncio.get_running_loop() + attempts: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + def response_for(request: httpx.Request) -> httpx.Response: + attempt: Final = next(attempts) + status_code: Final = 500 if attempt == 0 else 200 + body: Final = {"error": {"message": "retryable", "type": "server_error"}} if attempt == 0 else _CHAT_RESPONSE + return httpx.Response(status_code, json=body, request=request) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + session_id: Final = "retry-session" + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock(side_effect=response_for) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()], num_retries=1) + + response: Final = await router.acompletion( + model="gpu-model", + messages=[{"role": "user", "content": "hello"}], + metadata={"session_id": session_id}, + ) + hosts: Final = tuple(call.request.url.host for call in route.calls) + + assert response.choices[0].message.content == "ok" + assert hosts == _ranked_session_ips(session_id)[:2] + + +@pytest.mark.asyncio +async def test_positional_request_kwargs_reach_all_pod_resolver_decorators( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + class _Selector: + def __init__(self, discovery: KubernetesPodDiscovery) -> None: + self.kubernetes_pod_discovery: Final = discovery + + @resolve_pods_after + def sync(self, deployment: dict[str, object], request_kwargs: Mapping[str, object]) -> dict[str, object]: + return deployment + + @async_resolve_pods_after + async def async_call( + self, + deployment: dict[str, object], + request_kwargs: Mapping[str, object], + ) -> dict[str, object]: + return deployment + + class _BoundSelector: + def sync(self, deployment: dict[str, object], request_kwargs: Mapping[str, object]) -> dict[str, object]: + return deployment + + async def async_call( + self, + deployment: dict[str, object], + request_kwargs: Mapping[str, object], + ) -> dict[str, object]: + return deployment + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + session_id: Final = next( + candidate + for candidate in (f"positional-{index}" for index in range(100)) + if _ranked_session_ips(candidate)[0] != "10.0.0.1" + ) + request_kwargs: Final = {"metadata": {"session_id": session_id}} + sync_selector: Final = _Selector(KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment)) + async_selector: Final = _Selector(KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment)) + bound_selector: Final = _BoundSelector() + bound_sync: Final = resolve_pods_after_bound( + bound_selector.sync, + KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment), + ) + bound_async: Final = async_resolve_pods_after_bound( + bound_selector.async_call, + KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment), + ) + + sync_result: Final = sync_selector.sync(_deployment(), request_kwargs) + async_result: Final = await async_selector.async_call(_deployment(), request_kwargs) + bound_sync_result: Final = bound_sync(_deployment(), request_kwargs) + bound_async_result: Final = await bound_async(_deployment(), request_kwargs) + + assert ( + tuple( + httpx.URL(_api_base(result)).host + for result in (sync_result, async_result, bound_sync_result, bound_async_result) + ) + == (_ranked_session_ips(session_id)[0],) * 4 + ) + + +def test_direct_session_id_keyword_does_not_enable_session_affinity(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + request_kwargs: Final = {"session_id": "s1", "messages": [{"role": "user", "content": "hello"}]} + + selected_ips: Final = tuple( + httpx.URL(_api_base(discovery.resolve_deployment(_deployment(), request_kwargs))).host for _ in range(3) + ) + + assert selected_ips == ("10.0.0.1", "10.0.0.2", "10.0.0.3") + + +@pytest.mark.parametrize( + ("request_kwargs", "selection"), + [ + ({}, "round_robin"), + ({"metadata": {"session_id": "s1"}}, "session_affinity"), + ({"metadata": {"session_id": "s1", "request_retry_count": 1}}, "session_affinity_retry"), + ], +) +def test_resolved_deployment_records_pod_selection( + monkeypatch: pytest.MonkeyPatch, + request_kwargs: dict[str, object], + selection: str, +) -> None: + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + resolved: Final = discovery.resolve_deployment(_deployment(), request_kwargs) + pod_ip: Final = httpx.URL(_api_base(resolved)).host + + assert resolved[KUBERNETES_POD_ROUTING_KEY] == { + "service_host": httpx.URL(_SERVICE_URL).host, + "pod_ip": pod_ip, + "pod_count": 3, + "selection": selection, + } + + +@pytest.mark.parametrize("fallback", ["https", "no_proxy", "empty_ips", "dns_failure"]) +def test_unresolved_deployment_does_not_record_pod_selection( + monkeypatch: pytest.MonkeyPatch, + fallback: str, +) -> None: + if fallback == "empty_ips": + _stub_sync_dns(monkeypatch) + if fallback == "dns_failure": + + def failing_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + raise socket.gaierror(socket.EAI_AGAIN, "temporary failure") + + monkeypatch.setattr(socket, "getaddrinfo", failing_getaddrinfo) + if fallback == "no_proxy": + _stub_sync_dns(monkeypatch, "10.0.0.1") + + proxy_environment: Final = ( + _proxy_environment("vllm-headless.ns.svc.cluster.local") if fallback == "no_proxy" else _empty_proxy_environment + ) + api_base: Final = "https://vllm-headless.ns.svc.cluster.local:8000/v1" if fallback == "https" else _SERVICE_URL + discovery: Final = KubernetesPodDiscovery(proxy_environment=proxy_environment) + deployment: Final = _deployment(api_base=api_base) + + resolved: Final = discovery.resolve_deployment(deployment) + + assert resolved is deployment + assert KUBERNETES_POD_ROUTING_KEY not in resolved diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 1dec4de45f4..6403f00eb50 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -9978,6 +9978,94 @@ def test_update_kwargs_with_deployment_model_info_in_metadata(): assert model_info["output_cost_per_token"] == 0.0015 +def test_update_kwargs_with_deployment_clears_pod_routing_on_non_discovery_fallback() -> None: + from litellm.constants import KUBERNETES_POD_ROUTING_KEY + + routing_record: Final = { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "session_affinity", + } + discovery_deployment: Final = { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}, + "model_info": {"id": "discovery-id"}, + KUBERNETES_POD_ROUTING_KEY: routing_record, + } + fallback_deployment: Final = { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}, + "model_info": {"id": "fallback-id"}, + } + router: Final = litellm.Router(model_list=[discovery_deployment]) + kwargs: Final = {"metadata": {}} + + router._update_kwargs_with_deployment(deployment=discovery_deployment, kwargs=kwargs) + assert kwargs["metadata"][KUBERNETES_POD_ROUTING_KEY] == routing_record + + router._update_kwargs_with_deployment(deployment=fallback_deployment, kwargs=kwargs) + assert kwargs["metadata"][KUBERNETES_POD_ROUTING_KEY] is None + + +def test_update_kwargs_with_deployment_synchronizes_existing_litellm_metadata_pod_routing() -> None: + from litellm.constants import KUBERNETES_POD_ROUTING_KEY + + routing_record: Final = { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "session_affinity", + } + spoofed_record: Final = { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.9", + "pod_count": 3, + "selection": "round_robin", + } + deployment: Final = { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}, + "model_info": {"id": "discovery-id"}, + KUBERNETES_POD_ROUTING_KEY: routing_record, + } + original_litellm_metadata: Final = {KUBERNETES_POD_ROUTING_KEY: spoofed_record} + kwargs: Final = {"metadata": {}, "litellm_metadata": original_litellm_metadata} + router: Final = litellm.Router(model_list=[deployment]) + + router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, function_name="completion") + + assert kwargs["metadata"][KUBERNETES_POD_ROUTING_KEY] == routing_record + assert kwargs["litellm_metadata"][KUBERNETES_POD_ROUTING_KEY] == routing_record + assert kwargs["litellm_metadata"] is not original_litellm_metadata + assert original_litellm_metadata[KUBERNETES_POD_ROUTING_KEY] == spoofed_record + + +def test_update_kwargs_with_non_discovery_deployment_clears_other_pod_routing_bucket() -> None: + from litellm.constants import KUBERNETES_POD_ROUTING_KEY + + previous_record: Final = { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "session_affinity", + } + deployment: Final = { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}, + "model_info": {"id": "fallback-id"}, + } + original_litellm_metadata: Final = {KUBERNETES_POD_ROUTING_KEY: previous_record} + kwargs: Final = {"metadata": {}, "litellm_metadata": original_litellm_metadata} + router: Final = litellm.Router(model_list=[deployment]) + + router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, function_name="completion") + + assert kwargs["metadata"][KUBERNETES_POD_ROUTING_KEY] is None + assert kwargs["litellm_metadata"][KUBERNETES_POD_ROUTING_KEY] is None + assert original_litellm_metadata[KUBERNETES_POD_ROUTING_KEY] == previous_record + + def test_combine_fallback_usage(): """Test that _combine_fallback_usage merges partial and fallback usage.""" from litellm.router import Router diff --git a/ui/litellm-dashboard/src/components/logs/constants.ts b/ui/litellm-dashboard/src/components/logs/constants.ts index 9474c54d6b6..44231370521 100644 --- a/ui/litellm-dashboard/src/components/logs/constants.ts +++ b/ui/litellm-dashboard/src/components/logs/constants.ts @@ -33,6 +33,12 @@ export const CREDENTIAL_LABELS: Record = { false: "Configured key", }; +export const KUBERNETES_POD_ROUTING_LABELS = { + round_robin: "Round robin", + session_affinity: "Session affinity", + session_affinity_retry: "Session affinity (retry)", +} as const; + export const QUICK_SELECT_OPTIONS: { label: string; value: number; unit: string }[] = [ { label: "Last Minute", value: 1, unit: "minutes" }, { label: "Last 15 Minutes", value: 15, unit: "minutes" }, diff --git a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx index 6c2569710e6..7f39a0d8fe5 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx @@ -431,6 +431,37 @@ describe("LogDetailContent", () => { expect(screen.queryByText("Configured key")).not.toBeInTheDocument(); }); + it.each([ + { selection: "round_robin", label: "Round robin" }, + { selection: "session_affinity", label: "Session affinity" }, + { selection: "session_affinity_retry", label: "Session affinity (retry)" }, + ] as const)("shows the selected Kubernetes pod and routing mode", ({ selection, label }) => { + render( + , + ); + + expect(screen.getByText("Pod")).toBeInTheDocument(); + expect(screen.getByText(`10.0.0.2 · ${label} · 3 pods`)).toBeInTheDocument(); + }); + + it("omits the Pod row when routing metadata is absent", () => { + render(); + + expect(screen.queryByText("Pod")).not.toBeInTheDocument(); + }); + it("should display guardrail label when guardrail data exists", () => { render( + {podRouting && typeof podRouting === "object" && ( + + {podRouting.pod_ip} + {" · "} + {KUBERNETES_POD_ROUTING_LABELS[podRouting.selection]} + {" · "} + {podRouting.pod_count} pods + + )} {logEntry.requester_ip_address && ( {logEntry.requester_ip_address} )} diff --git a/ui/litellm-dashboard/src/components/logs/types.ts b/ui/litellm-dashboard/src/components/logs/types.ts index ce4a72a6d38..dc5fa4aaf5b 100644 --- a/ui/litellm-dashboard/src/components/logs/types.ts +++ b/ui/litellm-dashboard/src/components/logs/types.ts @@ -10,6 +10,13 @@ export const LOGS_SORT_FIELD_MAP = { export type LogsSortField = keyof typeof LOGS_SORT_FIELD_MAP; +export type KubernetesPodRouting = { + service_host: string; + pod_ip: string; + pod_count: number; + selection: "round_robin" | "session_affinity" | "session_affinity_retry"; +}; + export type LogEntry = { request_id: string; litellm_call_id?: string | null; @@ -29,7 +36,9 @@ export type LogEntry = { user?: string; end_user?: string; custom_llm_provider?: string; - metadata?: Record; + metadata?: Record & { + kubernetes_pod_routing?: KubernetesPodRouting | null; + }; cache_hit: string; cache_key?: string; request_tags?: Record; From f8f6a01caec7f43f0d9dfa919097d865247ed064 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:58:20 -0700 Subject: [PATCH 29/50] feat(ui): show kubernetes pods on the model page (#44666) Show discovered Kubernetes pods on the model page for admins Resolves LIT-9168 Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 1 + .../model_management_endpoints.py | 88 ++++++++ .../router_utils/kubernetes_pod_discovery.py | 31 +++ .../endpointaudit/coverage_allowlist.txt | 1 + tests/unit/proxy/auth/test_route_checks.py | 54 +++++ .../test_model_management_endpoints.py | 207 ++++++++++++++++++ .../test_kubernetes_pod_discovery.py | 172 +++++++++++++++ .../KubernetesPodsCard.integration.test.tsx | 118 ++++++++++ .../model_info/KubernetesPodsCard.tsx | 89 ++++++++ .../src/components/model_info_view.test.tsx | 88 ++++++++ .../src/components/model_info_view.tsx | 9 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 63 ++++++ 12 files changed, 920 insertions(+), 1 deletion(-) create mode 100644 ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.tsx diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c2cdd47304a..bc32978d477 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -668,6 +668,7 @@ class LiteLLMRoutes(enum.Enum): "/model/update", "/model/delete", "/model/info", + "/model/{model_id}/kubernetes_pods", "/jwt/key/mapping/new", "/jwt/key/mapping/update", "/jwt/key/mapping/delete", diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 7460522f53e..7eae193dd8a 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -168,6 +168,19 @@ if TYPE_CHECKING: from prisma import types as prisma_types router: Final = APIRouter() + + +class KubernetesPodsResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + model_id: str + service_host: str + port: int | None + pod_ips: tuple[str, ...] + pod_count: int + error: str | None + + CLEARABLE_LITELLM_PARAMS: Final = frozenset({"cache_control_injection_points", "litellm_credential_name"}) NULL_CLEARABLE_LITELLM_PARAMS: Final = frozenset((*SPECIAL_MODEL_INFO_PARAMS, *CLEARABLE_LITELLM_PARAMS)) @@ -1163,6 +1176,77 @@ def update_db_model( return prisma_compatible_model_dict +@router.get( + "/model/{model_id}/kubernetes_pods", + response_model=KubernetesPodsResponse, + tags=["model management"], + dependencies=[Depends(user_api_key_auth)], +) +async def get_kubernetes_pods( + model_id: str, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> KubernetesPodsResponse: + if user_api_key_dict.user_role not in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + raise ProxyException( + message="Only proxy admins can view Kubernetes pods", + type=ProxyErrorTypes.auth_error.value, + code=status.HTTP_403_FORBIDDEN, + param=None, + ) + + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + raise ProxyException( + message=CommonProxyErrors.no_llm_router.value, + type=ProxyErrorTypes.internal_server_error.value, + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + param=None, + ) + + deployment: Final[Mapping[str, object] | None] = _OPTIONAL_JSON_OBJECT.validate_python( + llm_router.get_model_info(id=model_id) # pyright: ignore[reportUnknownMemberType] # validated below + ) + if deployment is None: + raise ProxyException( + message=f"Model with id {model_id} not found", + type=ProxyErrorTypes.not_found_error.value, + code=status.HTTP_404_NOT_FOUND, + param="model_id", + ) + + lookup: Final = await llm_router.kubernetes_pod_discovery.async_lookup_pods(deployment) + if lookup is None: + raw_litellm_params: Final = deployment.get("litellm_params") + litellm_params: Final[Mapping[str, object] | None] = ( + _JSON_OBJECT.validate_python(raw_litellm_params) if isinstance(raw_litellm_params, Mapping) else None + ) + discovery_enabled: Final = litellm_params is not None and litellm_params.get("kubernetes_pod_discovery") is True + raise ProxyException( + message=( + "Kubernetes pod discovery is enabled for this model, but its api_base is not an http:// " + "service hostname, so no pods are resolved" + if discovery_enabled + else "Kubernetes pod discovery is not enabled for this model" + ), + type=ProxyErrorTypes.bad_request_error.value, + code=status.HTTP_400_BAD_REQUEST, + param="model_id", + ) + + return KubernetesPodsResponse( + model_id=model_id, + service_host=lookup.service_host, + port=lookup.port, + pod_ips=tuple(lookup.pod_ips), + pod_count=len(lookup.pod_ips), + error=lookup.error, + ) + + @router.patch( "/model/{model_id}/update", tags=["model management"], @@ -3257,6 +3341,10 @@ _deduplicate_litellm_router_models: Final = deduplicate_litellm_router_models _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_OPTIONAL_JSON_OBJECT: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter( + Mapping[str, object] | None, + config=ConfigDict(hide_input_in_errors=True), +) def model_info_as_mapping(model_info: object) -> Mapping[str, object] | None: diff --git a/litellm/router_utils/kubernetes_pod_discovery.py b/litellm/router_utils/kubernetes_pod_discovery.py index 05ba7be0344..34d99db1f6e 100644 --- a/litellm/router_utils/kubernetes_pod_discovery.py +++ b/litellm/router_utils/kubernetes_pod_discovery.py @@ -44,6 +44,14 @@ _PodSelection: TypeAlias = tuple[str, int, Literal["round_robin", "session_affin _DeploymentT: Final = TypeVar("_DeploymentT") +@dataclass(frozen=True, slots=True) +class KubernetesPodLookup: + service_host: str + port: int | None + pod_ips: tuple[str, ...] + error: str | None + + def _retry_count_from_metadata(metadata: object) -> int | None: if not isinstance(metadata, Mapping): return None @@ -188,6 +196,18 @@ class KubernetesPodDiscovery: finally: self._end_refresh(key) + async def async_lookup_pods(self, deployment: object) -> KubernetesPodLookup | None: + eligible: Final = self._eligible(deployment) + if eligible is None: + return None + _, (host, port), _ = eligible + result: Final = await self._async_getaddrinfo(host, port) + if isinstance(result, OSError): + if isinstance(result, socket.gaierror) and self._is_authoritative_no_pods(result): + return KubernetesPodLookup(service_host=host, port=port, pod_ips=(), error=None) + return KubernetesPodLookup(service_host=host, port=port, pod_ips=(), error=str(result)) + return KubernetesPodLookup(service_host=host, port=port, pod_ips=self._pod_ips(result), error=None) + @staticmethod def _session_id(request_kwargs: Mapping[str, object] | None) -> str | None: if request_kwargs is None: @@ -293,6 +313,17 @@ class KubernetesPodDiscovery: except OSError as error: return error + async def _async_getaddrinfo(self, host: str, port: int | None) -> Sequence[_AddressInfo] | OSError: + lookup: Final = asyncio.get_running_loop().getaddrinfo( + host, port, type=socket.SOCK_STREAM, flags=socket.AI_ADDRCONFIG + ) + try: + return await asyncio.wait_for(lookup, timeout=self.dns_timeout_seconds) + except (TimeoutError, asyncio.TimeoutError): + return socket.gaierror(socket.EAI_AGAIN, f"DNS lookup for {host} exceeded {self.dns_timeout_seconds}s") + except OSError as error: + return error + def _abandon_lookup( self, key: _CacheKey, diff --git a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt index 3a45eef3606..00f8f3b6031 100644 --- a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt +++ b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt @@ -74,6 +74,7 @@ GET /guardrails/ui/major_airlines GET /guardrails/ui/provider_specific_params GET /key/aliases GET /model/deprecations +GET /model/{model_id}/kubernetes_pods GET /search_tools/ui/available_providers GET /team/available GET /team/metadata_schema diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index a519a4e576a..6de54aa4af6 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -2472,6 +2472,7 @@ def test_proxy_admin_viewer_can_access_settings_read_endpoints(route): ADMIN_VIEWER_REPORTED_GET_ROUTES = [ "/health/latest", "/credentials", + "/model/model-pods-1/kubernetes_pods", "/v1/mcp/network/client-ip", "/claude-code/plugins", "/policy/templates", @@ -3615,6 +3616,59 @@ def test_internal_user_still_blocked_from_another_users_info(): assert "key not allowed to access this user's info" in str(exc_info.value.detail) +def test_internal_user_cannot_access_kubernetes_pod_discovery_route() -> None: + route: Final = "/model/model-pods-1/kubernetes_pods" + user_obj: Final = LiteLLM_UserTable( + user_id="internal_user", + user_email="user@example.com", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ) + valid_token: Final = UserAPIKeyAuth( + user_id="internal_user", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ) + request: Final = MagicMock(spec=Request) + request.method = "GET" + request.query_params = {} + + assert RouteChecks.is_management_route(route=route) + with pytest.raises(Exception, match="Only proxy admin can be used"): + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + + +def test_proxy_admin_viewer_can_access_kubernetes_pod_discovery_route() -> None: + route: Final = "/model/model-pods-1/kubernetes_pods" + user_obj: Final = LiteLLM_UserTable( + user_id="admin_viewer", + user_email="viewer@example.com", + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + ) + valid_token: Final = UserAPIKeyAuth( + user_id="admin_viewer", + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + ) + request: Final = MagicMock(spec=Request) + request.method = "GET" + request.query_params = {} + + assert RouteChecks.is_management_route(route=route) + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + + @pytest.mark.parametrize( "route", [ diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 7cd1107abce..bc714ad1cbc 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -2,6 +2,7 @@ import inspect import asyncio import contextlib import json +import socket from collections.abc import Iterator, Mapping from types import SimpleNamespace from typing import Dict, Final, Optional @@ -27,11 +28,13 @@ from litellm.proxy._types import ( ) from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.management_endpoints.model_management_endpoints import ( + KubernetesPodsResponse, ModelManagementAuthChecks, _get_team_deployments, _raise_if_rate_limits_required_but_missing, clear_cache, delete_team_models, + get_kubernetes_pods, patch_model, update_model, ) @@ -44,6 +47,210 @@ async def _passthrough_row(update_data): return update_data +def _kubernetes_pod_router( + discovery_enabled: bool | None = True, + api_base: str = "http://vllm-headless.ns.svc.cluster.local:8000/v1", +) -> Router: + return Router( + model_list=[ + { + "model_name": "gpu-llama", + "litellm_params": { + "model": "openai/test-model", + "api_base": api_base, + "api_key": "not-in-response", + **({"kubernetes_pod_discovery": discovery_enabled} if discovery_enabled is not None else {}), + }, + "model_info": {"id": "model-pods-1"}, + } + ], + ignore_invalid_deployments=True, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "user_role", + [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY], +) +async def test_get_kubernetes_pods_returns_sorted_read_only_response( + monkeypatch: pytest.MonkeyPatch, + user_role: LitellmUserRoles, +) -> None: + from litellm.proxy import proxy_server + + router: Final = _kubernetes_pod_router() + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[tuple[socket.AddressFamily, socket.SocketKind, int, str, tuple[str, int]]]: + return [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("10.0.0.3", 8000)), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("10.0.0.1", 8000)), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("10.0.0.1", 8000)), + ] + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + + response: Final = await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=user_role), + ) + + assert isinstance(response, KubernetesPodsResponse) + assert response.model_dump() == { + "model_id": "model-pods-1", + "service_host": "vllm-headless.ns.svc.cluster.local", + "port": 8000, + "pod_ips": ("10.0.0.1", "10.0.0.3"), + "pod_count": 2, + "error": None, + } + + +@pytest.mark.asyncio +async def test_get_kubernetes_pods_returns_dns_error_without_exposing_deployment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + router: Final = _kubernetes_pod_router() + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[tuple[socket.AddressFamily, socket.SocketKind, int, str, tuple[str, int]]]: + raise OSError("temporary DNS failure") + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + + response: Final = await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert response.model_dump() == { + "model_id": "model-pods-1", + "service_host": "vllm-headless.ns.svc.cluster.local", + "port": 8000, + "pod_ips": (), + "pod_count": 0, + "error": "temporary DNS failure", + } + + +@pytest.mark.asyncio +async def test_get_kubernetes_pods_returns_404_for_unknown_model(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", _kubernetes_pod_router()) + + with pytest.raises(ProxyException) as error: + await get_kubernetes_pods( + model_id="unknown", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert error.value.code == "404" + assert error.value.type == "not_found_error" + assert error.value.param == "model_id" + + +@pytest.mark.asyncio +async def test_get_kubernetes_pods_returns_500_without_router(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", None) + + with pytest.raises(ProxyException) as error: + await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert error.value.code == "500" + assert error.value.type == "internal_server_error" + + +@pytest.mark.asyncio +async def test_get_kubernetes_pods_returns_400_when_discovery_is_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", _kubernetes_pod_router(discovery_enabled=None)) + + with pytest.raises(ProxyException) as error: + await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert error.value.code == "400" + assert error.value.message == "Kubernetes pod discovery is not enabled for this model" + assert error.value.param == "model_id" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "api_base", + [ + "https://vllm-headless.ns.svc.cluster.local:8000/v1", + "http://192.0.2.10:8000/v1", + ], +) +async def test_get_kubernetes_pods_returns_400_for_ineligible_api_base( + monkeypatch: pytest.MonkeyPatch, + api_base: str, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", _kubernetes_pod_router(api_base=api_base)) + expected_message: Final = ( + "Kubernetes pod discovery is enabled for this model, but its api_base is not an http:// " + "service hostname, so no pods are resolved" + ) + + with pytest.raises(ProxyException) as error: + await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert error.value.code == "400" + assert error.value.message == expected_message + assert error.value.param == "model_id" + + +@pytest.mark.asyncio +async def test_get_kubernetes_pods_rejects_internal_users(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", _kubernetes_pod_router()) + + with pytest.raises(ProxyException) as error: + await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="internal-user", user_role=LitellmUserRoles.INTERNAL_USER), + ) + + assert error.value.code == "403" + assert error.value.type == "auth_error" + + @pytest.mark.asyncio @pytest.mark.parametrize("model", ["openai/test-model", "auto_router/complexity_router"]) @pytest.mark.parametrize("team_role", ["admin", "user"]) diff --git a/tests/unit/router_utils/test_kubernetes_pod_discovery.py b/tests/unit/router_utils/test_kubernetes_pod_discovery.py index 001b8863b17..115e2a49bc5 100644 --- a/tests/unit/router_utils/test_kubernetes_pod_discovery.py +++ b/tests/unit/router_utils/test_kubernetes_pod_discovery.py @@ -87,6 +87,178 @@ def _records(*ips: str, port: int = 8000) -> list[_AddrInfo]: return [_record(ip, port) for ip in ips] +@pytest.mark.asyncio +async def test_async_lookup_pods_returns_sorted_unique_addresses(monkeypatch: pytest.MonkeyPatch) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + assert host == "vllm-headless.ns.svc.cluster.local" + assert port == 8000 + assert type == socket.SOCK_STREAM + return _records("10.0.0.3", "10.0.0.1", "10.0.0.2", "10.0.0.1") + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + lookup: Final = await discovery.async_lookup_pods(_deployment()) + + assert lookup is not None + assert lookup.service_host == "vllm-headless.ns.svc.cluster.local" + assert lookup.port == 8000 + assert lookup.pod_ips == ("10.0.0.1", "10.0.0.2", "10.0.0.3") + assert lookup.error is None + + +@pytest.mark.asyncio +async def test_async_lookup_pods_treats_authoritative_no_pods_as_empty( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + raise socket.gaierror(socket.EAI_NONAME, "no ready pods") + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + lookup: Final = await discovery.async_lookup_pods(_deployment()) + + assert lookup is not None + assert lookup.pod_ips == () + assert lookup.error is None + + +@pytest.mark.asyncio +async def test_async_lookup_pods_returns_non_authoritative_dns_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + raise OSError("temporary DNS failure") + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + lookup: Final = await discovery.async_lookup_pods(_deployment()) + + assert lookup is not None + assert lookup.pod_ips == () + assert lookup.error == "temporary DNS failure" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("api_base", "discovery_enabled"), + [ + (_SERVICE_URL, False), + ("https://vllm-headless.ns.svc.cluster.local:8000/v1", True), + ("http://10.0.0.4:8000/v1", True), + ], +) +async def test_async_lookup_pods_skips_ineligible_deployments( + monkeypatch: pytest.MonkeyPatch, + api_base: str, + discovery_enabled: bool, +) -> None: + loop: Final = asyncio.get_running_loop() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.1") + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + lookup: Final = await discovery.async_lookup_pods( + _deployment(api_base=api_base, kubernetes_pod_discovery=discovery_enabled) + ) + + assert lookup is None + assert next(lookups) == 0 + + +@pytest.mark.asyncio +async def test_async_lookup_pods_does_not_cache_or_advance_round_robin_cursor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + async_lookups: Final = count() + sync_lookups: Final = count() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(async_lookups) + return _records("10.0.0.9") + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(sync_lookups) + return _records("10.0.0.1", "10.0.0.2") + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=30, + proxy_environment=_empty_proxy_environment, + ) + + first_lookup: Final = await discovery.async_lookup_pods(_deployment()) + first_resolution: Final = discovery.resolve_deployment(_deployment()) + second_lookup: Final = await discovery.async_lookup_pods(_deployment()) + second_resolution: Final = discovery.resolve_deployment(_deployment()) + + assert first_lookup is not None + assert first_lookup.pod_ips == ("10.0.0.9",) + assert second_lookup is not None + assert second_lookup.pod_ips == ("10.0.0.9",) + assert _api_base(first_resolution) == "http://10.0.0.1:8000/v1" + assert _api_base(second_resolution) == "http://10.0.0.2:8000/v1" + assert next(async_lookups) == 2 + assert next(sync_lookups) == 1 + + def _clock(ticks: tuple[float, ...]) -> Callable[[], float]: timestamps: Final = iter(ticks) diff --git a/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.integration.test.tsx b/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.integration.test.tsx new file mode 100644 index 00000000000..3725b1181ad --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.integration.test.tsx @@ -0,0 +1,118 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders, screen, testQueryClient } from "@/../tests/test-utils"; +import KubernetesPodsCard from "./KubernetesPodsCard"; + +const podResponse = { + model_id: "model-pods-1", + service_host: "vllm-headless.ns.svc.cluster.local", + port: 8000, + pod_ips: ["10.0.0.1", "10.0.0.2", "10.0.0.3"], + pod_count: 3, + error: null, +}; + +const lookupErrorResponse = { + ...podResponse, + pod_ips: [], + pod_count: 0, + error: "temporary DNS failure", +}; + +const network = vi.fn(); + +describe("KubernetesPodsCard", () => { + beforeEach(() => { + testQueryClient.clear(); + network.mockReset().mockResolvedValue(Response.json(podResponse)); + vi.stubGlobal("fetch", network); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + testQueryClient.clear(); + }); + + it("shows the service host, ready count, and all pod IPs", async () => { + renderWithProviders(); + + expect(await screen.findByText("vllm-headless.ns.svc.cluster.local:8000")).toBeInTheDocument(); + expect(screen.getByText("3 ready pods")).toBeInTheDocument(); + expect(screen.getByText("10.0.0.1")).toBeInTheDocument(); + expect(screen.getByText("10.0.0.2")).toBeInTheDocument(); + expect(screen.getByText("10.0.0.3")).toBeInTheDocument(); + }); + + it("shows an empty state when no pods are ready", async () => { + network.mockResolvedValueOnce( + Response.json({ + ...podResponse, + pod_ips: [], + pod_count: 0, + }), + ); + + renderWithProviders(); + + expect(await screen.findByText("No ready pods")).toBeInTheDocument(); + expect(screen.queryByText("0 ready pods")).not.toBeInTheDocument(); + }); + + it("shows a pod lookup error", async () => { + network.mockResolvedValueOnce(Response.json(lookupErrorResponse)); + + renderWithProviders(); + + expect(await screen.findByRole("alert")).toHaveTextContent("temporary DNS failure"); + expect(screen.queryByText("0 ready pods")).not.toBeInTheDocument(); + }); + + it("shows an API error", async () => { + network.mockResolvedValueOnce(Response.json({ detail: "DNS lookup unavailable" }, { status: 503 })); + + renderWithProviders(); + + expect(await screen.findByRole("alert")).toHaveTextContent("DNS lookup unavailable"); + }); + + it("shows the API message for an ineligible Kubernetes service host", async () => { + const message = + "Kubernetes pod discovery is enabled for this model, but its api_base is not an http:// " + + "service hostname, so no pods are resolved"; + network.mockResolvedValueOnce( + Response.json( + { + error: { + message, + type: "bad_request_error", + param: "model_id", + code: "400", + }, + }, + { status: 400 }, + ), + ); + + renderWithProviders(); + + expect(await screen.findByRole("alert")).toHaveTextContent(message); + }); + + it("refreshes the pod lookup when requested", async () => { + const user: ReturnType = userEvent.setup(); + renderWithProviders(); + + expect(await screen.findByText("10.0.0.1")).toBeInTheDocument(); + network.mockResolvedValueOnce( + Response.json({ + ...podResponse, + pod_ips: ["10.0.0.4"], + pod_count: 1, + }), + ); + await user.click(screen.getByRole("button", { name: "Refresh" })); + + expect(await screen.findByText("10.0.0.4")).toBeInTheDocument(); + expect(network).toHaveBeenCalledTimes(2); + }); +}); diff --git a/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.tsx b/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.tsx new file mode 100644 index 00000000000..054cd36254b --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.tsx @@ -0,0 +1,89 @@ +"use client"; + +import { $api } from "@/lib/http/api"; +import { Button } from "@/components/ui/button"; +import { Card } from "@/components/ui/card"; +import type { components } from "@/lib/http/schema"; + +interface KubernetesPodsCardProps { + modelId: string; +} + +interface KubernetesPodsCardContentProps { + data: components["schemas"]["KubernetesPodsResponse"] | undefined; + isError: boolean; + isPending: boolean; + apiErrorMessage: string; +} + +function KubernetesPodsCardContent({ data, isError, isPending, apiErrorMessage }: KubernetesPodsCardContentProps) { + const hasData = data !== undefined && !isError; + const hasLookupError = hasData && data.error !== null; + const showEmptyState = hasData && data.error === null && data.pod_ips.length === 0; + const showPodIps = hasData && data.error === null && data.pod_ips.length > 0; + const podCount = hasData && !hasLookupError ? data.pod_count : null; + + return ( + <> + {podCount !== null && podCount > 0 &&

{podCount} ready pods

} + {isPending && ( +

+ Loading Kubernetes pods +

+ )} + {isError && ( +

+ {apiErrorMessage} +

+ )} + {data && hasLookupError && ( +

+ {data.error} +

+ )} + {data && showEmptyState &&

No ready pods

} + {data && showPodIps && ( +
    + {data.pod_ips.map((ip) => ( +
  • {ip}
    • + ))} +
+ )} + + ); +} + +export default function KubernetesPodsCard({ modelId }: KubernetesPodsCardProps) { + const { data, error, isError, isPending, refetch } = $api.useQuery( + "get", + "/model/{model_id}/kubernetes_pods", + { params: { path: { model_id: modelId } } }, + { refetchInterval: 10_000 }, + ); + const apiErrorMessage = + error instanceof Error ? error.message : JSON.stringify(error) ?? "Failed to load Kubernetes pods"; + + return ( + +
+
+

Kubernetes pods

+ {data && ( +

+ {data.port === null ? data.service_host : `${data.service_host}:${data.port}`} +

+ )} +
+ +
+ + + ); +} diff --git a/ui/litellm-dashboard/src/components/model_info_view.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.test.tsx index 6e7bc9a86d7..2e85fab5b76 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.test.tsx @@ -6,6 +6,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import ModelInfoView from "./model_info_view"; import { toast } from "@/lib/toast"; import * as networking from "./networking"; +vi.mock("./model_info/KubernetesPodsCard", () => ({ + default: ({ modelId }: { modelId: string }) =>
{modelId}
, +})); vi.mock( "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults", async () => await import("../../tests/mocks/complexityScorerDefaults"), @@ -196,6 +199,91 @@ describe("ModelInfoView", () => { }); }); + it("shows Kubernetes pods for discovery-enabled models", async () => { + mockUseModelsInfo.mockReturnValue({ + data: { + data: [ + { + ...defaultModelData, + litellm_params: { ...defaultModelData.litellm_params, kubernetes_pod_discovery: true }, + }, + ], + }, + isLoading: false, + error: null, + }); + + render(, { wrapper }); + + expect(await screen.findByTestId("kubernetes-pods-card")).toHaveTextContent("123"); + }); + + it("shows Kubernetes pods for admin viewers", async () => { + mockUseModelsInfo.mockReturnValue({ + data: { + data: [ + { + ...defaultModelData, + litellm_params: { ...defaultModelData.litellm_params, kubernetes_pod_discovery: true }, + }, + ], + }, + isLoading: false, + error: null, + }); + + render(, { wrapper }); + + expect(await screen.findByTestId("kubernetes-pods-card")).toHaveTextContent("123"); + }); + + it("hides Kubernetes pods for non-admin users", async () => { + mockUseModelsInfo.mockReturnValue({ + data: { + data: [ + { + ...defaultModelData, + litellm_params: { ...defaultModelData.litellm_params, kubernetes_pod_discovery: true }, + }, + ], + }, + isLoading: false, + error: null, + }); + + render(, { wrapper }); + + expect(await screen.findByText("Model Settings")).toBeInTheDocument(); + expect(screen.queryByTestId("kubernetes-pods-card")).not.toBeInTheDocument(); + }); + + it("hides Kubernetes pods when discovery is disabled", async () => { + mockUseModelsInfo.mockReturnValue({ + data: { + data: [ + { + ...defaultModelData, + litellm_params: { ...defaultModelData.litellm_params, kubernetes_pod_discovery: false }, + }, + ], + }, + isLoading: false, + error: null, + }); + + render(, { wrapper }); + + expect(await screen.findByText("Model Settings")).toBeInTheDocument(); + expect(screen.queryByTestId("kubernetes-pods-card")).not.toBeInTheDocument(); + }); + + it("hides Kubernetes pods when discovery is not configured", async () => { + render(, { wrapper }); + + expect(await screen.findByText("Model Settings")).toBeInTheDocument(); + expect(screen.queryByTestId("kubernetes-pods-card")).not.toBeInTheDocument(); + }); + it("should display loading state when model data is loading", () => { mockUseModelsInfo.mockReturnValue({ data: null, diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index ae34b17d0e8..74c1e498594 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -23,7 +23,7 @@ import { isComplexityRouter as isComplexityRouterParams, } from "./add_model/auto_router_strategies"; import { canEditAutoRouter, canModifyModel } from "@/utils/modelPermissions"; -import { teamsUserCanAssign } from "@/utils/roles"; +import { isProxyAdminTierRole, teamsUserCanAssign } from "@/utils/roles"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import DeleteResourceModal from "./common_components/DeleteResourceModal"; import EditAutoRouterModal from "./edit_auto_router/edit_auto_router_modal"; @@ -50,6 +50,7 @@ import ModelInfoEditForm, { type ModelEditSubmitValues, type TouchedPricingField import { Tag } from "./tag_management/types"; import { getDisplayModelName } from "./view_model/model_name_display"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import KubernetesPodsCard from "./model_info/KubernetesPodsCard"; interface ModelInfoViewProps { modelId: string; @@ -680,6 +681,12 @@ export default function ModelInfoView({ + {modelData?.litellm_params?.kubernetes_pod_discovery === true && isProxyAdminTierRole(userRole ?? "") && ( +
+ +
+ )} + {/* Settings Card */}
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a111ed1d773..d697ccac98d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -10187,6 +10187,23 @@ export interface paths { patch?: never; trace?: never; }; + "/model/{model_id}/kubernetes_pods": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Kubernetes Pods */ + get: operations["get_kubernetes_pods_model__model_id__kubernetes_pods_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/model/{model_id}/update": { parameters: { query?: never; @@ -33764,6 +33781,21 @@ export interface components { */ tier: string; }; + /** KubernetesPodsResponse */ + KubernetesPodsResponse: { + /** Error */ + error: string | null; + /** Model Id */ + model_id: string; + /** Pod Count */ + pod_count: number; + /** Pod Ips */ + pod_ips: string[]; + /** Port */ + port: number | null; + /** Service Host */ + service_host: string; + }; /** LLMV2Calibration */ LLMV2Calibration: { capable: components["schemas"]["LLMV2ProbabilityCalibration"]; @@ -65038,6 +65070,37 @@ export interface operations { }; }; }; + get_kubernetes_pods_model__model_id__kubernetes_pods_get: { + parameters: { + query?: never; + header?: never; + path: { + model_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["KubernetesPodsResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; patch_model_model__model_id__update_patch: { parameters: { query?: never; From c82e8244a57862feecfb48218bdd20c72c2afe6d Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:02:56 +0000 Subject: [PATCH 30/50] fix(count_tokens)!: count tools the way DeepSeek V4 and Qwen3.5 chat templates render them (#47173) * fix(count_tokens): count tools the way DeepSeek V4 and Qwen3.5 chat templates render them * refactor(count_tokens): hold the tool counting wiring for a follow-up commit * refactor(count_tokens): hold the chat template tool sections for a follow-up commit * fix(count_tokens): count tools the way DeepSeek V4 and Qwen3.5 chat templates render them * fix(count_tokens): skip tool_choice and non-function tools for chat template families, leave out Fireworks V4.1 spelling * fix(count_tokens): count Anthropic tools in the shape the messages bridge sends them A missing description is no longer rendered as null and strict is counted when set, matching what DeepSeek V4 and Qwen3.5 deployments receive. Adds tests that pin each template's schema rendering and its system turn. * test(count_tokens): check the prompt caching threshold agrees with token_counter for DeepSeek V4 and Qwen3.5 tools * fix(count_tokens): skip Anthropic web search tools and count unmapped Anthropic tool keys for chat template families * test(token_counter): cover chat-template tool counts across count routes, budgets, ITPM and context edits Integration cells check that DeepSeek V4 and Qwen3.5 tools are counted the way their chat templates render them on /v1/messages/count_tokens, /utils/token_counter, /v1/responses/input_tokens, strict budget reservation, project ITPM on chat, messages and responses, and the clear_tool_uses trigger, with grok controls keeping OpenAI framing --- .../prompt_templates/chat_template_tools.py | 37 ++ litellm/litellm_core_utils/token_counter.py | 117 ++++-- .../test_chat_template_tool_count_wire.py | 383 ++++++++++++++++++ ...ls_prompt_templates_chat_template_tools.py | 31 ++ .../litellm_core_utils/test_token_counter.py | 189 ++++++++- 5 files changed, 728 insertions(+), 29 deletions(-) create mode 100644 litellm/litellm_core_utils/prompt_templates/chat_template_tools.py create mode 100644 tests/integration/providers/test_chat_template_tool_count_wire.py create mode 100644 tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py diff --git a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py new file mode 100644 index 00000000000..3c437ca7fbb --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py @@ -0,0 +1,37 @@ +import json +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Final + + +@dataclass(frozen=True, slots=True) +class ChatTemplateToolSection: + preamble_tokens: int + implicit_system_turn_tokens: int + render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str] + + +def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: + return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions) + + +def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: + return "".join( + "\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions + ) + + +_TOOL_SECTIONS: Final = ( + ( + re.compile(r"deepseek-v4(?![.\d]|p\d)", re.IGNORECASE), + ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas), + ), + (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)), +) + + +def chat_template_tool_section(model: str | None) -> ChatTemplateToolSection | None: + if not model: + return None + return next((section for pattern, section in _TOOL_SECTIONS if pattern.search(model)), None) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 7ac3e9c80c5..8aded69aed2 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -35,6 +35,10 @@ from litellm.constants import ( TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.prompt_templates.chat_template_tools import ( + ChatTemplateToolSection, + chat_template_tool_section, +) from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding if TYPE_CHECKING: @@ -347,6 +351,9 @@ EXTRAPOLATION_SAMPLES: Final = 16 T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") +_ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset( + ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming") +) def _count_offload_limiter_for_this_loop() -> anyio.CapacityLimiter: @@ -404,6 +411,7 @@ class _MessageCountParams: self.tokens_per_message = 3 self.tokens_per_name = 1 self.count_function = _get_count_function(model, custom_tokenizer) + self.tool_section = chat_template_tool_section(model) def token_counter( @@ -467,7 +475,9 @@ def token_counter( num_tokens = _count_messages(params, new_messages, use_default_image_token_count, default_token_count) if count_response_tokens is False: includes_system_message: Final = any(message.get("role", None) == "system" for message in new_messages) - num_tokens += _count_extra(params.count_function, tools, tool_choice, includes_system_message) + num_tokens += _count_extra( + params.count_function, tools, tool_choice, includes_system_message, params.tool_section + ) else: raise ValueError("Either text or messages must be provided") @@ -501,7 +511,8 @@ def messages_reach_token_count( _count_messages(params, [message], use_default_image_token_count, None) for message in new_messages ) running_totals: Final = accumulate( - per_message_counts, initial=_count_extra(params.count_function, tools, None, includes_system_message) + per_message_counts, + initial=_count_extra(params.count_function, tools, None, includes_system_message, params.tool_section), ) return any(total >= threshold for total in running_totals) @@ -592,6 +603,7 @@ def _count_extra( tools: list[ChatCompletionToolParam] | None, tool_choice: ChatCompletionNamedToolChoiceParam | None, includes_system_message: bool, + tool_section: ChatTemplateToolSection | None, ) -> int: """Count extra tokens for function definitions and tool choices. Args: @@ -599,26 +611,56 @@ def _count_extra( tools (Optional[List[ChatCompletionToolParam]]): The available tools. tool_choice (Optional[ChatCompletionNamedToolChoiceParam]): The tool choice. includes_system_message (bool): Whether the messages include a system message. + tool_section (Optional[ChatTemplateToolSection]): How the model's chat template renders tools, if known. """ + reply_primer_tokens: Final = 3 + tool_choice_tokens: Final = 0 if tool_section is not None else _count_tool_choice(count_function, tool_choice) + return ( + reply_primer_tokens + + _count_tool_definitions(count_function, tools, includes_system_message, tool_section) + + tool_choice_tokens + ) - num_tokens = 3 # every reply is primed with <|start|>assistant<|message|> - if tools: - num_tokens += count_function(_format_function_definitions(tools)) - num_tokens += 9 # Additional tokens for function definition of tools - # If there's a system message and tools are present, subtract four tokens - if tools and includes_system_message: - num_tokens -= 4 - # If tool_choice is 'none', add one token. - # If it's an object, add 4 + the number of tokens in the function name. - # If it's undefined or 'auto', don't add anything. +def _count_tool_definitions( + count_function: TokenCounterFunction, + tools: Sequence[object] | None, + includes_system_message: bool, + tool_section: ChatTemplateToolSection | None, +) -> int: + if not tools: + return 0 + if tool_section is not None: + return _count_chat_template_tools(count_function, tools, includes_system_message, tool_section) + system_message_discount: Final = 4 if includes_system_message else 0 + return count_function(_format_function_definitions(tools)) + 9 - system_message_discount + + +def _count_tool_choice( + count_function: TokenCounterFunction, + tool_choice: ChatCompletionNamedToolChoiceParam | None, +) -> int: if tool_choice == "none": - num_tokens += 1 - elif isinstance(tool_choice, dict): - num_tokens += 7 - num_tokens += count_function(str(tool_choice["function"]["name"])) + return 1 + if isinstance(tool_choice, dict): + return 7 + count_function(str(tool_choice["function"]["name"])) + return 0 - return num_tokens + +def _count_chat_template_tools( + count_function: TokenCounterFunction, + tools: Sequence[object], + includes_system_message: bool, + tool_section: ChatTemplateToolSection, +) -> int: + functions: Final = tuple( + function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name") + ) + if not functions: + return 0 + system_turn_tokens: Final = 0 if includes_system_message else tool_section.implicit_system_turn_tokens + schemas: Final = tool_section.render_tool_schemas(functions) + return tool_section.preamble_tokens + system_turn_tokens + count_function(schemas) def _get_extrapolating_count_function( @@ -1034,15 +1076,31 @@ def _format_function_definitions(tools: Sequence[object]) -> str: lines: Final = [] lines.append("namespace functions {") lines.append("") - for tool in tools: - if not isinstance(tool, Mapping): - continue - for function in _function_definitions_for_tool(cast(Mapping[str, object], tool)): - lines.extend(_format_single_function_definition(function)) + for function in _function_definitions(tools): + lines.extend(_format_single_function_definition(function)) lines.append("} // namespace functions") return "\n".join(lines) +def _tools_sent_as_functions(tools: Sequence[object]) -> tuple[Mapping[str, object], ...]: + return tuple(tool for tool in _tool_mappings(tools) if not _is_anthropic_web_search_tool(tool)) + + +def _is_anthropic_web_search_tool(tool: Mapping[str, object]) -> bool: + return tool.get("name") == "web_search" and "input_schema" not in tool and tool.get("type") != "function" + + +def _tool_mappings(tools: Sequence[object]) -> Iterable[Mapping[str, object]]: + for tool in tools: + if isinstance(tool, Mapping): + yield cast(Mapping[str, object], tool) + + +def _function_definitions(tools: Sequence[object]) -> Iterable[Mapping[str, object]]: + for tool in _tool_mappings(tools): + yield from _function_definitions_for_tool(tool) + + def _function_definitions_for_tool(tool: Mapping[str, object]) -> Iterable[Mapping[str, object]]: function: Final = tool.get("function") if isinstance(function, Mapping): @@ -1054,12 +1112,21 @@ def _function_definitions_for_tool(tool: Mapping[str, object]) -> Iterable[Mappi if isinstance(declaration, Mapping): yield declaration return - parameters: Final = tool.get("input_schema") or tool.get("parameters") or {} + input_schema: Final = tool.get("input_schema") + parameters: Final = input_schema or tool.get("parameters") or {} normalized_parameters: Final = parameters if isinstance(parameters, Mapping) else {} + unmapped_anthropic_keys: Final = ( + {key: value for key, value in tool.items() if key not in _ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE} + if input_schema is not None + else {} + ) + description: Final = tool.get("description") + strict: Final = tool.get("strict") yield { "name": tool.get("name"), - "description": tool.get("description"), - "parameters": normalized_parameters, + "parameters": {**normalized_parameters, **unmapped_anthropic_keys}, + **({"description": description} if description is not None else {}), + **({"strict": bool(strict)} if strict is not None else {}), } diff --git a/tests/integration/providers/test_chat_template_tool_count_wire.py b/tests/integration/providers/test_chat_template_tool_count_wire.py new file mode 100644 index 00000000000..d2b85e7168f --- /dev/null +++ b/tests/integration/providers/test_chat_template_tool_count_wire.py @@ -0,0 +1,383 @@ +import json +import re +import uuid +from collections.abc import Iterator, Mapping, Sequence +from contextlib import ExitStack +from dataclasses import dataclass +from itertools import chain +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + gateway_from_environment, + object_value, + string_value, +) +from integration._support.openai_wire import chat_reply +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from litellm.litellm_core_utils.default_encoding import encoding as cl100k_base +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +_API_KEY: Final = "scripted-provider-key" +_DEEPSEEK_V4: Final = "azure_ai/DeepSeek-V4-Flash-0731" +_QWEN3_5: Final = "databricks/databricks-qwen35-122b-a10b" +_GROK: Final = "azure_ai/grok-4-1" +_DEEPSEEK_V4_TOOL_PREAMBLE: Final = 222 +_QWEN3_5_TOOL_PREAMBLE: Final = 196 +_QWEN3_5_OPENED_SYSTEM_TURN: Final = 5 +_OPENAI_TOOL_OVERHEAD: Final = 9 +_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], +} +_FUNCTION: Final[dict[str, JsonValue]] = {"name": "lookup", "parameters": _SCHEMA} +_OPENAI_NAMESPACE: Final = "namespace functions {\n\ntype lookup = (_: {\nquery: string,\n}) => any;\n\n} // namespace functions" +_ANTHROPIC_TOOL: Final[dict[str, JsonValue]] = {"name": "lookup", "input_schema": _SCHEMA} +_OPENAI_TOOL: Final[dict[str, JsonValue]] = {"type": "function", "function": _FUNCTION} +_RESPONSES_TOOL: Final[dict[str, JsonValue]] = {"type": "function", **_FUNCTION} +_NAMED_TOOL_CHOICE: Final[dict[str, JsonValue]] = {"type": "function", "function": {"name": "lookup"}} +_WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = {"type": "web_search_20250305", "name": "web_search", "max_uses": 3} +_PROMPT: Final = "Find record 14." +_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": _PROMPT}] +_INPUT_PRICE: Final = 0.001 +_OUTPUT_PRICE: Final = 0.002 +_TINY_BUDGET: Final = 0.000000001 +_ESTIMATED_COST: Final = re.compile(r"Estimated request cost: ([^\s,]+)") + + +def _tokens(text: str) -> int: + return len(cl100k_base.encode(text)) + + +def _deepseek_v4_tool_tokens(function: Mapping[str, JsonValue]) -> int: + return _DEEPSEEK_V4_TOOL_PREAMBLE + _tokens(json.dumps(function, ensure_ascii=False)) + + +def _qwen3_5_tool_tokens(function: Mapping[str, JsonValue], *, with_system: bool) -> int: + opened_system_turn: Final = 0 if with_system else _QWEN3_5_OPENED_SYSTEM_TURN + schema: Final = json.dumps({"type": "function", "function": function}, ensure_ascii=False) + return _QWEN3_5_TOOL_PREAMBLE + opened_system_turn + _tokens("\n" + schema) + + +def _completion(_request: Request) -> Reply: + return chat_reply(f"chatcmpl-{uuid.uuid4().hex}", "scripted-model", "OK", stream=False) + + +def _deployment(scenario: Scenario, group: str, model: str, api_base: str) -> str: + name: Final = f"{group}-{uuid.uuid4().hex}" + created: Final = scenario.gateway.post( + "/model/new", + { + "model_name": name, + "litellm_params": {"model": model, "api_base": api_base, "api_key": _API_KEY}, + "model_info": {}, + }, + ) + scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"])) + return name + + +@dataclass(frozen=True, slots=True) +class _CountRoute: + path: str + field: str + tool: dict[str, JsonValue] + + def body(self, model: str, tools: Sequence[JsonValue]) -> dict[str, JsonValue]: + prompt: Final[dict[str, JsonValue]] = ( + {"input": _PROMPT} if self.path == "/v1/responses/input_tokens" else {"messages": _MESSAGES} + ) + return {"model": model, **prompt, **({"tools": list(tools)} if tools else {})} + + +_COUNT_ROUTES: Final[Mapping[str, _CountRoute]] = { + "messages-count-tokens": _CountRoute("/v1/messages/count_tokens", "input_tokens", _ANTHROPIC_TOOL), + "utils-token-counter": _CountRoute("/utils/token_counter", "total_tokens", _OPENAI_TOOL), + "responses-input-tokens": _CountRoute("/v1/responses/input_tokens", "input_tokens", _RESPONSES_TOOL), +} +_MESSAGES_COUNT: Final = _COUNT_ROUTES["messages-count-tokens"] + + +def _count(gateway: Gateway, route: _CountRoute, body: Mapping[str, JsonValue]) -> int: + response: Final = gateway.request("POST", route.path, body) + assert response.status_code == 200, response.text + counted: Final = JSON_OBJECT.validate_json(response.content)[route.field] + assert isinstance(counted, int), response.text + return counted + + +@pytest.mark.parametrize("route", sorted(_COUNT_ROUTES)) +def test_count_routes_count_deepseek_v4_tools_the_way_its_chat_template_renders_them( + gateway: Gateway, route: str +) -> None: + selected: Final = _COUNT_ROUTES[route] + with wire_server(_completion) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "count", _DEEPSEEK_V4, wire.url) + without_tools: Final = _count(gateway, selected, selected.body(model, ())) + with_tool: Final = _count(gateway, selected, selected.body(model, (selected.tool,))) + upstream: Final = wire.drain() + assert upstream == () + assert with_tool - without_tools == _deepseek_v4_tool_tokens(_FUNCTION) + + +@pytest.mark.parametrize("with_system", (False, True), ids=("no-system", "system")) +def test_messages_count_tokens_counts_qwen3_5_tools_the_way_its_chat_template_renders_them( + gateway: Gateway, with_system: bool +) -> None: + system: Final[dict[str, JsonValue]] = {"system": "Be brief."} if with_system else {} + with wire_server(_completion) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "count", _QWEN3_5, wire.url) + without_tools: Final = _count(gateway, _MESSAGES_COUNT, {**_MESSAGES_COUNT.body(model, ()), **system}) + with_tool: Final = _count( + gateway, _MESSAGES_COUNT, {**_MESSAGES_COUNT.body(model, (_ANTHROPIC_TOOL,)), **system} + ) + upstream: Final = wire.drain() + assert upstream == () + assert with_tool - without_tools == _qwen3_5_tool_tokens(_FUNCTION, with_system=with_system) + + +def test_messages_count_tokens_skips_web_search_and_counts_unmapped_tool_keys_for_deepseek_v4( + gateway: Gateway, +) -> None: + deferred_function: Final[dict[str, JsonValue]] = {"name": "lookup", "parameters": {**_SCHEMA, "defer_loading": True}} + with wire_server(_completion) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "count", _DEEPSEEK_V4, wire.url) + plain: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, (_ANTHROPIC_TOOL,))) + with_web_search: Final = _count( + gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, (_ANTHROPIC_TOOL, _WEB_SEARCH_TOOL)) + ) + deferred: Final = _count( + gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, ({**_ANTHROPIC_TOOL, "defer_loading": True},)) + ) + upstream: Final = wire.drain() + assert upstream == () + assert with_web_search == plain + assert deferred - plain == _deepseek_v4_tool_tokens(deferred_function) - _deepseek_v4_tool_tokens(_FUNCTION) + + +def test_messages_count_tokens_keeps_openai_tool_framing_for_other_models(gateway: Gateway) -> None: + with wire_server(_completion) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "count", _GROK, wire.url) + without_tools: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, ())) + plain: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, (_ANTHROPIC_TOOL,))) + deferred: Final = _count( + gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, ({**_ANTHROPIC_TOOL, "defer_loading": True},)) + ) + upstream: Final = wire.drain() + assert upstream == () + assert plain - without_tools == _OPENAI_TOOL_OVERHEAD + _tokens(_OPENAI_NAMESPACE) + assert deferred == plain + + +@dataclass(frozen=True, slots=True) +class _Endpoint: + path: str + body: dict[str, JsonValue] + tools: dict[str, JsonValue] + + +_ENDPOINTS: Final[Mapping[str, _Endpoint]] = { + "chat-completions": _Endpoint( + "/v1/chat/completions", + {"messages": _MESSAGES, "max_tokens": 1}, + {"tools": [_OPENAI_TOOL], "tool_choice": _NAMED_TOOL_CHOICE}, + ), + "messages": _Endpoint( + "/v1/messages", + {"messages": _MESSAGES, "max_tokens": 1}, + {"tools": [_ANTHROPIC_TOOL], "tool_choice": {"type": "auto"}}, + ), + "responses": _Endpoint( + "/v1/responses", + {"input": _PROMPT, "max_output_tokens": 16}, + {"tools": [_RESPONSES_TOOL]}, + ), +} + + +def _send(gateway: Gateway, endpoint: _Endpoint, group: str, key: str, *, tools: bool) -> httpx.Response: + return gateway.request( + "POST", + endpoint.path, + {"model": group, **endpoint.body, **(endpoint.tools if tools else {})}, + key=key, + ) + + +@dataclass(frozen=True, slots=True) +class _BudgetRig: + gateway: Gateway + wire: Wire + group: str + + +def _strict_budget_config(directory: Path, group: str, api_base: str) -> Path: + base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = { + **base, + "model_list": [ + { + "model_name": group, + "litellm_params": { + "model": _DEEPSEEK_V4, + "api_base": api_base, + "api_key": _API_KEY, + "input_cost_per_token": _INPUT_PRICE, + "output_cost_per_token": _OUTPUT_PRICE, + }, + } + ], + "general_settings": {**object_value(base["general_settings"]), "fail_closed_budget_enforcement": True}, + } + path: Final = directory / f"chat-template-tool-budget-{uuid.uuid4().hex[:8]}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def strict_budget(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_BudgetRig]: + directory: Final = tmp_path_factory.mktemp("chat-template-tool-budget") + group: Final = f"deepseek-v4-flash-{uuid.uuid4().hex}" + with ExitStack() as stack: + rig_gateway: Final = stack.enter_context(gateway_from_environment()) + wire: Final = stack.enter_context(wire_server(_completion)) + owned: Final = stack.enter_context( + owned_proxy_process(rig_gateway, directory, {}, config=_strict_budget_config(directory, group, wire.url)) + ) + yield _BudgetRig(owned.gateway, wire, group) + + +def _estimated_cost(response: httpx.Response) -> float: + assert response.status_code in (400, 422, 429), f"{response.status_code} {response.text}" + error: Final = object_value(JSON_OBJECT.validate_json(response.content)["error"]) + estimate: Final = _ESTIMATED_COST.search(string_value(error["message"])) + assert estimate is not None, response.text + return float(estimate[1]) + + +@pytest.mark.parametrize("endpoint", ("chat-completions", "messages")) +def test_budget_reservation_estimates_deepseek_v4_tools_the_way_its_chat_template_renders_them( + strict_budget: _BudgetRig, endpoint: str +) -> None: + selected: Final = _ENDPOINTS[endpoint] + with strict_budget.gateway.scenario() as scenario: + key: Final = scenario.key(models=[strict_budget.group], max_budget=_TINY_BUDGET) + without_tools: Final = _estimated_cost(_send(strict_budget.gateway, selected, strict_budget.group, key, tools=False)) + with_tools: Final = _estimated_cost(_send(strict_budget.gateway, selected, strict_budget.group, key, tools=True)) + assert strict_budget.wire.drain() == () + assert with_tools - without_tools == pytest.approx(_deepseek_v4_tool_tokens(_FUNCTION) * _INPUT_PRICE) + + +def _project_key(scenario: Scenario, group: str) -> str: + team: Final = scenario.team() + project: Final = scenario.project(team, model_itpm_limit={group: _deepseek_v4_tool_tokens(_FUNCTION)}) + return scenario.key(team_id=team, project_id=project) + + +@pytest.mark.parametrize("endpoint", sorted(_ENDPOINTS)) +def test_project_itpm_reserves_deepseek_v4_tools_the_way_its_chat_template_renders_them( + gateway: Gateway, endpoint: str +) -> None: + with wire_server(_completion) as wire, gateway.scenario() as scenario: + group: Final = _deployment(scenario, "deepseek-v4-flash", _DEEPSEEK_V4, wire.url) + response: Final = _send(gateway, _ENDPOINTS[endpoint], group, _project_key(scenario, group), tools=True) + upstream: Final = wire.drain() + assert response.status_code == 429, response.text + assert "model_per_project_itpm" in response.text, response.text + assert upstream == () + + +def test_project_itpm_keeps_openai_tool_framing_for_other_models(gateway: Gateway) -> None: + chat: Final = _ENDPOINTS["chat-completions"] + with wire_server(_completion) as wire, gateway.scenario() as scenario: + group: Final = _deployment(scenario, "grok-4-1", _GROK, wire.url) + response: Final = gateway.request( + "POST", + chat.path, + {"model": group, **chat.body, "tools": [_OPENAI_TOOL]}, + key=_project_key(scenario, group), + ) + upstream: Final = wire.drain() + assert response.status_code == 200, response.text + assert len(upstream) == 1, upstream + + +_FILES: Final = 3 +_READ_FILE_FUNCTION: Final[dict[str, JsonValue]] = { + "name": "read_file", + "parameters": {"type": "object", "properties": {"path": {"type": "string"}}}, +} +_READ_FILE_TOOL: Final[dict[str, JsonValue]] = { + "name": "read_file", + "input_schema": _READ_FILE_FUNCTION["parameters"], +} + + +def _read_file_round(marker: str, index: int) -> tuple[JsonValue, ...]: + return ( + {"role": "user", "content": f"Read file {index}."}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": f"toolu_{index}", "name": "read_file", "input": {"path": f"{index}.txt"}} + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": f"toolu_{index}", + "content": f"short file body.", + } + ], + }, + ) + + +def _agent_body(group: str, marker: str) -> dict[str, JsonValue]: + rounds: Final = tuple(chain.from_iterable(_read_file_round(marker, index) for index in range(_FILES))) + return { + "model": group, + "max_tokens": 16, + "tools": [_READ_FILE_TOOL], + "messages": [*rounds, {"role": "user", "content": "Summarize what you read."}], + "context_management": { + "edits": [ + { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "input_tokens", "value": _deepseek_v4_tool_tokens(_READ_FILE_FUNCTION)}, + "keep": {"type": "tool_uses", "value": 1}, + } + ] + }, + } + + +@pytest.mark.parametrize( + ("model", "files_kept"), + ((_DEEPSEEK_V4, (False, False, True)), (_GROK, (True, True, True))), + ids=("deepseek-v4", "grok-4-1"), +) +def test_messages_context_management_trigger_counts_tools_the_way_the_models_chat_template_renders_them( + gateway: Gateway, model: str, files_kept: tuple[bool, ...] +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_completion) as wire, gateway.scenario() as scenario: + group: Final = _deployment(scenario, "context", model, wire.url) + response: Final = gateway.request("POST", "/v1/messages", _agent_body(group, marker)) + upstream: Final = wire.drain() + assert response.status_code == 200, response.text + assert len(upstream) == 1, upstream + sent: Final = upstream[0].body.decode() + assert tuple(f"" in sent for index in range(_FILES)) == files_kept diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py new file mode 100644 index 00000000000..51249e01d4d --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py @@ -0,0 +1,31 @@ +from typing import Final + +import pytest + +from litellm.litellm_core_utils.prompt_templates.chat_template_tools import chat_template_tool_section + +_DESCRIBED_FUNCTION: Final = {"name": "f", "description": "é", "parameters": {}} +_BARE_FUNCTION: Final = {"name": "g", "parameters": {}} + + +# Tool schemas as rendered by render_tools in encoding/encoding_dsv4.py (deepseek-ai/DeepSeek-V4-Flash) and by +# chat_template.jinja (Qwen/Qwen3.5-122B-A10B), both read on Hugging Face on 2026-10-17 +@pytest.mark.parametrize( + ("model", "rendered"), + [ + ( + "DeepSeek-V4-Flash-0731", + '{"name": "f", "description": "é", "parameters": {}}\n{"name": "g", "parameters": {}}', + ), + ( + "databricks-qwen35-122b-a10b", + '\n{"type": "function", "function": {"name": "f", "description": "é", "parameters": {}}}' + '\n{"type": "function", "function": {"name": "g", "parameters": {}}}', + ), + ], +) +def test_chat_template_tool_section_renders_schemas_like_the_models_template(model: str, rendered: str): + section: Final = chat_template_tool_section(model) + + assert section is not None + assert section.render_tool_schemas((_DESCRIBED_FUNCTION, _BARE_FUNCTION)) == rendered diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 84c139e867b..6126d919180 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -9,7 +9,7 @@ import subprocess import sys import threading import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from concurrent.futures import Future, wait from itertools import accumulate, chain from pathlib import Path @@ -39,6 +39,7 @@ from litellm.litellm_core_utils.token_counter import ( offload_token_count, ) from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new +from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, @@ -515,6 +516,183 @@ def test_token_counter_skips_non_mapping_tools(): assert mixed_tokens == valid_tokens +_WEATHER_FUNCTION: Final = { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string", "description": "City name"}}, + "required": ["location"], + }, +} +_TIME_FUNCTION: Final = { + "name": "get_time", + "description": "Get the local time for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, +} +_WEB_SEARCH_FUNCTION: Final = { + "name": "web_search", + "description": "Search the web", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, +} + + +def _openai_tool(function: Mapping[str, object]) -> dict[str, object]: + return {"type": "function", "function": function} + + +def _anthropic_tool(function: Mapping[str, object]) -> dict[str, object]: + return {"name": function["name"], "description": function["description"], "input_schema": function["parameters"]} + + +_USER_MESSAGES: Final = ({"role": "user", "content": "What is the weather in Paris today?"},) +_SYSTEM_AND_USER_MESSAGES: Final = ({"role": "system", "content": "You are a helpful assistant."}, *_USER_MESSAGES) + + +@pytest.mark.parametrize("tool_shape", [_openai_tool, _anthropic_tool]) +@pytest.mark.parametrize( + ("model", "messages", "functions", "billed_prompt_tokens"), + [ + # usage.prompt_tokens billed for this request by Azure AI Foundry DeepSeek-V4-Flash-0731, + # Databricks databricks-qwen35-122b-a10b and DashScope qwen3.5-plus on 2026-10-17 + ("DeepSeek-V4-Flash-0731", _USER_MESSAGES, (_WEATHER_FUNCTION,), 295), + ("DeepSeek-V4-Flash-0731", _USER_MESSAGES, (_WEATHER_FUNCTION, _TIME_FUNCTION), 348), + ("DeepSeek-V4-Flash-0731", _SYSTEM_AND_USER_MESSAGES, (_WEATHER_FUNCTION,), 300), + ("databricks-qwen35-122b-a10b", _USER_MESSAGES, (_WEATHER_FUNCTION,), 289), + ("databricks-qwen35-122b-a10b", _USER_MESSAGES, (_WEATHER_FUNCTION, _TIME_FUNCTION), 352), + ("databricks-qwen35-122b-a10b", _SYSTEM_AND_USER_MESSAGES, (_WEATHER_FUNCTION,), 296), + ("qwen3.5-plus", _USER_MESSAGES, (_WEATHER_FUNCTION,), 284), + ("qwen3.5-plus", _USER_MESSAGES, (_WEATHER_FUNCTION, _TIME_FUNCTION), 342), + ], +) +def test_token_counter_tools_match_billed_prompt_for_json_tool_chat_templates( + tool_shape: Callable[[Mapping[str, object]], dict[str, object]], + model: str, + messages: tuple[dict[str, str], ...], + functions: tuple[Mapping[str, object], ...], + billed_prompt_tokens: int, +): + counted: Final = token_counter_new( + model=model, + messages=list(messages), + tools=[tool_shape(function) for function in functions], + ) + + assert abs(counted - billed_prompt_tokens) <= billed_prompt_tokens * 0.05 + + +@pytest.mark.parametrize( + ("model", "counted_like"), + [ + ("DeepSeek-V4-Pro", "DeepSeek-V4-Flash-0731"), + ("accounts/fireworks/models/deepseek-v4-flash", "DeepSeek-V4-Flash-0731"), + ("DeepSeek-V4.1-Flash", "deepseek-v3.2"), + ("accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v3.2"), + ("Qwen/Qwen3.5-122B-A10B", "databricks-qwen35-122b-a10b"), + ("qwen3-235b-a22b", "deepseek-v3.2"), + ], +) +def test_token_counter_counts_tools_by_the_models_chat_template_family(model: str, counted_like: str): + tools: Final = [_openai_tool(_WEATHER_FUNCTION), _openai_tool(_TIME_FUNCTION)] + + assert token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools) == token_counter_new( + model=counted_like, messages=list(_USER_MESSAGES), tools=tools + ) + + +@pytest.mark.parametrize("model", ["DeepSeek-V4-Flash-0731", "databricks-qwen35-122b-a10b"]) +@pytest.mark.parametrize( + "builtin_tool", + [ + {"type": "web_search_preview"}, + {"type": "web_search_20250305", "name": "web_search", "max_uses": 5}, + {"name": "web_search"}, + ], +) +def test_token_counter_skips_tools_not_sent_as_functions_for_chat_template_families( + model: str, builtin_tool: dict[str, object] +): + weather_tool: Final = _openai_tool(_WEATHER_FUNCTION) + + assert token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=[builtin_tool]) == token_counter_new( + model=model, messages=list(_USER_MESSAGES) + ) + assert token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=[builtin_tool, weather_tool] + ) == token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=[weather_tool]) + + +def _responses_tool(function: Mapping[str, object]) -> dict[str, object]: + return {"type": "function", **function} + + +def _gemini_tool(function: Mapping[str, object]) -> dict[str, object]: + return {"function_declarations": [function]} + + +@pytest.mark.parametrize("model", ["DeepSeek-V4-Flash-0731", "databricks-qwen35-122b-a10b"]) +@pytest.mark.parametrize("function", [_WEATHER_FUNCTION, _WEB_SEARCH_FUNCTION]) +@pytest.mark.parametrize("tool_shape", [_responses_tool, _gemini_tool]) +def test_token_counter_counts_function_tools_like_chat_completions_tools_for_chat_template_families( + model: str, function: Mapping[str, object], tool_shape: Callable[[Mapping[str, object]], dict[str, object]] +): + assert token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=[tool_shape(function)] + ) == token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool(function)]) + + +# Azure AI Foundry DeepSeek-V4-Flash-0731 billed 295 prompt tokens for the weather tool with each of +# these tool_choice values on 2026-10-17, the same as without one +@pytest.mark.parametrize("tool_choice", ["none", "required", {"type": "function", "function": {"name": "get_weather"}}]) +def test_token_counter_tool_choice_adds_nothing_for_deepseek_v4(tool_choice: object): + tools: Final = [_openai_tool(_WEATHER_FUNCTION)] + + assert token_counter_new( + model="DeepSeek-V4-Flash-0731", messages=list(_USER_MESSAGES), tools=tools, tool_choice=tool_choice + ) == token_counter_new(model="DeepSeek-V4-Flash-0731", messages=list(_USER_MESSAGES), tools=tools) + + +@pytest.mark.parametrize("model", ["DeepSeek-V4-Flash-0731", "databricks-qwen35-122b-a10b"]) +@pytest.mark.parametrize( + "anthropic_tool", + [ + {"name": "get_weather", "input_schema": _WEATHER_FUNCTION["parameters"]}, + _anthropic_tool(_WEATHER_FUNCTION), + {**_anthropic_tool(_WEATHER_FUNCTION), "strict": True}, + {**_anthropic_tool(_WEATHER_FUNCTION), "defer_loading": True, "input_examples": [{"location": "Paris"}]}, + _anthropic_tool(_WEB_SEARCH_FUNCTION), + ], +) +def test_token_counter_counts_anthropic_tools_as_sent_to_chat_template_families( + model: str, anthropic_tool: dict[str, object] +): + sent_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai([anthropic_tool]) # pyright: ignore[reportArgumentType] # raw Anthropic tool dict + + assert token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=[anthropic_tool]) == token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=sent_tools + ) + + +# Qwen3.5's chat_template.jinja opens a system turn for the tool schemas when the request has none, 5 tokens that a +# request with a system message already pays, while DeepSeek V4's encoding_dsv4.py appends them to the first turn +# either way (both read on Hugging Face on 2026-10-17) +@pytest.mark.parametrize( + ("model", "implicit_system_turn_tokens"), [("DeepSeek-V4-Flash-0731", 0), ("databricks-qwen35-122b-a10b", 5)] +) +def test_token_counter_counts_the_tools_system_turn_only_without_a_system_message( + model: str, implicit_system_turn_tokens: int +): + tools: Final = [_openai_tool(_WEATHER_FUNCTION)] + system_cost_with_tools: Final = token_counter_new( + model=model, messages=list(_SYSTEM_AND_USER_MESSAGES), tools=tools + ) - token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools) + system_cost_without_tools: Final = token_counter_new( + model=model, messages=list(_SYSTEM_AND_USER_MESSAGES) + ) - token_counter_new(model=model, messages=list(_USER_MESSAGES)) + + assert system_cost_with_tools == system_cost_without_tools - implicit_system_turn_tokens + + class NeedsToleranceUpdateError(Exception): """Custom exception to mark tests that have improved""" @@ -1770,14 +1948,17 @@ _THRESHOLD_TEST_TOOLS: Final = [ ] -def test_messages_reach_token_count_agrees_with_token_counter_at_every_threshold() -> None: +@pytest.mark.parametrize( + "model", ["claude-3-5-sonnet-20240620", "DeepSeek-V4-Flash-0731", "databricks-qwen35-122b-a10b"] +) +def test_messages_reach_token_count_agrees_with_token_counter_at_every_threshold(model: str) -> None: """The threshold check is the same arithmetic as token_counter(...) >= threshold, including the tools and system-message adjustments, so the boundary values must agree exactly.""" from litellm.litellm_core_utils.token_counter import messages_reach_token_count messages = _threshold_test_messages(turns=12) total = token_counter_new( - model="claude-3-5-sonnet-20240620", + model=model, messages=messages, tools=_THRESHOLD_TEST_TOOLS, use_default_image_token_count=True, @@ -1785,7 +1966,7 @@ def test_messages_reach_token_count_agrees_with_token_counter_at_every_threshold assert total > 100 for threshold in (0, 1, total - 1, total, total + 1, 10 * total): assert messages_reach_token_count( - model="claude-3-5-sonnet-20240620", + model=model, messages=messages, threshold=threshold, tools=_THRESHOLD_TEST_TOOLS, From 90af729933fe04626d233ed7790752ff1720cc57 Mon Sep 17 00:00:00 2001 From: Jon Varley Date: Mon, 19 Oct 2026 18:37:25 +0100 Subject: [PATCH 31/50] fix(cost-map): enable function calling + tool choice for fireworks deepseek-v4p1-flash-1017 (#47326) Co-authored-by: Cursor --- litellm/model_prices_and_context_window_backup.json | 4 ++-- model_prices_and_context_window.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e0ace9c4f39..5b68eeb85e3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -75966,11 +75966,11 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", - "supports_function_calling": false, + "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": false, + "supports_tool_choice": true, "supports_vision": true }, "github_copilot/claude-sonnet-4.6": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e0ace9c4f39..5b68eeb85e3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -75966,11 +75966,11 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", - "supports_function_calling": false, + "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": false, + "supports_tool_choice": true, "supports_vision": true }, "github_copilot/claude-sonnet-4.6": { From 2f49e7f1c3e77dc676f0aacf501c99e3884e1f2a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:58:49 -0700 Subject: [PATCH 32/50] test(cost): add tier 1 cost tracking cases and wire bridge legs for 2026-10-19 (#47342) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cost_calculation/cases/gemini.py | 21 +++++--- .../cost_calculation/cases/openai.py | 3 ++ .../cost_calculation/cases/vertex_ai.py | 53 ++++++++++++++++++- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/tests/integration/cost_calculation/cases/gemini.py b/tests/integration/cost_calculation/cases/gemini.py index bc6665c423f..8a50e91ec01 100644 --- a/tests/integration/cost_calculation/cases/gemini.py +++ b/tests/integration/cost_calculation/cases/gemini.py @@ -3494,6 +3494,10 @@ GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES: Final = bridge_pairs( (GEMINI_GEMINI_4_ARGON_INPUT_TEXT, GEMINI_GEMINI_4_ARGON_STREAM), ("/v1/responses", "/v1/messages"), ) +GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES: Final = bridge_pairs( + (GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT, GEMINI_GEMINI_3_8_FLASH_STREAM), + ("/v1/responses", "/v1/messages"), +) GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES: Final = bridge_pairs( (GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM), ("/v1/responses", "/v1/messages"), @@ -3572,6 +3576,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_GEMINI_3_1_PRO_PREVIEW_TOOL_CALL, GEMINI_GEMINI_3_1_PRO_PREVIEW_SERVICE_TIER_PRIORITY, *(pair.bridged for pair in GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES), + *(pair.bridged for pair in GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES), *(pair.bridged for pair in GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES), ) @@ -3606,6 +3611,14 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( plain=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[1].bridged, streamed=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[3].bridged, ), + StreamParityTestCase( + plain=GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES[0].bridged, + streamed=GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES[2].bridged, + ), + StreamParityTestCase( + plain=GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES[1].bridged, + streamed=GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES[3].bridged, + ), StreamParityTestCase( plain=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[0].bridged, streamed=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[2].bridged, @@ -3618,14 +3631,10 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( *bridge_pairs( - ( - GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT, - GEMINI_GEMINI_3_8_FLASH_CACHE_READ, - GEMINI_GEMINI_3_8_FLASH_REASONING, - GEMINI_GEMINI_3_8_FLASH_STREAM, - ), + (GEMINI_GEMINI_3_8_FLASH_CACHE_READ, GEMINI_GEMINI_3_8_FLASH_REASONING), ("/v1/responses", "/v1/messages"), ), + *GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES, *GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES, *GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES, ) diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index f610fd9591a..acba66223e5 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -12008,6 +12008,9 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GPT_6_ASTRA_RESPONSES_NATIVE_STREAM, GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM, GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON, + GPT_6_1_SOL_MESSAGES_BRIDGED_JSON, + GPT_6_LUNA_MESSAGES_BRIDGED_JSON, + GPT_6_ASTRA_MESSAGES_BRIDGED_JSON, GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM, GPT_6_LUNA_MESSAGES_BRIDGED_STREAM, GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM, diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index b512df46d1d..3b825bfe7b2 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -3146,6 +3146,41 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_READ: Final = _vertex_claude_shape_case( cache_read_cost=0.00351039, ), ) +VERTEX_AI_CLAUDE_HAIKU_5_5_TIERED_INPUT_ABOVE_100K: Final = _vertex_claude_shape_case( + VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, + "vertex_ai/claude-haiku-5-5-tiered_input_above_100k", + "vertex_ai/claude-haiku-5-5", + _vertex_claude_usage(input_tokens=150000, output_tokens=480), + ExactExpected( + spend=0.9906, + input_cost=0.975, + output_cost=0.0156, + prompt_tokens=150000, + completion_tokens=480, + ), + response_content=[ + {"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"}, + {"type": "text", "text": "scripted answer for cost fixtures"}, + ], +) +VERTEX_AI_CLAUDE_HAIKU_5_5_TIERED_CACHE_READ_ABOVE_100K: Final = _vertex_claude_shape_case( + VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_READ, + "vertex_ai/claude-haiku-5-5-tiered_cache_read_above_100k", + "vertex_ai/claude-haiku-5-5", + _vertex_claude_usage(input_tokens=4096, output_tokens=480, cache_read_input_tokens=110592), + ExactExpected( + spend=0.1141088, + input_cost=0.0985088, + output_cost=0.0156, + prompt_tokens=114688, + completion_tokens=480, + cache_read_cost=0.0718848, + ), + response_content=[ + {"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"}, + {"type": "text", "text": "scripted answer for cost fixtures"}, + ], +) VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M: Final = _vertex_claude_shape_case( VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_WRITE_5M, "vertex_ai/claude-haiku-5-5-cache_write_5m", @@ -3530,6 +3565,10 @@ GEMINI_4_ARGON_ROUTE_BRIDGES: Final = bridge_pairs( (GEMINI_4_ARGON_INPUT_TEXT, GEMINI_4_ARGON_STREAM), ("/v1/responses", "/v1/messages"), ) +GEMINI_3_8_FLASH_ROUTE_BRIDGES: Final = bridge_pairs( + (GEMINI_3_8_FLASH_INPUT_TEXT, GEMINI_3_8_FLASH_STREAM), + ("/v1/responses", "/v1/messages"), +) GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES: Final = bridge_pairs( (GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_3_1_PRO_PREVIEW_STREAM), ("/v1/responses", "/v1/messages"), @@ -3604,6 +3643,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, VERTEX_AI_CLAUDE_HAIKU_5_5_STREAM, VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_READ, + VERTEX_AI_CLAUDE_HAIKU_5_5_TIERED_INPUT_ABOVE_100K, + VERTEX_AI_CLAUDE_HAIKU_5_5_TIERED_CACHE_READ_ABOVE_100K, VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M, VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_1H, VERTEX_AI_CLAUDE_HAIKU_5_5_THINKING, @@ -3633,6 +3674,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_4_ARGON_REASONING, GEMINI_4_ARGON_TOOL_CALL, *(pair.bridged for pair in GEMINI_4_ARGON_ROUTE_BRIDGES), + *(pair.bridged for pair in GEMINI_3_8_FLASH_ROUTE_BRIDGES), *(pair.bridged for pair in GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES), ) @@ -3699,6 +3741,14 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( plain=GEMINI_4_ARGON_ROUTE_BRIDGES[1].bridged, streamed=GEMINI_4_ARGON_ROUTE_BRIDGES[3].bridged, ), + StreamParityTestCase( + plain=GEMINI_3_8_FLASH_ROUTE_BRIDGES[0].bridged, + streamed=GEMINI_3_8_FLASH_ROUTE_BRIDGES[2].bridged, + ), + StreamParityTestCase( + plain=GEMINI_3_8_FLASH_ROUTE_BRIDGES[1].bridged, + streamed=GEMINI_3_8_FLASH_ROUTE_BRIDGES[3].bridged, + ), StreamParityTestCase( plain=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[0].bridged, streamed=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[2].bridged, @@ -3711,9 +3761,10 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( *bridge_pairs( - (GEMINI_3_8_FLASH_INPUT_TEXT, GEMINI_3_8_FLASH_CACHE_READ, GEMINI_3_8_FLASH_STREAM), + (GEMINI_3_8_FLASH_CACHE_READ, GEMINI_3_8_FLASH_REASONING), ("/v1/responses", "/v1/messages"), ), + *GEMINI_3_8_FLASH_ROUTE_BRIDGES, *GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES, *GEMINI_4_ARGON_ROUTE_BRIDGES, *VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS, From 0197b961716f2086ea4cb7dfb1dc89a05010db48 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:17:21 -0700 Subject: [PATCH 33/50] chore(pricing): add github_copilot/claude-opus-4.8 deprecation date (#47370) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 1 + model_prices_and_context_window.json | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5b68eeb85e3..6675bb170b8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -32524,6 +32524,7 @@ "github_copilot/claude-opus-4.8": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-11-16", "input_cost_per_token": 5e-06, "litellm_provider": "github_copilot", "max_input_tokens": 200000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5b68eeb85e3..6675bb170b8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -32524,6 +32524,7 @@ "github_copilot/claude-opus-4.8": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-11-16", "input_cost_per_token": 5e-06, "litellm_provider": "github_copilot", "max_input_tokens": 200000, From 037e805b970230e4fb5c94784ea1b95e009a2b59 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:19:43 +0000 Subject: [PATCH 34/50] build(deps): bump aiohttp to 3.14.5 to clear osv-scan (#47369) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- uv.lock | 475 +++++++++++++++++++++++++++----------------------------- 1 file changed, 226 insertions(+), 249 deletions(-) diff --git a/uv.lock b/uv.lock index cb77c470885..beb66fe42b2 100644 --- a/uv.lock +++ b/uv.lock @@ -85,7 +85,7 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.14.3" +version = "3.14.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, @@ -98,126 +98,127 @@ dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.13'" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/4c/bdccd81e9ee225b69c60e7766c9a5b05364f118f4d383713b89a682d772d/aiohttp-3.14.5.tar.gz", hash = "sha256:5558a7f5a05af9ecf744af91e5baefc436f93c9333e656c27ec253f9a6bbe178", size = 8078052, upload-time = "2026-10-11T01:05:12.408Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/4d/4a99fb425c5e0cad715eea7bd190aff46f38b959a0a2dadb993705d34b26/aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b", size = 765848, upload-time = "2026-07-23T01:52:08.217Z" }, - { url = "https://files.pythonhosted.org/packages/74/e8/43b85dc55b8e950dc644babe762add781319ea881b57b33d2cce12017d12/aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a", size = 517476, upload-time = "2026-07-23T01:52:10.846Z" }, - { url = "https://files.pythonhosted.org/packages/7f/9e/73b582c4dbbc3c12ef4473822475effaabf1f934b56f14f5b03fe5d3a2af/aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5", size = 515334, upload-time = "2026-07-23T01:52:12.636Z" }, - { url = "https://files.pythonhosted.org/packages/79/03/e98c3c9e05a5bdf97defe5ff9169baba4f0ec9a901f2d60e0f060c2f051e/aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f", size = 1708830, upload-time = "2026-07-23T01:52:14.538Z" }, - { url = "https://files.pythonhosted.org/packages/d7/2c/26e60b694844dfd2176c57f913a22d0cd6a16f9ff202cbda7580d0328b98/aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43", size = 1674012, upload-time = "2026-07-23T01:52:16.486Z" }, - { url = "https://files.pythonhosted.org/packages/38/65/672df92e3172cd876aacfa97a952ac560877eb169384b2991ac5b273de4c/aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9", size = 1767015, upload-time = "2026-07-23T01:52:18.28Z" }, - { url = "https://files.pythonhosted.org/packages/9e/c5/228dec7bfec1c373cc2217cdeb47d6456dcd7a13a4c55144930a75ae3851/aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8", size = 1858700, upload-time = "2026-07-23T01:52:20.08Z" }, - { url = "https://files.pythonhosted.org/packages/bd/ff/cb36724e8c8d17f90ada567a9ff3efe1d6e9b549fba697a242aece180f21/aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479", size = 1714075, upload-time = "2026-07-23T01:52:22.071Z" }, - { url = "https://files.pythonhosted.org/packages/9f/3a/296a4135c6366376263aeef54b15caca1f07676c2ae0c525d7832f2f808a/aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b", size = 1588234, upload-time = "2026-07-23T01:52:23.757Z" }, - { url = "https://files.pythonhosted.org/packages/7d/81/9d5d853ef892dc066d1eb6db0e87a47348b920c1c879aa554612fdbd9d79/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d", size = 1677300, upload-time = "2026-07-23T01:52:25.861Z" }, - { url = "https://files.pythonhosted.org/packages/68/96/021d386ae32d9b26d4b88df2e794546232ff56bb6be952bf6be227c0bbc7/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d", size = 1691501, upload-time = "2026-07-23T01:52:28Z" }, - { url = "https://files.pythonhosted.org/packages/29/9f/af66adce26a14af135c003cbd0f44ccaa68cebd30ff8ac99ca47fb4958f7/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2", size = 1735113, upload-time = "2026-07-23T01:52:29.995Z" }, - { url = "https://files.pythonhosted.org/packages/2f/90/28c390d4c9851effe52ac25b5a2e1d92246acd00728b4fc7975dafb67484/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48", size = 1577486, upload-time = "2026-07-23T01:52:31.937Z" }, - { url = "https://files.pythonhosted.org/packages/db/c2/00e23a1bf2abb70dd353f6987db7e7f2491d0261f7363997738c71c98f95/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f", size = 1751353, upload-time = "2026-07-23T01:52:33.688Z" }, - { url = "https://files.pythonhosted.org/packages/6e/7d/d51a706a8cbfa57f0611127daf61ab3ae02ab8420b0407412079227d1c65/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32", size = 1698681, upload-time = "2026-07-23T01:52:38.167Z" }, - { url = "https://files.pythonhosted.org/packages/ec/b0/90bd5cd9fdd9787cb4211d284d1fb8401339a933cb0227a15b71e789232f/aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e", size = 456733, upload-time = "2026-07-23T01:52:41.823Z" }, - { url = "https://files.pythonhosted.org/packages/d8/15/fe5b8f6a71ae112bc677163d0b0701bda5dc15005249582258ede0eb88c7/aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c", size = 480460, upload-time = "2026-07-23T01:52:43.905Z" }, - { url = "https://files.pythonhosted.org/packages/54/00/45e98b6645cd7f00a4b78b749ebd309094b0eaeb2d2e96157eadbc0d0050/aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb", size = 453479, upload-time = "2026-07-23T01:52:46.075Z" }, - { url = "https://files.pythonhosted.org/packages/f8/5c/b3e4ff8ad43a8afef9602c5e90285936da1beaea8b029016b793891f03c3/aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3", size = 764250, upload-time = "2026-07-23T01:52:48.525Z" }, - { url = "https://files.pythonhosted.org/packages/0e/da/f1b384465e51449d844056b75070461da03a9a23e6c1747003695bf4172a/aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a", size = 516281, upload-time = "2026-07-23T01:52:51.047Z" }, - { url = "https://files.pythonhosted.org/packages/b9/3f/01264f820ee2e3712a827892b1cd6ff80f3300c1fcbffbb45714a915d47a/aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8", size = 514742, upload-time = "2026-07-23T01:52:53.779Z" }, - { url = "https://files.pythonhosted.org/packages/9e/8d/a71c6f2db52ac1ed142b133f7feddaa6b70539c3f4de24d7e226c95b794c/aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239", size = 1780613, upload-time = "2026-07-23T01:52:56.948Z" }, - { url = "https://files.pythonhosted.org/packages/a5/11/3dd9b3fb3a170f6ec9011b5291d876a6fab4086714c9e158600edf01b4fd/aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f", size = 1737688, upload-time = "2026-07-23T01:52:59.294Z" }, - { url = "https://files.pythonhosted.org/packages/6d/3e/834c26918be7d88068822b40e0db30fca50b5f4fe79104aa16a93f1d74e6/aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06", size = 1845742, upload-time = "2026-07-23T01:53:01.641Z" }, - { url = "https://files.pythonhosted.org/packages/cc/c9/49ab8572df7d66bc13d11e31f781292badb04180dd87ba98733066c6aed7/aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929", size = 1928412, upload-time = "2026-07-23T01:53:04.018Z" }, - { url = "https://files.pythonhosted.org/packages/a5/b9/2b8f0c0ce09c87a1daf80fd483431b56b1435d3f62789bc86f572e1245de/aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db", size = 1786220, upload-time = "2026-07-23T01:53:06.481Z" }, - { url = "https://files.pythonhosted.org/packages/85/00/9c45f81de11710460edfa1dc81317b6e882703b160926c879a9d20da9fcc/aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce", size = 1637231, upload-time = "2026-07-23T01:53:10.258Z" }, - { url = "https://files.pythonhosted.org/packages/19/ce/967d628e910756f3539c6107cb7844a1b69440dcb3029a5ee7871b09ab63/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c", size = 1753161, upload-time = "2026-07-23T01:53:13.817Z" }, - { url = "https://files.pythonhosted.org/packages/11/b2/0c3d4114f0aee4f580f5b3b4eb71b24d7a23b834ea506a4dfebe76513f35/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15", size = 1756356, upload-time = "2026-07-23T01:53:16.211Z" }, - { url = "https://files.pythonhosted.org/packages/63/5d/99e7d91c82f1399d1ae2a854e080bd1493fbc31e5e959dbc4ec33dac3bec/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c", size = 1819846, upload-time = "2026-07-23T01:53:18.289Z" }, - { url = "https://files.pythonhosted.org/packages/ad/05/d5e1cb6480eeffd3f901d40a2c5e2d1e7effdc797837da3b490272699f13/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae", size = 1628531, upload-time = "2026-07-23T01:53:23.86Z" }, - { url = "https://files.pythonhosted.org/packages/c9/90/b934682bcaefae18a9e04f3dff5b68522ba810906358ae5029b68110ea3b/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910", size = 1832712, upload-time = "2026-07-23T01:53:27.551Z" }, - { url = "https://files.pythonhosted.org/packages/21/df/6061679faaf81fac746e7307c7adb71e858071a5d34c27583afefc64f543/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7", size = 1775014, upload-time = "2026-07-23T01:53:30.223Z" }, - { url = "https://files.pythonhosted.org/packages/8a/1d/f854878bbc69b88faefe924b619a34a6f59ec05fd387c77690667eaa75eb/aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa", size = 456006, upload-time = "2026-07-23T01:53:34.97Z" }, - { url = "https://files.pythonhosted.org/packages/73/0c/2af9d1674baccd1dbd47282a93d660a22e57ef6167c856deb24b4214fbab/aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d", size = 481069, upload-time = "2026-07-23T01:53:39.673Z" }, - { url = "https://files.pythonhosted.org/packages/8e/76/88401ff3fc95e85c5fc38d588f36f55e61ecb64343b2bc8d69326f453cc0/aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39", size = 453021, upload-time = "2026-07-23T01:53:43.749Z" }, - { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" }, - { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" }, - { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" }, - { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" }, - { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" }, - { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" }, - { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" }, - { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" }, - { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" }, - { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" }, - { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" }, - { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" }, - { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" }, - { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" }, - { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" }, - { url = "https://files.pythonhosted.org/packages/57/be/5afd201cc0ab139029aadb75392efe85a293403d9dd3a3226161c21ce00c/aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86", size = 506269, upload-time = "2026-07-23T01:54:49.075Z" }, - { url = "https://files.pythonhosted.org/packages/22/09/dec8189d62b45ade009f6792a2264b942a90cb88aeaf181239933cd72c3c/aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627", size = 515166, upload-time = "2026-07-23T01:54:51.894Z" }, - { url = "https://files.pythonhosted.org/packages/28/24/2854869d29ed8a8b19d74f9ec6629515f7e04d02dd329d9d179201e58e47/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82", size = 486263, upload-time = "2026-07-23T01:54:54.223Z" }, - { url = "https://files.pythonhosted.org/packages/d4/dd/57187c8be2a35aea65eaee3bd2c3dcbbcf0204f5106c89637e3610380cd1/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c", size = 492299, upload-time = "2026-07-23T01:54:56.236Z" }, - { url = "https://files.pythonhosted.org/packages/b9/11/06ae6ed8f0d414edf4068861e233d8fe23ee699bfd4b3ceb8663db948a62/aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f", size = 502235, upload-time = "2026-07-23T01:54:58.377Z" }, - { url = "https://files.pythonhosted.org/packages/7e/a3/559639c34a345d2cf7c52dff6838119f2eaf29eb508227b5b83f573af813/aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80", size = 750883, upload-time = "2026-07-23T01:55:00.65Z" }, - { url = "https://files.pythonhosted.org/packages/91/cd/41e131f13afd1e7b0172a9d9eda085ef90eb8439f41f0d279db81ed3ae60/aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0", size = 508473, upload-time = "2026-07-23T01:55:02.945Z" }, - { url = "https://files.pythonhosted.org/packages/bc/6b/e7f13410d391c6e55b4c007a8de024355389d7d459e3d64c42b2d33617e5/aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf", size = 509190, upload-time = "2026-07-23T01:55:05.173Z" }, - { url = "https://files.pythonhosted.org/packages/97/21/6464573e53d69672cc1eada3e5c5cb2d2efa82701e8305a0f2047a576967/aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd", size = 1761478, upload-time = "2026-07-23T01:55:07.383Z" }, - { url = "https://files.pythonhosted.org/packages/1a/81/d217043a4c17fbce360905e3b2bdd20139ebc9a2de836d035d179c4da006/aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807", size = 1735092, upload-time = "2026-07-23T01:55:09.803Z" }, - { url = "https://files.pythonhosted.org/packages/a1/66/e13a02d0eeb1a9a502402a977abb4e4abff9fe4051c26f80558c57a7c975/aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8", size = 1800546, upload-time = "2026-07-23T01:55:12.012Z" }, - { url = "https://files.pythonhosted.org/packages/26/5e/57d42fca1d18cb5acc1cad945d017fabc5d6ae71d8a08ad66be8dc3ee544/aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24", size = 1895250, upload-time = "2026-07-23T01:55:14.357Z" }, - { url = "https://files.pythonhosted.org/packages/ca/1c/7da8d08e74d56f00070822f9638ff3f1c563f8ad87d1efa996c87bfc8644/aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5", size = 1789289, upload-time = "2026-07-23T01:55:16.668Z" }, - { url = "https://files.pythonhosted.org/packages/cd/0f/cf16bcf56896981c1a0319f5d5db9337994b5165730c48a8fa07e9b34be6/aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4", size = 1586706, upload-time = "2026-07-23T01:55:18.913Z" }, - { url = "https://files.pythonhosted.org/packages/fe/6f/76eac12a7f2480e1e304f842efdb07db33256b0d9165b866b6ef0806c202/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9", size = 1724652, upload-time = "2026-07-23T01:55:21.296Z" }, - { url = "https://files.pythonhosted.org/packages/39/b6/19c8c592baeeb94b75f966547d40c02ac7590902306ec5863d5c027cf506/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1", size = 1756239, upload-time = "2026-07-23T01:55:23.705Z" }, - { url = "https://files.pythonhosted.org/packages/dc/c9/4e9383150296f97f873b680c4de8fb2cd88608fb9f48c79edcb111611abc/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371", size = 1769161, upload-time = "2026-07-23T01:55:26.082Z" }, - { url = "https://files.pythonhosted.org/packages/aa/1e/147bdc6cc5de5f3ab011be8bf5d6e786633249f22c20bae06f85e45f5387/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde", size = 1578759, upload-time = "2026-07-23T01:55:28.846Z" }, - { url = "https://files.pythonhosted.org/packages/fd/31/78388a9d6040ece2e11df62ea229a822cf5e52d238374b220ae9975b2623/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e", size = 1792025, upload-time = "2026-07-23T01:55:31.457Z" }, - { url = "https://files.pythonhosted.org/packages/03/51/a3d29fdf2c25d796746af8ad6fe56a45d6256c38b0a8a2ed752e1160b3a2/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71", size = 1768477, upload-time = "2026-07-23T01:55:33.87Z" }, - { url = "https://files.pythonhosted.org/packages/29/a6/442e18b5afeade534d877a2dc3c3e392aff8d49787890b0cf84790410267/aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0", size = 451069, upload-time = "2026-07-23T01:55:36.121Z" }, - { url = "https://files.pythonhosted.org/packages/9d/69/3d876ac02659f271cf7f6769f14a8e3de5b6e888ed8b5a7e998086a4cec8/aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883", size = 476518, upload-time = "2026-07-23T01:55:38.303Z" }, - { url = "https://files.pythonhosted.org/packages/b2/0e/50d6e6471cd31edce8b282bdec59375a3a69124d8a989a0b1313355cae52/aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2", size = 447676, upload-time = "2026-07-23T01:55:40.451Z" }, - { url = "https://files.pythonhosted.org/packages/c8/20/887fdcf832326571b370ffc347b3e70abe101096f3720126aac161b1d872/aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062", size = 509067, upload-time = "2026-07-23T01:55:42.618Z" }, - { url = "https://files.pythonhosted.org/packages/ad/a3/92cec936f78cc4bf0fa5554ebe593b73459d94e3c62303e1902a4cccb6f7/aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6", size = 514774, upload-time = "2026-07-23T01:55:44.937Z" }, - { url = "https://files.pythonhosted.org/packages/29/ba/2a0c38df3fc557620b6a5acd98364af050053b6285b4dc7ee74100c63c18/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919", size = 488134, upload-time = "2026-07-23T01:55:47.135Z" }, - { url = "https://files.pythonhosted.org/packages/48/d6/d51b7d4bf309af3693940d8ffd2b9ed0b682434ef85959b7c9c137f60cf8/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7", size = 494201, upload-time = "2026-07-23T01:55:49.451Z" }, - { url = "https://files.pythonhosted.org/packages/3f/5a/8f624384e5f1efabb5229b94157eb966b021e97bdb188c62860c2ae243c2/aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0", size = 502766, upload-time = "2026-07-23T01:55:51.656Z" }, - { url = "https://files.pythonhosted.org/packages/a6/26/4ff0164370deec18fb19254ee4ab10b7a73304ac0c860b13f5f84663759b/aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924", size = 756557, upload-time = "2026-07-23T01:55:53.964Z" }, - { url = "https://files.pythonhosted.org/packages/97/a3/7056b86dc0d9ec709ea9777eae3b0161428f943372f8b98c01c11593b682/aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646", size = 510168, upload-time = "2026-07-23T01:55:56.22Z" }, - { url = "https://files.pythonhosted.org/packages/85/ed/0357a015892fd68058bf2d39d3fd1958e459b997a7db30aaa6aaa434ae96/aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b", size = 512957, upload-time = "2026-07-23T01:55:58.437Z" }, - { url = "https://files.pythonhosted.org/packages/47/d1/8aba53f15ccb2238405f5e9d30e2a8ca44f93878c26e7165ade00d374b1c/aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30", size = 1750149, upload-time = "2026-07-23T01:56:00.856Z" }, - { url = "https://files.pythonhosted.org/packages/49/bd/40c3fee327529284375c6701cbb0fa4600cc2e8432af1378f897e2ef7d3a/aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9", size = 1707685, upload-time = "2026-07-23T01:56:03.371Z" }, - { url = "https://files.pythonhosted.org/packages/2a/a3/ca0cc6724cca8114b05694abd916060758c79894c3aa5b012cdadc1bc28e/aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f", size = 1803911, upload-time = "2026-07-23T01:56:05.817Z" }, - { url = "https://files.pythonhosted.org/packages/95/b5/85b099c299c3ffd38ad9b3e43694c8a346934e4a30c88c4fd5a841234f77/aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d", size = 1876929, upload-time = "2026-07-23T01:56:08.413Z" }, - { url = "https://files.pythonhosted.org/packages/d5/b7/1da684a04175473fa4cddbf9a2f572e79514c3fd27a74597f43057d4f3da/aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147", size = 1761112, upload-time = "2026-07-23T01:56:10.918Z" }, - { url = "https://files.pythonhosted.org/packages/d1/16/bc4b55e3e5cb175fd69c53c90d60d2f47797cb343da5106e23863dc4dba4/aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c", size = 1583500, upload-time = "2026-07-23T01:56:13.613Z" }, - { url = "https://files.pythonhosted.org/packages/2a/e8/13a9d957a1ee40837f46aa30f0f4c657e673ad86a2e6362a9f9be20d26d9/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a", size = 1713940, upload-time = "2026-07-23T01:56:15.969Z" }, - { url = "https://files.pythonhosted.org/packages/38/05/d33c680c1bcf1c7e130f9cbfc1fc02fe8bb0c4af2a94a53dd5fb56131e5c/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0", size = 1724413, upload-time = "2026-07-23T01:56:18.591Z" }, - { url = "https://files.pythonhosted.org/packages/85/1d/af798d306f7a74b6a632dbcabcf62a4c91391b7582d2a8c6d7712e2cc54e/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661", size = 1770748, upload-time = "2026-07-23T01:56:21.074Z" }, - { url = "https://files.pythonhosted.org/packages/a8/92/ad720d472556a995049206867765e9410969684f86ee09423ff9969044c1/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22", size = 1577564, upload-time = "2026-07-23T01:56:23.475Z" }, - { url = "https://files.pythonhosted.org/packages/60/ad/0ed7586cbef7a884e23a752fa2bb987a122e6a5dd50dab109258d0a95193/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41", size = 1782080, upload-time = "2026-07-23T01:56:25.994Z" }, - { url = "https://files.pythonhosted.org/packages/97/ea/dbaed0d73e8a69aad653b045dab451c67c2454bb731a37b45a86593e9422/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf", size = 1745813, upload-time = "2026-07-23T01:56:28.604Z" }, - { url = "https://files.pythonhosted.org/packages/81/1b/6893d4bc57e434fc93a6c9217c637d967a0b651d989f6e3265179375754a/aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da", size = 455872, upload-time = "2026-07-23T01:56:31.031Z" }, - { url = "https://files.pythonhosted.org/packages/f5/8b/c7baa1ba1eda4db6989baefe5de6d99834921b84ebd7918624febcb9f290/aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100", size = 481030, upload-time = "2026-07-23T01:56:33.365Z" }, - { url = "https://files.pythonhosted.org/packages/22/8c/c29d067df825a2df88ca432db848aa2fe8199598359cc06c12b09320cac9/aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc", size = 453669, upload-time = "2026-07-23T01:56:35.731Z" }, - { url = "https://files.pythonhosted.org/packages/6a/a4/9c033beb355d39b6147980597ec9645e4729243f686ee4dc73945de72030/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b", size = 791403, upload-time = "2026-07-23T01:56:37.972Z" }, - { url = "https://files.pythonhosted.org/packages/80/ca/87c32a0a7704583cfc49660bd817889bae5b830bf53b5dcb4e92145ac2da/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0", size = 526413, upload-time = "2026-07-23T01:56:40.523Z" }, - { url = "https://files.pythonhosted.org/packages/9e/d8/8ec0e471248c500acdce2be3f46db8fb62b5eb60efef072529cc85ee1d26/aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e", size = 532135, upload-time = "2026-07-23T01:56:42.876Z" }, - { url = "https://files.pythonhosted.org/packages/fe/45/f8919fd936e8b79fcd9bda7b6d8e62613462a713f4f17987fd7c34399142/aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716", size = 1922742, upload-time = "2026-07-23T01:56:45.528Z" }, - { url = "https://files.pythonhosted.org/packages/f6/ec/9ca76b28a27525b0cc53e20842e0228b022f301ce1f436b7d814b4aaf2df/aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f", size = 1787371, upload-time = "2026-07-23T01:56:48.045Z" }, - { url = "https://files.pythonhosted.org/packages/b1/04/6acdbf17315f7b55f1937e3387acb89a3cddeb4995689553d064af8e92ab/aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553", size = 1912623, upload-time = "2026-07-23T01:56:50.605Z" }, - { url = "https://files.pythonhosted.org/packages/86/e6/438b0c79ca6f45eb9fd9817dd4c01a91919a38c0de5ee9e05e2b4dc0ece7/aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100", size = 2005515, upload-time = "2026-07-23T01:56:53.153Z" }, - { url = "https://files.pythonhosted.org/packages/bb/6b/62cbd6577758699525f5c712d1ddef57d9875fbab0ae8d5f5a202fd598f8/aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85", size = 1879906, upload-time = "2026-07-23T01:56:55.818Z" }, - { url = "https://files.pythonhosted.org/packages/00/95/18bcbf830a21dc3aae24d8f6b6feaf3db1d2090242d00a7868db2ffb0b67/aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33", size = 1675849, upload-time = "2026-07-23T01:56:58.861Z" }, - { url = "https://files.pythonhosted.org/packages/a9/19/47f4968659c5e23606c3790c80fc624e691c153d036148449ee84d31b287/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f", size = 1843496, upload-time = "2026-07-23T01:57:01.591Z" }, - { url = "https://files.pythonhosted.org/packages/64/af/38c33c4dd82fddcb4e56c4653b6f1072a8edbc6b7fa15809f14932c41e2d/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0", size = 1827746, upload-time = "2026-07-23T01:57:05.131Z" }, - { url = "https://files.pythonhosted.org/packages/a1/9d/0537cda4885ac8f5b7053d164dd06312f4c483a4edcb8ee5b8aaf2a989bf/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098", size = 1853810, upload-time = "2026-07-23T01:57:08.043Z" }, - { url = "https://files.pythonhosted.org/packages/19/fe/26f9c5e6458385aa86497836b0dea6fb2f027827d63f37c7856cce9286ee/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25", size = 1668895, upload-time = "2026-07-23T01:57:10.837Z" }, - { url = "https://files.pythonhosted.org/packages/ec/4c/618b1db9b9ba079b8875d2cdf78e7c4a3bf72903bd5850fee7dd9544600a/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9", size = 1883833, upload-time = "2026-07-23T01:57:13.672Z" }, - { url = "https://files.pythonhosted.org/packages/94/c6/bd959bd1e4771f9fd944e9e436224c48c77b018b73b519b5aad346335bcc/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb", size = 1844251, upload-time = "2026-07-23T01:57:16.593Z" }, - { url = "https://files.pythonhosted.org/packages/5e/19/08d41839658bdd44a0ed2480f3891705ecb487ce28c0dde62c9040c997e0/aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963", size = 474180, upload-time = "2026-07-23T01:57:19.306Z" }, - { url = "https://files.pythonhosted.org/packages/99/5d/3cd6ef0a2b2851f7ab913b5b079334781bd50ff56a323e4454063377a080/aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b", size = 500528, upload-time = "2026-07-23T01:57:21.762Z" }, - { url = "https://files.pythonhosted.org/packages/a4/37/cfd1ed540a4d318da025590d96b728e63713c09e9377950fc655dadeb856/aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7", size = 469280, upload-time = "2026-07-23T01:57:24.241Z" }, + { url = "https://files.pythonhosted.org/packages/ca/82/f7f0652e3a845e095d8505a2807a0fe5782abb71f99b2f516c93ff0be492/aiohttp-3.14.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef692a24087a699c0a4a26af45e746e0c1eae2116f6d8a5ff91d8aae2b867b45", size = 834351, upload-time = "2026-10-11T00:59:07.668Z" }, + { url = "https://files.pythonhosted.org/packages/95/c5/10cb0a5195bc3ec05a74ee494f8431a15e559800f2e869f3526ad4a16f04/aiohttp-3.14.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1220353657ad49493551f089ce02f1a348fd57ffd585bfec77f2f3c4fe3a7346", size = 560663, upload-time = "2026-10-11T00:59:09.853Z" }, + { url = "https://files.pythonhosted.org/packages/32/dc/b7fefdd7dd64d080da2d69d609344191cc94b0e2eba70200c371014c9b36/aiohttp-3.14.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:330900acd0dc4cb8b27f9c127fbaad770964845338493e7906ae3822e82dbf8d", size = 553263, upload-time = "2026-10-11T00:59:11.5Z" }, + { url = "https://files.pythonhosted.org/packages/61/5f/3271cad1b34347e8bf27abee20d501ebdc56cec97e8ae59cd3fabb23d917/aiohttp-3.14.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0684952aeae1f5dbfe02d46039338513b94009baecd15d8e4098a357c4c4a2a6", size = 1843813, upload-time = "2026-10-11T00:59:13.143Z" }, + { url = "https://files.pythonhosted.org/packages/67/af/2011b30fe61889af112ad380e6fd44a93cca4b51296f612cac666c177882/aiohttp-3.14.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d94e44be379e569758fee8a9a58431cfc3c2598c708b92b1cfe96c66b4c94aef", size = 1811243, upload-time = "2026-10-11T00:59:15.091Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ba/72f3ac523ed48d38cdfb169f1b2a2e93ce06d9fcf4f46d0f5692e9539bcc/aiohttp-3.14.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5af42135fdfebdadbc2bcd9c0842a48ccf0d62794c36a260b21dc4b94d1e0119", size = 1898815, upload-time = "2026-10-11T00:59:17.224Z" }, + { url = "https://files.pythonhosted.org/packages/e6/69/bbf24f3b555ea1befa7db1e2510c728c8b0e80e61ce360beb7d94bc562b7/aiohttp-3.14.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4f07fe3ac408d8b3f768be471dc3f56d43843c47d97c66120534467a15ead197", size = 2022390, upload-time = "2026-10-11T00:59:19.244Z" }, + { url = "https://files.pythonhosted.org/packages/5d/38/01f5fc732d5373d6c099db0343279cd4e6607287d336bab6ff6ae9085a0d/aiohttp-3.14.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f375db73a39f5cf83696d500e21a67f418dc9a988955756f254be8f03b7b3651", size = 1856329, upload-time = "2026-10-11T00:59:21.19Z" }, + { url = "https://files.pythonhosted.org/packages/15/85/1f86fbd8fe34ad447c63e679404a1d5be435fe256a9d281b791fc1ed0d35/aiohttp-3.14.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8df7d481654ac96fe1ba9a02a9f67770fdd367823e0d5ef01b922725c4bd2cfa", size = 1712763, upload-time = "2026-10-11T00:59:22.982Z" }, + { url = "https://files.pythonhosted.org/packages/39/1e/ced51cf47e427e7aa6142c3bf6f33022472703bc9061289329532d3c40c0/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e95c8def4b81c5d68d5cf1f54c07acd7c0d2577af244e5b6da802120825737c6", size = 1812125, upload-time = "2026-10-11T00:59:25.032Z" }, + { url = "https://files.pythonhosted.org/packages/6a/f1/9610b4e263b554ca4191501ba72b952efe06556b647da74887abc5d41049/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f2ebb54b3f932210503072f09974b4fb574d823e497a944adfdcd140a6a00255", size = 1829550, upload-time = "2026-10-11T00:59:27.078Z" }, + { url = "https://files.pythonhosted.org/packages/ac/8a/1d1fb27cf9fa99b13133de33044c4d10d7705be0d4173d6d56abad1b67f9/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:038c2c7e8caa26b6c8423779b5eaf1893904048a512c19b32fe841ffa5592b50", size = 1868179, upload-time = "2026-10-11T00:59:29.315Z" }, + { url = "https://files.pythonhosted.org/packages/24/9f/3cfafa86ff025623dcca0d30335db7b8a6f3fa348148441208147e827e04/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b7806e804889231b0e06469fd4a5c06313d1c0a3377322b6d9237fa5e0fe4167", size = 1697721, upload-time = "2026-10-11T00:59:31.161Z" }, + { url = "https://files.pythonhosted.org/packages/62/1d/86439a23bf32296e715d8c75535f0cd701821d36719324baef65bcd6af9e/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9bab2045550c4fe0f7baf89574db1b455c195750702ba96fef1f16972b146617", size = 1887000, upload-time = "2026-10-11T00:59:33.329Z" }, + { url = "https://files.pythonhosted.org/packages/1d/0f/6044f2d02b3874966d84cd119f9b8eda46aa2991230078aea4dd17750cc9/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96a2e584f0b9ed8f1fa33211397dcf67bb7069866402cb405d191c2f0defb9a3", size = 1842020, upload-time = "2026-10-11T00:59:35.384Z" }, + { url = "https://files.pythonhosted.org/packages/18/64/b49d2a34428ff937631b07dde74d5879a7a2c8559aab39331d40769e8cb6/aiohttp-3.14.5-cp310-cp310-win32.whl", hash = "sha256:602c1e9b718a3275c580149f947e7fac65044c0a20e599553fb12e9700da9eca", size = 489282, upload-time = "2026-10-11T00:59:37.061Z" }, + { url = "https://files.pythonhosted.org/packages/a6/7c/1114e723f1c4a65597820b605a0ddcd7f3b28fdfde3ac8d520d517ca582c/aiohttp-3.14.5-cp310-cp310-win_amd64.whl", hash = "sha256:bea559ad70218d230663e4210875735076a9bfea5994cef34a55a25faeaf2544", size = 512021, upload-time = "2026-10-11T00:59:38.7Z" }, + { url = "https://files.pythonhosted.org/packages/48/97/e08c646f4bad63d77876b24591d013d7f33f02fda7c931eafe3224f38467/aiohttp-3.14.5-cp310-cp310-win_arm64.whl", hash = "sha256:dca3fa8d8a0a26679862eccb0b1a9151b2b9f1cd2c212e7a6335778faaff5833", size = 493016, upload-time = "2026-10-11T00:59:40.525Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f3/8997f18890a92c79f77fcfbb4f78ca17c2d4ab109e9eb6d31b4e9de194a0/aiohttp-3.14.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d51db97c96384fbfcaf8f4c65922183a68b94f891c3c10c862ef5f6df2adbb1f", size = 832358, upload-time = "2026-10-11T00:59:42.195Z" }, + { url = "https://files.pythonhosted.org/packages/8e/42/084651e9efb5cadb99265f786df60f2a7b353cead7bfd2c87003cb4867ad/aiohttp-3.14.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ae53924aa853a7a2ca20ed4142c7c6b56338e4d4cd999e2980075b9efc2e257a", size = 559636, upload-time = "2026-10-11T00:59:43.677Z" }, + { url = "https://files.pythonhosted.org/packages/3d/fe/92838944e601f0fe195fd3f3ada37e29fbee3d19bfd323fe15937ec79d36/aiohttp-3.14.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2c473a355f9239efcb72c92d5abfd8fcdb0cc78c8e9af607e72ca12dbb36593", size = 552588, upload-time = "2026-10-11T00:59:45.592Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b8/dd9b95c5c20ceae1b47738e656bd79e9883e3117a9cc8b143901eb604e9a/aiohttp-3.14.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32e8fa6644e541fcd7e02430588c7fc93b602c1778ea0bc345505db76b61cfb4", size = 1928224, upload-time = "2026-10-11T00:59:47.294Z" }, + { url = "https://files.pythonhosted.org/packages/c6/47/70010cd2ba8746968d53d433ff328c33f06a66c99da5f3de759c64a6b2eb/aiohttp-3.14.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:579f97d5120f2971876d2ddca2968135f6944d00c44c3a6590ad7d86ca9b403f", size = 1877674, upload-time = "2026-10-11T00:59:49.472Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a1/b026f071dbd87f0bdba86b92d47e46f3899e3ad7143b6e0fe7e7887090b4/aiohttp-3.14.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:24409db442e2fb6e766bc7f3943851a8381dec3098140e43bb2e843b79e31b12", size = 1985850, upload-time = "2026-10-11T00:59:51.296Z" }, + { url = "https://files.pythonhosted.org/packages/36/d1/f7b6f6f8c3cb5a71b53baeddc2e70c224c28b12a23a55c42994c504019b7/aiohttp-3.14.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5e8f97c0488ffda3082766ac0f2c8150a9a58c4d05788330e479cfd449b37939", size = 2105471, upload-time = "2026-10-11T00:59:53.191Z" }, + { url = "https://files.pythonhosted.org/packages/42/74/2a2b22953de15c6c8a80debf26db3047e4e8a5a5db6dd7c2c6c01eec0605/aiohttp-3.14.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50a195903119008fe9cc68710535eb37f556ffffd6a7759afe70a2c145587045", size = 1929915, upload-time = "2026-10-11T00:59:55.609Z" }, + { url = "https://files.pythonhosted.org/packages/33/ad/f80e8d33933d0eacd0217efa3b7fb32c9f48ed480ed0db53cf9948bf474f/aiohttp-3.14.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0133c3c3b54a0bf1e71fa5c1ad95c93f07fd54e24ef1fe182f5122e1573d2bf1", size = 1773832, upload-time = "2026-10-11T00:59:57.611Z" }, + { url = "https://files.pythonhosted.org/packages/04/a4/0273d239f3e3bb69438f209c64c82e1f98dc60d5db264795e41f7dd05dcb/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c172db893e516e1358e65a95ee20b7ce7173963eefe318b6ab2a2220688b999e", size = 1901001, upload-time = "2026-10-11T01:00:00.605Z" }, + { url = "https://files.pythonhosted.org/packages/16/ed/7dd439d26c654645bc34ca3c9a822fc11c1c51801fed1cefdfe1cc41c1f6/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f2a7966bda23dd85051f1661ce0ace38d6890e05ec6c357ecae9d2479cba377e", size = 1907218, upload-time = "2026-10-11T01:00:02.699Z" }, + { url = "https://files.pythonhosted.org/packages/32/dd/86249c3b8248562a17fd3e22ee0c164378d65766f01aceef62cf52783710/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4887d130a7bbfed3a85493bb5a25e5b5b558d40c1d986dd16970d2bb26d63793", size = 1958650, upload-time = "2026-10-11T01:00:04.851Z" }, + { url = "https://files.pythonhosted.org/packages/bc/0b/ca4d53d68f683ce195807fd0aeb063bf6d1c6b39343fff234c9524e8d5a9/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:225c579c23b68b343cccea27a7e06e3bd8ec23a09c30b427eb3f1e4ca6239b20", size = 1760057, upload-time = "2026-10-11T01:00:06.978Z" }, + { url = "https://files.pythonhosted.org/packages/6b/42/005437ffd7fa56c2ce3347add40404b654a32754ce91f2e2a14b9e5ab2bc/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ab52d8f1fc1b64821c1fbad64a647ed6203627004059a6d1ed4f0858a1499703", size = 1978081, upload-time = "2026-10-11T01:00:08.868Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/3a5b66fe1b7b23d3c6524350817ed55de5feb7fbb4cccba7cbe044d118dd/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:cb131d775a1573c1aee66656bd78b023577bbdb6cb8349a07773bd4f73e68a6e", size = 1922954, upload-time = "2026-10-11T01:00:11.035Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/d08a554f281d42fd7c6b2b6dd7738f3e3a7a9f657ddd87e11ca2091cdfe3/aiohttp-3.14.5-cp311-cp311-win32.whl", hash = "sha256:e87046c8ff77a8decdb6a41d8ab25824b47531b2da933aeab0c1e21c7acff329", size = 488101, upload-time = "2026-10-11T01:00:12.933Z" }, + { url = "https://files.pythonhosted.org/packages/0f/15/52b5e65f02b33686ae49f1518548ae4f5a7adaf31d9d3c54fa12a143137b/aiohttp-3.14.5-cp311-cp311-win_amd64.whl", hash = "sha256:6f275c11d1aa6d4c458e05a68be084efe3c55a113d99e3f46a318098e52948fc", size = 512575, upload-time = "2026-10-11T01:00:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b9/bb75012635f3defb0f7cbb7c4ae391e36bb9cb3fbdca3b9944e2ebebf743/aiohttp-3.14.5-cp311-cp311-win_arm64.whl", hash = "sha256:b032a0023eb41d768ce77d83210ab2a3c389bc0b09313273c7e1eca48c10a755", size = 493179, upload-time = "2026-10-11T01:00:16.961Z" }, + { url = "https://files.pythonhosted.org/packages/d5/94/6ba86efddcb616c811b40e6a0dfdd862738f647e4e3860a961075d5e9ed8/aiohttp-3.14.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df37b620684e19b5e25724412518ccafc3b1a49cdac706fdbd2f983fad943450", size = 817499, upload-time = "2026-10-11T01:00:18.901Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e1/7bca6d84dabd228aa8eb4b7f9feac2586aaa9be5505d7d65bf287a615c43/aiohttp-3.14.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ef60869969180ec2464f1349aff07138ae35ca2200f0946cb3552e49e8f301a8", size = 549467, upload-time = "2026-10-11T01:00:20.854Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/11b89f45ca486252dd67dd5f3231fec04bf5518da39a95cb3997619f17fb/aiohttp-3.14.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d079c0a0135c36e7beb6f1c88087c8f108dc5891cdd0b5eafa778421bda70ed2", size = 548067, upload-time = "2026-10-11T01:00:22.659Z" }, + { url = "https://files.pythonhosted.org/packages/22/ff/c6615806c14aab34f82b9424ccde8ce6e417315fd57ce1c5b4d4747888e1/aiohttp-3.14.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abfda5cb094a829f7bc25216a32f7db2e85cc65bd59910f8e7b40b3d9b224764", size = 1896245, upload-time = "2026-10-11T01:00:24.638Z" }, + { url = "https://files.pythonhosted.org/packages/39/b2/25a8c971ae6a92c8394d77e42422d7f38989e05cf41a5ceb92d73d67ab7e/aiohttp-3.14.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9cc882cf8619109583c906b4d4a85d6a111a98afa34b7a450d1e08118d016820", size = 1872180, upload-time = "2026-10-11T01:00:26.838Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d5/99f93ea36cc5205e47c1e5a803e087f2ad21b5430b5db2e942cb6e988a37/aiohttp-3.14.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7457580535e019e1247ea35d6a02bf081ad30c26d0cbc210c93f6c3ab67a0835", size = 1937261, upload-time = "2026-10-11T01:00:28.74Z" }, + { url = "https://files.pythonhosted.org/packages/40/a6/9ac9c9e6695040bd73d2584a1b59a9388f6433c76d5294a7bf591e21ffe5/aiohttp-3.14.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c5ed596aedb9c42afd3fe0aae3117725378ac73d2cc5ddc735056fbdb96c5d02", size = 2064994, upload-time = "2026-10-11T01:00:30.623Z" }, + { url = "https://files.pythonhosted.org/packages/da/e4/aa172eb534b7f02f1f8ff1c3213347eaf3cf218db91a727c6863c22f1035/aiohttp-3.14.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20f085697d7e911f1f73c43ed03fafbed1e7121797e2eb5428efa80398060584", size = 1919630, upload-time = "2026-10-11T01:00:32.548Z" }, + { url = "https://files.pythonhosted.org/packages/06/7d/4eedafc5bababa8932636c141e346806966eade12c0b7e5946d43bf8218b/aiohttp-3.14.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:74b0a9c8270f9b0a11410e124ff8d4f18bfc1f1837440ec84da5ae7b50927b5d", size = 1710016, upload-time = "2026-10-11T01:00:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/b2/94/eee018537ba19da0ceb2ac79cab83faed4ac49568e08376e2799043f2538/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:19e2ba471507c34f8252402ab50f5ab512398b9ea8c8f1cb26beb3f75793ba30", size = 1858112, upload-time = "2026-10-11T01:00:36.581Z" }, + { url = "https://files.pythonhosted.org/packages/68/76/354653a306547238f3427283905972d796ba7c292ba9977abec9f2b6f260/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d418ce2af40c6bb685b3f663e9e8de27cb0a22431d8e88a167348d7f01878073", size = 1896371, upload-time = "2026-10-11T01:00:38.478Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ec/63e8c7136b570e356345ad3174e3820fdc973ea10712cb6c649bf875755a/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:70cb4008ac2ed1e0ca9e824deb4b53d3aa0d939109698ebf1e723a84337bd794", size = 1905836, upload-time = "2026-10-11T01:00:40.527Z" }, + { url = "https://files.pythonhosted.org/packages/57/d8/11365bda144b127928cd42533d0eff78a55613c9e28f81941bd6630ea887/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a23fe35d776bc03cb495938b9594450d047e3bc08c5255315a82323e9cb7d2dd", size = 1696534, upload-time = "2026-10-11T01:00:42.686Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3d/82df0461b18e00b2998f205c03e0d3010222478c43640aceb8e03dcd7e8f/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3e0eb43bed3c6801a6cee315195377789e90b2a72c2277a475b578535312488d", size = 1933970, upload-time = "2026-10-11T01:00:44.71Z" }, + { url = "https://files.pythonhosted.org/packages/03/ad/6ddfe0aacd931c17b53533336d97e9d11a98b96d6ae815a9da0b19f82ccf/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be7dd397d64ca3e1869626fa9318aaebb54b7bf93bc72d7a205448d83e4f748", size = 1899501, upload-time = "2026-10-11T01:00:46.629Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8e/189bdd9ae4793059bb09f6dc880f211a6c6c7859cebcbf33912dbfab7dd7/aiohttp-3.14.5-cp312-cp312-win32.whl", hash = "sha256:eb324e2009fb54db30a071dad7caf6998ee2879c4704007efb244514dad1fec1", size = 480370, upload-time = "2026-10-11T01:00:48.468Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ce/1f08114679d49655b30a6e0a29858375c94b82c1de1a0bd0a20c2fee8b02/aiohttp-3.14.5-cp312-cp312-win_amd64.whl", hash = "sha256:2cc38a4f2b516bef1714e690df87a0e043faf1a7693c82d860091684453d5111", size = 506141, upload-time = "2026-10-11T01:00:50.272Z" }, + { url = "https://files.pythonhosted.org/packages/79/d4/c7b4f60b16a1b7e43249fa9031ae05e7e8c341ba4b7d1866f914dafeaa0e/aiohttp-3.14.5-cp312-cp312-win_arm64.whl", hash = "sha256:a63afd1f757de949028387e65a7127b61ad0f775432dbb0e62816ae619fe69ac", size = 488108, upload-time = "2026-10-11T01:00:52.321Z" }, + { url = "https://files.pythonhosted.org/packages/d3/e1/2841e020ebb7aefae5513586193e011e06313d9a6bdbd296622afbbce204/aiohttp-3.14.5-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:9ad7e6aa38c20da1be697874349c4c273c8a03b7887169665081706398d0439a", size = 538932, upload-time = "2026-10-11T01:00:54.3Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a6/7fb8ea8fe96bcc7b7c7a36d10f021d99d01a8dc8a4b3f0ddacecfad9a80e/aiohttp-3.14.5-cp313-cp313-android_24_x86_64.whl", hash = "sha256:f59c7673465908cbe506117176156c127f29f917677afceada34957179221d91", size = 553329, upload-time = "2026-10-11T01:00:56.442Z" }, + { url = "https://files.pythonhosted.org/packages/de/64/d056e3c27647dc25af1a592cf356245382ea7c808171b9dac7677afedfc8/aiohttp-3.14.5-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1b5416552740edf07234cc9437d0706f2acb67b93c198670b1a68e1b2b587dec", size = 521367, upload-time = "2026-10-11T01:00:58.345Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e4/95226147e11d4db916fd1d495dcf85af8e3816e38333e42718241196e848/aiohttp-3.14.5-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:43351bdb5e4c3cb7d1772368e988534e869a74db7778079a83782c11c69535c7", size = 528322, upload-time = "2026-10-11T01:01:00.211Z" }, + { url = "https://files.pythonhosted.org/packages/17/cd/1d3c9192cafdb51cad62b2d3ded96cff9cc8af51893210aadd448a325389/aiohttp-3.14.5-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c8c4478bef6d57fcfda15dae461ea3c9f06aa7b257c58df3f2300174ccbb185a", size = 539874, upload-time = "2026-10-11T01:01:02.06Z" }, + { url = "https://files.pythonhosted.org/packages/f6/0c/dfa33aecc7d4d1dc75e05248f5eac5a0edf4d09e7b44d93ab62529b0c1db/aiohttp-3.14.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c2c30484dd1417ef98b51021ffa2cc0d7f3c78918adaaaab7e70817335ab3e02", size = 813836, upload-time = "2026-10-11T01:01:04.01Z" }, + { url = "https://files.pythonhosted.org/packages/33/17/4a63738052d20567d55529d6daa1b9480d906fd52930fbcf6d3fbed618f0/aiohttp-3.14.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dab9ac5a67c8d1f070c00fa8fccb7cbd1b8dcc1a8d6b42f37540df9b3d4cc603", size = 548070, upload-time = "2026-10-11T01:01:06.035Z" }, + { url = "https://files.pythonhosted.org/packages/15/e5/b57e58695a757fd4c02497c033fced96a69c631b13866c43d336530c9670/aiohttp-3.14.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e1cc2bfaee8c214f06080a7c7d5772419b8a1108e8e5349236189811823fb02a", size = 545786, upload-time = "2026-10-11T01:01:07.817Z" }, + { url = "https://files.pythonhosted.org/packages/9a/68/8c2c67a3aedf46e00f3c42f04fbc6983de80d4ed5786151e33681ba45883/aiohttp-3.14.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74efb69332b85675b1eabd760a8cfc2e2cf42c60607c66f88014c1bdfb40942d", size = 1886492, upload-time = "2026-10-11T01:01:09.834Z" }, + { url = "https://files.pythonhosted.org/packages/ab/4b/74aab5e8d28c62e8f795b4fe8f38cf5586fd264a9a27bd2141ef6490333d/aiohttp-3.14.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:42b5e616946dbaf505e2bff18c9af2cd4ef9e7ef300ee58a6e951a5b7cf147ae", size = 1870057, upload-time = "2026-10-11T01:01:12.045Z" }, + { url = "https://files.pythonhosted.org/packages/a8/f7/eafc3b1988302b1815d9fd4a21071be5c360d616c0a430d02fd92dc97688/aiohttp-3.14.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f9033b43f511f27547c557dcaba0177649e10a3725336ccd2cce0fdc1dc4850d", size = 1925119, upload-time = "2026-10-11T01:01:14.09Z" }, + { url = "https://files.pythonhosted.org/packages/a1/04/78d8f294f74dd570f3898ff20402349fce524176045df98ba727d6846a68/aiohttp-3.14.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:15a310d3c71398e3d7bfc93a1a73fbe664315cd9e9016b8efc1cff85eeab7155", size = 2053137, upload-time = "2026-10-11T01:01:16.344Z" }, + { url = "https://files.pythonhosted.org/packages/32/51/395d225ef36f5a50d8e548dcd3141bfdbcd31fb6eed859022c573d2c4d66/aiohttp-3.14.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1ffa3a523a36d8628f98c06492ae16a31a23d14c0b4ec721757b477319f656d6", size = 1911038, upload-time = "2026-10-11T01:01:18.653Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a4/2aec1aa06d82e8a244843b5dae31d78061e5e76744270a86dd0ee051c889/aiohttp-3.14.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fb6a6e919bfb703227bc1ce6579281b84b1a2ba57deb9794dfdbec7dcd1e40c", size = 1702205, upload-time = "2026-10-11T01:01:20.904Z" }, + { url = "https://files.pythonhosted.org/packages/16/27/6051bfde7b6f418f70edd60d655fa426abb3355fa0981764739d87ecf160/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:43e1b7994a8b038125f722bff07492ef501110722c2727c408995d9fb864c421", size = 1848588, upload-time = "2026-10-11T01:01:22.918Z" }, + { url = "https://files.pythonhosted.org/packages/9e/44/55efc06fc26c4e6e1c095f231b4c222bf2d86d64a8eebbc24b2bb5958ea8/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:5f3e96071686755d9cd3600c3880183eb94b012178e92746d68101800f0ed8a3", size = 1887540, upload-time = "2026-10-11T01:01:25.278Z" }, + { url = "https://files.pythonhosted.org/packages/48/dc/1502bfdc2a65760d386ac6a00090b0addaa8a3c9c60b3f8127fad3a9afb2/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:cd88b01f3d37b7a2a34f91d98f14720206f1ea3d540843fab2d649dd5fb91fec", size = 1895030, upload-time = "2026-10-11T01:01:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/93/7e/44174bb6288264418c9eec07a5e35180969c0d5a796c7af544db3cb8a33a/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f2ed8b64dc0c651c0f5a9c926777719770021251b8f336d97c4b80b660836ce1", size = 1693244, upload-time = "2026-10-11T01:01:29.39Z" }, + { url = "https://files.pythonhosted.org/packages/47/dd/b507d64e50db23888582fff08eda13998b12f9dea70c072edaac19218380/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a9918e58faf62ba2c7147927d06057aec78f42475aff5048047ec47e7265a600", size = 1926308, upload-time = "2026-10-11T01:01:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/98/4b/5b51b4f63e3f2793151f4aea49c48fe1e00baeb7cec9c7a206de499f8de0/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:42f320d4a5b00b9af0bddcfec5407dc6f2d9816f006b2f79ebbaa31f16895df3", size = 1888406, upload-time = "2026-10-11T01:01:33.715Z" }, + { url = "https://files.pythonhosted.org/packages/ff/13/d5e818a5eaba9f822016727299f41c0e1075799f82a6433ec508a93ab867/aiohttp-3.14.5-cp313-cp313-win32.whl", hash = "sha256:3ae800a20947e2c2e53088047d021e6bf7d51560cc49f6a0737a1f79d2e3a13c", size = 479970, upload-time = "2026-10-11T01:01:35.677Z" }, + { url = "https://files.pythonhosted.org/packages/8d/d0/8eca2c65aa467320990d78fb2005f38ed3588944280c39ff9deb6423fef1/aiohttp-3.14.5-cp313-cp313-win_amd64.whl", hash = "sha256:d05e94cdfe0d15d0206f970722d2554780ce562787b21b218b275447f8751319", size = 505985, upload-time = "2026-10-11T01:01:37.574Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f8/4cdd65305d2fca14b886bea9ed2abb1fe726287872524e56e3d26692b47d/aiohttp-3.14.5-cp313-cp313-win_arm64.whl", hash = "sha256:f001b571ead90ca1770f1e616db255351a1703317f20374c361ef22f12c06d09", size = 487629, upload-time = "2026-10-11T01:01:39.477Z" }, + { url = "https://files.pythonhosted.org/packages/43/be/3184a1d34a8be665569eadb7e9e764b4629e4f3413e241cb2e4d6fecf3b3/aiohttp-3.14.5-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:939042d5cda21d41a6f512e7cc8b8e33a2aebff863352251da495fbd91b673b5", size = 540673, upload-time = "2026-10-11T01:01:41.354Z" }, + { url = "https://files.pythonhosted.org/packages/60/2a/d35f3ba4cf157b072e3b674bf9983047ca5ea5173c995d32d877e1191d36/aiohttp-3.14.5-cp314-cp314-android_24_x86_64.whl", hash = "sha256:6da32b5ff3fd78d244e37300463434c7145162bfd2b6e9e915ab164da37f7343", size = 554087, upload-time = "2026-10-11T01:01:43.719Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d2/61a33880ca4eaca95a9c60ca3f6beed15555af1028652dfaac601627787b/aiohttp-3.14.5-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1b438b73c38111818d0c9d6a5c2bfed8584c8e503a49ef085d70e874ec846738", size = 522613, upload-time = "2026-10-11T01:01:46.154Z" }, + { url = "https://files.pythonhosted.org/packages/31/1d/de579b299d2225dc2c6fd99d579d91f16c02a913fb5af9a3cf2fe9bd88ba/aiohttp-3.14.5-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:755933b107ea7a6a9ac916f635a70595a5b1a32fac10a8ff0b9f2ab88555550c", size = 529682, upload-time = "2026-10-11T01:01:48.477Z" }, + { url = "https://files.pythonhosted.org/packages/f4/4a/ddb923564e15e053b6e060b0036e1694dcadcb13aa476c5a87dcad20e336/aiohttp-3.14.5-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:b3cc509327c7b27f6f4727a8830f4004f6df7766e179f2f4b8e54e65c0bec5d3", size = 540120, upload-time = "2026-10-11T01:01:50.474Z" }, + { url = "https://files.pythonhosted.org/packages/3d/36/a640fbecaa53727a5900b892bdbe17b5f3e8cc88903e22864fe41b654def/aiohttp-3.14.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:7bd8ac754ebd6733a3e2a0dd1674c4d8ab086196803fd8dcd776f07b4e2607d9", size = 818795, upload-time = "2026-10-11T01:01:52.665Z" }, + { url = "https://files.pythonhosted.org/packages/ef/b6/d52ca608859e271b5fa7944074802dc45f60e52c318a4ddc34edbf73586e/aiohttp-3.14.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e724a7b6091f0b1ac064f9d1b15ff9ec52e6033a86cdae649e5f086e32a3c0db", size = 549667, upload-time = "2026-10-11T01:01:54.65Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b5/05b8ac39a76ff4bca89f42a4c2471560c71f894ff6e4bc16158c951874e3/aiohttp-3.14.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c32e26310cc10e547f53cd13d39a369034f69dcb7d749d5cb0e5f67bc196b6ba", size = 548922, upload-time = "2026-10-11T01:01:56.547Z" }, + { url = "https://files.pythonhosted.org/packages/19/b0/5aa186d56ce2334dabe29b70bd99dc8ae926ee44184c0de64a53d415a4f3/aiohttp-3.14.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1eb8167961ec4dfcc8cb9dd50bd0ee72519f7ef496be95203e49e27b01618382", size = 1882422, upload-time = "2026-10-11T01:01:59.258Z" }, + { url = "https://files.pythonhosted.org/packages/db/f7/7d5c91bb9620db300c8ddb05337a9014301acb626223faff3abcb8ea47d7/aiohttp-3.14.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c1d60eafd9c7e8e74abd03a5b00df44e7febfe6d9b89b559c0a6551eef0699d4", size = 1843621, upload-time = "2026-10-11T01:02:01.417Z" }, + { url = "https://files.pythonhosted.org/packages/30/0a/b208953b96d8f24b75f6da704f508e6c5cf3022f52b60c61933df082e89c/aiohttp-3.14.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:137351bf20bbed9a65e839f4a4452ac377389bdb2f2857d2acffef38f5e9f2d1", size = 1927915, upload-time = "2026-10-11T01:02:03.697Z" }, + { url = "https://files.pythonhosted.org/packages/f6/79/90ebcccb55e2d1e11a1fed581d83bb966e38fb35fb4b2577fdc980f8707a/aiohttp-3.14.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fba47bc2c3d7303c3d027c6cf4d07626c37b1314ac81f5820c31032e0ca1f677", size = 2036339, upload-time = "2026-10-11T01:02:06.046Z" }, + { url = "https://files.pythonhosted.org/packages/a6/66/55a8904b3a129fafdf94f9cc0a2e4ca09a3c914650be52355db7ad0bbdb6/aiohttp-3.14.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94684b879ac1d71e4238850c99b62dc1b28d9086b156a2555f082010b85a865c", size = 1888041, upload-time = "2026-10-11T01:02:08.384Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b8/96b25da7329a52e42c812b1e8b076386039ec4fc312afa043d173d8147fc/aiohttp-3.14.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:56572c42e3ecd636de8d2c3dd54cf5fc939cb5c32eb56297f176a0d366fac622", size = 1703849, upload-time = "2026-10-11T01:02:10.903Z" }, + { url = "https://files.pythonhosted.org/packages/1b/43/fbf976e3ae4c038d6f5c84945ab2150298c2d71201674d4e53d158063e75/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a95529a92a446db351675f4aab518feaf5e99842f63f5dd17160c2b74f382db3", size = 1849703, upload-time = "2026-10-11T01:02:13.15Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7d/218e912f4c1d89bde7ac551409be57ad2f6121638e56942d395a7cb1fa58/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:3edbece0379b8b4aaa67619b8aa2399bb66fce372cd5911098a434ea77220aa0", size = 1852695, upload-time = "2026-10-11T01:02:15.295Z" }, + { url = "https://files.pythonhosted.org/packages/5a/42/252a1b9287e3b6e393a1c3bd1776f36af30f5f25f071bbf2b0cb7eba9116/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:56d9828f204331a5ca8850fcfe2bcce95a149f1f223f60cc7216e5524978e480", size = 1893493, upload-time = "2026-10-11T01:02:17.93Z" }, + { url = "https://files.pythonhosted.org/packages/78/97/71cae83d5100556fad1521684f7cd1e3e578432644f850245ed3bd969310/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:20064a177a070d789ee64a50b01a9161d3468e989baacfc6c714aa685c4b332f", size = 1693468, upload-time = "2026-10-11T01:02:20.666Z" }, + { url = "https://files.pythonhosted.org/packages/1f/69/73d88e97a8b5f0ca7a946d0011c0de99fb188b1687c7948ecd0553dc5bf0/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:81c2b3dfd56c62bee6108e4852d5970b4cf9086390b6983f52b666e878c1f115", size = 1915398, upload-time = "2026-10-11T01:02:23.011Z" }, + { url = "https://files.pythonhosted.org/packages/05/f0/881644bcb15d4b258daea9b720a0af9dc4330496cc8d6ade9090cdd0cffc/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:09ec102b4b8c9a920275733bbc11fdbb615efe6f9231a06007c0218d336fb77a", size = 1871470, upload-time = "2026-10-11T01:02:25.278Z" }, + { url = "https://files.pythonhosted.org/packages/7f/de/19d9ebbcce5aedaa3242d8a99ff8816a60bdb7629fb0084bf4a45bfd1f62/aiohttp-3.14.5-cp314-cp314-win32.whl", hash = "sha256:9c428eb2bd8817588d16a0ab898aa4eb5d141f896aa2b394cc79a4cf61d9a8e2", size = 485517, upload-time = "2026-10-11T01:02:27.579Z" }, + { url = "https://files.pythonhosted.org/packages/a9/74/8cdaf0e58c2588371670d5a9a8215bbb971d36940b6e5d051967dd05c07d/aiohttp-3.14.5-cp314-cp314-win_amd64.whl", hash = "sha256:6f967dde489ca6a8c02d093ab245d2cbf50ccb5c36adf0188b17b0ca39d24b67", size = 511755, upload-time = "2026-10-11T01:02:29.685Z" }, + { url = "https://files.pythonhosted.org/packages/1a/6b/e0100e25502430a531c7cf1482a378d0b65bf728ab60c01ee270e56bc469/aiohttp-3.14.5-cp314-cp314-win_arm64.whl", hash = "sha256:1d2d981b53dd09a319e3570ef8cc3bbc3ef86f5a7abef0f6b2bff3867db3a9e7", size = 495151, upload-time = "2026-10-11T01:02:32.163Z" }, + { url = "https://files.pythonhosted.org/packages/9d/c2/ca2ead7b655688c53c03aeeb6e96e6851c9ff08d6be7b13802f53a6ae8fd/aiohttp-3.14.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9ce66feae6ac65327379460380549bf1b8df8e17c4e25df2a2bcf168272e3bed", size = 855396, upload-time = "2026-10-11T01:02:34.443Z" }, + { url = "https://files.pythonhosted.org/packages/a7/70/22206fea409255a240c926ce11de48de354ae2bb90ca44f709c05497585d/aiohttp-3.14.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:27c2322e03f66101acb09869ce1cf1efc04994ee95e1735b69827bf8c8b9d781", size = 567734, upload-time = "2026-10-11T01:02:36.644Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e5/79a36c118308b56f8667d67e05d2fb6dc638ab45985704cdb199631bedaa/aiohttp-3.14.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ff75a7537413a86e7cafe98e0e1d6e3dc4b15c6349896e7d5c6b881bfdb6d550", size = 567379, upload-time = "2026-10-11T01:02:38.757Z" }, + { url = "https://files.pythonhosted.org/packages/63/a3/2ebec7dece3b1f02c30d2e484647f6f7b13952b1bb40a4cb285b849e8432/aiohttp-3.14.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c061aa954daaf57d2a4b8374f9fca621ef0e1b603584431c220c22458c59b6d", size = 2050686, upload-time = "2026-10-11T01:02:41.169Z" }, + { url = "https://files.pythonhosted.org/packages/3a/d2/7e4d093db2f4450482652e7ef19a9e19919028f5135aa52bc4078c3beb80/aiohttp-3.14.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1612fa5857b37bf32e5c1eaeefb96e3b01e9c70679eec81f0934e8a600080863", size = 1915077, upload-time = "2026-10-11T01:02:43.563Z" }, + { url = "https://files.pythonhosted.org/packages/a6/88/bd40d09958442a0a1df67da81de496361d6e2afc04f9db2950d835da750b/aiohttp-3.14.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:adbeee7d6fd4cf5fe0aece2fb3edc4243615d3180430ba8149d01a90670cac99", size = 2027204, upload-time = "2026-10-11T01:02:46.185Z" }, + { url = "https://files.pythonhosted.org/packages/63/eb/3a601c1f8d3103c1a60ea981f20924855da9f575d2007fc38f8898b792fb/aiohttp-3.14.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b2966998927d7bed9db12c0a4647b0c7b179755878fc9c357fe1ffd3e3b0c1a5", size = 2149520, upload-time = "2026-10-11T01:02:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/22/d0/4e41bfe1b1ce1cb6f6d2e59fa7a88ef5cf92c402b2d07e9018778ba9edbc/aiohttp-3.14.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e317e0fb6b16212c881d2205a7d87414c29acd69320b3aa6dce9d9c7b86fe4f", size = 2002039, upload-time = "2026-10-11T01:02:51.293Z" }, + { url = "https://files.pythonhosted.org/packages/6f/5e/72067019545c502b881b031153c437752ecef48d7d213bc0218ccebb4bfb/aiohttp-3.14.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:50343c1757b4b6f6708eeaf24534b32f19dfb99fb1b762c00420867a62fc81e0", size = 1786164, upload-time = "2026-10-11T01:02:53.533Z" }, + { url = "https://files.pythonhosted.org/packages/d9/fe/7741efd6119bfb7a00827fe6f7b84b4409de58d888ae21adc5a9a6824992/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:083673c7a94c3ea035caaa5ca04288bdb44887abfe1f5ba23294e6a4b03efd2d", size = 1970689, upload-time = "2026-10-11T01:02:55.888Z" }, + { url = "https://files.pythonhosted.org/packages/43/e7/342a13bf67f34d269bf2f7e870ecd72b99c832cc6f0a271a9210c0ebfb84/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2528cb4c6b92008c76ac9ac6298624069bb2db91ff4929905512d1d84485f658", size = 1951967, upload-time = "2026-10-11T01:02:58.467Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d4/fdb3b27340617e5e64df18a70fa89097778652ea5c7c7e2f79def76999c3/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b1b8ece1e71132d2afba4dbc0c3d62c766e25165990b25db1196c04969eb3d84", size = 1968511, upload-time = "2026-10-11T01:03:00.883Z" }, + { url = "https://files.pythonhosted.org/packages/83/b2/e8f88298de78d1a951f36f9f966d38ec6ed1d4721303ba02064a545e8aa6/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:fce9523df31cea6284f3e2c479876750d7687cf671d7b25d32b19effc0e86441", size = 1778497, upload-time = "2026-10-11T01:03:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/22/68/9ccdb93d664345c546be7f34480b921774d8c0d98f47e70d7e03b115d475/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:09e0eb18c7e0c8777e2f9149de63799195b9b3ca1b5c81ba6f32f2c6b8628210", size = 1999676, upload-time = "2026-10-11T01:03:05.829Z" }, + { url = "https://files.pythonhosted.org/packages/57/4a/a33cfa6dcb00e94194ae4fe710432ca4ed111e016356b4ba4d2f4c3a124c/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6774814fd5c338e72ee0da5cbb9432816df450e69c019f72b5d29bdec2a1792d", size = 1963437, upload-time = "2026-10-11T01:03:08.196Z" }, + { url = "https://files.pythonhosted.org/packages/f4/20/eacbecfea3b5c3dcbfc9b023e3a5460f43e5dda16d06a2877ebe7184c3f3/aiohttp-3.14.5-cp314-cp314t-win32.whl", hash = "sha256:33f706574e32c6e694f352a856e05caf18f7f2c871b3e87b41c55ea452b409ab", size = 503973, upload-time = "2026-10-11T01:03:10.481Z" }, + { url = "https://files.pythonhosted.org/packages/ff/78/18eec294f6c8c5dc845dcf6d730a0147d8d0f17e86138a7bdb85e43a30fa/aiohttp-3.14.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5ba14a839fbe87cf7c12a6b5661c05f324a296eb8363141edb3944ba63d4c9d3", size = 531190, upload-time = "2026-10-11T01:03:12.716Z" }, + { url = "https://files.pythonhosted.org/packages/9d/39/e53f8169acc85271ebd12b5b32ad7f1541b35639ccbe0f49034c64785d10/aiohttp-3.14.5-cp314-cp314t-win_arm64.whl", hash = "sha256:1061b364556e8172e8d46b0b183adeeb73e8c42d30ebc745591e1bd89acad52e", size = 510360, upload-time = "2026-10-11T01:03:15.08Z" }, + { url = "https://files.pythonhosted.org/packages/68/30/173960c42b05a6c59f7558e4b12a4b0d9ba376cf6aa9bde7f9e08a30ca8d/aiohttp-3.14.5-py3-none-any.whl", hash = "sha256:efc21a454892828368b11c2c780de0ff8bc991f73f6b99c6b66e56205470929b", size = 279517, upload-time = "2026-10-11T01:05:08.523Z" }, ] [[package]] @@ -10792,142 +10793,118 @@ wheels = [ [[package]] name = "yarl" -version = "1.23.0" +version = "1.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "multidict" }, { name = "propcache" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/23/6e/beb1beec874a72f23815c1434518bfc4ed2175065173fb138c3705f658d4/yarl-1.23.0.tar.gz", hash = "sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5", size = 194676, upload-time = "2026-03-01T22:07:53.373Z" } +sdist = { url = "https://files.pythonhosted.org/packages/75/16/e8be8e2fb175bbf41a0680381a319f1199fae256588241a2ac8677eafb49/yarl-1.25.1.tar.gz", hash = "sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7", size = 246245, upload-time = "2026-09-15T19:35:02.264Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/0d/9cc638702f6fc3c7a3685bcc8cf2a9ed7d6206e932a49f5242658047ef51/yarl-1.23.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cff6d44cb13d39db2663a22b22305d10855efa0fa8015ddeacc40bc59b9d8107", size = 123764, upload-time = "2026-03-01T22:04:09.7Z" }, - { url = "https://files.pythonhosted.org/packages/7a/35/5a553687c5793df5429cd1db45909d4f3af7eee90014888c208d086a44f0/yarl-1.23.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c53f8347cd4200f0d70a48ad059cabaf24f5adc6ba08622a23423bc7efa10d", size = 86282, upload-time = "2026-03-01T22:04:11.892Z" }, - { url = "https://files.pythonhosted.org/packages/68/2e/c5a2234238f8ce37a8312b52801ee74117f576b1539eec8404a480434acc/yarl-1.23.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a6940a074fb3c48356ed0158a3ca5699c955ee4185b4d7d619be3c327143e05", size = 86053, upload-time = "2026-03-01T22:04:13.292Z" }, - { url = "https://files.pythonhosted.org/packages/74/3f/bbd8ff36fb038622797ffbaf7db314918bb4d76f1cc8a4f9ca7a55fe5195/yarl-1.23.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ed5f69ce7be7902e5c70ea19eb72d20abf7d725ab5d49777d696e32d4fc1811d", size = 99395, upload-time = "2026-03-01T22:04:15.133Z" }, - { url = "https://files.pythonhosted.org/packages/77/04/9516bc4e269d2a3ec9c6779fcdeac51ce5b3a9b0156f06ac7152e5bba864/yarl-1.23.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:389871e65468400d6283c0308e791a640b5ab5c83bcee02a2f51295f95e09748", size = 92143, upload-time = "2026-03-01T22:04:16.829Z" }, - { url = "https://files.pythonhosted.org/packages/c7/63/88802d1f6b1cb1fc67d67a58cd0cf8a1790de4ce7946e434240f1d60ab4a/yarl-1.23.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dda608c88cf709b1d406bdfcd84d8d63cff7c9e577a403c6108ce8ce9dcc8764", size = 107643, upload-time = "2026-03-01T22:04:18.519Z" }, - { url = "https://files.pythonhosted.org/packages/8e/db/4f9b838f4d8bdd6f0f385aed8bbf21c71ed11a0b9983305c302cbd557815/yarl-1.23.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c4fe09e0780c6c3bf2b7d4af02ee2394439d11a523bbcf095cf4747c2932007", size = 108700, upload-time = "2026-03-01T22:04:20.373Z" }, - { url = "https://files.pythonhosted.org/packages/50/12/95a1d33f04a79c402664070d43b8b9f72dc18914e135b345b611b0b1f8cc/yarl-1.23.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c9921eb8bd12633b41ad27686bbb0b1a2a9b8452bfdf221e34f311e9942ed4", size = 102769, upload-time = "2026-03-01T22:04:23.055Z" }, - { url = "https://files.pythonhosted.org/packages/86/65/91a0285f51321369fd1a8308aa19207520c5f0587772cfc2e03fc2467e90/yarl-1.23.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5f10fd85e4b75967468af655228fbfd212bdf66db1c0d135065ce288982eda26", size = 101114, upload-time = "2026-03-01T22:04:25.031Z" }, - { url = "https://files.pythonhosted.org/packages/58/80/c7c8244fc3e5bc483dc71a09560f43b619fab29301a0f0a8f936e42865c7/yarl-1.23.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dbf507e9ef5688bada447a24d68b4b58dd389ba93b7afc065a2ba892bea54769", size = 98883, upload-time = "2026-03-01T22:04:27.281Z" }, - { url = "https://files.pythonhosted.org/packages/86/e7/71ca9cc9ca79c0b7d491216177d1aed559d632947b8ffb0ee60f7d8b23e3/yarl-1.23.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:85e9beda1f591bc73e77ea1c51965c68e98dafd0fec72cdd745f77d727466716", size = 94172, upload-time = "2026-03-01T22:04:28.554Z" }, - { url = "https://files.pythonhosted.org/packages/6a/3f/6c6c8a0fe29c26fb2db2e8d32195bb84ec1bfb8f1d32e7f73b787fcf349b/yarl-1.23.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0e1fdaa14ef51366d7757b45bde294e95f6c8c049194e793eedb8387c86d5993", size = 107010, upload-time = "2026-03-01T22:04:30.385Z" }, - { url = "https://files.pythonhosted.org/packages/56/38/12730c05e5ad40a76374d440ed8b0899729a96c250516d91c620a6e38fc2/yarl-1.23.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:75e3026ab649bf48f9a10c0134512638725b521340293f202a69b567518d94e0", size = 100285, upload-time = "2026-03-01T22:04:31.752Z" }, - { url = "https://files.pythonhosted.org/packages/34/92/6a7be9239f2347234e027284e7a5f74b1140cc86575e7b469d13fba1ebfe/yarl-1.23.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:80e6d33a3d42a7549b409f199857b4fb54e2103fc44fb87605b6663b7a7ff750", size = 108230, upload-time = "2026-03-01T22:04:33.844Z" }, - { url = "https://files.pythonhosted.org/packages/5e/81/4aebccfa9376bd98b9d8bfad20621a57d3e8cfc5b8631c1fa5f62cdd03f4/yarl-1.23.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5ec2f42d41ccbd5df0270d7df31618a8ee267bfa50997f5d720ddba86c4a83a6", size = 103008, upload-time = "2026-03-01T22:04:35.856Z" }, - { url = "https://files.pythonhosted.org/packages/38/0f/0b4e3edcec794a86b853b0c6396c0a888d72dfce19b2d88c02ac289fb6c1/yarl-1.23.0-cp310-cp310-win32.whl", hash = "sha256:debe9c4f41c32990771be5c22b56f810659f9ddf3d63f67abfdcaa2c6c9c5c1d", size = 83073, upload-time = "2026-03-01T22:04:38.268Z" }, - { url = "https://files.pythonhosted.org/packages/a0/71/ad95c33da18897e4c636528bbc24a1dd23fe16797de8bc4ec667b8db0ba4/yarl-1.23.0-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f043cb8a2d71c981c09c510da013bc79fd661f5c60139f00dd3c3cc4f2ffb", size = 87328, upload-time = "2026-03-01T22:04:39.558Z" }, - { url = "https://files.pythonhosted.org/packages/e2/14/dfa369523c79bccf9c9c746b0a63eb31f65db9418ac01275f7950962e504/yarl-1.23.0-cp310-cp310-win_arm64.whl", hash = "sha256:263cd4f47159c09b8b685890af949195b51d1aa82ba451c5847ca9bc6413c220", size = 82463, upload-time = "2026-03-01T22:04:41.454Z" }, - { url = "https://files.pythonhosted.org/packages/a2/aa/60da938b8f0997ba3a911263c40d82b6f645a67902a490b46f3355e10fae/yarl-1.23.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b35d13d549077713e4414f927cdc388d62e543987c572baee613bf82f11a4b99", size = 123641, upload-time = "2026-03-01T22:04:42.841Z" }, - { url = "https://files.pythonhosted.org/packages/24/84/e237607faf4e099dbb8a4f511cfd5efcb5f75918baad200ff7380635631b/yarl-1.23.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cbb0fef01f0c6b38cb0f39b1f78fc90b807e0e3c86a7ff3ce74ad77ce5c7880c", size = 86248, upload-time = "2026-03-01T22:04:44.757Z" }, - { url = "https://files.pythonhosted.org/packages/b2/0d/71ceabc14c146ba8ee3804ca7b3d42b1664c8440439de5214d366fec7d3a/yarl-1.23.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc52310451fc7c629e13c4e061cbe2dd01684d91f2f8ee2821b083c58bd72432", size = 85988, upload-time = "2026-03-01T22:04:46.365Z" }, - { url = "https://files.pythonhosted.org/packages/8c/6c/4a90d59c572e46b270ca132aca66954f1175abd691f74c1ef4c6711828e2/yarl-1.23.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2c6b50c7b0464165472b56b42d4c76a7b864597007d9c085e8b63e185cf4a7a", size = 100566, upload-time = "2026-03-01T22:04:47.639Z" }, - { url = "https://files.pythonhosted.org/packages/49/fb/c438fb5108047e629f6282a371e6e91cf3f97ee087c4fb748a1f32ceef55/yarl-1.23.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:aafe5dcfda86c8af00386d7781d4c2181b5011b7be3f2add5e99899ea925df05", size = 92079, upload-time = "2026-03-01T22:04:48.925Z" }, - { url = "https://files.pythonhosted.org/packages/d9/13/d269aa1aed3e4f50a5a103f96327210cc5fa5dd2d50882778f13c7a14606/yarl-1.23.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ee33b875f0b390564c1fb7bc528abf18c8ee6073b201c6ae8524aca778e2d83", size = 108741, upload-time = "2026-03-01T22:04:50.838Z" }, - { url = "https://files.pythonhosted.org/packages/85/fb/115b16f22c37ea4437d323e472945bea97301c8ec6089868fa560abab590/yarl-1.23.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4c41e021bc6d7affb3364dc1e1e5fa9582b470f283748784bd6ea0558f87f42c", size = 108099, upload-time = "2026-03-01T22:04:52.499Z" }, - { url = "https://files.pythonhosted.org/packages/9a/64/c53487d9f4968045b8afa51aed7ca44f58b2589e772f32745f3744476c82/yarl-1.23.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99c8a9ed30f4164bc4c14b37a90208836cbf50d4ce2a57c71d0f52c7fb4f7598", size = 102678, upload-time = "2026-03-01T22:04:55.176Z" }, - { url = "https://files.pythonhosted.org/packages/85/59/cd98e556fbb2bf8fab29c1a722f67ad45c5f3447cac798ab85620d1e70af/yarl-1.23.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f2af5c81a1f124609d5f33507082fc3f739959d4719b56877ab1ee7e7b3d602b", size = 100803, upload-time = "2026-03-01T22:04:56.588Z" }, - { url = "https://files.pythonhosted.org/packages/9e/c0/b39770b56d4a9f0bb5f77e2f1763cd2d75cc2f6c0131e3b4c360348fcd65/yarl-1.23.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6b41389c19b07c760c7e427a3462e8ab83c4bb087d127f0e854c706ce1b9215c", size = 100163, upload-time = "2026-03-01T22:04:58.492Z" }, - { url = "https://files.pythonhosted.org/packages/e7/64/6980f99ab00e1f0ff67cb84766c93d595b067eed07439cfccfc8fb28c1a6/yarl-1.23.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:1dc702e42d0684f42d6519c8d581e49c96cefaaab16691f03566d30658ee8788", size = 93859, upload-time = "2026-03-01T22:05:00.268Z" }, - { url = "https://files.pythonhosted.org/packages/38/69/912e6c5e146793e5d4b5fe39ff5b00f4d22463dfd5a162bec565ac757673/yarl-1.23.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:0e40111274f340d32ebcc0a5668d54d2b552a6cca84c9475859d364b380e3222", size = 108202, upload-time = "2026-03-01T22:05:02.273Z" }, - { url = "https://files.pythonhosted.org/packages/59/97/35ca6767524687ad64e5f5c31ad54bc76d585585a9fcb40f649e7e82ffed/yarl-1.23.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:4764a6a7588561a9aef92f65bda2c4fb58fe7c675c0883862e6df97559de0bfb", size = 99866, upload-time = "2026-03-01T22:05:03.597Z" }, - { url = "https://files.pythonhosted.org/packages/d3/1c/1a3387ee6d73589f6f2a220ae06f2984f6c20b40c734989b0a44f5987308/yarl-1.23.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:03214408cfa590df47728b84c679ae4ef00be2428e11630277be0727eba2d7cc", size = 107852, upload-time = "2026-03-01T22:05:04.986Z" }, - { url = "https://files.pythonhosted.org/packages/a4/b8/35c0750fcd5a3f781058bfd954515dd4b1eab45e218cbb85cf11132215f1/yarl-1.23.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:170e26584b060879e29fac213e4228ef063f39128723807a312e5c7fec28eff2", size = 102919, upload-time = "2026-03-01T22:05:06.397Z" }, - { url = "https://files.pythonhosted.org/packages/e5/1c/9a1979aec4a81896d597bcb2177827f2dbee3f5b7cc48b2d0dadb644b41d/yarl-1.23.0-cp311-cp311-win32.whl", hash = "sha256:51430653db848d258336cfa0244427b17d12db63d42603a55f0d4546f50f25b5", size = 82602, upload-time = "2026-03-01T22:05:08.444Z" }, - { url = "https://files.pythonhosted.org/packages/93/22/b85eca6fa2ad9491af48c973e4c8cf6b103a73dbb271fe3346949449fca0/yarl-1.23.0-cp311-cp311-win_amd64.whl", hash = "sha256:bf49a3ae946a87083ef3a34c8f677ae4243f5b824bfc4c69672e72b3d6719d46", size = 87461, upload-time = "2026-03-01T22:05:10.145Z" }, - { url = "https://files.pythonhosted.org/packages/93/95/07e3553fe6f113e6864a20bdc53a78113cda3b9ced8784ee52a52c9f80d8/yarl-1.23.0-cp311-cp311-win_arm64.whl", hash = "sha256:b39cb32a6582750b6cc77bfb3c49c0f8760dc18dc96ec9fb55fbb0f04e08b928", size = 82336, upload-time = "2026-03-01T22:05:11.554Z" }, - { url = "https://files.pythonhosted.org/packages/88/8a/94615bc31022f711add374097ad4144d569e95ff3c38d39215d07ac153a0/yarl-1.23.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860", size = 124737, upload-time = "2026-03-01T22:05:12.897Z" }, - { url = "https://files.pythonhosted.org/packages/e3/6f/c6554045d59d64052698add01226bc867b52fe4a12373415d7991fdca95d/yarl-1.23.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069", size = 87029, upload-time = "2026-03-01T22:05:14.376Z" }, - { url = "https://files.pythonhosted.org/packages/19/2a/725ecc166d53438bc88f76822ed4b1e3b10756e790bafd7b523fe97c322d/yarl-1.23.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25", size = 86310, upload-time = "2026-03-01T22:05:15.71Z" }, - { url = "https://files.pythonhosted.org/packages/99/30/58260ed98e6ff7f90ba84442c1ddd758c9170d70327394a6227b310cd60f/yarl-1.23.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8", size = 97587, upload-time = "2026-03-01T22:05:17.384Z" }, - { url = "https://files.pythonhosted.org/packages/76/0a/8b08aac08b50682e65759f7f8dde98ae8168f72487e7357a5d684c581ef9/yarl-1.23.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072", size = 92528, upload-time = "2026-03-01T22:05:18.804Z" }, - { url = "https://files.pythonhosted.org/packages/52/07/0b7179101fe5f8385ec6c6bb5d0cb9f76bd9fb4a769591ab6fb5cdbfc69a/yarl-1.23.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8", size = 105339, upload-time = "2026-03-01T22:05:20.235Z" }, - { url = "https://files.pythonhosted.org/packages/d3/8a/36d82869ab5ec829ca8574dfcb92b51286fcfb1e9c7a73659616362dc880/yarl-1.23.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7", size = 105061, upload-time = "2026-03-01T22:05:22.268Z" }, - { url = "https://files.pythonhosted.org/packages/66/3e/868e5c3364b6cee19ff3e1a122194fa4ce51def02c61023970442162859e/yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51", size = 100132, upload-time = "2026-03-01T22:05:23.638Z" }, - { url = "https://files.pythonhosted.org/packages/cf/26/9c89acf82f08a52cb52d6d39454f8d18af15f9d386a23795389d1d423823/yarl-1.23.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67", size = 99289, upload-time = "2026-03-01T22:05:25.749Z" }, - { url = "https://files.pythonhosted.org/packages/6f/54/5b0db00d2cb056922356104468019c0a132e89c8d3ab67d8ede9f4483d2a/yarl-1.23.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7", size = 96950, upload-time = "2026-03-01T22:05:27.318Z" }, - { url = "https://files.pythonhosted.org/packages/f6/40/10fa93811fd439341fad7e0718a86aca0de9548023bbb403668d6555acab/yarl-1.23.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d", size = 93960, upload-time = "2026-03-01T22:05:28.738Z" }, - { url = "https://files.pythonhosted.org/packages/bc/d2/8ae2e6cd77d0805f4526e30ec43b6f9a3dfc542d401ac4990d178e4bf0cf/yarl-1.23.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760", size = 104703, upload-time = "2026-03-01T22:05:30.438Z" }, - { url = "https://files.pythonhosted.org/packages/2f/0c/b3ceacf82c3fe21183ce35fa2acf5320af003d52bc1fcf5915077681142e/yarl-1.23.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2", size = 98325, upload-time = "2026-03-01T22:05:31.835Z" }, - { url = "https://files.pythonhosted.org/packages/9d/e0/12900edd28bdab91a69bd2554b85ad7b151f64e8b521fe16f9ad2f56477a/yarl-1.23.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86", size = 105067, upload-time = "2026-03-01T22:05:33.358Z" }, - { url = "https://files.pythonhosted.org/packages/15/61/74bb1182cf79c9bbe4eb6b1f14a57a22d7a0be5e9cedf8e2d5c2086474c3/yarl-1.23.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34", size = 100285, upload-time = "2026-03-01T22:05:35.4Z" }, - { url = "https://files.pythonhosted.org/packages/69/7f/cd5ef733f2550de6241bd8bd8c3febc78158b9d75f197d9c7baa113436af/yarl-1.23.0-cp312-cp312-win32.whl", hash = "sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d", size = 82359, upload-time = "2026-03-01T22:05:36.811Z" }, - { url = "https://files.pythonhosted.org/packages/f5/be/25216a49daeeb7af2bec0db22d5e7df08ed1d7c9f65d78b14f3b74fd72fc/yarl-1.23.0-cp312-cp312-win_amd64.whl", hash = "sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e", size = 87674, upload-time = "2026-03-01T22:05:38.171Z" }, - { url = "https://files.pythonhosted.org/packages/d2/35/aeab955d6c425b227d5b7247eafb24f2653fedc32f95373a001af5dfeb9e/yarl-1.23.0-cp312-cp312-win_arm64.whl", hash = "sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9", size = 81879, upload-time = "2026-03-01T22:05:40.006Z" }, - { url = "https://files.pythonhosted.org/packages/9a/4b/a0a6e5d0ee8a2f3a373ddef8a4097d74ac901ac363eea1440464ccbe0898/yarl-1.23.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:16c6994ac35c3e74fb0ae93323bf8b9c2a9088d55946109489667c510a7d010e", size = 123796, upload-time = "2026-03-01T22:05:41.412Z" }, - { url = "https://files.pythonhosted.org/packages/67/b6/8925d68af039b835ae876db5838e82e76ec87b9782ecc97e192b809c4831/yarl-1.23.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4a42e651629dafb64fd5b0286a3580613702b5809ad3f24934ea87595804f2c5", size = 86547, upload-time = "2026-03-01T22:05:42.841Z" }, - { url = "https://files.pythonhosted.org/packages/ae/50/06d511cc4b8e0360d3c94af051a768e84b755c5eb031b12adaaab6dec6e5/yarl-1.23.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7c6b9461a2a8b47c65eef63bb1c76a4f1c119618ffa99ea79bc5bb1e46c5821b", size = 85854, upload-time = "2026-03-01T22:05:44.85Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f4/4e30b250927ffdab4db70da08b9b8d2194d7c7b400167b8fbeca1e4701ca/yarl-1.23.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2569b67d616eab450d262ca7cb9f9e19d2f718c70a8b88712859359d0ab17035", size = 98351, upload-time = "2026-03-01T22:05:46.836Z" }, - { url = "https://files.pythonhosted.org/packages/86/fc/4118c5671ea948208bdb1492d8b76bdf1453d3e73df051f939f563e7dcc5/yarl-1.23.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e9d9a4d06d3481eab79803beb4d9bd6f6a8e781ec078ac70d7ef2dcc29d1bea5", size = 92711, upload-time = "2026-03-01T22:05:48.316Z" }, - { url = "https://files.pythonhosted.org/packages/56/11/1ed91d42bd9e73c13dc9e7eb0dd92298d75e7ac4dd7f046ad0c472e231cd/yarl-1.23.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f514f6474e04179d3d33175ed3f3e31434d3130d42ec153540d5b157deefd735", size = 106014, upload-time = "2026-03-01T22:05:50.028Z" }, - { url = "https://files.pythonhosted.org/packages/ce/c9/74e44e056a23fbc33aca71779ef450ca648a5bc472bdad7a82339918f818/yarl-1.23.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fda207c815b253e34f7e1909840fd14299567b1c0eb4908f8c2ce01a41265401", size = 105557, upload-time = "2026-03-01T22:05:51.416Z" }, - { url = "https://files.pythonhosted.org/packages/66/fe/b1e10b08d287f518994f1e2ff9b6d26f0adeecd8dd7d533b01bab29a3eda/yarl-1.23.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34b6cf500e61c90f305094911f9acc9c86da1a05a7a3f5be9f68817043f486e4", size = 101559, upload-time = "2026-03-01T22:05:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/72/59/c5b8d94b14e3d3c2a9c20cb100119fd534ab5a14b93673ab4cc4a4141ea5/yarl-1.23.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d7504f2b476d21653e4d143f44a175f7f751cd41233525312696c76aa3dbb23f", size = 100502, upload-time = "2026-03-01T22:05:54.954Z" }, - { url = "https://files.pythonhosted.org/packages/77/4f/96976cb54cbfc5c9fd73ed4c51804f92f209481d1fb190981c0f8a07a1d7/yarl-1.23.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:578110dd426f0d209d1509244e6d4a3f1a3e9077655d98c5f22583d63252a08a", size = 98027, upload-time = "2026-03-01T22:05:56.409Z" }, - { url = "https://files.pythonhosted.org/packages/63/6e/904c4f476471afdbad6b7e5b70362fb5810e35cd7466529a97322b6f5556/yarl-1.23.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:609d3614d78d74ebe35f54953c5bbd2ac647a7ddb9c30a5d877580f5e86b22f2", size = 95369, upload-time = "2026-03-01T22:05:58.141Z" }, - { url = "https://files.pythonhosted.org/packages/9d/40/acfcdb3b5f9d68ef499e39e04d25e141fe90661f9d54114556cf83be8353/yarl-1.23.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4966242ec68afc74c122f8459abd597afd7d8a60dc93d695c1334c5fd25f762f", size = 105565, upload-time = "2026-03-01T22:06:00.286Z" }, - { url = "https://files.pythonhosted.org/packages/5e/c6/31e28f3a6ba2869c43d124f37ea5260cac9c9281df803c354b31f4dd1f3c/yarl-1.23.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e0fd068364a6759bc794459f0a735ab151d11304346332489c7972bacbe9e72b", size = 99813, upload-time = "2026-03-01T22:06:01.712Z" }, - { url = "https://files.pythonhosted.org/packages/08/1f/6f65f59e72d54aa467119b63fc0b0b1762eff0232db1f4720cd89e2f4a17/yarl-1.23.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:39004f0ad156da43e86aa71f44e033de68a44e5a31fc53507b36dd253970054a", size = 105632, upload-time = "2026-03-01T22:06:03.188Z" }, - { url = "https://files.pythonhosted.org/packages/a3/c4/18b178a69935f9e7a338127d5b77d868fdc0f0e49becd286d51b3a18c61d/yarl-1.23.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e5723c01a56c5028c807c701aa66722916d2747ad737a046853f6c46f4875543", size = 101895, upload-time = "2026-03-01T22:06:04.651Z" }, - { url = "https://files.pythonhosted.org/packages/8f/54/f5b870b5505663911dba950a8e4776a0dbd51c9c54c0ae88e823e4b874a0/yarl-1.23.0-cp313-cp313-win32.whl", hash = "sha256:1b6b572edd95b4fa8df75de10b04bc81acc87c1c7d16bcdd2035b09d30acc957", size = 82356, upload-time = "2026-03-01T22:06:06.04Z" }, - { url = "https://files.pythonhosted.org/packages/7a/84/266e8da36879c6edcd37b02b547e2d9ecdfea776be49598e75696e3316e1/yarl-1.23.0-cp313-cp313-win_amd64.whl", hash = "sha256:baaf55442359053c7d62f6f8413a62adba3205119bcb6f49594894d8be47e5e3", size = 87515, upload-time = "2026-03-01T22:06:08.107Z" }, - { url = "https://files.pythonhosted.org/packages/00/fd/7e1c66efad35e1649114fa13f17485f62881ad58edeeb7f49f8c5e748bf9/yarl-1.23.0-cp313-cp313-win_arm64.whl", hash = "sha256:fb4948814a2a98e3912505f09c9e7493b1506226afb1f881825368d6fb776ee3", size = 81785, upload-time = "2026-03-01T22:06:10.181Z" }, - { url = "https://files.pythonhosted.org/packages/9c/fc/119dd07004f17ea43bb91e3ece6587759edd7519d6b086d16bfbd3319982/yarl-1.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:aecfed0b41aa72b7881712c65cf764e39ce2ec352324f5e0837c7048d9e6daaa", size = 130719, upload-time = "2026-03-01T22:06:11.708Z" }, - { url = "https://files.pythonhosted.org/packages/e6/0d/9f2348502fbb3af409e8f47730282cd6bc80dec6630c1e06374d882d6eb2/yarl-1.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a41bcf68efd19073376eb8cf948b8d9be0af26256403e512bb18f3966f1f9120", size = 89690, upload-time = "2026-03-01T22:06:13.429Z" }, - { url = "https://files.pythonhosted.org/packages/50/93/e88f3c80971b42cfc83f50a51b9d165a1dbf154b97005f2994a79f212a07/yarl-1.23.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cde9a2ecd91668bcb7f077c4966d8ceddb60af01b52e6e3e2680e4cf00ad1a59", size = 89851, upload-time = "2026-03-01T22:06:15.53Z" }, - { url = "https://files.pythonhosted.org/packages/1c/07/61c9dd8ba8f86473263b4036f70fb594c09e99c0d9737a799dfd8bc85651/yarl-1.23.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5023346c4ee7992febc0068e7593de5fa2bf611848c08404b35ebbb76b1b0512", size = 95874, upload-time = "2026-03-01T22:06:17.553Z" }, - { url = "https://files.pythonhosted.org/packages/9e/e9/f9ff8ceefba599eac6abddcfb0b3bee9b9e636e96dbf54342a8577252379/yarl-1.23.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d1009abedb49ae95b136a8904a3f71b342f849ffeced2d3747bf29caeda218c4", size = 88710, upload-time = "2026-03-01T22:06:19.004Z" }, - { url = "https://files.pythonhosted.org/packages/eb/78/0231bfcc5d4c8eec220bc2f9ef82cb4566192ea867a7c5b4148f44f6cbcd/yarl-1.23.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a8d00f29b42f534cc8aa3931cfe773b13b23e561e10d2b26f27a8d309b0e82a1", size = 101033, upload-time = "2026-03-01T22:06:21.203Z" }, - { url = "https://files.pythonhosted.org/packages/cd/9b/30ea5239a61786f18fd25797151a17fbb3be176977187a48d541b5447dd4/yarl-1.23.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:95451e6ce06c3e104556d73b559f5da6c34a069b6b62946d3ad66afcd51642ea", size = 100817, upload-time = "2026-03-01T22:06:22.738Z" }, - { url = "https://files.pythonhosted.org/packages/62/e2/a4980481071791bc83bce2b7a1a1f7adcabfa366007518b4b845e92eeee3/yarl-1.23.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531ef597132086b6cf96faa7c6c1dcd0361dd5f1694e5cc30375907b9b7d3ea9", size = 97482, upload-time = "2026-03-01T22:06:24.21Z" }, - { url = "https://files.pythonhosted.org/packages/e5/1e/304a00cf5f6100414c4b5a01fc7ff9ee724b62158a08df2f8170dfc72a2d/yarl-1.23.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88f9fb0116fbfcefcab70f85cf4b74a2b6ce5d199c41345296f49d974ddb4123", size = 95949, upload-time = "2026-03-01T22:06:25.697Z" }, - { url = "https://files.pythonhosted.org/packages/68/03/093f4055ed4cae649ac53bca3d180bd37102e9e11d048588e9ab0c0108d0/yarl-1.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e7b0460976dc75cb87ad9cc1f9899a4b97751e7d4e77ab840fc9b6d377b8fd24", size = 95839, upload-time = "2026-03-01T22:06:27.309Z" }, - { url = "https://files.pythonhosted.org/packages/b9/28/4c75ebb108f322aa8f917ae10a8ffa4f07cae10a8a627b64e578617df6a0/yarl-1.23.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:115136c4a426f9da976187d238e84139ff6b51a20839aa6e3720cd1026d768de", size = 90696, upload-time = "2026-03-01T22:06:29.048Z" }, - { url = "https://files.pythonhosted.org/packages/23/9c/42c2e2dd91c1a570402f51bdf066bfdb1241c2240ba001967bad778e77b7/yarl-1.23.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:ead11956716a940c1abc816b7df3fa2b84d06eaed8832ca32f5c5e058c65506b", size = 100865, upload-time = "2026-03-01T22:06:30.525Z" }, - { url = "https://files.pythonhosted.org/packages/74/05/1bcd60a8a0a914d462c305137246b6f9d167628d73568505fce3f1cb2e65/yarl-1.23.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:fe8f8f5e70e6dbdfca9882cd9deaac058729bcf323cf7a58660901e55c9c94f6", size = 96234, upload-time = "2026-03-01T22:06:32.692Z" }, - { url = "https://files.pythonhosted.org/packages/90/b2/f52381aac396d6778ce516b7bc149c79e65bfc068b5de2857ab69eeea3b7/yarl-1.23.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:a0e317df055958a0c1e79e5d2aa5a5eaa4a6d05a20d4b0c9c3f48918139c9fc6", size = 100295, upload-time = "2026-03-01T22:06:34.268Z" }, - { url = "https://files.pythonhosted.org/packages/e5/e8/638bae5bbf1113a659b2435d8895474598afe38b4a837103764f603aba56/yarl-1.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6f0fd84de0c957b2d280143522c4f91a73aada1923caee763e24a2b3fda9f8a5", size = 97784, upload-time = "2026-03-01T22:06:35.864Z" }, - { url = "https://files.pythonhosted.org/packages/80/25/a3892b46182c586c202629fc2159aa13975d3741d52ebd7347fd501d48d5/yarl-1.23.0-cp313-cp313t-win32.whl", hash = "sha256:93a784271881035ab4406a172edb0faecb6e7d00f4b53dc2f55919d6c9688595", size = 88313, upload-time = "2026-03-01T22:06:37.39Z" }, - { url = "https://files.pythonhosted.org/packages/43/68/8c5b36aa5178900b37387937bc2c2fe0e9505537f713495472dcf6f6fccc/yarl-1.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:dd00607bffbf30250fe108065f07453ec124dbf223420f57f5e749b04295e090", size = 94932, upload-time = "2026-03-01T22:06:39.579Z" }, - { url = "https://files.pythonhosted.org/packages/c6/cc/d79ba8292f51f81f4dc533a8ccfb9fc6992cabf0998ed3245de7589dc07c/yarl-1.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ac09d42f48f80c9ee1635b2fcaa819496a44502737660d3c0f2ade7526d29144", size = 84786, upload-time = "2026-03-01T22:06:41.988Z" }, - { url = "https://files.pythonhosted.org/packages/90/98/b85a038d65d1b92c3903ab89444f48d3cee490a883477b716d7a24b1a78c/yarl-1.23.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:21d1b7305a71a15b4794b5ff22e8eef96ff4a6d7f9657155e5aa419444b28912", size = 124455, upload-time = "2026-03-01T22:06:43.615Z" }, - { url = "https://files.pythonhosted.org/packages/39/54/bc2b45559f86543d163b6e294417a107bb87557609007c007ad889afec18/yarl-1.23.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:85610b4f27f69984932a7abbe52703688de3724d9f72bceb1cca667deff27474", size = 86752, upload-time = "2026-03-01T22:06:45.425Z" }, - { url = "https://files.pythonhosted.org/packages/24/f9/e8242b68362bffe6fb536c8db5076861466fc780f0f1b479fc4ffbebb128/yarl-1.23.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23f371bd662cf44a7630d4d113101eafc0cfa7518a2760d20760b26021454719", size = 86291, upload-time = "2026-03-01T22:06:46.974Z" }, - { url = "https://files.pythonhosted.org/packages/ea/d8/d1cb2378c81dd729e98c716582b1ccb08357e8488e4c24714658cc6630e8/yarl-1.23.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a80f77dc1acaaa61f0934176fccca7096d9b1ff08c8ba9cddf5ae034a24319", size = 99026, upload-time = "2026-03-01T22:06:48.459Z" }, - { url = "https://files.pythonhosted.org/packages/0a/ff/7196790538f31debe3341283b5b0707e7feb947620fc5e8236ef28d44f72/yarl-1.23.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:bd654fad46d8d9e823afbb4f87c79160b5a374ed1ff5bde24e542e6ba8f41434", size = 92355, upload-time = "2026-03-01T22:06:50.306Z" }, - { url = "https://files.pythonhosted.org/packages/c1/56/25d58c3eddde825890a5fe6aa1866228377354a3c39262235234ab5f616b/yarl-1.23.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:682bae25f0a0dd23a056739f23a134db9f52a63e2afd6bfb37ddc76292bbd723", size = 106417, upload-time = "2026-03-01T22:06:52.1Z" }, - { url = "https://files.pythonhosted.org/packages/51/8a/882c0e7bc8277eb895b31bce0138f51a1ba551fc2e1ec6753ffc1e7c1377/yarl-1.23.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a82836cab5f197a0514235aaf7ffccdc886ccdaa2324bc0aafdd4ae898103039", size = 106422, upload-time = "2026-03-01T22:06:54.424Z" }, - { url = "https://files.pythonhosted.org/packages/42/2b/fef67d616931055bf3d6764885990a3ac647d68734a2d6a9e1d13de437a2/yarl-1.23.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c57676bdedc94cd3bc37724cf6f8cd2779f02f6aba48de45feca073e714fe52", size = 101915, upload-time = "2026-03-01T22:06:55.895Z" }, - { url = "https://files.pythonhosted.org/packages/18/6a/530e16aebce27c5937920f3431c628a29a4b6b430fab3fd1c117b26ff3f6/yarl-1.23.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c7f8dc16c498ff06497c015642333219871effba93e4a2e8604a06264aca5c5c", size = 100690, upload-time = "2026-03-01T22:06:58.21Z" }, - { url = "https://files.pythonhosted.org/packages/88/08/93749219179a45e27b036e03260fda05190b911de8e18225c294ac95bbc9/yarl-1.23.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5ee586fb17ff8f90c91cf73c6108a434b02d69925f44f5f8e0d7f2f260607eae", size = 98750, upload-time = "2026-03-01T22:06:59.794Z" }, - { url = "https://files.pythonhosted.org/packages/d9/cf/ea424a004969f5d81a362110a6ac1496d79efdc6d50c2c4b2e3ea0fc2519/yarl-1.23.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:17235362f580149742739cc3828b80e24029d08cbb9c4bda0242c7b5bc610a8e", size = 94685, upload-time = "2026-03-01T22:07:01.375Z" }, - { url = "https://files.pythonhosted.org/packages/e2/b7/14341481fe568e2b0408bcf1484c652accafe06a0ade9387b5d3fd9df446/yarl-1.23.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:0793e2bd0cf14234983bbb371591e6bea9e876ddf6896cdcc93450996b0b5c85", size = 106009, upload-time = "2026-03-01T22:07:03.151Z" }, - { url = "https://files.pythonhosted.org/packages/0a/e6/5c744a9b54f4e8007ad35bce96fbc9218338e84812d36f3390cea616881a/yarl-1.23.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3650dc2480f94f7116c364096bc84b1d602f44224ef7d5c7208425915c0475dd", size = 100033, upload-time = "2026-03-01T22:07:04.701Z" }, - { url = "https://files.pythonhosted.org/packages/0c/23/e3bfc188d0b400f025bc49d99793d02c9abe15752138dcc27e4eaf0c4a9e/yarl-1.23.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f40e782d49630ad384db66d4d8b73ff4f1b8955dc12e26b09a3e3af064b3b9d6", size = 106483, upload-time = "2026-03-01T22:07:06.231Z" }, - { url = "https://files.pythonhosted.org/packages/72/42/f0505f949a90b3f8b7a363d6cbdf398f6e6c58946d85c6d3a3bc70595b26/yarl-1.23.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94f8575fbdf81749008d980c17796097e645574a3b8c28ee313931068dad14fe", size = 102175, upload-time = "2026-03-01T22:07:08.4Z" }, - { url = "https://files.pythonhosted.org/packages/aa/65/b39290f1d892a9dd671d1c722014ca062a9c35d60885d57e5375db0404b5/yarl-1.23.0-cp314-cp314-win32.whl", hash = "sha256:c8aa34a5c864db1087d911a0b902d60d203ea3607d91f615acd3f3108ac32169", size = 83871, upload-time = "2026-03-01T22:07:09.968Z" }, - { url = "https://files.pythonhosted.org/packages/a9/5b/9b92f54c784c26e2a422e55a8d2607ab15b7ea3349e28359282f84f01d43/yarl-1.23.0-cp314-cp314-win_amd64.whl", hash = "sha256:63e92247f383c85ab00dd0091e8c3fa331a96e865459f5ee80353c70a4a42d70", size = 89093, upload-time = "2026-03-01T22:07:11.501Z" }, - { url = "https://files.pythonhosted.org/packages/e0/7d/8a84dc9381fd4412d5e7ff04926f9865f6372b4c2fd91e10092e65d29eb8/yarl-1.23.0-cp314-cp314-win_arm64.whl", hash = "sha256:70efd20be968c76ece7baa8dafe04c5be06abc57f754d6f36f3741f7aa7a208e", size = 83384, upload-time = "2026-03-01T22:07:13.069Z" }, - { url = "https://files.pythonhosted.org/packages/dd/8d/d2fad34b1c08aa161b74394183daa7d800141aaaee207317e82c790b418d/yarl-1.23.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9a18d6f9359e45722c064c97464ec883eb0e0366d33eda61cb19a244bf222679", size = 131019, upload-time = "2026-03-01T22:07:14.903Z" }, - { url = "https://files.pythonhosted.org/packages/19/ff/33009a39d3ccf4b94d7d7880dfe17fb5816c5a4fe0096d9b56abceea9ac7/yarl-1.23.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2803ed8b21ca47a43da80a6fd1ed3019d30061f7061daa35ac54f63933409412", size = 89894, upload-time = "2026-03-01T22:07:17.372Z" }, - { url = "https://files.pythonhosted.org/packages/0c/f1/dab7ac5e7306fb79c0190766a3c00b4cb8d09a1f390ded68c85a5934faf5/yarl-1.23.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:394906945aa8b19fc14a61cf69743a868bb8c465efe85eee687109cc540b98f4", size = 89979, upload-time = "2026-03-01T22:07:19.361Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b1/08e95f3caee1fad6e65017b9f26c1d79877b502622d60e517de01e72f95d/yarl-1.23.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71d006bee8397a4a89f469b8deb22469fe7508132d3c17fa6ed871e79832691c", size = 95943, upload-time = "2026-03-01T22:07:21.266Z" }, - { url = "https://files.pythonhosted.org/packages/c0/cc/6409f9018864a6aa186c61175b977131f373f1988e198e031236916e87e4/yarl-1.23.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:62694e275c93d54f7ccedcfef57d42761b2aad5234b6be1f3e3026cae4001cd4", size = 88786, upload-time = "2026-03-01T22:07:23.129Z" }, - { url = "https://files.pythonhosted.org/packages/76/40/cc22d1d7714b717fde2006fad2ced5efe5580606cb059ae42117542122f3/yarl-1.23.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a31de1613658308efdb21ada98cbc86a97c181aa050ba22a808120bb5be3ab94", size = 101307, upload-time = "2026-03-01T22:07:24.689Z" }, - { url = "https://files.pythonhosted.org/packages/8f/0d/476c38e85ddb4c6ec6b20b815bdd779aa386a013f3d8b85516feee55c8dc/yarl-1.23.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb1e8b8d66c278b21d13b0a7ca22c41dd757a7c209c6b12c313e445c31dd3b28", size = 100904, upload-time = "2026-03-01T22:07:26.287Z" }, - { url = "https://files.pythonhosted.org/packages/72/32/0abe4a76d59adf2081dcb0397168553ece4616ada1c54d1c49d8936c74f8/yarl-1.23.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50f9d8d531dfb767c565f348f33dd5139a6c43f5cbdf3f67da40d54241df93f6", size = 97728, upload-time = "2026-03-01T22:07:27.906Z" }, - { url = "https://files.pythonhosted.org/packages/b7/35/7b30f4810fba112f60f5a43237545867504e15b1c7647a785fbaf588fac2/yarl-1.23.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575aa4405a656e61a540f4a80eaa5260f2a38fff7bfdc4b5f611840d76e9e277", size = 95964, upload-time = "2026-03-01T22:07:30.198Z" }, - { url = "https://files.pythonhosted.org/packages/2d/86/ed7a73ab85ef00e8bb70b0cb5421d8a2a625b81a333941a469a6f4022828/yarl-1.23.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:041b1a4cefacf65840b4e295c6985f334ba83c30607441ae3cf206a0eed1a2e4", size = 95882, upload-time = "2026-03-01T22:07:32.132Z" }, - { url = "https://files.pythonhosted.org/packages/19/90/d56967f61a29d8498efb7afb651e0b2b422a1e9b47b0ab5f4e40a19b699b/yarl-1.23.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d38c1e8231722c4ce40d7593f28d92b5fc72f3e9774fe73d7e800ec32299f63a", size = 90797, upload-time = "2026-03-01T22:07:34.404Z" }, - { url = "https://files.pythonhosted.org/packages/72/00/8b8f76909259f56647adb1011d7ed8b321bcf97e464515c65016a47ecdf0/yarl-1.23.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d53834e23c015ee83a99377db6e5e37d8484f333edb03bd15b4bc312cc7254fb", size = 101023, upload-time = "2026-03-01T22:07:35.953Z" }, - { url = "https://files.pythonhosted.org/packages/ac/e2/cab11b126fb7d440281b7df8e9ddbe4851e70a4dde47a202b6642586b8d9/yarl-1.23.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2e27c8841126e017dd2a054a95771569e6070b9ee1b133366d8b31beb5018a41", size = 96227, upload-time = "2026-03-01T22:07:37.594Z" }, - { url = "https://files.pythonhosted.org/packages/c2/9b/2c893e16bfc50e6b2edf76c1a9eb6cb0c744346197e74c65e99ad8d634d0/yarl-1.23.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:76855800ac56f878847a09ce6dba727c93ca2d89c9e9d63002d26b916810b0a2", size = 100302, upload-time = "2026-03-01T22:07:39.334Z" }, - { url = "https://files.pythonhosted.org/packages/28/ec/5498c4e3a6d5f1003beb23405671c2eb9cdbf3067d1c80f15eeafe301010/yarl-1.23.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e09fd068c2e169a7070d83d3bde728a4d48de0549f975290be3c108c02e499b4", size = 98202, upload-time = "2026-03-01T22:07:41.717Z" }, - { url = "https://files.pythonhosted.org/packages/fe/c3/cd737e2d45e70717907f83e146f6949f20cc23cd4bf7b2688727763aa458/yarl-1.23.0-cp314-cp314t-win32.whl", hash = "sha256:73309162a6a571d4cbd3b6a1dcc703c7311843ae0d1578df6f09be4e98df38d4", size = 90558, upload-time = "2026-03-01T22:07:43.433Z" }, - { url = "https://files.pythonhosted.org/packages/e1/19/3774d162f6732d1cfb0b47b4140a942a35ca82bb19b6db1f80e9e7bdc8f8/yarl-1.23.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4503053d296bc6e4cbd1fad61cf3b6e33b939886c4f249ba7c78b602214fabe2", size = 97610, upload-time = "2026-03-01T22:07:45.773Z" }, - { url = "https://files.pythonhosted.org/packages/51/47/3fa2286c3cb162c71cdb34c4224d5745a1ceceb391b2bd9b19b668a8d724/yarl-1.23.0-cp314-cp314t-win_arm64.whl", hash = "sha256:44bb7bef4ea409384e3f8bc36c063d77ea1b8d4a5b2706956c0d6695f07dcc25", size = 86041, upload-time = "2026-03-01T22:07:49.026Z" }, - { url = "https://files.pythonhosted.org/packages/69/68/c8739671f5699c7dc470580a4f821ef37c32c4cb0b047ce223a7f115757f/yarl-1.23.0-py3-none-any.whl", hash = "sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f", size = 48288, upload-time = "2026-03-01T22:07:51.388Z" }, + { url = "https://files.pythonhosted.org/packages/85/92/55fa9ee84cb8ec9930a910b5753936a926f918d8cc8965bdac571479c095/yarl-1.25.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:142c06c4d6a35ee3ec5da08499805e879cb3ca7c1fbfbecb0140fe72403818d6", size = 144695, upload-time = "2026-09-15T19:29:53.114Z" }, + { url = "https://files.pythonhosted.org/packages/16/b4/9edc8e605b16b2eb5bd0c2ccc73e2b15aab583862460767fb43f91f11839/yarl-1.25.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:24ce942011a61953e7d313438038f4d32ff21387b775f58a957f7a07dd55ef95", size = 104332, upload-time = "2026-09-15T19:29:55.443Z" }, + { url = "https://files.pythonhosted.org/packages/ae/de/4204e6646e278b1cd4a9cf73ebe8b7cfcf16693f1249b324eb273d67602a/yarl-1.25.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e23c82b63cd7652fc24d33ed6cc17099d607aa3b4fc4ddc75e95062f3d82df4", size = 104449, upload-time = "2026-09-15T19:29:57.239Z" }, + { url = "https://files.pythonhosted.org/packages/1d/fc/21d74789198ad68a23a209ab0d73fe5a52d39ebbd9945041e72827acdd77/yarl-1.25.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ee202350cf57abf0e9502a41601841019c25d3db7ff52d980aaf31446254059", size = 116828, upload-time = "2026-09-15T19:29:59.031Z" }, + { url = "https://files.pythonhosted.org/packages/8f/31/90891ddb61848c067ebd1fb505ab5902a6e7d4707c2f3823f67ff37883d3/yarl-1.25.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5df89f769cc8ff94c3d7e7603386fba309d25ce5240132d26c15baa8d0e96c4c", size = 107150, upload-time = "2026-09-15T19:30:00.868Z" }, + { url = "https://files.pythonhosted.org/packages/a1/15/f3d18743d3688b5f01aa83034b6c00665692ab20e1d0c89a29000d4979ea/yarl-1.25.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:83e9f4a25085bd4b7214701a0794ff1f50fc633ffb8bdfebf07abdd81c2db126", size = 124704, upload-time = "2026-09-15T19:30:03.128Z" }, + { url = "https://files.pythonhosted.org/packages/99/37/718789d8004775d6a2db86b2afc72b1d156e0590b88a1b8634ddd4bdb8ee/yarl-1.25.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e636b64d24fd9c38053c5e389a1174c66361fa49dcfd220f4dd35b4abde7cb89", size = 129246, upload-time = "2026-09-15T19:30:05.014Z" }, + { url = "https://files.pythonhosted.org/packages/28/b9/7818ac6dec7fbcd16be2d19495807c01e2c38834e57d41c63356a75e786b/yarl-1.25.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e5637ca8d0bd7fb72648a6c7934af4baaccb697657f7438c9d264fc2abb8b0b1", size = 118071, upload-time = "2026-09-15T19:30:06.977Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e3/8d098cafb30a64df7283b5a5dd0d32d22a71b18a1b9ec071d0776ffe5c2a/yarl-1.25.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:683e362b8ba453080f7489c66f4ea794e751c35b72e7eab3575ef784c2fbc7fb", size = 116180, upload-time = "2026-09-15T19:30:08.836Z" }, + { url = "https://files.pythonhosted.org/packages/20/9c/fbe9432478d87e00eef80242e15179639df162cb3d5d8d978fc56dcb6c51/yarl-1.25.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:df23df54b5114a17c2d0ef192433e2e5a9f0c5178c32375e90b7cfc965f349d0", size = 116578, upload-time = "2026-09-15T19:30:10.782Z" }, + { url = "https://files.pythonhosted.org/packages/b1/8e/b3dfd03732236b86b0bcf06a72a37de412ff6f82e719a947c3c651d4dece/yarl-1.25.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326", size = 108886, upload-time = "2026-09-15T19:30:12.951Z" }, + { url = "https://files.pythonhosted.org/packages/83/e1/94dacb650d4963d5f844bc3df7ba70288cb7b68ed5a4d070da71acb19b14/yarl-1.25.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:b8075fe90bc08e40b8b8a1874fab42ee4c7b56af05c5886e9cc841397f916908", size = 124098, upload-time = "2026-09-15T19:30:14.963Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3f/c93f76a218258c7bfc60dd27fd56e9102bdbc625517f4a6ee673295fe4e4/yarl-1.25.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:a8c2b841478068440d8b733005d13a5ef535b9928cbc05f17182d410f32ba449", size = 115768, upload-time = "2026-09-15T19:30:16.736Z" }, + { url = "https://files.pythonhosted.org/packages/a8/79/4d93f13c3b05cda3c962805dec28cbc255c50239b3457808abc5633a00c2/yarl-1.25.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:ca32926d7d77bcc8838425c4c95e040a3ace1cb7dfdae599013458dcda2607ca", size = 122461, upload-time = "2026-09-15T19:30:18.901Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8e/e2ba83b3a9bfc1d3b882ad35fa8abc04a7af4d9356f2e329a23d0c8d889d/yarl-1.25.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:192a866877a49993949ef1975864ad8728bea28ee810f6abe1a0729c2b500426", size = 118295, upload-time = "2026-09-15T19:30:21.07Z" }, + { url = "https://files.pythonhosted.org/packages/42/67/cb5ea1baa0c0ac60bda44ce59f601133154af2a2e67722d3517f8793855d/yarl-1.25.1-cp310-cp310-win_amd64.whl", hash = "sha256:3f4d48a6112712973e676bd792121fee470e432d749177162d9949d5c9460a1b", size = 102929, upload-time = "2026-09-15T19:30:23.122Z" }, + { url = "https://files.pythonhosted.org/packages/2f/85/9a1e98de10fc0e738d517efd2a9984c3e7db6cf35fa72b2b373988a9e9b7/yarl-1.25.1-cp310-cp310-win_arm64.whl", hash = "sha256:48796ea00a303961507dc6c8437c4b325a6fc3f95f7c36c71b91ea9a8150963c", size = 98854, upload-time = "2026-09-15T19:30:25.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/b3/2cea721d495ca57f8f414aa4867ee263f486b274e07463158cf52f02ba7f/yarl-1.25.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9d693bf4bf534e9ba3ae2780cfd577f5135629f7b5ac653490859d0b77864865", size = 143794, upload-time = "2026-09-15T19:30:26.946Z" }, + { url = "https://files.pythonhosted.org/packages/55/e6/cd145cff8e5cf60b8b3c41fbecfa2a45028a8dec3fbc52bec03595ff3d3b/yarl-1.25.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ab2054c5531af2a9ba7b69b8ec91e4f884420e83a8c5e579b013084cb57e5e5d", size = 103993, upload-time = "2026-09-15T19:30:28.661Z" }, + { url = "https://files.pythonhosted.org/packages/ce/7c/fbb40fe2d53747c40aa36a9e2bd2178a202f942bda0b670f3306d4aefbce/yarl-1.25.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:564fdc7085d2245ab84f88882fdb1d6ac0723124bff6ded35bfb1c00f812630d", size = 104010, upload-time = "2026-09-15T19:30:31.069Z" }, + { url = "https://files.pythonhosted.org/packages/f0/0b/5a516f70641092283f57cf3670bdb75e7327bcc0dcb5038697e4dfbfd569/yarl-1.25.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420", size = 116444, upload-time = "2026-09-15T19:30:32.894Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8a/d1a627f827b0a404ae0f5647cab0534959081cde13b0756a783469fb3b5e/yarl-1.25.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fb2a01ba8cd9c5d2c5dc1ec35e0fc951d04b4f037541d4ac090c993ce58b3d7", size = 107565, upload-time = "2026-09-15T19:30:35.188Z" }, + { url = "https://files.pythonhosted.org/packages/aa/cf/c8e0aaec886840a6c4480fe44eaec7cd4319f79a563b79321473be79c56f/yarl-1.25.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e92b6bcc741b86d67606c40d3cb9c7cc8e6c737f81e31f4a94efc204456c92e3", size = 125006, upload-time = "2026-09-15T19:30:37.1Z" }, + { url = "https://files.pythonhosted.org/packages/50/26/0cce366d54a93cdc8342965dc7663385e4db161625cc1e8b18e786753d24/yarl-1.25.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:72c34ac7ad4314c19362d5ce27626dcc8429bd30bbf8c179f4234078851f9492", size = 128717, upload-time = "2026-09-15T19:30:38.904Z" }, + { url = "https://files.pythonhosted.org/packages/95/0c/a71501bbc1a674ff72c4d6c2b75f4d9a5af819f5244c3a7558080a8802c5/yarl-1.25.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5add7b4ca7afeea91d52e4d4e4db3b1fe9885b71f07054560d8c4296b7441a2", size = 117728, upload-time = "2026-09-15T19:30:40.809Z" }, + { url = "https://files.pythonhosted.org/packages/e7/6f/c3267ca01defeed9ed9c4ff9b17bd54915432c405945233265b707475d1c/yarl-1.25.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:def538065f9e4d4cf1ae164bd59aba00dfa84f03923e0de4c3788f252d6bcd17", size = 116224, upload-time = "2026-09-15T19:30:42.818Z" }, + { url = "https://files.pythonhosted.org/packages/8f/69/fad57ee52d648431718ee0f1f68966a99c1352c3924688ffbdcc9d3fe51a/yarl-1.25.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de", size = 116299, upload-time = "2026-09-15T19:30:44.966Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d4/6a8c1e29f33338687ca278cba0a8fbf6525a322c2c02a9a500ccbe041152/yarl-1.25.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:71f42c5b9a948c113bbdebfa544598321431d064ff959d32e99b1feb61d68345", size = 108625, upload-time = "2026-09-15T19:30:46.874Z" }, + { url = "https://files.pythonhosted.org/packages/e8/d2/3a35ae791c9cb6522c106923ff25c3230d999091e5e65511bca23bbd9914/yarl-1.25.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:72849d892954be4d09e569b8b831ac39ce58417fedc767d4308a0fe542018a40", size = 124515, upload-time = "2026-09-15T19:30:49.082Z" }, + { url = "https://files.pythonhosted.org/packages/be/fd/2b022109a6b4af0f7dc371cf7500af380b0d4f034010243e1b0ce218dc93/yarl-1.25.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:efb01a106f971cb3752856bca2318bbdf7f01bd8823779c461586cbe5ffd5258", size = 115711, upload-time = "2026-09-15T19:30:51.208Z" }, + { url = "https://files.pythonhosted.org/packages/18/59/f7586271136c3ddb0126bbfe661844699369b76b555fe38c4efe86870b2e/yarl-1.25.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a1daf47cd95a7c3a63456336bc5aaa8c86dd3a47d07ed3d0e76132ae4666a5a1", size = 122751, upload-time = "2026-09-15T19:30:53.535Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0b/07f7a2d881f7e16c385b47fc1753382600847cad305dcc7fa0c25828acf8/yarl-1.25.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9489e6abf47ba37f332075a91444c7cfedb03e6ce99fbb2f116bfe1ce810da3b", size = 117983, upload-time = "2026-09-15T19:30:55.277Z" }, + { url = "https://files.pythonhosted.org/packages/aa/9d/8cdceec66a9b940700cb45931741403f045b162afd79bcb93c41cadd0972/yarl-1.25.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7306dee25b8a0e737363f347362b875094b4dc4e367311470656ae420fdbf8e", size = 102894, upload-time = "2026-09-15T19:30:57.606Z" }, + { url = "https://files.pythonhosted.org/packages/17/f1/7ec357db1d3ad2863542d71e8fe64a126bbec17b134cd7d30951196809a5/yarl-1.25.1-cp311-cp311-win_arm64.whl", hash = "sha256:abb1384477f5901d436b5d2e5465954de46ea6098f59163d243660b5c4461d35", size = 98609, upload-time = "2026-09-15T19:30:59.705Z" }, + { url = "https://files.pythonhosted.org/packages/75/b3/cd32ac66ae622b854c2df0ac52106dda220d361b65a64fde7d5b3684aa3f/yarl-1.25.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4", size = 144798, upload-time = "2026-09-15T19:31:01.821Z" }, + { url = "https://files.pythonhosted.org/packages/61/fb/a2c52a8007c2051ba74662afb112ecf3d00346af4c25e33df9d80fd14fb8/yarl-1.25.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d", size = 104583, upload-time = "2026-09-15T19:31:04.05Z" }, + { url = "https://files.pythonhosted.org/packages/be/dd/ee38aec8e09fdf957e50d4085453fbe202f56c6c3b4cf07b81cdb4f09ee9/yarl-1.25.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6", size = 104325, upload-time = "2026-09-15T19:31:06.338Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b3/058dbfb1857b484c9cf9cc135659f50b85ce66e03c99e44dc2f7b6161f55/yarl-1.25.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58", size = 115358, upload-time = "2026-09-15T19:31:08.593Z" }, + { url = "https://files.pythonhosted.org/packages/db/39/29693446cf0cf6b15a0e2f75a5d40f93c56819b05b0622196f45e95b5cc0/yarl-1.25.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad", size = 107658, upload-time = "2026-09-15T19:31:10.802Z" }, + { url = "https://files.pythonhosted.org/packages/86/b3/3c4dd7e1af43b931fba95e0a722737f2ea94a6d199c802585282831d7abd/yarl-1.25.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40", size = 122660, upload-time = "2026-09-15T19:31:12.84Z" }, + { url = "https://files.pythonhosted.org/packages/bd/b5/1b60dbc3cfc9c5712b15148c206748f2bc93953ffdbe25ea75b63dfc89c9/yarl-1.25.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c", size = 126506, upload-time = "2026-09-15T19:31:15.088Z" }, + { url = "https://files.pythonhosted.org/packages/bc/7b/ca212cbe170ac8b96e45317ecbcf9c3c3ecf0cdec98d5b088a9c4088929b/yarl-1.25.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d", size = 117050, upload-time = "2026-09-15T19:31:17.241Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c3/72b4938cdbe619ad71ac156182faef4908846b84dc3ca4dbb4c4e6f84014/yarl-1.25.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54", size = 114174, upload-time = "2026-09-15T19:31:19.294Z" }, + { url = "https://files.pythonhosted.org/packages/e8/43/268717870f9ba0cc9701a95181587f6dc8c5f387aab4aeecc83158f38a79/yarl-1.25.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f", size = 114944, upload-time = "2026-09-15T19:31:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/da/84/baa5bf504d51fe062c4bcaf62936da97fffb43285978d0b39984824231fd/yarl-1.25.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc", size = 108263, upload-time = "2026-09-15T19:31:23.388Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/779a2ed9e0152a601a27039bed9aead3f0b79797a67e2c44bfa444622dd8/yarl-1.25.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d", size = 122184, upload-time = "2026-09-15T19:31:25.343Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1f/118e9e5b8f07694d63fd3222e801d7782270003f1a222aa798df3f8d5933/yarl-1.25.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1", size = 114001, upload-time = "2026-09-15T19:31:27.465Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/a4cf1cf372313734b17996d4007f9f73596e7a178b9485802e5494ecf484/yarl-1.25.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68", size = 120565, upload-time = "2026-09-15T19:31:29.47Z" }, + { url = "https://files.pythonhosted.org/packages/05/79/ad94f93ca731bc9e44d321833ab96b82a4f9f5f63cf773f81a4aeea5ecc1/yarl-1.25.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707", size = 117060, upload-time = "2026-09-15T19:31:31.367Z" }, + { url = "https://files.pythonhosted.org/packages/bb/cc/51a7b4abf4ac593b8e7eb3794b28e5a35ae26eed8bc04787628d215af82f/yarl-1.25.1-cp312-cp312-win_amd64.whl", hash = "sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03", size = 102593, upload-time = "2026-09-15T19:31:33.329Z" }, + { url = "https://files.pythonhosted.org/packages/9d/21/0941a6b93a58b59a1ec75e5333bf06929b671309c43c0cd201c172d9c39f/yarl-1.25.1-cp312-cp312-win_arm64.whl", hash = "sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85", size = 97697, upload-time = "2026-09-15T19:31:35.855Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/2f3129bbcc9a5c8ba12cc2b29d8060a3bab9c8043c456cfd4b5ca3188890/yarl-1.25.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:25868beca8b6765f8f7d0e11fe6dd7c66dd4b0793b9500286d20cc92352126a5", size = 143623, upload-time = "2026-09-15T19:31:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/17/e1/f1bc3390fdca352826676b531d0712736f156919090206700421d46b2c37/yarl-1.25.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:10b2fd95332f0d716d5eee3c9fb2ce8eada19082de7fee83d32e37992fd75c26", size = 104011, upload-time = "2026-09-15T19:31:40.25Z" }, + { url = "https://files.pythonhosted.org/packages/a8/aa/50acc5c3e5da04172ae3c281c75405af4d2ca911e16120ab0563f4dffb66/yarl-1.25.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f12afda4eea8c8994a76d4df1875c765194f5fbe8a9d197929ea303caee29ec", size = 103677, upload-time = "2026-09-15T19:31:42.46Z" }, + { url = "https://files.pythonhosted.org/packages/30/d2/7d1e0ab9f8390e1fbcede5a6dbf70d23c96ad09b8c5567f3a514d1ddb0e2/yarl-1.25.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:14b79a30a93a3ce2e8832603fd0ab780ada281b0ba5110b519a634f2d7d7d1fc", size = 115392, upload-time = "2026-09-15T19:31:44.371Z" }, + { url = "https://files.pythonhosted.org/packages/71/e1/5ba1e3a2a22139213655e760919038e8ed7e2d4a99826d0bbddb3beb96e5/yarl-1.25.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4bd6340d20ae2c7ca719b87b426e808e90743b676d05d4c26c4fb5ca71f41184", size = 107493, upload-time = "2026-09-15T19:31:46.273Z" }, + { url = "https://files.pythonhosted.org/packages/f5/53/780653d5e0f73831f467cf13548912e5eec97f21dc49fc8daf21da027df4/yarl-1.25.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:126a2533570c554719ca40a1288fdee1700b6bc82e7131aa69fa85252d92e651", size = 122537, upload-time = "2026-09-15T19:31:48.654Z" }, + { url = "https://files.pythonhosted.org/packages/03/92/d54fa70236c6036271c9c9c09fd978df5cbe3ef49ef6c46e9b833476d215/yarl-1.25.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a3faadac7d812ddac258feb57b9846b60c1b437c4f4b9ad42595c6f6fe4390df", size = 126170, upload-time = "2026-09-15T19:31:50.872Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b7/a82a49bf88340b837ef6972b508a1604ae377b9e6904b46b10cf5f1cf925/yarl-1.25.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be80550d9bfe83d9b62398a37081a90434e6df2d978ec345c3d2820de6beddab", size = 117012, upload-time = "2026-09-15T19:31:53.189Z" }, + { url = "https://files.pythonhosted.org/packages/ef/78/5d684b411e3f3602464ee9b538db48205038f8605872985f61efb809ced0/yarl-1.25.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e07595c7d6f4db270ceede356a1bd1c07a34f1c26f958d1ed0cd7b48e0d2bba3", size = 114950, upload-time = "2026-09-15T19:31:55.694Z" }, + { url = "https://files.pythonhosted.org/packages/2f/11/51d82b852c64f7fad0fc7a7ff3031517204887e874c722bbca839c0b23ac/yarl-1.25.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eb96ed1ae6c7d072d60840c0434aef07a2df611812810807fbc54263a6053e9a", size = 115428, upload-time = "2026-09-15T19:31:57.966Z" }, + { url = "https://files.pythonhosted.org/packages/e4/49/9d1978049bf646b9ea918313926453c6901b71c92f097467777d47d36a88/yarl-1.25.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3feb99222553a8cbedfa52c2f59dd84c3f50d5b582c728d522caf8d72769a54b", size = 108428, upload-time = "2026-09-15T19:32:00.048Z" }, + { url = "https://files.pythonhosted.org/packages/43/35/7b8f1ebb45d7ec3dda7d1909bf44f458de41ef91e2937f107733582a5166/yarl-1.25.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a2ed0ba415ccdf08f14bf544cb78346d0f76086707ffee24921a2c84dbf1305a", size = 121961, upload-time = "2026-09-15T19:32:02.436Z" }, + { url = "https://files.pythonhosted.org/packages/63/d6/d8b689ab7ca26edeb85f6ff28812aac7a25376eefc1780e303a7bfbaceff/yarl-1.25.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2b49375d22299b0a834c2bca72f39aaecc270d96fb24c30424899676f487b22a", size = 114961, upload-time = "2026-09-15T19:32:04.456Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d5/1a1798ea4dc6b7ee3260010a27907ebc697c95dae99817d817ed446d24aa/yarl-1.25.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ef74070ac553c59eb4f04258722066d6c6135b7baa03b2e9f2da65c096e96d98", size = 120036, upload-time = "2026-09-15T19:32:06.5Z" }, + { url = "https://files.pythonhosted.org/packages/91/8d/b1b35ed7903da6669b1d367cb2c09436acd4ff508029b4f39a0c0c2058fc/yarl-1.25.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0a66db89ea473abeac4b70523cafd94db3772380e565f9d28af7a179b7af71fa", size = 117276, upload-time = "2026-09-15T19:32:09.401Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8f/4db01cef62caff0d7a4593ed694fb8a41a27a11158cab80d290221f13e57/yarl-1.25.1-cp313-cp313-win_amd64.whl", hash = "sha256:1f51020b2eb8a003c84925638ec63c21a750a4bddd3a22ec8eac6a742dadf1b9", size = 101945, upload-time = "2026-09-15T19:32:11.545Z" }, + { url = "https://files.pythonhosted.org/packages/c0/5e/3ce00497c5c0babb74d4130c10c3828ccd215b4819d12020c42429f991ac/yarl-1.25.1-cp313-cp313-win_arm64.whl", hash = "sha256:b10dd0557ba422715b5206b3743192135a6022acca8baec51aa127d0a75db8fe", size = 97270, upload-time = "2026-09-15T19:32:14.127Z" }, + { url = "https://files.pythonhosted.org/packages/80/cf/54023edfab7aa773b860503db0c56e962ccab0922803ee97988c176ea090/yarl-1.25.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e", size = 143975, upload-time = "2026-09-15T19:32:16.416Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a8/e6c1be0e6761d0f2d10bbf33a3e1e02b99dc83874d92945d7b461a72481e/yarl-1.25.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f", size = 104018, upload-time = "2026-09-15T19:32:18.364Z" }, + { url = "https://files.pythonhosted.org/packages/6e/bb/dda344765ffd3430afe1a1c66c866a57fae67786537d4f14607df6505ac1/yarl-1.25.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac", size = 104156, upload-time = "2026-09-15T19:32:20.459Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5f/ed1538bcd06009fe990d6d283dd7667f639e62a81e35c6d8c6ef6c08fb3c/yarl-1.25.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c", size = 116025, upload-time = "2026-09-15T19:32:22.766Z" }, + { url = "https://files.pythonhosted.org/packages/a2/af/2185daf56b99830d3356ecfada46faaa49945de6626e842b7728088d4980/yarl-1.25.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c", size = 106985, upload-time = "2026-09-15T19:32:24.749Z" }, + { url = "https://files.pythonhosted.org/packages/c1/65/bc1ae564fb4b04a30b6a8f250e787772581c57e4c3d5cf07ac3359de3103/yarl-1.25.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743", size = 123030, upload-time = "2026-09-15T19:32:27.084Z" }, + { url = "https://files.pythonhosted.org/packages/6a/3e/e2afcde10d74e53b3fa889960991efb3019beda2b1682a01de720a302056/yarl-1.25.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f", size = 126765, upload-time = "2026-09-15T19:32:29.332Z" }, + { url = "https://files.pythonhosted.org/packages/a2/be/415b00c0fe5a0615b062a456b26623d7ec91c2bee20faea1a14045aa0469/yarl-1.25.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57", size = 117199, upload-time = "2026-09-15T19:32:31.49Z" }, + { url = "https://files.pythonhosted.org/packages/97/27/3d8c63ddd3e8bcfd033748ab93876678ce59bacd66e4cb1ed851c9c5b37e/yarl-1.25.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453", size = 115187, upload-time = "2026-09-15T19:32:34.137Z" }, + { url = "https://files.pythonhosted.org/packages/39/b7/7a81d0be1a502a26a0d4326c6f2ecb736c824f570ea1c6529f2b0b227b50/yarl-1.25.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc", size = 116085, upload-time = "2026-09-15T19:32:36.438Z" }, + { url = "https://files.pythonhosted.org/packages/f0/69/39fff459916aa0fab42215dc47b759586fd80f94aa56dfc4a7c15ba6e0dc/yarl-1.25.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f", size = 107996, upload-time = "2026-09-15T19:32:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/c0/39/80b9a55a3335590451d9ecf3eb593a8c635351f4c905ef056d7e8a8fd9e7/yarl-1.25.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8", size = 122549, upload-time = "2026-09-15T19:32:41.151Z" }, + { url = "https://files.pythonhosted.org/packages/42/7d/a179c6757818bb59372a4adafd09f7f26a3b4a0f04c3ae404b544c0b0c82/yarl-1.25.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868", size = 115107, upload-time = "2026-09-15T19:32:43.072Z" }, + { url = "https://files.pythonhosted.org/packages/32/2b/a773ac867e4ab53a98ed98e5cefe3bae31e6f550252ca9d1de266f1a40c5/yarl-1.25.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62", size = 120666, upload-time = "2026-09-15T19:32:45.061Z" }, + { url = "https://files.pythonhosted.org/packages/bc/41/52be6505e85b0f76b4f85b01b5de7e06a0512201abc2c95e14e099549174/yarl-1.25.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233", size = 117505, upload-time = "2026-09-15T19:32:47.177Z" }, + { url = "https://files.pythonhosted.org/packages/f5/01/349c0386caedbbe488d519f252df54efac8a1459282d466c474bdd84a620/yarl-1.25.1-cp314-cp314-win_amd64.whl", hash = "sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b", size = 103446, upload-time = "2026-09-15T19:32:49.615Z" }, + { url = "https://files.pythonhosted.org/packages/5c/f0/8ec63180f77912f0dc4e5a42760cb8c08d20da1d5ace3578a01b84d1f3d8/yarl-1.25.1-cp314-cp314-win_arm64.whl", hash = "sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414", size = 99159, upload-time = "2026-09-15T19:32:51.686Z" }, + { url = "https://files.pythonhosted.org/packages/47/7d/92d2220d6886b70ab1ed8579533ac2af2dfac716d5d929001daff7986df9/yarl-1.25.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa", size = 150071, upload-time = "2026-09-15T19:32:53.911Z" }, + { url = "https://files.pythonhosted.org/packages/64/fc/b245e448124bcda9340df38e3553fa222b50260fca027a84095e9bd8642d/yarl-1.25.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287", size = 106780, upload-time = "2026-09-15T19:32:56.443Z" }, + { url = "https://files.pythonhosted.org/packages/51/e2/9a6ce2e334ebf218a30335ae76fb1696459430d42f733b8cb0d7d65b84d3/yarl-1.25.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94", size = 107361, upload-time = "2026-09-15T19:32:58.827Z" }, + { url = "https://files.pythonhosted.org/packages/ed/70/66e8c76b569b450d16e190f15071c916c3df70b0e33927e415ac497cf0c2/yarl-1.25.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9", size = 114396, upload-time = "2026-09-15T19:33:02.24Z" }, + { url = "https://files.pythonhosted.org/packages/73/23/0d82838a05c57fdc05bc8b66e8c92dcc0df15e27463a5f163142d521c682/yarl-1.25.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0", size = 104882, upload-time = "2026-09-15T19:33:04.494Z" }, + { url = "https://files.pythonhosted.org/packages/86/d4/ea08615c4edaa6049a13a2f1128944d068d1893abda7d708d4d7ea01599a/yarl-1.25.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa", size = 119485, upload-time = "2026-09-15T19:33:06.583Z" }, + { url = "https://files.pythonhosted.org/packages/1a/82/0898bdce9b1ae403b308b9c733d0d24af4a3464270c2c081f457b16c3e0d/yarl-1.25.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b", size = 122490, upload-time = "2026-09-15T19:33:08.653Z" }, + { url = "https://files.pythonhosted.org/packages/d1/38/97d79b81c342b78246cfedb74809e68841f3198d21653e10d3232bd9c622/yarl-1.25.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6", size = 115336, upload-time = "2026-09-15T19:33:11.056Z" }, + { url = "https://files.pythonhosted.org/packages/8e/9d/2577896554cd310dc470adb6da0b7dd0b435cb63e2565204a7ac240e504c/yarl-1.25.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee", size = 111825, upload-time = "2026-09-15T19:33:13.204Z" }, + { url = "https://files.pythonhosted.org/packages/29/6b/7ac49d8ba84a5c4bd73415a4c949d22c749cb3762579b3d50e48019a78aa/yarl-1.25.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0", size = 114655, upload-time = "2026-09-15T19:33:15.553Z" }, + { url = "https://files.pythonhosted.org/packages/e5/18/e5942a16723f5b72f9b1297fd5a85a54f6300cd15c0dcb5005b90cd89156/yarl-1.25.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7", size = 106395, upload-time = "2026-09-15T19:33:17.599Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2d/549fa46240781513ebc47ae7eb418df428a163a2a3d644cc9cbb3ecb7846/yarl-1.25.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b", size = 119277, upload-time = "2026-09-15T19:33:19.973Z" }, + { url = "https://files.pythonhosted.org/packages/76/16/4763f78dcdc0b3b9fb3842b04afe72b9320857c6a69300c62a0eab03d119/yarl-1.25.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d", size = 112504, upload-time = "2026-09-15T19:33:22.464Z" }, + { url = "https://files.pythonhosted.org/packages/ae/b4/974e3edfe0d188393ce1cb9de400111c63fe61f4eb3b772a500d84c970d1/yarl-1.25.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3", size = 116243, upload-time = "2026-09-15T19:33:24.788Z" }, + { url = "https://files.pythonhosted.org/packages/b0/aa/157b940428da80c104ca09666a740e51c94963df65d5b112e06b52e4d7a8/yarl-1.25.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3", size = 115822, upload-time = "2026-09-15T19:33:26.886Z" }, + { url = "https://files.pythonhosted.org/packages/7e/af/19fbdce41412e1b96825544cc52cd7029d3724655d0988237972f078bd29/yarl-1.25.1-cp314-cp314t-win_amd64.whl", hash = "sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371", size = 107386, upload-time = "2026-09-15T19:33:29.009Z" }, + { url = "https://files.pythonhosted.org/packages/2a/99/f6431c8968e89be608d74b28ae2d024521b2953f27dd44e0dece5e04f67a/yarl-1.25.1-cp314-cp314t-win_arm64.whl", hash = "sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6", size = 102094, upload-time = "2026-09-15T19:33:31.151Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/318c7980066769c6bcd9221ed2248294f5698811da099013098c670565ed/yarl-1.25.1-py3-none-any.whl", hash = "sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3", size = 63617, upload-time = "2026-09-15T19:34:59.616Z" }, ] [[package]] From 714d84acb609d873b4f86de49ef89d48b49b2c1d Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:20:46 -0700 Subject: [PATCH 35/50] fix(cost-map): add fireworks minimax m3.1 flash standard prices (#47371) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 5 ++++- model_prices_and_context_window.json | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6675bb170b8..de76b6e1bdf 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76118,15 +76118,18 @@ ] }, "fireworks_ai/accounts/fireworks/models/minimax-m3p1-flash": { + "cache_read_input_token_cost": 1.5e-08, "cache_read_input_token_cost_priority": 2.25e-08, + "input_cost_per_token": 1.5e-07, "input_cost_per_token_priority": 2.25e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "max_tokens": 1048576, "mode": "chat", + "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 9e-07, - "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "source": "https://platform.minimax.io/docs/guides/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6675bb170b8..de76b6e1bdf 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76118,15 +76118,18 @@ ] }, "fireworks_ai/accounts/fireworks/models/minimax-m3p1-flash": { + "cache_read_input_token_cost": 1.5e-08, "cache_read_input_token_cost_priority": 2.25e-08, + "input_cost_per_token": 1.5e-07, "input_cost_per_token_priority": 2.25e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "max_tokens": 1048576, "mode": "chat", + "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 9e-07, - "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "source": "https://platform.minimax.io/docs/guides/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, From 40e4ade6a6d5824a072e90a391cf8b2d32675a48 Mon Sep 17 00:00:00 2001 From: Justin Israel Date: Tue, 20 Oct 2026 07:40:38 +1300 Subject: [PATCH 36/50] feat(proxy): opt-in self-serve budget edits for key owners on /key/update (#44763) * feat(proxy): opt-in self-serve budget edits for key owners on /key/update Adds general_settings.self_serve_budget_policy (disabled by default) so a non-admin can change max_budget or budget_limits on a key they created and still own. lower_only allows adding or lowering limits. ceiling also allows raising or removing them when the owner could create that key with the new budget via /key/generate (same member permissions and delegation ceiling). soft_budget and spend stay admin-only * fix(proxy): sync dashboard API types and cover the UI session cap on update * test(proxy): cover team role restriction for owner budget raises Drop two unreachable branches from the non-raising key generation check. * fix(proxy): close self-serve budget bypasses for linked budgets, null caps and retained limits Keys with a linked budget_id need an admin to change max_budget, owners cannot leave a key uncapped when an upperbound max_budget is configured, and the ceiling check now runs on the budget the key would carry after the update. * fix(proxy): keep open key budget windows and the admin path on self-serve budget edits A resubmitted budget_limits window now keeps its stored reset_at while that period is still running, so adding or editing one window no longer restarts the others. Team and org admins answer False on a ceiling loosen so they fall through to _check_key_admin_access as before. The self-serve tests feed the team through the cache and prisma mocks instead of patching get_team_object, and every helper is typed * perf(proxy): group key budget windows by duration in one pass The per-duration minimum used to rescan the whole window list once per distinct duration, so a request with thousands of windows pinned the event loop before any permission check ran. Sort once and take each group's minimum instead * refactor(proxy): take budget windows as a Sequence in the delegation ceiling checks * test(proxy): assert allowed self-serve budget edits return no denial Also drop the unreachable assert_never arm from the exhaustive budget change match. --------- Co-authored-by: ryan-crabbe-berri --- litellm/proxy/_types.py | 4 + .../management_endpoints/key_budget_change.py | 135 +++++ .../key_management_endpoints.py | 280 ++++++++--- .../team_member_permission_checks.py | 17 +- .../test_key_budget_change.py | 467 ++++++++++++++++++ .../test_key_management_endpoints.py | 45 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 7 files changed, 874 insertions(+), 79 deletions(-) create mode 100644 litellm/proxy/management_endpoints/key_budget_change.py create mode 100644 tests/unit/proxy/management_endpoints/test_key_budget_change.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index bc32978d477..e4f77ac0005 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2793,6 +2793,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="Fine-grained control over which object types to load from the database when store_model_in_db is True. Available types: 'models', 'mcp', 'guardrails', 'vector_stores', 'pass_through_endpoints', 'prompts', 'model_cost_map', 'tools', 'config_overrides'. If not set, all objects are loaded (default behavior).", ) + self_serve_budget_policy: Literal["disabled", "lower_only", "ceiling"] | None = Field( + None, + description="Lets a non-admin change max_budget or budget_limits on a key they created and still own via /key/update. 'disabled' (default) requires an admin for any budget change. 'lower_only' lets the owner add or lower limits. 'ceiling' also lets the owner raise or remove limits when they could create that key with the new budget via /key/generate.", + ) user_mcp_management_mode: UserMCPManagementMode | None = Field( None, description="Controls how non-admin users interact with MCP servers in the dashboard. 'restricted' shows only accessible servers, 'view_all' lists every server in read-only mode.", diff --git a/litellm/proxy/management_endpoints/key_budget_change.py b/litellm/proxy/management_endpoints/key_budget_change.py new file mode 100644 index 00000000000..3143ef53d76 --- /dev/null +++ b/litellm/proxy/management_endpoints/key_budget_change.py @@ -0,0 +1,135 @@ +"""Classifies a /key/update request's budget edits as tightening or loosening the key's stored limits.""" + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from itertools import groupby +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_proxy_logger +from litellm.models.team import BudgetLimitEntry +from litellm.models.verification_token import LiteLLM_VerificationToken +from litellm.proxy._types import UpdateKeyRequest + + +@dataclass(frozen=True, slots=True) +class KeyBudgetUnchanged: + kind: Literal["unchanged"] = "unchanged" + + +@dataclass(frozen=True, slots=True) +class KeyBudgetTightened: + kind: Literal["tightened"] = "tightened" + + +@dataclass(frozen=True, slots=True) +class KeyBudgetLoosened: + kind: Literal["loosened"] = "loosened" + + +@dataclass(frozen=True, slots=True) +class KeyBudgetAdminOnly: + kind: Literal["admin_only"] = "admin_only" + + +KeyBudgetChange: TypeAlias = KeyBudgetUnchanged | KeyBudgetTightened | KeyBudgetLoosened | KeyBudgetAdminOnly + +_FieldChange: TypeAlias = Literal["unchanged", "tightened", "loosened"] + +SELF_SERVE_BUDGET_POLICY_SETTING: Final = "self_serve_budget_policy" +SelfServeBudgetPolicy: TypeAlias = Literal["disabled", "lower_only", "ceiling"] + +_STORED_WINDOWS: Final = TypeAdapter(list[BudgetLimitEntry]) +_POLICY: Final = TypeAdapter(SelfServeBudgetPolicy) + + +def resolve_self_serve_budget_policy(general_settings: Mapping[str, object]) -> SelfServeBudgetPolicy: + configured: Final = general_settings.get(SELF_SERVE_BUDGET_POLICY_SETTING, "disabled") + try: + return _POLICY.validate_python(configured) + except ValidationError: + verbose_proxy_logger.warning( + "general_settings.%s=%r is not one of disabled, lower_only, ceiling; using disabled", + SELF_SERVE_BUDGET_POLICY_SETTING, + configured, + ) + return "disabled" + + +def _max_budget_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> _FieldChange: + if "max_budget" not in data.model_fields_set or data.max_budget == existing.max_budget: + return "unchanged" + if data.max_budget is None: + return "loosened" + if existing.max_budget is None or data.max_budget < existing.max_budget: + return "tightened" + return "loosened" + + +def _duration(window: BudgetLimitEntry) -> str: + return window.budget_duration + + +def _caps_by_duration(windows: Sequence[BudgetLimitEntry]) -> Mapping[str, float]: + grouped: Final = groupby(sorted(windows, key=_duration), key=_duration) + return MappingProxyType({duration: min(w.max_budget for w in group) for duration, group in grouped}) + + +def _stored_caps(existing: LiteLLM_VerificationToken) -> Mapping[str, float] | None: + try: + return _caps_by_duration(_STORED_WINDOWS.validate_python(existing.budget_limits or [])) + except ValidationError: + return None + + +def _budget_limits_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> _FieldChange: + """ + Window spend counters are keyed by the exact `budget_duration` string, so a window only + counts as kept when the same string survives with a cap no higher than before. + """ + if "budget_limits" not in data.model_fields_set: + return "unchanged" + stored: Final = _stored_caps(existing) + if stored is None: + return "loosened" + requested: Final = _caps_by_duration(data.budget_limits or ()) + kept_or_lowered: Final = all(d in requested and requested[d] <= cap for d, cap in stored.items()) + return "tightened" if kept_or_lowered else "loosened" + + +@dataclass(frozen=True, slots=True) +class EffectiveKeyBudget: + max_budget: float | None + budget_limits: tuple[BudgetLimitEntry, ...] + + +def stored_budget_windows(existing: LiteLLM_VerificationToken) -> tuple[BudgetLimitEntry, ...]: + """The key's stored windows, or none when the column holds something the window model cannot read.""" + try: + return tuple(_STORED_WINDOWS.validate_python(existing.budget_limits or [])) + except ValidationError: + return () + + +def effective_key_budget(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> EffectiveKeyBudget: + """The budget the key would carry after the update, with omitted fields keeping their stored values.""" + max_budget: Final = data.max_budget if "max_budget" in data.model_fields_set else existing.max_budget + if "budget_limits" in data.model_fields_set: + return EffectiveKeyBudget(max_budget=max_budget, budget_limits=tuple(data.budget_limits or ())) + return EffectiveKeyBudget(max_budget=max_budget, budget_limits=stored_budget_windows(existing)) + + +def classify_key_budget_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> KeyBudgetChange: + overrides_linked_budget: Final = ( + existing.budget_id is not None and _max_budget_change(data, existing) != "unchanged" + ) + if data.spend is not None or "soft_budget" in data.model_fields_set or overrides_linked_budget: + return KeyBudgetAdminOnly() + changes: Final = frozenset((_max_budget_change(data, existing), _budget_limits_change(data, existing))) + if "loosened" in changes: + return KeyBudgetLoosened() + if "tightened" in changes: + return KeyBudgetTightened() + return KeyBudgetUnchanged() diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index e87ca70035c..022eba499a4 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -106,6 +106,17 @@ from litellm.proxy.management_endpoints.common_utils import ( # noqa: F401 # l validate_budget_duration, validate_finite_spend, ) +from litellm.proxy.management_endpoints.key_budget_change import ( + KeyBudgetAdminOnly, + KeyBudgetChange, + KeyBudgetLoosened, + KeyBudgetTightened, + KeyBudgetUnchanged, + classify_key_budget_change, + effective_key_budget, + resolve_self_serve_budget_policy, + stored_budget_windows, +) from litellm.proxy.management_endpoints.model_management_endpoints import ( # noqa: F401 # legacy module exports _add_model_to_db, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export add_model_to_db, @@ -190,6 +201,7 @@ from litellm.types.router import Deployment from litellm.types.utils import ( BudgetConfig, PersonalUIKeyGenerationConfig, + StandardKeyGenerationConfig, TeamUIKeyGenerationConfig, ) @@ -848,6 +860,33 @@ def key_generation_check( return _personal_key_generation_check(user_api_key_dict=user_api_key_dict, data=data) +def _caller_may_generate_key( + team_table: LiteLLM_TeamTableCachedObj | None, + user_api_key_dict: UserAPIKeyAuth, +) -> bool: + """Non-raising mirror of the role and member-permission gates in `key_generation_check`.""" + settings: Final = litellm.key_generation_settings or StandardKeyGenerationConfig() + if team_table is None: + personal: Final = settings.get("personal_key_generation") + return ( + personal is None + or "allowed_user_roles" not in personal + or user_api_key_dict.user_role in personal["allowed_user_roles"] + ) + team_settings: Final = settings.get( + "team_key_generation", TeamUIKeyGenerationConfig(allowed_team_member_roles=["admin", "user"]) + ) + caller_team_role: Final = _get_caller_team_role(team_table=team_table, user_api_key_dict=user_api_key_dict) + if caller_team_role is None or ( + "allowed_team_member_roles" in team_settings + and caller_team_role not in team_settings["allowed_team_member_roles"] + ): + return False + return caller_team_role == "admin" or TeamMemberPermissionChecks.is_route_granted_to_team_members( + team_table=team_table, route=KeyManagementRoutes.KEY_GENERATE.value + ) + + def raise_on_invalid_key_logging_config(metadata: Mapping[str, object] | None) -> None: """Key-level logging writes go through key metadata, not /team/callback. @@ -1072,7 +1111,7 @@ _check_permissions_caller_permission: Final = check_permissions_caller_permissio def _check_budget_limits_delegation_ceiling( - budget_limits: list[BudgetLimitEntry] | None, + budget_limits: Sequence[BudgetLimitEntry] | None, delegation_ceiling: float | None, user_api_key_dict: UserAPIKeyAuth, is_ui_session_team_key: bool, @@ -1122,6 +1161,67 @@ def _check_budget_limits_delegation_ceiling( ) +def _enforce_budget_delegation_ceiling( + requested_max_budget: float | None, + requested_team_id: str | None, + budget_limits: Sequence[BudgetLimitEntry] | None, + team_table: LiteLLM_TeamTableCachedObj | None, + user_api_key_dict: UserAPIKeyAuth, +) -> None: + # Delegated-authority ceiling (GHSA-q775-qw9r-2r4g): a non-admin caller + # cannot grant a key a higher budget than their own authority. + # UI session personal keys are capped by user_max_budget when it is available. + is_ui_session_token: Final = user_api_key_dict.team_id == UI_SESSION_TOKEN_TEAM_ID + is_ui_session_team_key: Final = is_ui_session_token and requested_team_id is not None + if ( + user_api_key_dict.is_session_token + and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value + and not is_ui_session_team_key + and requested_max_budget is not None + and team_table is None + ): + raise HTTPException( + status_code=400, + detail={ + "error": ( + f"max_budget ({requested_max_budget}) cannot be set without " + "specifying team_id when using a CLI session token." + ) + }, + ) + delegation_ceiling: Final = ( + user_api_key_dict.user_max_budget + if is_ui_session_token and user_api_key_dict.user_max_budget is not None + else user_api_key_dict.max_budget + if user_api_key_dict.max_budget is not None + else (team_table.max_budget if user_api_key_dict.is_session_token and team_table is not None else None) + ) + if ( + user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value + and not is_ui_session_team_key + and requested_max_budget is not None + and delegation_ceiling is not None + and requested_max_budget > delegation_ceiling + ): + raise HTTPException( + status_code=400, + detail={ + "error": ( + f"max_budget ({requested_max_budget}) cannot exceed the caller's " + f"own max_budget ({delegation_ceiling})." + ) + }, + ) + + _check_budget_limits_delegation_ceiling( + budget_limits=budget_limits, + delegation_ceiling=delegation_ceiling, + user_api_key_dict=user_api_key_dict, + is_ui_session_team_key=is_ui_session_team_key, + team_table=team_table, + ) + + async def validate_team_id_used_in_service_account_request( team_id: str | None, prisma_client: PrismaClient | None, @@ -1364,57 +1464,12 @@ async def _common_key_generation_helper( # check if user set upperbound key/generate params on config.yaml _enforce_upperbound_key_params(data, fill_defaults=True) - # Delegated-authority ceiling (GHSA-q775-qw9r-2r4g): a non-admin caller - # cannot grant a key a higher budget than their own authority. - # UI session personal keys are capped by user_max_budget when it is available. - is_ui_session_token: Final = user_api_key_dict.team_id == UI_SESSION_TOKEN_TEAM_ID - is_ui_session_team_key = is_ui_session_token and _requested_team_id is not None - if ( - user_api_key_dict.is_session_token - and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value - and not is_ui_session_team_key - and _requested_max_budget is not None - and team_table is None - ): - raise HTTPException( - status_code=400, - detail={ - "error": ( - f"max_budget ({_requested_max_budget}) cannot be set without " - "specifying team_id when using a CLI session token." - ) - }, - ) - delegation_ceiling: Final = ( - user_api_key_dict.user_max_budget - if is_ui_session_token and user_api_key_dict.user_max_budget is not None - else user_api_key_dict.max_budget - if user_api_key_dict.max_budget is not None - else (team_table.max_budget if user_api_key_dict.is_session_token and team_table is not None else None) - ) - if ( - user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value - and not is_ui_session_team_key - and _requested_max_budget is not None - and delegation_ceiling is not None - and _requested_max_budget > delegation_ceiling - ): - raise HTTPException( - status_code=400, - detail={ - "error": ( - f"max_budget ({_requested_max_budget}) cannot exceed the caller's " - f"own max_budget ({delegation_ceiling})." - ) - }, - ) - - _check_budget_limits_delegation_ceiling( + _enforce_budget_delegation_ceiling( + requested_max_budget=_requested_max_budget, + requested_team_id=_requested_team_id, budget_limits=data.budget_limits, - delegation_ceiling=delegation_ceiling, - user_api_key_dict=user_api_key_dict, - is_ui_session_team_key=is_ui_session_team_key, team_table=team_table, + user_api_key_dict=user_api_key_dict, ) check_permissions_caller_permission( data=data, @@ -2750,16 +2805,12 @@ async def prepare_key_update_data( non_default_values["budget_duration"] = budget_duration if "budget_limits" in non_default_values: - raw_windows: Final = non_default_values["budget_limits"] - if raw_windows: - from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time - - initialized_windows: Final = [] - for window in raw_windows: - w = window if isinstance(window, dict) else window.model_dump() - w["reset_at"] = get_budget_reset_time(budget_duration=w["budget_duration"]).isoformat() - initialized_windows.append(w) - non_default_values["budget_limits"] = json.dumps(initialized_windows) + requested_windows: Final = _budget_limit_windows(data.budget_limits) + if requested_windows: + stored_windows: Final = stored_budget_windows(existing_key_row) + non_default_values["budget_limits"] = json.dumps( + [_window_with_reset_at(window, stored_windows) for window in requested_windows] + ) else: # [] / None clears the field; prisma-client-py has no DbNull # sentinel for Json? columns, so store the JSON literal null @@ -3327,6 +3378,62 @@ async def _acting_as_team_admin_for_key_update( return True +async def _owner_may_change_key_budget( + budget_change: KeyBudgetChange, + data: UpdateKeyRequest, + existing_key_row: LiteLLM_VerificationToken, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, +) -> bool: + """ + Whether the key's owner may make this budget change without admin rights, per + general_settings.self_serve_budget_policy. `ceiling` lets the owner loosen only when they + could have created the key with the new budget anyway (same generate permissions and + delegation ceiling as /key/generate). Team and org admins answer False on a loosen so they + keep the admin path, which has no ceiling. + """ + policy: Final = resolve_self_serve_budget_policy(_general_settings()) + if policy == "disabled": + return False + match budget_change: + case KeyBudgetUnchanged() | KeyBudgetTightened(): + return True + case KeyBudgetAdminOnly(): + return False + case KeyBudgetLoosened(): + if policy == "lower_only": + return False + team_id: Final = data.team_id if "team_id" in data.model_fields_set else existing_key_row.team_id + team_table: Final = ( + None + if team_id is None + else await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + ) + if team_table is not None and await get_team_access().allows( + user_api_key_dict, team_table, TEAM_OR_ORG_ADMIN + ): + return False + if not _caller_may_generate_key(team_table=team_table, user_api_key_dict=user_api_key_dict): + return False + effective: Final = effective_key_budget(data=data, existing=existing_key_row) + upperbound: Final = litellm.upperbound_key_generate_params + if effective.max_budget is None and upperbound is not None and upperbound.max_budget is not None: + return False + _enforce_budget_delegation_ceiling( + requested_max_budget=effective.max_budget, + requested_team_id=team_id, + budget_limits=effective.budget_limits, + team_table=team_table, + user_api_key_dict=user_api_key_dict, + ) + return True + + async def _validate_update_key_data( data: UpdateKeyRequest, existing_key_row: LiteLLM_VerificationToken, @@ -3420,22 +3527,18 @@ async def _validate_update_key_data( # - Anyone else (non-PROXY_ADMIN, not the owner, not a team member # on a team key): must pass _check_key_admin_access (PROXY_ADMIN # / key-owner / team-admin / org-admin of the key). - # - max_budget / spend / budget_limits: always require the admin - # check, even for the key owner or a team member (matches the - # existing admin-only budget semantics). budget_limits uses - # model_fields_set because an explicit null/[] clears the field - # and must gate the same as setting or changing it. + # - max_budget / budget_limits: admin check unless + # general_settings.self_serve_budget_policy lets the key owner make the + # change (_owner_may_change_key_budget). + # - soft_budget: admin check, since it writes the key's budget row, + # which can be a shared /budget/new tier. # - spend gates on presence alone (not a value diff): the DB spend # lags the live cross-pod counter, so letting an "unchanged" spend # through the non-admin path would let a key owner / team member # overwrite the live counter below real usage and silently weaken # enforcement. - _is_budget_change: Final = ( - (data.max_budget is not None and data.max_budget != existing_key_row.max_budget) - or data.spend is not None - or "budget_limits" in data.model_fields_set - or "soft_budget" in data.model_fields_set - ) + budget_change: Final = classify_key_budget_change(data=data, existing=existing_key_row) + _is_budget_change: Final = not isinstance(budget_change, KeyBudgetUnchanged) _existing_metadata: Final = getattr(existing_key_row, "metadata", None) _existing_throttle: Final = ( @@ -3484,8 +3587,19 @@ async def _validate_update_key_data( # non-budget change means the caller was authorized — skip the redundant # _check_key_admin_access that would otherwise require team/org admin status. _key_is_team_key: Final = getattr(existing_key_row, "team_id", None) is not None - can_skip_admin_check: Final = (caller_is_creator or _key_is_team_key) and not ( - _is_budget_change or is_project_change + budget_change_allowed: Final = not _is_budget_change or ( + caller_is_creator + and await _owner_may_change_key_budget( + budget_change=budget_change, + data=data, + existing_key_row=existing_key_row, + user_api_key_dict=user_api_key_dict, + prisma_client=checked_prisma_client, + user_api_key_cache=user_api_key_cache, + ) + ) + can_skip_admin_check: Final = ( + (caller_is_creator or _key_is_team_key) and budget_change_allowed and not is_project_change ) if (not _is_proxy_admin) and not can_skip_admin_check: hashed_key: Final = existing_key_row.token @@ -6459,6 +6573,26 @@ def _budget_limit_windows(budget_limits: Sequence[object] | str | None) -> tuple return tuple(raw_window if isinstance(raw_window, dict) else raw_window.model_dump() for raw_window in raw_windows) +def _window_with_reset_at( + window: Mapping[str, object], stored_windows: Sequence[BudgetLimitEntry] +) -> Mapping[str, object]: + """A resubmitted window keeps its running period, so editing one window never restarts the others.""" + duration: Final = str(window["budget_duration"]) + now: Final = datetime.now(timezone.utc) + open_reset_at: Final = next( + (w.reset_at for w in stored_windows if w.budget_duration == duration and _is_open_reset_at(w.reset_at, now)), + None, + ) + reset_at: Final = open_reset_at if open_reset_at is not None else get_budget_reset_time(budget_duration=duration) + return {**window, "reset_at": reset_at.isoformat()} + + +def _is_open_reset_at(reset_at: datetime | None, now: datetime) -> bool: + if reset_at is None: + return False + return (reset_at if reset_at.tzinfo is not None else reset_at.replace(tzinfo=timezone.utc)) > now + + def _advance_one_key_budget_window(window: Mapping[str, object]) -> Mapping[str, object]: """Restart one budget window from now, by advancing its `reset_at`. diff --git a/litellm/proxy/management_helpers/team_member_permission_checks.py b/litellm/proxy/management_helpers/team_member_permission_checks.py index 3e6d0e47265..834b032b04f 100644 --- a/litellm/proxy/management_helpers/team_member_permission_checks.py +++ b/litellm/proxy/management_helpers/team_member_permission_checks.py @@ -101,6 +101,13 @@ class TeamMemberPermissionChecks: code=401, ) + @staticmethod + def is_route_granted_to_team_members(team_table: LiteLLM_TeamTableCachedObj, route: str) -> bool: + granted_routes: Final = TeamMemberPermissionChecks._get_list_of_route_enum_as_str( + TeamMemberPermissionChecks.get_permissions_for_team_member(team_table=team_table) + ) + return RouteChecks.check_route_access(route=route, allowed_routes=granted_routes) + @staticmethod def does_team_member_have_permissions_for_endpoint( team_member_role: Literal["admin", "user"] | None, @@ -118,12 +125,10 @@ class TeamMemberPermissionChecks: if team_member_role == "admin": return True - _team_member_permissions: Final = TeamMemberPermissionChecks.get_permissions_for_team_member( - team_table=team_table, - ) - team_member_permissions = TeamMemberPermissionChecks._get_list_of_route_enum_as_str(_team_member_permissions) - - if not RouteChecks.check_route_access(route=route, allowed_routes=team_member_permissions): + if not TeamMemberPermissionChecks.is_route_granted_to_team_members(team_table=team_table, route=route): + team_member_permissions: Final = TeamMemberPermissionChecks._get_list_of_route_enum_as_str( + TeamMemberPermissionChecks.get_permissions_for_team_member(team_table=team_table) + ) raise ProxyException( message=f"Team member does not have permissions for endpoint: {route}. You only have access to the following endpoints: {team_member_permissions} for team {team_table.team_id}. To create keys for this team, please ask your proxy admin to check the team member permission settings and update the settings to allow team member users to create keys.", type=ProxyErrorTypes.team_member_permission_error, diff --git a/tests/unit/proxy/management_endpoints/test_key_budget_change.py b/tests/unit/proxy/management_endpoints/test_key_budget_change.py new file mode 100644 index 00000000000..baf8d870117 --- /dev/null +++ b/tests/unit/proxy/management_endpoints/test_key_budget_change.py @@ -0,0 +1,467 @@ +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import HTTPException + +from litellm.constants import UI_SESSION_TOKEN_TEAM_ID +from litellm.models.verification_token import LiteLLM_VerificationToken +from litellm.proxy._types import ( + LiteLLM_TeamTableCachedObj, + LitellmUserRoles, + Member, + ProxyException, + UpdateKeyRequest, + UserAPIKeyAuth, +) +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.management_endpoints.key_budget_change import ( + KeyBudgetAdminOnly, + KeyBudgetChange, + KeyBudgetLoosened, + KeyBudgetTightened, + KeyBudgetUnchanged, + classify_key_budget_change, + resolve_self_serve_budget_policy, +) +from litellm.proxy.management_endpoints.key_management_endpoints import _validate_update_key_data +from litellm.types.proxy.management_endpoints.ui_sso import LiteLLM_UpperboundKeyGenerateParams + +_OWNER: Final = "owner-1" +_TEAM_ADMIN: Final = "team-admin" +_TEAM: Final = "team-1" +_WEEKLY: Final[Mapping[str, object]] = MappingProxyType( + {"budget_duration": "7d", "max_budget": 50.0, "reset_at": "2026-10-08T00:00:00+00:00"} +) + + +def _key( + *, + user_id: str = _OWNER, + created_by: str = _OWNER, + team_id: str | None = None, + max_budget: float | None = None, + budget_limits: Sequence[Mapping[str, object]] | None = None, + budget_id: str | None = None, +) -> LiteLLM_VerificationToken: + return LiteLLM_VerificationToken( + token="hashed-owner-key", + user_id=user_id, + created_by=created_by, + team_id=team_id, + max_budget=max_budget, + budget_limits=None if budget_limits is None else [dict(window) for window in budget_limits], + budget_id=budget_id, + metadata={}, + ) + + +def _window(duration: str, cap: float) -> Mapping[str, object]: + return MappingProxyType({"budget_duration": duration, "max_budget": cap}) + + +@pytest.mark.parametrize( + ("update", "stored", "expected"), + [ + (UpdateKeyRequest(key="sk-1", key_alias="renamed"), _key(max_budget=10.0), KeyBudgetUnchanged()), + (UpdateKeyRequest(key="sk-1", max_budget=10.0), _key(max_budget=10.0), KeyBudgetUnchanged()), + (UpdateKeyRequest(key="sk-1", max_budget=5.0), _key(max_budget=10.0), KeyBudgetTightened()), + (UpdateKeyRequest(key="sk-1", max_budget=5.0), _key(), KeyBudgetTightened()), + (UpdateKeyRequest(key="sk-1", max_budget=20.0), _key(max_budget=10.0), KeyBudgetLoosened()), + (UpdateKeyRequest(key="sk-1", max_budget=None), _key(max_budget=10.0), KeyBudgetLoosened()), + (UpdateKeyRequest(key="sk-1", budget_limits=[_window("24h", 5.0)]), _key(), KeyBudgetTightened()), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_WEEKLY, _window("24h", 5.0)]), + _key(budget_limits=[_WEEKLY]), + KeyBudgetTightened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 50.0)]), + _key(budget_limits=[_WEEKLY]), + KeyBudgetTightened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 40.0)]), + _key(budget_limits=[_WEEKLY]), + KeyBudgetTightened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 60.0)]), + _key(budget_limits=[_WEEKLY]), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("24h", 5.0)]), + _key(budget_limits=[_WEEKLY]), + KeyBudgetLoosened(), + ), + (UpdateKeyRequest(key="sk-1", budget_limits=[]), _key(budget_limits=[_WEEKLY]), KeyBudgetLoosened()), + (UpdateKeyRequest(key="sk-1", budget_limits=None), _key(budget_limits=[_WEEKLY]), KeyBudgetLoosened()), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("1d", 5.0)]), + _key(budget_limits=[_window("24h", 5.0)]), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 40.0)]), + _key(budget_limits=[_WEEKLY, _window("7d", 30.0)]), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 40.0), _window("24h", 5.0)]), + _key(budget_limits=[_window("7d", 30.0), _window("24h", 5.0), _window("7d", 50.0)]), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("24h", 5.0)]), + _key(budget_limits=[{"budget_duration": "24h"}]), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", max_budget=5.0, budget_limits=[]), + _key(max_budget=10.0, budget_limits=[_WEEKLY]), + KeyBudgetLoosened(), + ), + (UpdateKeyRequest(key="sk-1", spend=0.0), _key(), KeyBudgetAdminOnly()), + (UpdateKeyRequest(key="sk-1", soft_budget=1.0), _key(), KeyBudgetAdminOnly()), + (UpdateKeyRequest(key="sk-1", soft_budget=None, max_budget=5.0), _key(max_budget=10.0), KeyBudgetAdminOnly()), + ], +) +def test_classify_key_budget_change( + update: UpdateKeyRequest, stored: LiteLLM_VerificationToken, expected: KeyBudgetChange +) -> None: + assert classify_key_budget_change(update, stored) == expected + + +@pytest.mark.parametrize( + ("settings", "expected"), + [ + ({}, "disabled"), + ({"self_serve_budget_policy": "lower_only"}, "lower_only"), + ({"self_serve_budget_policy": "ceiling"}, "ceiling"), + ({"self_serve_budget_policy": "everything"}, "disabled"), + ({"self_serve_budget_policy": True}, "disabled"), + ], +) +def test_resolve_self_serve_budget_policy(settings: Mapping[str, object], expected: str) -> None: + assert resolve_self_serve_budget_policy(settings) == expected + + +def _team(member_permissions: Sequence[str]) -> LiteLLM_TeamTableCachedObj: + return LiteLLM_TeamTableCachedObj( + team_id=_TEAM, + members_with_roles=[Member(user_id=_TEAM_ADMIN, role="admin"), Member(user_id=_OWNER, role="user")], + team_member_permissions=list(member_permissions), + ) + + +def _caller( + *, + user_id: str = _OWNER, + max_budget: float | None = None, + team_id: str | None = None, + user_max_budget: float | None = None, +) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + user_id=user_id, + user_role=LitellmUserRoles.INTERNAL_USER, + max_budget=max_budget, + team_id=team_id, + user_max_budget=user_max_budget, + ) + + +async def _update( + monkeypatch: pytest.MonkeyPatch, + *, + policy: str | None, + data: UpdateKeyRequest, + existing: LiteLLM_VerificationToken, + team: LiteLLM_TeamTableCachedObj | None = None, + caller: UserAPIKeyAuth | None = None, +) -> None: + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {} if policy is None else {"self_serve_budget_policy": policy}, + ) + prisma: Final = MagicMock() + prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=existing) + prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team) + prisma.writer_db.litellm_teamtable.find_unique = AsyncMock(return_value=team) + await _validate_update_key_data( + data=data, + existing_key_row=existing, + user_api_key_dict=caller or _caller(), + llm_router=None, + premium_user=False, + prisma_client=prisma, + user_api_key_cache=UserApiKeyCache(), + ) + + +async def _denial( + monkeypatch: pytest.MonkeyPatch, + *, + policy: str | None, + data: UpdateKeyRequest, + existing: LiteLLM_VerificationToken, + team: LiteLLM_TeamTableCachedObj | None = None, + caller: UserAPIKeyAuth | None = None, +) -> HTTPException | None: + try: + await _update(monkeypatch, policy=policy, data=data, existing=existing, team=team, caller=caller) + except HTTPException as exc: + return exc + return None + + +_ADD_DAILY_WINDOW: Final = UpdateKeyRequest(key="sk-1", budget_limits=[_WEEKLY, _window("24h", 5.0)]) +_RAISE_WEEKLY_WINDOW: Final = UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 80.0)]) + + +def _team_key_with_weekly(*, user_id: str = _OWNER, created_by: str = _OWNER) -> LiteLLM_VerificationToken: + return _key(user_id=user_id, created_by=created_by, team_id=_TEAM, budget_limits=[_WEEKLY]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("policy", "data", "member_permissions"), + [ + ("lower_only", _ADD_DAILY_WINDOW, ["/key/update"]), + ("ceiling", _ADD_DAILY_WINDOW, ["/key/update"]), + ("ceiling", _RAISE_WEEKLY_WINDOW, ["/key/update", "/key/generate"]), + ], +) +async def test_owner_changes_team_key_budget_when_policy_allows( + monkeypatch: pytest.MonkeyPatch, policy: str, data: UpdateKeyRequest, member_permissions: Sequence[str] +) -> None: + assert ( + await _denial( + monkeypatch, + policy=policy, + data=data, + existing=_team_key_with_weekly(), + team=_team(member_permissions), + ) + is None + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("policy", "data", "member_permissions"), + [ + (None, _ADD_DAILY_WINDOW, ["/key/update", "/key/generate"]), + ("disabled", _ADD_DAILY_WINDOW, ["/key/update", "/key/generate"]), + ("lower_only", _RAISE_WEEKLY_WINDOW, ["/key/update", "/key/generate"]), + ("ceiling", _RAISE_WEEKLY_WINDOW, ["/key/update"]), + ("ceiling", UpdateKeyRequest(key="sk-1", soft_budget=1.0), ["/key/update", "/key/generate"]), + ("ceiling", UpdateKeyRequest(key="sk-1", spend=0.0), ["/key/update", "/key/generate"]), + ], +) +async def test_owner_team_key_budget_change_needs_admin( + monkeypatch: pytest.MonkeyPatch, policy: str | None, data: UpdateKeyRequest, member_permissions: Sequence[str] +) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy=policy, + data=data, + existing=_team_key_with_weekly(), + team=_team(member_permissions), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_key_created_by_someone_else_needs_admin_even_to_tighten(monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=_ADD_DAILY_WINDOW, + existing=_team_key_with_weekly(created_by=_TEAM_ADMIN), + team=_team(["/key/update", "/key/generate"]), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_team_member_without_update_permission_is_still_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.raises(ProxyException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=_ADD_DAILY_WINDOW, + existing=_team_key_with_weekly(), + team=_team(["/key/generate"]), + ) + assert str(exc.value.code) == "401" + + +@pytest.mark.asyncio +async def test_ceiling_lets_owner_raise_personal_key_budget(monkeypatch: pytest.MonkeyPatch) -> None: + assert ( + await _denial( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=30.0), + existing=_key(max_budget=10.0), + ) + is None + ) + + +@pytest.mark.asyncio +async def test_ceiling_caps_owner_raise_at_callers_own_budget(monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=30.0), + existing=_key(max_budget=10.0), + caller=_caller(max_budget=20.0), + ) + assert exc.value.status_code == 400 + assert "cannot exceed the caller's own max_budget (20.0)" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_ceiling_does_not_cap_a_team_admin_raising_their_own_team_key(monkeypatch: pytest.MonkeyPatch) -> None: + assert ( + await _denial( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=30.0), + existing=_key(user_id=_TEAM_ADMIN, created_by=_TEAM_ADMIN, team_id=_TEAM, max_budget=10.0), + team=_team(["/key/update"]), + caller=_caller(user_id=_TEAM_ADMIN, max_budget=20.0), + ) + is None + ) + + +@pytest.mark.asyncio +async def test_ceiling_caps_ui_session_owner_raise_at_user_budget(monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=30.0), + existing=_key(max_budget=10.0), + caller=_caller(team_id=UI_SESSION_TOKEN_TEAM_ID, user_max_budget=20.0), + ) + assert exc.value.status_code == 400 + assert "cannot exceed the caller's own max_budget (20.0)" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_ceiling_respects_personal_key_generation_role_restriction(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "litellm.key_generation_settings", + {"personal_key_generation": {"allowed_user_roles": [LitellmUserRoles.PROXY_ADMIN.value]}}, + ) + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=None), + existing=_key(max_budget=10.0), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_lower_only_lets_owner_lower_personal_key_budget_without_generate_rights( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "litellm.key_generation_settings", + {"personal_key_generation": {"allowed_user_roles": [LitellmUserRoles.PROXY_ADMIN.value]}}, + ) + assert ( + await _denial( + monkeypatch, + policy="lower_only", + data=UpdateKeyRequest(key="sk-1", max_budget=5.0), + existing=_key(max_budget=10.0), + ) + is None + ) + + +@pytest.mark.asyncio +async def test_ceiling_respects_team_key_generation_role_restriction(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "litellm.key_generation_settings", + {"team_key_generation": {"allowed_team_member_roles": ["admin"]}}, + ) + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=_RAISE_WEEKLY_WINDOW, + existing=_team_key_with_weekly(), + team=_team(["/key/update", "/key/generate"]), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["lower_only", "ceiling"]) +async def test_owner_cannot_override_a_cap_inherited_from_a_linked_budget( + monkeypatch: pytest.MonkeyPatch, policy: str +) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy=policy, + data=UpdateKeyRequest(key="sk-1", max_budget=100.0), + existing=_key(max_budget=None, budget_id="shared-tier-10"), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_ceiling_cannot_leave_a_key_uncapped_when_an_upperbound_is_configured( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("litellm.upperbound_key_generate_params", LiteLLM_UpperboundKeyGenerateParams(max_budget=20.0)) + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=None), + existing=_key(max_budget=10.0), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_ceiling_checks_the_retained_max_budget_when_only_windows_change( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", budget_limits=[]), + existing=_key(max_budget=100.0, budget_limits=[_WEEKLY]), + caller=_caller(max_budget=20.0), + ) + assert exc.value.status_code == 400 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", [None, "disabled"]) +async def test_owner_cannot_clear_max_budget_by_default(monkeypatch: pytest.MonkeyPatch, policy: str | None) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy=policy, + data=UpdateKeyRequest(key="sk-1", max_budget=None), + existing=_key(max_budget=10.0), + ) + assert exc.value.status_code == 403 diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index dffd9b9b513..e451c38a251 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -2513,6 +2513,51 @@ async def test_prepare_key_update_data_budget_limits_serializes_windows(): assert windows[0]["reset_at"] is not None +def _key_with_weekly_window(reset_at: datetime) -> LiteLLM_VerificationToken: + return LiteLLM_VerificationToken( + token="test-token", + user_id="test-user", + metadata={}, + budget_limits=[{"budget_duration": "7d", "max_budget": 50.0, "reset_at": reset_at.isoformat()}], + ) + + +async def _updated_windows_by_duration( + existing_key: LiteLLM_VerificationToken, budget_limits: list[dict[str, object]] +) -> dict[str, dict[str, object]]: + result: Final = await prepare_key_update_data( + data=UpdateKeyRequest.model_validate({"key": "test-token", "budget_limits": budget_limits}), + existing_key_row=existing_key, + ) + return {window["budget_duration"]: window for window in json.loads(result["budget_limits"])} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("weekly_cap", [50.0, 40.0, 60.0]) +async def test_prepare_key_update_data_keeps_a_resubmitted_window_mid_period(weekly_cap: float) -> None: + open_reset_at: Final = datetime.now(timezone.utc) + timedelta(days=3) + + windows: Final = await _updated_windows_by_duration( + _key_with_weekly_window(open_reset_at), + [{"budget_duration": "7d", "max_budget": weekly_cap}, {"budget_duration": "1d", "max_budget": 5.0}], + ) + + assert datetime.fromisoformat(windows["7d"]["reset_at"]) == open_reset_at + assert windows["7d"]["max_budget"] == weekly_cap + assert datetime.fromisoformat(windows["1d"]["reset_at"]) > datetime.now(timezone.utc) + + +@pytest.mark.asyncio +async def test_prepare_key_update_data_restarts_a_resubmitted_window_whose_period_ended() -> None: + stale_reset_at: Final = datetime.now(timezone.utc) - timedelta(days=1) + + windows: Final = await _updated_windows_by_duration( + _key_with_weekly_window(stale_reset_at), [{"budget_duration": "7d", "max_budget": 50.0}] + ) + + assert datetime.fromisoformat(windows["7d"]["reset_at"]) > datetime.now(timezone.utc) + + @pytest.mark.asyncio async def test_prepare_key_update_data_disable_global_guardrails_false_no_premium( monkeypatch, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d697ccac98d..daffc3c0c41 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30213,6 +30213,11 @@ export interface components { * @default false */ search_tool_deny_by_default: boolean; + /** + * Self Serve Budget Policy + * @description Lets a non-admin change max_budget or budget_limits on a key they created and still own via /key/update. 'disabled' (default) requires an admin for any budget change. 'lower_only' lets the owner add or lower limits. 'ceiling' also lets the owner raise or remove limits when they could create that key with the new budget via /key/generate. + */ + self_serve_budget_policy?: ("disabled" | "lower_only" | "ceiling") | null; /** @description Daily check of the spend LiteLLM captured against the provider's own bill (OpenAI via OPENAI_ADMIN_KEY). Publishes litellm_spend_capture_rate per provider and alerts when the ratio over the lookback window falls under the threshold (default 0.9). Off unless set. */ spend_capture_rate_check?: components["schemas"]["SpendCaptureRateCheckSettings"] | null; /** @description Which keys of LiteLLM_SpendLogs.metadata are written to the database. Set exactly one of 'include' (write only these keys) or 'exclude' (drop these keys). 'status' and 'cold_storage_object_key' are always written. Daily spend tables, budgets and logging callbacks still see every key. Unset writes every key */ From d7620c77a1ebaf30bb55f31030418b9da2f8cd21 Mon Sep 17 00:00:00 2001 From: Paul El Khoury <82139421+pOpO1-9@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:40:56 +0200 Subject: [PATCH 37/50] fix(openai): drop unsupported reasoning effort for gpt-6.1-sol (#43935) * fix(openai): drop unsupported reasoning effort for gpt-6.1-sol gpt-6.1-sol accepts low, medium, high, xhigh, and max. none and minimal were still forwarded, because chat completions ignored an explicit supports_none_reasoning_effort=false and the Responses mapper never applied that contract. Co-authored-by: Cursor * fix(openai): mark the stripped reasoning object as outgoing JSON Co-authored-by: Cursor * Fix Azure responses reading the OpenAI effort map gpt-6-astra on Azure accepts reasoning.effort none. The responses gate was checking the OpenAI entry, which rejects it. Signed-off-by: Paul El Khoury <82139421+pOpO1-9@users.noreply.github.com> Co-authored-by: Cursor * fix(openai): gate every reasoning_effort level from the cost map on chat and Responses One gate reads the model's cost-map row for whatever effort level a request asks for, on Chat Completions and the Responses API, for OpenAI and Azure. xhigh stays opt-in on chat (Azure also keeps none opt-in); every other level is refused only when its supports__reasoning_effort flag is false. A refused level is dropped under drop_params or answered with a local 400, and Responses keeps the rest of the reasoning object when it drops effort. The guards that follow judge temperature by the level the provider will see. Tests move into the mapped test files and derive the expected outcome for each level from the map row, so a new GPT-6 model needs only its row. * test(openai): pin gpt-6.1-sol's refused effort levels and prove the row flags alone drive the gate * fix(cost-map): azure gpt-6-astra refuses none and honors max on Responses, as live calls show * test(reasoning_effort): pin the astra max and none flips through the adapter and Foundry paths * fix(reasoning_effort): keep max out of the route gates and correct the gpt-5.3-codex rows The same model honors max on Responses and refuses it on Chat Completions, so one row flag cannot gate it on either route. The gates now skip max, the Azure gpt-6-astra rows keep max off so the /v1/messages adapter degrades to xhigh and /model_group/info stops short of it as on main, and chat still forwards max for the Responses bridge. The gpt-5.3-codex rows said none and xhigh off and minimal on; the live model answers the other way round (2026-10-01), and an opt-out gate would have dropped the levels Codex CLI sends. * fix(reasoning_effort): bridge function tools to Responses when the chat gate drops none * docs(reasoning_effort): correct the max advertisement note * fix(azure_ai): read the Foundry row on the Responses effort gate and turn none off for Azure gpt-6.1-sol * fix(azure): gate a prefixed Foundry deployment's Responses effort off its row litellm.responses(model="azure_ai/", api_base=) is re-routed to the azure provider with the azure_ai/ prefix intact, so the Azure Responses config saw a prefixed name, skipped it as another provider's pass-through, and forwarded none and minimal to Azure even under drop_params. The Azure gpt-5 resolver also read no row at all for a prefixed name. The Azure Responses gate now detects the family the way the Azure chat gate does, and an azure_ai/ name on the Azure OpenAI configs reads its azure_ai/ row when the map has one, else the azure/ twin, the rule the Foundry chat config already applies to the same deployment. * fix(cost-map): turn none off on azure/eu/gpt-6-astra like its sibling rows * docs(reasoning_effort): say how each gate reads none and shorten the module docstring * fix(azure_ai): call the public effort-level check from the Foundry Responses gate * test(reasoning_effort): fold the Foundry Responses tests into the mapped file and type the row helpers --------- Signed-off-by: Paul El Khoury <82139421+pOpO1-9@users.noreply.github.com> Co-authored-by: Cursor Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: ryan-crabbe-berri --- .../llms/azure/chat/gpt_5_transformation.py | 59 +++----- .../llms/azure/responses/transformation.py | 8 + .../llms/azure_ai/responses/transformation.py | 13 ++ .../llms/openai/chat/gpt_5_transformation.py | 102 ++++++++----- .../llms/openai/responses/transformation.py | 39 ++++- ...odel_prices_and_context_window_backup.json | 22 +-- litellm/responses/bridge_check.py | 26 +++- .../reasoning_effort_capability.py | 25 ++- model_prices_and_context_window.json | 22 +-- ..._handler_reasoning_effort_normalization.py | 26 +++- .../chat/test_azure_gpt5_transformation.py | 143 ++++++++++++++++-- .../response/test_azure_transformation.py | 125 ++++++++++++++- .../chat/test_azure_ai_transformation.py | 17 ++- .../test_azure_ai_responses_transformation.py | 74 +++++++++ .../test_openai_responses_transformation.py | 109 +++++++++++++ .../llms/openai/test_gpt5_transformation.py | 101 +++++++++++++ .../test_reasoning_effort_capability.py | 15 +- tests/unit/test_main.py | 49 +++++- 18 files changed, 822 insertions(+), 153 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index a23fffb5744..a9b6c27a0df 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -1,12 +1,10 @@ """Support for Azure OpenAI gpt-5 model family.""" -from typing import Final +from typing import ClassVar, Final import litellm -from litellm.exceptions import UnsupportedParamsError from litellm.llms.openai.chat.gpt_5_transformation import ( OpenAIGPT5Config, - get_effort_level, is_gpt_reasoning_series_name, ) from litellm.types.llms.openai import AllMessageValues @@ -15,9 +13,16 @@ from .gpt_transformation import AzureOpenAIConfig class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): - """Azure specific handling for gpt-5 models.""" + """Azure specific handling for gpt-5 models. + + reasoning_effort none joins xhigh as opt-in here: Azure rejected it on every gpt-5 + deployment before gpt-5.1 (https://github.com/BerriAI/litellm/issues/16704), and a deployment + name the map never heard of must not forward it. See + https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning + """ GPT5_SERIES_ROUTE = "gpt5_series/" + OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none")) @classmethod def _model_map_lookup_name(cls, model: str) -> str: @@ -27,11 +32,18 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): model_prices_and_context_window.json; ``azure/gpt-5.1`` is. Overriding the shared resolver rather than one lookup means the supports, explicitly-disabled and default-effort answers all read the same entry. + + An ``azure_ai/`` name reaches this config when ``get_llm_provider`` re-routes a Foundry + deployment on an OpenAI-v1 host to the azure provider with the prefix intact. It is the + same product the azure_ai config reads ``azure_ai/`` for, so that row answers when + the map carries it, and the azure/ twin when it does not. """ if model.startswith(cls.GPT5_SERIES_ROUTE): return "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :] if model.startswith("azure/"): return model + if model.startswith("azure_ai/"): + return model if model in litellm.model_cost else "azure/" + model.removeprefix("azure_ai/") return "azure/" + model @classmethod @@ -75,33 +87,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params: bool, api_version: str = "", ) -> dict: - reasoning_effort_value = non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") - effective_effort: Final = get_effort_level(reasoning_effort_value) - - # gpt-5.1/5.2/5.4 support reasoning_effort='none', but other gpt-5 models don't - # See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning - supports_none: Final = self._supports_reasoning_effort_level(model, "none") - - if effective_effort == "none" and not supports_none: - if litellm.drop_params is True or (drop_params is not None and drop_params is True): - non_default_params = non_default_params.copy() - optional_params = optional_params.copy() - if get_effort_level(non_default_params.get("reasoning_effort")) == "none": - non_default_params.pop("reasoning_effort") - if get_effort_level(optional_params.get("reasoning_effort")) == "none": - optional_params.pop("reasoning_effort") - else: - raise UnsupportedParamsError( - status_code=400, - message=( - "Azure OpenAI does not support reasoning_effort='none' for this model. " - "To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n" - "`litellm_settings:\n drop_params: true`\n" - "Issue: https://github.com/BerriAI/litellm/issues/16704" - ), - ) - - result: Final = OpenAIGPT5Config.map_openai_params( + return OpenAIGPT5Config.map_openai_params( self, non_default_params=non_default_params, optional_params=optional_params, @@ -109,17 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params=drop_params, ) - # Only drop reasoning_effort='none' for models that don't support it - result_effort: Final = get_effort_level(result.get("reasoning_effort")) - if result_effort == "none" and not supports_none: - result.pop("reasoning_effort") - - # Azure gpt-5.4+ with tools + reasoning_effort is now routed to the - # Responses API bridge (same as OpenAI), so we no longer need to drop - # reasoning_effort here. See: responses_api_bridge_check() in main.py. - - return result - def transform_request( self, model: str, diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 75714d5253b..74e65572809 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -65,10 +65,18 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.AZURE + @staticmethod + def _is_gpt_5_model(model: str) -> bool: + return AzureOpenAIGPT5Config.is_model_gpt_5_model(model) + @staticmethod def _supports_reasoning_effort_none(model: str) -> bool: return AzureOpenAIGPT5Config.supports_reasoning_effort_level(model, "none") + @staticmethod + def _effort_level_is_disabled(model: str, level: str) -> bool: + return AzureOpenAIGPT5Config.row_disables_reasoning_effort_level(model, level) + @staticmethod def _effort_resolves_to_none(model: str, effort: str | None) -> bool: return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort) diff --git a/litellm/llms/azure_ai/responses/transformation.py b/litellm/llms/azure_ai/responses/transformation.py index 2721f26e0fb..d23de05e148 100644 --- a/litellm/llms/azure_ai/responses/transformation.py +++ b/litellm/llms/azure_ai/responses/transformation.py @@ -3,6 +3,7 @@ from typing import Final import httpx from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig +from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config from litellm.llms.azure_ai.common_utils import ( AzureFoundryModelInfo, api_key_header_for_base, @@ -27,6 +28,18 @@ class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.AZURE_AI + @staticmethod + def _supports_reasoning_effort_none(model: str) -> bool: + return AzureAIGPT5Config.supports_reasoning_effort_level(model, "none") + + @staticmethod + def _effort_level_is_disabled(model: str, level: str) -> bool: + return AzureAIGPT5Config.row_disables_reasoning_effort_level(model, level) + + @staticmethod + def _effort_resolves_to_none(model: str, effort: str | None) -> bool: + return AzureAIGPT5Config.effort_resolves_to_none(model, effort) + def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict: params: Final = litellm_params or GenericLiteLLMParams() auth_headers: Final = get_azure_ai_auth_headers( diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 5bf19844287..8d68e7bbad5 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -1,7 +1,7 @@ """Support for OpenAI gpt-5 model family.""" import re -from typing import Final +from typing import ClassVar, Final import litellm from litellm.utils import ( @@ -71,6 +71,10 @@ _get_effort_level = get_effort_level GPT_REASONING_SERIES_MARKERS: Final = ("gpt-5", "gpt-6") +def unsupported_reasoning_effort_message(param: str, level: str, model: str) -> str: + return f"{param}={level} is not supported for {model}. To drop unsupported params set `litellm.drop_params = True`" + + def is_gpt_reasoning_series_name(model: str) -> bool: normalized: Final = model.split("/")[-1] return any(marker in model for marker in GPT_REASONING_SERIES_MARKERS) and "gpt-5-chat" not in normalized @@ -86,6 +90,9 @@ class OpenAIGPT5Config(OpenAIGPTConfig): - Support for GPT-5-Codex models optimized for code generation. """ + OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh",)) + ROUTE_SPECIFIC_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("max",)) + @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: return is_gpt_reasoning_series_name(model) @@ -205,15 +212,10 @@ class OpenAIGPT5Config(OpenAIGPTConfig): return False @classmethod - def _is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool: + def is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool: """Return True only when the model map explicitly sets the capability to False. - Unlike ``_supports_reasoning_effort_level`` (which requires an explicit True), - this method returns True only when ``supports_{level}_reasoning_effort`` is - explicitly set to ``False`` in the model map. A missing key is treated as - supported (i.e. this method returns False = not disabled). - - Use this for opt-out checks where unknown models should be allowed through. + A missing key is not a disable. Unknown models pass through. """ return is_explicitly_disabled_factory( model=cls._model_map_lookup_name(model), @@ -221,6 +223,39 @@ class OpenAIGPT5Config(OpenAIGPTConfig): key=f"supports_{level}_reasoning_effort", ) + @classmethod + def _is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool: + return cls.is_reasoning_effort_level_explicitly_disabled(model, level) + + @classmethod + def refuses_reasoning_effort_level(cls, model: str, level: str) -> bool: + """The chat gate for any requested effort level, read off the model's map row alone. + + An opt-in level (xhigh here; Azure adds none) is refused unless the row says true, since + the providers rejected it on most of the family when it arrived. Every other level is + refused only when the row says false: a model the map never heard of, or a row without the + flag, keeps forwarding what it was sent, and a level with no flag at all (medium, high) is + never refused. So a new model in the family needs its map row and no code change, the + row's supports__reasoning_effort flags being the whole contract. + + A route-specific level (max) is never refused by either gate: the same model honors it + on Responses and refuses it on Chat Completions, so one row flag cannot gate it on + either route. Chat forwards it for the Responses bridge and a plain chat call gets the + provider's own 400, exactly as before the gate read every level. + """ + if level in cls.ROUTE_SPECIFIC_REASONING_EFFORTS: + return False + if level in cls.OPT_IN_REASONING_EFFORTS: + return not cls._supports_reasoning_effort_level(model, level) + return cls.is_reasoning_effort_level_explicitly_disabled(model, level) + + @classmethod + def row_disables_reasoning_effort_level(cls, model: str, level: str) -> bool: + """The Responses gate: opt-out for every level except the route-specific ones.""" + if level in cls.ROUTE_SPECIFIC_REASONING_EFFORTS: + return False + return cls.is_reasoning_effort_level_explicitly_disabled(model, level) + def get_supported_openai_params(self, model: str) -> list: if self.is_model_gpt_5_search_model(model): return [ @@ -281,12 +316,12 @@ class OpenAIGPT5Config(OpenAIGPTConfig): drop_params=drop_params, ) - # Get raw reasoning_effort and effective effort level for all guards. - # Use effective_effort (extracted string) for xhigh validation, "none" checks, and - # tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"} - # must be treated as effort="none" to avoid incorrect tool-drop or sampling errors. - raw_reasoning_effort = non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") - effective_effort: Final = get_effort_level(raw_reasoning_effort) + # Dict inputs like {"effort": "none", "summary": "detailed"} are judged by their effort + # string so the gate, the sampling guard, and the temperature guard all see one level. + raw_reasoning_effort: Final = non_default_params.get("reasoning_effort") or optional_params.get( + "reasoning_effort" + ) + requested_effort: Final = get_effort_level(raw_reasoning_effort) # Normalize dict reasoning_effort to string for Chat Completions API. # Example: {"effort": "high", "summary": "detailed"} -> "high" @@ -298,31 +333,20 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if "reasoning_effort" in optional_params: optional_params["reasoning_effort"] = normalized - if effective_effort == "xhigh": - # xhigh is an opt-in capability: only allow if model explicitly supports it. - if not self._supports_reasoning_effort_level(model, effective_effort): - if litellm.drop_params or drop_params: - non_default_params.pop("reasoning_effort", None) - optional_params.pop("reasoning_effort", None) - else: - raise litellm.utils.UnsupportedParamsError( - message=(f"reasoning_effort={effective_effort} is not supported for this model."), - status_code=400, - ) - elif effective_effort in ("minimal", "low"): - # minimal/low are opt-out: unknown models pass through; only block when - # the model map explicitly sets supports_{level}_reasoning_effort=false. - # Example: gpt-5.5-pro only accepts {medium, high, xhigh}, so it sets - # supports_low_reasoning_effort=false (and supports_minimal=false). - if self._is_reasoning_effort_level_explicitly_disabled(model, effective_effort): - if litellm.drop_params or drop_params: - non_default_params.pop("reasoning_effort", None) - optional_params.pop("reasoning_effort", None) - else: - raise litellm.utils.UnsupportedParamsError( - message=(f"reasoning_effort={effective_effort} is not supported for this model."), - status_code=400, - ) + if requested_effort is not None and self.refuses_reasoning_effort_level(model, requested_effort): + if litellm.drop_params or drop_params: + non_default_params.pop("reasoning_effort", None) + optional_params.pop("reasoning_effort", None) + else: + raise litellm.utils.UnsupportedParamsError( + message=unsupported_reasoning_effort_message("reasoning_effort", requested_effort, model), + status_code=400, + ) + + # The level the provider will see: None once dropped, so the guards below judge by the model's default effort + effective_effort: Final = get_effort_level( + non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") + ) ################################################################ # max_tokens is not supported for gpt-5 models on OpenAI API diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 768d89b9a97..464dc799c16 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -25,7 +25,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig -from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name +from litellm.llms.openai.chat.gpt_5_transformation import ( + is_gpt_reasoning_series_name, + unsupported_reasoning_effort_message, +) from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE from litellm.responses.litellm_completion_transformation.reasoning_items import is_litellm_minted_reasoning_item from litellm.responses.litellm_completion_transformation.tool_search import ( @@ -278,6 +281,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return supports_none_reasoning_effort(model=model, custom_llm_provider=None) + @staticmethod + def _effort_level_is_disabled(model: str, level: str) -> bool: + """Whether the model's cost-map row turns this effort level off. + + The same row the chat gate reads, so a level the row refuses is refused on both wires + and a new model needs its row, not code. This surface stays opt-out for every level, + xhigh included, since it never refused a level the row omits; max is the exception, a + level the row cannot gate because the same model takes it here and refuses it on chat. + Azure overrides this so a bare deployment name reads the azure/ entry. + """ + from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config + + return OpenAIGPT5Config.row_disables_reasoning_effort_level(model, level) + @staticmethod def _effort_resolves_to_none(model: str, effort: str | None) -> bool: """Whether this request's reasoning effort ends up as "none", the one condition @@ -384,7 +401,25 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): lookup_name: Final = self._model_map_lookup_name(model) if self._is_gpt_5_model(model=lookup_name): reasoning: Final = params.get("reasoning") or {} - effort: Final = reasoning.get("effort") if isinstance(reasoning, dict) else None + requested_effort: Final = reasoning.get("effort") if isinstance(reasoning, dict) else None + if ( + isinstance(reasoning, dict) + and isinstance(requested_effort, str) + and self._effort_level_is_disabled(lookup_name, requested_effort) + ): + if not (drop_params or litellm.drop_params): + raise litellm.UnsupportedParamsError( + message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model), + status_code=400, + ) + remaining: Final = reasoning.copy() + remaining.pop("effort", None) + if remaining: + params["reasoning"] = remaining + else: + params.pop("reasoning", None) + forwarded_reasoning: Final = params.get("reasoning") + effort: Final = forwarded_reasoning.get("effort") if isinstance(forwarded_reasoning, dict) else None supports_none: Final = self._supports_reasoning_effort_none(model=lookup_name) effort_is_none: Final = supports_none and self._effort_resolves_to_none(lookup_name, effort) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index de76b6e1bdf..6e573402427 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3895,7 +3895,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_cache_breakpoint": true, @@ -8409,7 +8409,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8458,7 +8458,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -9252,7 +9252,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -33352,9 +33352,9 @@ "supports_response_schema": true, "supports_vision": true, "supports_reasoning": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.3-codex-spark": { "litellm_provider": "chatgpt", @@ -37807,9 +37807,9 @@ "supports_tool_search": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "deprecation_date": "2027-04-01" }, "gpt-5.3-chat-latest": { @@ -69624,7 +69624,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 120a0d7c828..60318d6b184 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -5,6 +5,7 @@ from urllib.parse import urlsplit import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, foundry_chat_rejects_function_tools_while_reasoning, @@ -133,6 +134,16 @@ def chat_target_accepts_reasoning_summary(model: str, custom_llm_provider: str | return chat_config is not None and chat_config.accepts_reasoning_summary_in_reasoning_effort +def _chat_gate_refuses_reasoning_effort_none(model: str, custom_llm_provider: str) -> bool: + match custom_llm_provider: + case "azure": + return litellm.AzureOpenAIGPT5Config.refuses_reasoning_effort_level(model, "none") + case "azure_ai": + return AzureAIGPT5Config.refuses_reasoning_effort_level(model, "none") + case _: + return OpenAIGPT5Config.refuses_reasoning_effort_level(model, "none") + + def responses_api_bridge_check( model: str, custom_llm_provider: str, @@ -183,7 +194,9 @@ def responses_api_bridge_check( # server-side), and Chat Completions rejects function tools whenever reasoning is # on ("Function tools with reasoning_effort are not supported ... use # /v1/responses or set reasoning_effort to 'none'"), so only an explicit - # ``"none"`` keeps the request chat-servable. Custom (grammar) tools are served + # ``"none"`` the chat gate forwards keeps the request chat-servable: a ``none`` + # the model's map row refuses is dropped (or refused) before the call, which + # leaves the provider's default effort in force. Custom (grammar) tools are served # natively by Chat Completions with reasoning on, so custom-only requests stay on # chat and keep their native custom tool_call response shape. # - The UNSET-effort arm only fires against endpoints known to enforce that @@ -213,10 +226,13 @@ def responses_api_bridge_check( (tool.get("type") if isinstance(tool, dict) else getattr(tool, "type", None)) == "tool_search" for tool in (tools or ()) ) - if isinstance(reasoning_effort, dict): - reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None - else: - reasoning_active = reasoning_effort != "none" + requested_effort: Final = reasoning_effort.get("effort") if isinstance(reasoning_effort, dict) else reasoning_effort + requested_summary: Final = reasoning_effort.get("summary") if isinstance(reasoning_effort, dict) else None + reasoning_active: Final = ( + requested_effort != "none" + or requested_summary is not None + or _chat_gate_refuses_reasoning_effort_none(model, custom_llm_provider) + ) # The reasoning+tools constraint is enforced by the real OpenAI backend behind any api.openai.com # host (the default URL or a PrivateLink hostname such as .privatelink.api.openai.com) and # by Azure OpenAI through the azure provider. Resolve the effective OpenAI base arg>global>env>default diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 200de29b664..d660adf05a6 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -1,20 +1,17 @@ """Resolve which reasoning_effort values a deployment, and by intersection a model group, accepts. -An entry that states its levels outright in reasoning_effort_levels is read first and wins -whole, for a model whose set the per-level flags cannot express: Kimi K3 takes low, high and max, -and no flag can drop medium because medium has none. Every other entry answers through the -supports_*_reasoning_effort flags below, whose polarity mirrors how a request path reads that same -flag. medium and high are unconditional for a reasoning model. minimal and low are opt-out: -openai/chat/gpt_5_transformation.py refuses them only when the map says false. xhigh and max are -opt-in. none is opt-out everywhere except the azure gpt-5 family, whose config raises -UnsupportedParamsError without an explicit true. +An entry that states its levels outright in reasoning_effort_levels wins whole, for a model whose set +the per-level flags cannot express (Kimi K3 takes low, high and max, and no flag can drop medium). +Every other entry answers through its supports_*_reasoning_effort flags, read with the polarity the +chat gate uses: medium and high are unconditional, minimal and low are opt-out (refused only on an +explicit false), xhigh is opt-in, and none is opt-out except on the azure gpt-5 family, whose chat +config refuses it without an explicit true. -xhigh is gated on the request path by the openai and azure gpt-5 configs. max is not gated there at -all: outside the gpt-6-astra rows every entry carrying supports_max_reasoning_effort is Claude-family, -and anthropic/chat/transformation.py gates max on the output_config path while its reasoning_effort -path maps any level to a thinking budget. Making max opt-in is a deliberate trade, then, since an -explicit flag is the only signal that the tier is a real one rather than litellm rounding the level -to a budget, and a missing flag costs advisory metadata rather than a rejected request. +The Responses gate reads none, minimal, low and xhigh as opt-out. Neither gate judges max, which +Chat Completions refuses and Responses honors on the gpt-6 family, so its flag is advisory and +advertised opt-in: outside the gpt-6 rows every entry carrying supports_max_reasoning_effort is +Claude-family, where anthropic/chat/transformation.py gates max on the output_config path, so an +explicit flag is the only signal that the tier is real. A deployment the map describes with no effort flags at all resolves to None rather than to the opt-out defaults. 689 of the map's 854 reasoning entries carry no flag, and the o-series, xai and diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index de76b6e1bdf..6e573402427 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3895,7 +3895,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_cache_breakpoint": true, @@ -8409,7 +8409,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8458,7 +8458,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -9252,7 +9252,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -33352,9 +33352,9 @@ "supports_response_schema": true, "supports_vision": true, "supports_reasoning": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.3-codex-spark": { "litellm_provider": "chatgpt", @@ -37807,9 +37807,9 @@ "supports_tool_search": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "deprecation_date": "2027-04-01" }, "gpt-5.3-chat-latest": { @@ -69624,7 +69624,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py index 389398dcbf4..7a4b5badb3a 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py @@ -321,6 +321,23 @@ class TestTheNormalizedTierIsTheTierSent: ): assert _reasoning_effort_sent(model, provider, effort) == expected + @pytest.mark.parametrize( + "model, provider", + [ + ("gpt-5.5", "azure_ai"), + ("azure_ai/gpt-5.5", "azure_ai"), + ("gpt-5.6", "azure"), + ("us/gpt-5.6", "azure"), + ], + ) + def test_an_azure_hosted_deployment_without_the_flag_drops_to_the_tier_it_accepts( + self, local_model_cost_map, model, provider + ): + """Chat Completions answers ``max`` on gpt-5.6 with a 400 naming ``none`` through ``xhigh`` + (OpenAI, 2026-10-01) and none of these Azure rows declares the level, so the adapter sends + the tier below instead of the rejected one.""" + assert _reasoning_effort_sent(model, provider, "max") == "xhigh" + @pytest.mark.parametrize( "model, provider", [ @@ -328,13 +345,16 @@ class TestTheNormalizedTierIsTheTierSent: ("azure_ai/gpt-6-astra", "azure_ai"), ("gpt-6-astra", "azure"), ("us/gpt-6-astra", "azure"), + ("eu/gpt-6-astra", "azure"), ], ) - def test_an_azure_hosted_astra_deployment_drops_to_the_tier_it_accepts( + def test_an_azure_hosted_astra_deployment_drops_to_the_tier_its_chat_route_accepts( self, local_model_cost_map: None, model: str, provider: str ) -> None: - """The deployment answers ``max`` with a 400 naming ``none`` through ``xhigh``, so the rows - say so and the adapter sends the tier below instead of the rejected one.""" + """A Foundry gpt-6-astra answers a Chat Completions ``max`` with a 400 naming low through + xhigh while honoring the same level on the Responses route (2026-10-01). The row's flag is + route-blind, so it keeps max off and the adapter, which cannot know which route the call + takes, sends the tier below instead of the level chat rejects.""" assert _reasoning_effort_sent(model, provider, "max") == "xhigh" def test_the_openai_hosted_twin_still_sends_max(self, local_model_cost_map): diff --git a/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py index 5fcd1f41789..80328022f61 100644 --- a/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final import pytest @@ -369,17 +370,22 @@ def test_azure_gpt_6_astra_takes_the_reasoning_series_request_shape(): model="gpt-6-astra", custom_llm_provider="azure", max_tokens=100, - reasoning_effort="max", + reasoning_effort="xhigh", ) assert params["max_completion_tokens"] == 100 assert "max_tokens" not in params - assert params["reasoning_effort"] == "max" + assert params["reasoning_effort"] == "xhigh" -@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra"]) -def test_azure_gpt6_astra_reasoning_effort_none_unlocks_temperature(config: AzureOpenAIGPT5Config, model: str): - """Foundry's gpt-6-astra accepts reasoning_effort='none' and, only then, a non-default - temperature (verified live against a Foundry deployment), unlike OpenAI's gpt-6-astra.""" +@pytest.mark.parametrize("model", ["azure/gpt-6-sol", "azure/us/gpt-6-sol"]) +def test_azure_reasoning_effort_none_unlocks_temperature_off_the_azure_row( + config: AzureOpenAIGPT5Config, monkeypatch: pytest.MonkeyPatch, model: str +): + """A deployment whose azure/ row takes none accepts a non-default temperature with it, read off + that row even when OpenAI's twin row says the model refuses none.""" + monkeypatch.setitem( + litellm.model_cost, "gpt-6-sol", {**litellm.model_cost["gpt-6-sol"], "supports_none_reasoning_effort": False} + ) params = config.map_openai_params( non_default_params={"temperature": 0.2, "reasoning_effort": "none"}, optional_params={}, @@ -391,14 +397,127 @@ def test_azure_gpt6_astra_reasoning_effort_none_unlocks_temperature(config: Azur assert params["reasoning_effort"] == "none" -@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra"]) -def test_azure_gpt6_astra_rejects_reasoning_effort_minimal(config: AzureOpenAIGPT5Config, model: str): - """Foundry's gpt-6-astra lists none, low, medium, high, xhigh and max but not minimal.""" - with pytest.raises(litellm.utils.UnsupportedParamsError): - config.map_openai_params( - non_default_params={"reasoning_effort": "minimal"}, +def _azure_chat_row_forwards(row: Mapping[str, object], level: str) -> bool: + """Azure chat polarity: max is never gated, xhigh and none need an explicit true, every other + level only has to not be false.""" + if level == "max": + return True + flag: Final = row.get(f"supports_{level}_reasoning_effort") + return flag is True if level in ("xhigh", "none") else flag is not False + + +@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra", "azure/eu/gpt-6-astra"]) +@pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) +def test_azure_gpt6_astra_forwards_exactly_the_effort_levels_its_map_row_allows( + config: AzureOpenAIGPT5Config, model: str, level: str +): + """The azure/ row is the whole contract for a Foundry deployment. A live Foundry gpt-6-astra + (model gpt-6-astra-2026-09-03) answered reasoning_effort none with a 400 naming low, medium, + high and xhigh on 2026-10-01, so its row turns none and minimal off. The row keeps max off too, + since chat refuses it, but no gate reads that flag: the Responses route honors max, so chat + forwards it for the bridge and a plain chat call gets the provider's own 400.""" + forwarded: Final = _azure_chat_row_forwards(litellm.model_cost[model], level) + dropped: Final = config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model=model, + drop_params=True, + api_version="2025-04-01-preview", + ) + assert ("reasoning_effort" in dropped) is forwarded + if forwarded: + kept: Final = config.map_openai_params( + non_default_params={"reasoning_effort": level}, optional_params={}, model=model, drop_params=False, api_version="2025-04-01-preview", ) + assert kept["reasoning_effort"] == level + else: + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model=model, + drop_params=False, + api_version="2025-04-01-preview", + ) + + +@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra", "azure/eu/gpt-6-astra"]) +def test_azure_gpt6_astra_row_turns_none_and_minimal_off(model: str): + """Pinned on purpose so the row-derived test above cannot go vacuous: a Foundry gpt-6-astra + deployment refused none on chat and on Responses on 2026-10-01 (400 unsupported_value naming + low, medium, high, xhigh), and minimal the same way; max it refused on chat while honoring it + on Responses, so the route-blind flag stays off. When Azure adds a level, flip the row and + this assertion together.""" + row: Final = litellm.model_cost[model] + assert row["supports_none_reasoning_effort"] is False + assert row["supports_minimal_reasoning_effort"] is False + assert row["supports_max_reasoning_effort"] is False + + +def test_azure_chat_gate_reads_the_foundry_row_when_azure_ai_prefix_survives_provider_remap( + config: AzureOpenAIGPT5Config, monkeypatch: pytest.MonkeyPatch +): + """A Foundry deployment on an OpenAI-v1 host is re-routed to the azure provider with its + azure_ai/ prefix intact, and this config then gates it. The rows are made to disagree on low, + a level neither real row flags, so the prefixed name has to read azure_ai/gpt-6-sol while the + bare deployment name keeps reading azure/gpt-6-sol.""" + monkeypatch.setitem( + litellm.model_cost, + "azure_ai/gpt-6-sol", + {**litellm.model_cost["azure_ai/gpt-6-sol"], "supports_low_reasoning_effort": False}, + ) + monkeypatch.setitem( + litellm.model_cost, + "azure/gpt-6-sol", + {**litellm.model_cost["azure/gpt-6-sol"], "supports_low_reasoning_effort": True}, + ) + prefixed: Final = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="azure_ai/gpt-6-sol", + drop_params=True, + api_version="2025-04-01-preview", + ) + assert "reasoning_effort" not in prefixed + bare: Final = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="gpt-6-sol", + drop_params=True, + api_version="2025-04-01-preview", + ) + assert bare["reasoning_effort"] == "low" + + +def test_azure_chat_gate_falls_back_to_the_azure_row_for_an_azure_ai_name_the_map_lacks( + config: AzureOpenAIGPT5Config, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.delitem(litellm.model_cost, "azure_ai/gpt-6-sol") + monkeypatch.setitem( + litellm.model_cost, + "azure/gpt-6-sol", + {**litellm.model_cost["azure/gpt-6-sol"], "supports_low_reasoning_effort": False}, + ) + dropped: Final = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="azure_ai/gpt-6-sol", + drop_params=True, + api_version="2025-04-01-preview", + ) + assert "reasoning_effort" not in dropped + + +@pytest.mark.parametrize("key", ["azure/gpt-6.1-sol", "azure/gpt-6.1-sol-2026-09-29", "azure_ai/gpt-6.1-sol"]) +def test_azure_gpt61_sol_rows_turn_none_off(key: str): + """Pinned on purpose, as the astra row above: a Foundry gpt-6.1-sol deployment (model + gpt-6.1-sol-2026-09-29) refused none on chat and on Responses on 2026-10-02 (400 unsupported_value + naming low, medium, high, xhigh, and max on Responses), and minimal the same way, while these rows + still said none was on. When Azure adds a level, flip the row and this assertion together.""" + row: Final = litellm.model_cost[key] + assert row["supports_none_reasoning_effort"] is False + assert row["supports_minimal_reasoning_effort"] is False diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index 9b4c618c78b..7ed98f6c705 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -727,16 +727,21 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: litellm.add_known_models(model_cost_map=litellm.model_cost) -def test_azure_responses_gpt6_astra_reasoning_effort_none_unlocks_temperature(local_model_cost_map: None): - """Foundry's gpt-6-astra accepts reasoning.effort='none' with a non-default temperature - while OpenAI's gpt-6-astra does not, so the gate must read the azure/ cost-map entry - for the bare deployment name rather than OpenAI's.""" +def test_azure_responses_reasoning_effort_none_unlocks_temperature_off_the_azure_row( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +): + """The gate reads the azure/ cost-map entry for the bare deployment name rather than OpenAI's + twin: with OpenAI's gpt-6-sol row turned off for none, azure/gpt-6-sol's row still lets none + through and, only then, a non-default temperature.""" + monkeypatch.setitem( + litellm.model_cost, "gpt-6-sol", {**litellm.model_cost["gpt-6-sol"], "supports_none_reasoning_effort": False} + ) params = AzureOpenAIResponsesAPIConfig().map_openai_params( response_api_optional_params=ResponsesAPIOptionalRequestParams( temperature=0.2, reasoning={"effort": "none"}, ), - model="gpt-6-astra", + model="gpt-6-sol", drop_params=False, ) assert params["temperature"] == 0.2 @@ -755,6 +760,116 @@ def test_azure_responses_gpt6_astra_rejects_temperature_while_reasoning(local_mo ) +@pytest.mark.parametrize("model", ["gpt-6-astra", "gpt-6.1-sol"]) +@pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) +def test_azure_responses_gpt6_effort_levels_follow_the_azure_row(local_model_cost_map: None, model: str, level: str): + """A bare deployment name reads the azure/ row, opt-out for every level but max, so the none + and minimal Foundry refuses on this route (live 400s on 2026-10-01 for gpt-6-astra and 2026-10-02 + for gpt-6.1-sol) are dropped or refused here while the max it honors goes through whatever the + route-blind flag says.""" + forwarded = ( + level == "max" or litellm.model_cost[f"azure/{model}"].get(f"supports_{level}_reasoning_effort") is not False + ) + dropped = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": level}), + model=model, + drop_params=True, + ) + assert ("reasoning" in dropped) is forwarded + if forwarded: + kept = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": level}), + model=model, + drop_params=False, + ) + assert kept["reasoning"] == {"effort": level} + else: + with pytest.raises(litellm.UnsupportedParamsError): + AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": level}), + model=model, + drop_params=False, + ) + + +def test_azure_gpt6_astra_keeps_max_on_this_route_whatever_its_row_says(local_model_cost_map: None): + """Pinned on purpose: the row-derived test above exempts max by hand, so this one has to fail + if the gate ever reads the flag. A Foundry gpt-6-astra deployment answered reasoning.effort max + through /v1/responses with 200, the effort echoed back and reasoning tokens billed, on + 2026-10-01, while chat refuses the same level, as OpenAI's chat does. The row keeps max off for + the adapter and the picker, and this gate still forwards it even under drop_params.""" + assert litellm.model_cost["azure/gpt-6-astra"]["supports_max_reasoning_effort"] is False + kept = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": "max"}), + model="gpt-6-astra", + drop_params=True, + ) + assert kept["reasoning"] == {"effort": "max"} + + +@pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) +def test_azure_responses_effort_gate_fires_when_azure_ai_prefix_survives_provider_remap( + local_model_cost_map: None, level: str +): + """litellm.responses(model="azure_ai/gpt-6.1-sol", api_base=) is re-routed to the + azure provider with the prefix intact, so this config sees the prefixed name. The gate used to + skip every prefixed name as another provider's pass-through, and a live Foundry gpt-6.1-sol + answered the forwarded none and minimal with 400 on 2026-10-02 even under drop_params. The + prefixed name reads the azure_ai/ row and drops or refuses exactly what that row turns off.""" + forwarded = ( + level == "max" + or litellm.model_cost["azure_ai/gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is not False + ) + dropped = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + max_output_tokens=60, reasoning={"effort": level} + ), + model="azure_ai/gpt-6.1-sol", + drop_params=True, + ) + assert ("reasoning" in dropped) is forwarded + if forwarded: + return + with pytest.raises(litellm.UnsupportedParamsError): + AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + max_output_tokens=60, reasoning={"effort": level} + ), + model="azure_ai/gpt-6.1-sol", + drop_params=False, + ) + + +def test_azure_responses_prefixed_name_reads_the_foundry_row_not_the_azure_twin( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +): + """The rows are made to disagree on low, a level neither real row flags, so the prefixed name + proving it reads azure_ai/gpt-6-sol cannot pass by reading azure/gpt-6-sol, which the bare + deployment name keeps reading.""" + monkeypatch.setitem( + litellm.model_cost, + "azure_ai/gpt-6-sol", + {**litellm.model_cost["azure_ai/gpt-6-sol"], "supports_low_reasoning_effort": False}, + ) + monkeypatch.setitem( + litellm.model_cost, + "azure/gpt-6-sol", + {**litellm.model_cost["azure/gpt-6-sol"], "supports_low_reasoning_effort": True}, + ) + prefixed = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": "low"}), + model="azure_ai/gpt-6-sol", + drop_params=True, + ) + assert "reasoning" not in prefixed + bare = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": "low"}), + model="gpt-6-sol", + drop_params=True, + ) + assert bare["reasoning"] == {"effort": "low"} + + _COMPLETED_RESPONSE: Final = { "id": "resp_dated_chat_deployment", "object": "response", diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 634228da1bf..5186541a62b 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -306,17 +306,30 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url)) -def test_foundry_gpt_6_astra_keeps_sampling_params_when_reasoning_effort_is_none(_local_model_cost_map): +def test_foundry_gpt_6_sol_keeps_sampling_params_when_reasoning_effort_is_none(_local_model_cost_map): + """gpt-6-sol's Foundry row still says none, so the sampling params that ride on it stay. gpt-6-astra + used to be the model here until its row turned none off (a live Foundry deployment answered none + with a 400 naming low through xhigh on 2026-10-01); the test below pins that flip.""" optional_params = AzureAIStudioConfig().map_openai_params( non_default_params={"reasoning_effort": "none", "temperature": 0.2, "top_p": 0.9}, optional_params={}, - model="gpt-6-astra", + model="gpt-6-sol", drop_params=False, ) assert optional_params == {"reasoning_effort": "none", "temperature": 0.2, "top_p": 0.9} +def test_foundry_gpt_6_astra_refuses_reasoning_effort_none_before_the_call(_local_model_cost_map): + with pytest.raises(litellm.utils.UnsupportedParamsError): + AzureAIStudioConfig().map_openai_params( + non_default_params={"reasoning_effort": "none"}, + optional_params={}, + model="gpt-6-astra", + drop_params=False, + ) + + def test_azure_ai_grok_stop_parameter_handling(): diff --git a/tests/unit/llms/azure_ai/responses/test_azure_ai_responses_transformation.py b/tests/unit/llms/azure_ai/responses/test_azure_ai_responses_transformation.py index bae956eb061..26076f8bb34 100644 --- a/tests/unit/llms/azure_ai/responses/test_azure_ai_responses_transformation.py +++ b/tests/unit/llms/azure_ai/responses/test_azure_ai_responses_transformation.py @@ -1,12 +1,15 @@ import json +from typing import Final import httpx import pytest import respx import litellm +from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig from litellm.llms.azure_ai.responses.transformation import AzureAIResponsesAPIConfig from litellm.responses.main import _will_bridge_to_chat_completions +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager @@ -309,3 +312,74 @@ async def test_aresponses_serverless_host_stays_on_chat_bridge(): assert chat_route.called assert not responses_route.called assert chat_route.calls.last.request.headers["Authorization"] == "Bearer fake-key" + + +def _responses_params(**fields: object) -> ResponsesAPIOptionalRequestParams: + return ResponsesAPIOptionalRequestParams(**fields) # pyright: ignore[reportArgumentType] # TypedDict kwargs from a test helper + + +def test_foundry_responses_effort_gate_reads_the_azure_ai_row( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +): + """A Foundry deployment and its Azure OpenAI namesake are different products, so the Foundry + Responses route answers from the azure_ai/ row the Foundry chat route already reads, never from + the azure/ twin. The rows are made to disagree on low, a level neither real row flags, so a + leak from the wrong row cannot hide behind rows that happen to agree.""" + monkeypatch.setitem( + litellm.model_cost, + "azure_ai/gpt-6-sol", + {**litellm.model_cost["azure_ai/gpt-6-sol"], "supports_low_reasoning_effort": False}, + ) + monkeypatch.setitem( + litellm.model_cost, + "azure/gpt-6-sol", + {**litellm.model_cost["azure/gpt-6-sol"], "supports_low_reasoning_effort": True}, + ) + + dropped: Final = AzureAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=_responses_params(reasoning={"effort": "low"}), + model="gpt-6-sol", + drop_params=True, + ) + assert "reasoning" not in dropped + + with pytest.raises(litellm.UnsupportedParamsError): + AzureAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=_responses_params(reasoning={"effort": "low"}), + model="gpt-6-sol", + drop_params=False, + ) + + kept_by_azure_openai: Final = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=_responses_params(reasoning={"effort": "low"}), + model="gpt-6-sol", + drop_params=True, + ) + assert kept_by_azure_openai["reasoning"] == {"effort": "low"} + + +def test_foundry_responses_none_unlocks_temperature_off_the_azure_ai_row( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +): + """The sampling-param unlock that rides on none reads the same azure_ai/ row as the effort gate: + with the azure/ twin turned off for none, the Foundry row still lets none through and, only + then, a non-default temperature.""" + monkeypatch.setitem( + litellm.model_cost, + "azure_ai/gpt-6-sol", + {**litellm.model_cost["azure_ai/gpt-6-sol"], "supports_none_reasoning_effort": True}, + ) + monkeypatch.setitem( + litellm.model_cost, + "azure/gpt-6-sol", + {**litellm.model_cost["azure/gpt-6-sol"], "supports_none_reasoning_effort": False}, + ) + + params: Final = AzureAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=_responses_params(temperature=0.2, reasoning={"effort": "none"}), + model="gpt-6-sol", + drop_params=False, + ) + + assert params["temperature"] == 0.2 + assert params["reasoning"] == {"effort": "none"} diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index f85e4db447b..020f491dcfd 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -2303,6 +2303,115 @@ class TestResponsesSurfaceSharesTheEffortRule: ) assert ("top_p" in mapped) is top_p_survives + @pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) + def test_effort_levels_the_map_row_turns_off_are_dropped_or_refused(self, local_model_cost_map, level): + """Same row as the chat gate, read opt-out for every level but max. gpt-6.1-sol's row turns + none and minimal off: OpenAI lists low, medium, high, xhigh and max for it + (https://developers.openai.com/api/docs/models/gpt-6.1-sol, via #43910), and a local proxy + got OpenAI's 400 for none and minimal through /v1/responses on 2026-09-30.""" + forwarded = ( + level == "max" or litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is not False + ) + dropped = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-6.1-sol", + drop_params=True, + ) + assert ("reasoning" in dropped) is forwarded + if forwarded: + kept = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-6.1-sol", + drop_params=False, + ) + assert kept["reasoning"] == {"effort": level} + else: + with pytest.raises(litellm.UnsupportedParamsError): + OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-6.1-sol", + drop_params=False, + ) + + @pytest.mark.parametrize( + ("flags", "level", "forwarded"), + [ + ({}, "none", True), + ({"supports_none_reasoning_effort": False}, "none", False), + ({"supports_none_reasoning_effort": True}, "none", True), + ({"supports_minimal_reasoning_effort": False}, "minimal", False), + ({"supports_max_reasoning_effort": False}, "max", True), + ({}, "xhigh", True), + ({"supports_xhigh_reasoning_effort": False}, "xhigh", False), + ({"supports_none_reasoning_effort": False}, "medium", True), + ], + ) + def test_the_row_flags_alone_decide_whether_a_level_is_forwarded( + self, local_model_cost_map, monkeypatch, flags, level, forwarded + ): + """Same model name, rewritten row: this surface reads every level opt-out, xhigh included, + so the gate flips with the flags alone. max is the one level no flag can turn off here, + since the same model honors it on this route and refuses it on chat. The chat test module + pins the real row's facts.""" + real_row = litellm.model_cost["gpt-6.1-sol"] + without_flags = { + field: value + for field, value in real_row.items() + if not (field.startswith("supports_") and field.endswith("_reasoning_effort")) + } + monkeypatch.setitem(litellm.model_cost, "gpt-6.1-sol", {**without_flags, **flags}) + dropped = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-6.1-sol", + drop_params=True, + ) + assert ("reasoning" in dropped) is forwarded + if not forwarded: + with pytest.raises(litellm.UnsupportedParamsError): + OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-6.1-sol", + drop_params=False, + ) + + def test_dropping_a_refused_effort_keeps_the_rest_of_the_reasoning_object(self, local_model_cost_map): + refused = next( + level + for level in ("none", "minimal", "low") + if litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is False + ) + mapped = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": refused, "summary": "auto"}}, + model="gpt-6.1-sol", + drop_params=True, + ) + assert mapped["reasoning"] == {"summary": "auto"} + + @pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) + def test_gpt5_3_codex_forwards_exactly_the_effort_levels_its_map_row_allows(self, local_model_cost_map, level): + """The gate turns a stale false into a local refusal, so the Responses-only codex row has to + say what the route does. On 2026-10-01 /v1/responses answered gpt-5.3-codex none and xhigh + with 200 (effort echoed back) and minimal and max with 400 unsupported_value naming none, + low, medium, high and xhigh, so the row takes none and xhigh and turns minimal off.""" + forwarded = ( + level == "max" or litellm.model_cost["gpt-5.3-codex"].get(f"supports_{level}_reasoning_effort") is not False + ) + mapped = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-5.3-codex", + drop_params=True, + ) + assert ("reasoning" in mapped) is forwarded + + def test_gpt5_3_codex_row_takes_none_and_xhigh(self, local_model_cost_map): + """Pinned on purpose so the row-derived test above cannot go vacuous: the live answers of + 2026-10-01 listed above. When OpenAI changes the set, flip the row and this assertion + together.""" + row = litellm.model_cost["gpt-5.3-codex"] + assert row["supports_none_reasoning_effort"] is True + assert row["supports_xhigh_reasoning_effort"] is True + assert row["supports_minimal_reasoning_effort"] is False + def test_top_p_raises_without_drop_params(self, local_model_cost_map): with pytest.raises(litellm.UnsupportedParamsError): OpenAIResponsesAPIConfig().map_openai_params( diff --git a/tests/unit/llms/openai/test_gpt5_transformation.py b/tests/unit/llms/openai/test_gpt5_transformation.py index 41f5816600f..7b10411ba20 100644 --- a/tests/unit/llms/openai/test_gpt5_transformation.py +++ b/tests/unit/llms/openai/test_gpt5_transformation.py @@ -1,4 +1,6 @@ import re +from collections.abc import Mapping +from typing import Final import pytest @@ -431,6 +433,105 @@ def test_gpt5_drops_reasoning_effort_minimal_when_requested(config: OpenAIConfig assert "reasoning_effort" not in params +def _chat_row_forwards(row: Mapping[str, object], level: str) -> bool: + """Chat polarity: max is never gated, xhigh needs an explicit true, every other level only has + to not be false.""" + if level == "max": + return True + flag: Final = row.get(f"supports_{level}_reasoning_effort") + return flag is True if level == "xhigh" else flag is not False + + +@pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) +def test_gpt6_1_sol_forwards_exactly_the_effort_levels_its_map_row_allows(config: OpenAIConfig, level: str): + """The map row is the whole contract, so a new GPT-6 model is gated by its row with no code + change. gpt-6.1-sol's row turns none and minimal off: OpenAI lists low, medium, high, xhigh + and max for it (https://developers.openai.com/api/docs/models/gpt-6.1-sol, via #43910), and a + local proxy got OpenAI's 400 for none and minimal on both endpoints on 2026-09-30.""" + forwarded: Final = _chat_row_forwards(litellm.model_cost["gpt-6.1-sol"], level) + dropped: Final = config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model="gpt-6.1-sol", + drop_params=True, + ) + assert ("reasoning_effort" in dropped) is forwarded + if forwarded: + kept: Final = config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model="gpt-6.1-sol", + drop_params=False, + ) + assert kept["reasoning_effort"] == level + else: + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model="gpt-6.1-sol", + drop_params=False, + ) + + +def test_gpt6_1_sol_row_turns_none_and_minimal_off(): + """Pinned on purpose so the row-derived test above cannot go vacuous: OpenAI lists low, medium, + high, xhigh and max for gpt-6.1-sol and neither none nor minimal + (https://developers.openai.com/api/docs/models/gpt-6.1-sol, read 2026-09-30 via #43910), and a + local proxy got OpenAI's 400 for both on 2026-09-30. When OpenAI adds a level, flip the row and + this assertion together.""" + row: Final = litellm.model_cost["gpt-6.1-sol"] + assert row["supports_none_reasoning_effort"] is False + assert row["supports_minimal_reasoning_effort"] is False + + +def _row_without_effort_flags(row: Mapping[str, object]) -> dict[str, object]: + return { + field: value + for field, value in row.items() + if not (field.startswith("supports_") and field.endswith("_reasoning_effort")) + } + + +@pytest.mark.parametrize( + ("flags", "level", "forwarded"), + [ + ({}, "none", True), + ({"supports_none_reasoning_effort": False}, "none", False), + ({"supports_none_reasoning_effort": True}, "none", True), + ({"supports_minimal_reasoning_effort": False}, "minimal", False), + ({"supports_max_reasoning_effort": False}, "max", True), + ({}, "max", True), + ({}, "xhigh", False), + ({"supports_xhigh_reasoning_effort": True}, "xhigh", True), + ({"supports_none_reasoning_effort": False}, "medium", True), + ], +) +def test_the_row_flags_alone_decide_whether_a_chat_level_is_forwarded( + config: OpenAIConfig, monkeypatch: pytest.MonkeyPatch, flags: dict[str, bool], level: str, forwarded: bool +): + """Same model name, rewritten row: the gate flips with the flags and nothing else, so breaking the + gate fails here while the pinned test above catches the real row losing a flag.""" + monkeypatch.setitem( + litellm.model_cost, "gpt-6.1-sol", {**_row_without_effort_flags(litellm.model_cost["gpt-6.1-sol"]), **flags} + ) + dropped: Final = config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model="gpt-6.1-sol", + drop_params=True, + ) + assert ("reasoning_effort" in dropped) is forwarded + if not forwarded: + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model="gpt-6.1-sol", + drop_params=False, + ) + + def test_gpt5_minimal_dict_triggers_validation(config: OpenAIConfig): """Dict with effort='minimal' triggers minimal model-support validation.""" with pytest.raises(litellm.utils.UnsupportedParamsError): diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index 6a0c274f149..2226b1f39df 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -395,22 +395,25 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels: [ ("azure/gpt-6-astra", "azure"), ("azure/us/gpt-6-astra", "azure"), + ("azure/eu/gpt-6-astra", "azure"), ("azure_ai/gpt-6-astra", "azure_ai"), ], ) - def test_an_azure_hosted_deployment_advertises_none_but_not_max( + def test_an_azure_hosted_deployment_advertises_the_levels_both_its_routes_accept( self, local_model_cost_map, model, custom_llm_provider ): - """Microsoft hosts the same model with a different level set than OpenAI does. Verified live - on both Azure routes: none returns 200 with zero reasoning tokens and unlocks temperature, - which OpenAI's API rejects, while max returns 400 unsupported_value naming none through - xhigh as the levels it does take.""" + """Verified live on 2026-10-01 against a Foundry gpt-6-astra deployment (model + gpt-6-astra-2026-09-03): none returns 400 unsupported_value on chat and on Responses, naming + low, medium, high and xhigh as the levels it takes; Responses also honors max with reasoning + tokens billed while chat refuses it, as OpenAI's chat does. The flag is route-blind, so the + row keeps max off and the advertised set stops at the level every route accepts. + Earlier evidence (2026-09-05, commit e79f3ec5205) had none accepted on chat; the deployment + no longer behaves that way.""" from litellm.utils import get_model_info_helper model_info = dict(get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)) assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ( - "none", "low", "medium", "high", diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 7c292ea8f08..63f440d4aba 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -1305,6 +1305,50 @@ def test_responses_api_bridge_check_gpt_5_4_tools_with_reasoning_none_stays_chat assert model_info.get("mode") != "responses" +@pytest.mark.parametrize( + "model, custom_llm_provider, reasoning_effort", + [ + ("gpt-6.1-sol", "openai", "none"), + ("gpt-6.1-sol", "openai", {"effort": "none"}), + ("gpt-6-astra", "azure", "none"), + ], +) +def test_responses_api_bridge_check_tools_with_a_none_the_row_refuses_routes_to_responses( + local_model_cost_map: None, model: str, custom_llm_provider: str, reasoning_effort: object +) -> None: + """The ``none`` escape hatch only holds when the chat gate forwards it. These rows refuse + ``none``, so the gate drops it (or refuses the call) and the provider runs its default effort, + under which Chat Completions rejects function tools; the bridge has to fire the way it does + for an unset effort. Row-derived: the test follows the flag, never the vendor's answer.""" + from litellm.main import responses_api_bridge_check + + row_key: Final = model if custom_llm_provider == "openai" else f"{custom_llm_provider}/{model}" + assert litellm.model_cost[row_key]["supports_none_reasoning_effort"] is False + + model_info, _ = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=reasoning_effort, + ) + + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_a_none_the_row_refuses_stays_chat_without_tools( + local_model_cost_map: None, +) -> None: + """Without function tools nothing forces the Responses route: the dropped ``none`` leaves a + plain chat call at the default effort, as for any other dropped level.""" + from litellm.main import responses_api_bridge_check + + model_info, _ = responses_api_bridge_check( + model="gpt-6.1-sol", custom_llm_provider="openai", reasoning_effort="none" + ) + + assert model_info.get("mode") != "responses" + + def test_responses_api_bridge_check_reasoning_none_with_summary_still_routes_to_responses(): """A reasoning summary is Responses-only regardless of effort value.""" from litellm.main import responses_api_bridge_check @@ -1712,11 +1756,15 @@ _FOUNDRY_FUNCTION_TOOL: Final = ({"type": "function", "function": {"name": "get_ pytest.param("gpt-6-astra", "https://myresource.openai.azure.com", None, id="gpt-6-azure-openai-host"), pytest.param("gpt-5.6-sol", _FOUNDRY_API_BASE, "low", id="gpt-5.6-explicit-effort"), pytest.param("gpt-5.6-sol", _FOUNDRY_API_BASE, {"effort": "high"}, id="gpt-5.6-explicit-effort-dict"), + pytest.param("gpt-6-astra", _FOUNDRY_API_BASE, "none", id="gpt-6-row-refused-none"), ], ) def test_responses_api_bridge_check_azure_ai_foundry_rejected_tools_route_to_responses( model_name, api_base, reasoning_effort ): + """The explicit ``none`` only keeps a Foundry gpt-6 request on chat while the deployment takes + it; the ``azure_ai/gpt-6-astra`` row says it does not, so the gate drops it and the call runs at + the default effort, under which Foundry rejects function tools from gpt-6 on.""" from litellm.main import responses_api_bridge_check model_info, model = responses_api_bridge_check( @@ -1734,7 +1782,6 @@ def test_responses_api_bridge_check_azure_ai_foundry_rejected_tools_route_to_res @pytest.mark.parametrize( "model_name, api_base, reasoning_effort", [ - pytest.param("gpt-6-astra", _FOUNDRY_API_BASE, "none", id="explicit-none-stays-chat"), pytest.param("gpt-5.6-sol", _FOUNDRY_API_BASE, None, id="gpt-5.6-unset-effort-stays-chat"), pytest.param("gpt-5.6-sol", _FOUNDRY_API_BASE, "none", id="gpt-5.6-explicit-none-stays-chat"), pytest.param("gpt-5.5", _FOUNDRY_API_BASE, "high", id="gpt-5.5-explicit-effort-stays-chat"), From f07596a5fdf9cefb9e49212477b08ebc1d68444d Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 19 Oct 2026 12:21:18 -0700 Subject: [PATCH 38/50] fix(helm): stop litellm-helm from injecting placeholder models into every install (#47056) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- helm/litellm-helm/README.md | 13 +++++++--- helm/litellm-helm/tests/deployment_tests.yaml | 26 +++++++++++++++++++ helm/litellm-helm/values.yaml | 12 +-------- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/helm/litellm-helm/README.md b/helm/litellm-helm/README.md index ff432c451e5..e059d45e037 100644 --- a/helm/litellm-helm/README.md +++ b/helm/litellm-helm/README.md @@ -70,11 +70,18 @@ proxyConfigMap: proxy_config: general_settings: master_key: os.environ/PROXY_MASTER_KEY + model_list: [] +``` + +The chart ships no models, so add them under `proxy_config.model_list` or from the Admin UI with `store_model_in_db: true` + +``` +proxy_config: model_list: - - model_name: gpt-3.5-turbo + - model_name: gpt-5.6 litellm_params: - model: gpt-3.5-turbo - api_key: eXaMpLeOnLy + model: openai/gpt-5.6 + api_key: os.environ/OPENAI_API_KEY ``` #### Example using existing `proxyConfigMap` instead of creating it: diff --git a/helm/litellm-helm/tests/deployment_tests.yaml b/helm/litellm-helm/tests/deployment_tests.yaml index ee946038202..292eeadfd13 100644 --- a/helm/litellm-helm/tests/deployment_tests.yaml +++ b/helm/litellm-helm/tests/deployment_tests.yaml @@ -270,6 +270,32 @@ tests: - matchRegex: path: data["config.yaml"] pattern: '(?m)^\s*health_check_details:\s*false$' + - it: should not render any placeholder models when the user sets proxy_config without model_list + template: configmap-litellm.yaml + set: + proxy_config.general_settings.store_model_in_db: true + asserts: + - notMatchRegex: + path: data["config.yaml"] + pattern: 'model_name:' + - matchRegex: + path: data["config.yaml"] + pattern: 'store_model_in_db: true' + - matchRegex: + path: data["config.yaml"] + pattern: 'master_key: os.environ/PROXY_MASTER_KEY' + - it: should render a user-supplied model_list as the only models + template: configmap-litellm.yaml + set: + proxy_config.model_list: + - model_name: my-model + litellm_params: + model: openai/my-model + api_key: os.environ/OPENAI_API_KEY + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: 'model_list:\n- litellm_params:\n api_key: os.environ/OPENAI_API_KEY\n model: openai/my-model\n model_name: my-model\n$' - it: should allow overriding liveness, readiness, and startup probes template: deployment.yaml set: diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 199c4595149..8450c5f2840 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -180,17 +180,7 @@ proxyConfigMap: # Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml # Reference: https://docs.litellm.ai/docs/proxy/configs proxy_config: - model_list: - # At least one model must exist for the proxy to start. - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo - api_key: eXaMpLeOnLy - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + model_list: [] general_settings: master_key: os.environ/PROXY_MASTER_KEY From 2e427c6247903b133bd8e3db9076bf5d70d3f9f0 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 12:37:47 -0700 Subject: [PATCH 39/50] chore(deps): require anyio>=4.14.0 to rule out the lock-waiter deadlock (#47378) anyio below 4.14.0 can leave a cancelled waiter queued on an asyncio Lock or Semaphore, so the next task to acquire waits forever on a lock nobody holds (agronholm/anyio#1145, fixed in 4.14.0). httpcore takes these locks for every httpx request, including the prisma query-engine client litellm imports anyio directly but never declared it, so pip and uv kept an already installed anyio 4.13.0 when installing litellm and accepted an explicit anyio==4.13.0 next to it. uv.lock already resolves 4.14.2, so Docker and CI installs do not change. Both the litellm and litellm-core manifests get the floor, and the lock only records the new direct dependency Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- packaging/litellm-core/pyproject.toml | 1 + pyproject.toml | 1 + tests/unit/test_package_metadata.py | 17 +++++++++++++++++ uv.lock | 2 ++ 4 files changed, 21 insertions(+) diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index cf87cc6b936..2add6868bd9 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", + "anyio>=4.14.0,<5.0", "async-timeout>=4.0.3,<6.0; python_version < '3.11'", "pydantic>=2.11.0,<3.0.0; python_version < '3.14'", "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", diff --git a/pyproject.toml b/pyproject.toml index f6697a1b57a..6c71140e6b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", + "anyio>=4.14.0,<5.0", "async-timeout>=4.0.3,<6.0; python_version < '3.11'", "pydantic>=2.11.0,<3.0.0; python_version < '3.14'", "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", diff --git a/tests/unit/test_package_metadata.py b/tests/unit/test_package_metadata.py index fb6069291d8..dc6e3dca64e 100644 --- a/tests/unit/test_package_metadata.py +++ b/tests/unit/test_package_metadata.py @@ -5,6 +5,7 @@ from typing import Final from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet from packaging.utils import NormalizedName, canonicalize_name +from packaging.version import Version if sys.version_info >= (3, 11): import tomllib @@ -18,6 +19,8 @@ OPTIONAL_DEPENDENCIES: Final = PYPROJECT["project"]["optional-dependencies"] AIOHTTP_POOL_POISONING_RANGE: Final = SpecifierSet(">=3.14.0,<3.14.2") AIOHTTP_POOL_POISONING_RELEASES: Final = ("3.14.0", "3.14.1") AIOHTTP_POOL_POISONING_SOURCE: Final = "aio-libs/aiohttp#12953, fixed in aiohttp 3.14.2" +ANYIO_STRANDED_LOCK_WAITER_FIX: Final = Version("4.14.0") +ANYIO_STRANDED_LOCK_WAITER_SOURCE: Final = "agronholm/anyio#1145, fixed in anyio 4.14.0 released 2026-06-15" CLI_RUNTIME_DEPENDENCIES: Final = frozenset({"rich", "pyyaml", "requests"}) PROXY_SERVER_ONLY_DEPENDENCIES: Final = frozenset( { @@ -92,3 +95,17 @@ def test_locked_aiohttp_version_is_not_pool_poisoning() -> None: locked: Final = _locked_package("aiohttp")["version"] assert isinstance(locked, str) assert not AIOHTTP_POOL_POISONING_RANGE.contains(locked), AIOHTTP_POOL_POISONING_SOURCE + + +def test_declared_anyio_floor_is_at_or_above_stranded_lock_waiter_fix() -> None: + declared: Final = tuple( + Requirement(requirement) + for requirement in PYPROJECT["project"]["dependencies"] + if canonicalize_name(Requirement(requirement).name) == "anyio" + ) + assert len(declared) == 1, ANYIO_STRANDED_LOCK_WAITER_SOURCE + assert declared[0].marker is None, ANYIO_STRANDED_LOCK_WAITER_SOURCE + assert any( + specifier.operator == ">=" and Version(specifier.version) >= ANYIO_STRANDED_LOCK_WAITER_FIX + for specifier in declared[0].specifier + ), ANYIO_STRANDED_LOCK_WAITER_SOURCE diff --git a/uv.lock b/uv.lock index beb66fe42b2..69d9f777687 100644 --- a/uv.lock +++ b/uv.lock @@ -4508,6 +4508,7 @@ version = "1.107.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, + { name = "anyio" }, { name = "async-timeout", marker = "python_full_version < '3.11'" }, { name = "boto3" }, { name = "click" }, @@ -4761,6 +4762,7 @@ requires-dist = [ { name = "a2a-sdk", marker = "extra == 'extra-proxy'", specifier = ">=1.1.0,<2.0" }, { name = "aiohttp", specifier = ">=3.14.2,<4.0" }, { name = "anthropic", extras = ["vertex"], marker = "extra == 'proxy-runtime'", specifier = ">=0.84.0,<1.0" }, + { name = "anyio", specifier = ">=4.14.0,<5.0" }, { name = "apscheduler", marker = "extra == 'proxy'", specifier = ">=3.11.2,<4.0" }, { name = "async-timeout", marker = "python_full_version < '3.11'", specifier = ">=4.0.3,<6.0" }, { name = "audioread", marker = "extra == 'stt-nvidia-riva'", specifier = ">=3.0.1" }, From 3e18a4b55d5dcab3a5514c554a890ed0f9441392 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 13:18:20 -0700 Subject: [PATCH 40/50] fix(router): record cooled deployments on the all-cooled failure row (#47120) * fix(router): record cooled deployments on the all-cooled failure row A RouterRateLimitError now carries the cooled deployments of the route the request resolves to for the caller's team (model group, alias, wildcard or team route), each with its masked api_base. get_error_information copies them into error_information.error_cooldown_deployments so the SpendLogs failure row and every logging callback can tell which deployments were cooling * fix(router): name cooled deployments from the router's own resolved route Use the deployments the router resolved for the request, before health and cooldown filtering, instead of re-deriving the route. Alias chains, the provider-model fallback, wildcards and team filtering now match routing by construction. A routing-strategy refusal records null. * fix(router): name only the cooled deployments tag and region routing allow The refusal's cooled-deployment list now leaves out deployments the request's tags or the end user's allowed region keep it from, so a tenant's refusal row no longer shows another tenant's api_base * fix(router): keep failure logging working for refusal subclasses that skip init A RouterRateLimitError subclass that never calls its parent init now logs a null cooled-deployment list instead of breaking the failure payload * test(router): call the refusal routing helpers directly The router coverage check needs each router function called by name from a unit test --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 3 + litellm/router.py | 104 +++++ litellm/router_utils/cooldown_handlers.py | 24 +- litellm/router_utils/handle_error.py | 12 +- litellm/types/router.py | 4 + litellm/types/utils.py | 6 + .../spend/test_router_cooldown_failure_row.py | 358 +++++++++++++++++ .../test_litellm_logging.py | 44 +++ .../router_utils/test_cooldown_handlers.py | 361 +++++++++++++++++- 9 files changed, 913 insertions(+), 3 deletions(-) create mode 100644 tests/integration/spend/test_router_cooldown_failure_row.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0ac216c2029..3a1dafa331c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -151,6 +151,7 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse +from litellm.types.router import RouterRateLimitError from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, CachingDetails, @@ -6692,6 +6693,7 @@ class StandardLoggingPayloadSetup: rate_limit_category: Final = validate_rate_limit_category(getattr(original_exception, "category", None)) rate_limit_type: Final = validate_rate_limit_type(getattr(original_exception, "rate_limit_type", None)) budget_error: Final = original_exception if isinstance(original_exception, BudgetExceededError) else None + router_error: Final = original_exception if isinstance(original_exception, RouterRateLimitError) else None provider_request_id: Final = _get_provider_request_id(original_exception) if original_exception else None return StandardLoggingPayloadErrorInformation( @@ -6707,6 +6709,7 @@ class StandardLoggingPayloadSetup: error_budget_entity_id=budget_error.entity_id if budget_error else None, error_budget_limit=budget_error.max_budget if budget_error else None, error_budget_spend=budget_error.current_cost if budget_error else None, + error_cooldown_deployments=router_error.cooldown_deployments if router_error else None, normalized_error=normalize_error(original_exception, error_status, error_message), ) diff --git a/litellm/router.py b/litellm/router.py index 113565fecaa..af79fa763c4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -222,6 +222,7 @@ from litellm.router_utils.cooldown_handlers import ( _first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper across router_utils submodules, matching the other cooldown_handlers imports on this line async_get_cooldown_deployments, async_get_cooldown_deployments_with_debug_info, + get_cooldown_deployment_info, get_cooldown_deployments, is_advisor_orchestration_failure, is_background_response_poll_not_found, @@ -12598,6 +12599,80 @@ class Router: if isinstance(model_info, Mapping) and model_info.get("id") is not None ) + def _region_routable_deployments( + self, + deployments: Sequence[DeploymentTypedDict], + request_kwargs: Mapping[str, object] | None, + has_request_input: bool, + ) -> tuple[Mapping[str, Any], ...]: + """ + `deployments` without the ones `_pre_call_checks` keeps this request away from for being outside its allowed + region, typed for the tag routing filter + """ + allowed_model_region: Final = None if request_kwargs is None else request_kwargs.get("allowed_model_region") + if allowed_model_region is None or not self.enable_pre_call_checks or not has_request_input: + return tuple(deployments) + if not isinstance(allowed_model_region, str): + return () + return tuple( + deployment + for deployment in deployments + if is_region_allowed( + litellm_params=LiteLLM_Params.model_validate(deployment["litellm_params"]), + allowed_model_region=allowed_model_region, + ) + ) + + async def _async_routable_cooled_deployment_ids( + self, + model: str, + request_deployments: Sequence[DeploymentTypedDict], + cooldown_deployments: Iterable[str], + request_kwargs: Mapping[str, object], + has_request_input: bool, + ) -> frozenset[str]: + """ + Ids of the cooling deployments in `request_deployments` that region and tag routing would send this request + to, each tag routing pick counted along with the picks it falls back to once that one is unavailable, judged + without recording a tag match on the request + """ + cooled_ids: Final = frozenset(cooldown_deployments) + metadata_variable_name: Final = get_metadata_variable_name_from_kwargs(request_kwargs) + metadata: Final = request_kwargs.get(metadata_variable_name) + if metadata_variable_name in request_kwargs and not isinstance(metadata, Mapping): + return frozenset() + tag_request: Final[Mapping[str, Any]] = ( + {**request_kwargs, metadata_variable_name: {**metadata}} + if isinstance(metadata, Mapping) + else request_kwargs + ) + routable_ids = frozenset[str]() # rebind-ok: gains each pass's tag routing picks + remaining = tuple( # rebind-ok: each pass drops what tag routing just picked + deployment + for deployment in self._region_routable_deployments(request_deployments, request_kwargs, has_request_input) + if self._deployment_ids((deployment,)) <= cooled_ids + ) + while remaining: + try: + picked = await get_deployments_for_tag( + llm_router_instance=self, + model=model, + healthy_deployments=remaining, + request_kwargs=tag_request, + metadata_variable_name=metadata_variable_name, + ) + except ValueError: # tag routing sends this request to none of the rest + break + picked_identities = frozenset(id(deployment) for deployment in picked) + rest = tuple(deployment for deployment in remaining if id(deployment) not in picked_identities) + if len(rest) == len(remaining): + break + routable_ids |= self._deployment_ids( + tuple(deployment for deployment in remaining if id(deployment) in picked_identities) + ) + remaining = rest + return routable_ids + def has_model_id(self, candidate_id: str) -> bool: """ O(1) membership check for a deployment ID without allocating large lists. @@ -14457,6 +14532,13 @@ class Router: litellm_router_instance=self, model=model, parent_otel_span=parent_otel_span, + route_model_ids=await self._async_routable_cooled_deployment_ids( + model=model, + request_deployments=request_deployments, + cooldown_deployments=cooldown_deployments, + request_kwargs=cast(Mapping[str, object], request_kwargs), # cast-ok: str-keyed request kwargs + has_request_input=messages is not None or input is not None, + ), ) raise exception @@ -15656,6 +15738,17 @@ class Router: enable_pre_call_checks=self.enable_pre_call_checks, cooldown_list=_cooldown_list, model_ids=model_ids, + cooldown_deployments=get_cooldown_deployment_info( + self, + self._deployment_ids( + self._region_routable_deployments( + request_deployments, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: str-keyed request kwargs + messages is not None or input is not None, + ) + ), + _cooldown_list, + ), ) if strategy == "simple-shuffle": @@ -15830,6 +15923,17 @@ class Router: enable_pre_call_checks=self.enable_pre_call_checks, cooldown_list=_cooldown_list, model_ids=pass_through_model_ids, + cooldown_deployments=get_cooldown_deployment_info( + self, + self._deployment_ids( + self._region_routable_deployments( + request_deployments, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: str-keyed request kwargs + messages is not None or input is not None, + ) + ), + _cooldown_list, + ), ) # 6. Apply load balancing strategy diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index bbd895320bd..7ed4864ebc3 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -8,7 +8,7 @@ Router cooldown handlers import asyncio import math -from collections.abc import Mapping +from collections.abc import Collection, Mapping, Sequence from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -28,12 +28,14 @@ from litellm.constants import ( ) from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCacheValue from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback from litellm.types.router import Deployment from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN, + CooldownDeploymentInfo, InternalCallOrigin, ) @@ -644,6 +646,26 @@ def get_cooldown_deployments(litellm_router_instance: LitellmRouter, parent_otel _get_cooldown_deployments = get_cooldown_deployments +def _deployment_api_base(litellm_router_instance: LitellmRouter, model_id: str) -> str | None: + deployment: Final = litellm_router_instance.get_deployment(model_id=model_id) + api_base: Final = deployment.litellm_params.api_base if deployment else None + return None if api_base is None else mask_api_base_credentials(api_base) + + +def get_cooldown_deployment_info( + litellm_router_instance: LitellmRouter, route_model_ids: Collection[str], cooldown_list: Sequence[str] +) -> tuple[CooldownDeploymentInfo, ...]: + """ + The deployments the router resolved for this request (`route_model_ids`) that are in `cooldown_list`, each + with its masked api_base + """ + return tuple( + CooldownDeploymentInfo(model_id=model_id, api_base=_deployment_api_base(litellm_router_instance, model_id)) + for model_id in cooldown_list + if model_id in route_model_ids + ) + + def should_cooldown_based_on_allowed_fails_policy( litellm_router_instance: LitellmRouter, deployment: str, diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index 14ed6ed2adc..38d9ef05706 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -1,9 +1,11 @@ +from collections.abc import Collection from typing import TYPE_CHECKING, Any, Final from litellm._logging import redact_secrets, verbose_router_logger from litellm.constants import MAX_EXCEPTION_MESSAGE_LENGTH from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments_with_debug_info, + get_cooldown_deployment_info, ) from litellm.types.integrations.slack_alerting import AlertType from litellm.types.router import RouterRateLimitError @@ -69,7 +71,10 @@ async def send_llm_exception_alert( async def async_raise_no_deployment_exception( - litellm_router_instance: LitellmRouter, model: str, parent_otel_span: Span | None + litellm_router_instance: LitellmRouter, + model: str, + parent_otel_span: Span | None, + route_model_ids: Collection[str] | None = None, ): """ Raises a RouterRateLimitError if no deployment is found for the given model. @@ -94,4 +99,9 @@ async def async_raise_no_deployment_exception( enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks, cooldown_list=cooldown_list_ids, model_ids=model_ids, + cooldown_deployments=( + None + if route_model_ids is None + else get_cooldown_deployment_info(litellm_router_instance, route_model_ids, cooldown_list_ids) + ), ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 68dbc701c9e..46617e1b753 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -43,6 +43,7 @@ from .llms.bedrock import AwsSessionTag from .llms.openai import OpenAIFileObject from .search import SearchProvider from .utils import ( + CooldownDeploymentInfo, CustomPricingLiteLLMParams, MirroredPricingParams, ModelResponse, @@ -1106,6 +1107,7 @@ class RouterErrorTypes(str, enum.Enum): class RouterRateLimitError(ValueError): status_code: Final = 429 + cooldown_deployments: tuple[CooldownDeploymentInfo, ...] | None = None def __init__( self, @@ -1114,11 +1116,13 @@ class RouterRateLimitError(ValueError): enable_pre_call_checks: bool, cooldown_list: list, model_ids: Sequence[str] = (), + cooldown_deployments: Sequence[CooldownDeploymentInfo] | None = None, ) -> None: self.model = model self.cooldown_time = cooldown_time self.enable_pre_call_checks = enable_pre_call_checks self.cooldown_list = cooldown_list + self.cooldown_deployments = None if cooldown_deployments is None else tuple(cooldown_deployments) self.all_deployments_in_cooldown = bool(model_ids) and frozenset(model_ids) <= frozenset(cooldown_list) self.type = ( RouterErrorTypes.all_deployments_in_cooldown.value diff --git a/litellm/types/utils.py b/litellm/types/utils.py index fd1d58adafb..2e05107f82d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3386,6 +3386,11 @@ class StandardLoggingZeroCostDiagnostic(TypedDict): missing_pricing_keys: ReadOnly[tuple[str, ...]] +class CooldownDeploymentInfo(TypedDict): + model_id: ReadOnly[str] + api_base: ReadOnly[str | None] + + class StandardLoggingPayloadErrorInformation(TypedDict, total=False): error_code: str | None error_class: str | None @@ -3414,6 +3419,7 @@ class StandardLoggingPayloadErrorInformation(TypedDict, total=False): error_budget_entity_id: str | None error_budget_limit: float | None error_budget_spend: float | None + error_cooldown_deployments: ReadOnly[tuple[CooldownDeploymentInfo, ...] | None] normalized_error: ReadOnly[str | None] diff --git a/tests/integration/spend/test_router_cooldown_failure_row.py b/tests/integration/spend/test_router_cooldown_failure_row.py new file mode 100644 index 00000000000..04e36785823 --- /dev/null +++ b/tests/integration/spend/test_router_cooldown_failure_row.py @@ -0,0 +1,358 @@ +import asyncio +import json +import uuid +from collections.abc import Callable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +from dataclasses import dataclass +from itertools import product +from pathlib import Path +from typing import Final + +import anthropic +import httpx +import openai +import pytest +import yaml +from integration._support.client import GATEWAY_LIMITS, Gateway, eventually, gateway_from_environment, object_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.redis_process import OwnedRedis, owned_redis +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue +from redis import Redis + +pytestmark: Final = pytest.mark.timeout(300) + +COOLDOWN_SECONDS: Final = 120 +NO_DEPLOYMENTS: Final = "No deployments available" +ROW_SECONDS: Final = 70 +CHAT: Final = "/v1/chat/completions" +MESSAGES: Final = "/v1/messages" +RESPONSES: Final = "/v1/responses" +FAILURE_ROW: Final = ( + "SELECT status, model_group, model_id, api_base, metadata::jsonb -> 'error_information' AS error " + 'FROM "LiteLLM_SpendLogs" WHERE request_id = %s' +) +ENDPOINT_CELLS: Final = tuple((path, stream) for path in (CHAT, MESSAGES, RESPONSES) for stream in (False, True)) +ENDPOINT_GROUPS: Final = tuple( + f"{path.rsplit('/', 1)[-1]}-{'stream' if stream else 'json'}" for path, stream in ENDPOINT_CELLS +) +SDK_CELLS: Final = ("openai-sync", "openai-async-stream", "anthropic-sync") +TAGGED: Final = "tagged" +REGIONAL: Final = "regional" +GROUPS: Final = (*ENDPOINT_GROUPS, *SDK_CELLS, "neighbour", "scoped", "burst", TAGGED, REGIONAL) +LANES: Final = ("a", "b") +LANE_PARAMS: Final[Mapping[tuple[str, str], Mapping[str, JsonValue]]] = { + (TAGGED, "a"): {"tags": ["tenant-a"]}, + (TAGGED, "b"): {"tags": ["default"]}, + (REGIONAL, "a"): {"region_name": "eu"}, + (REGIONAL, "b"): {"region_name": "us"}, +} +TENANT_A: Final[Mapping[str, JsonValue]] = {"metadata": {"tags": ["tenant-a"]}} +UNTAGGED: Final[Mapping[str, JsonValue]] = {} + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + cache: OwnedRedis + prefix: str + wire: Wire + + def group(self, name: str) -> str: + return f"{self.prefix}-{name}" + + def deployments(self, name: str) -> Mapping[str, str]: + return {f"{self.group(name)}-{lane}": f"{self.wire.url}/{self.group(name)}/{lane}/v1" for lane in LANES} + + +@dataclass(frozen=True, slots=True) +class Outcome: + status: int + detail: str + call_id: str + + +def is_model_probe(request: Request) -> bool: + return (request.method, request.target) == ("GET", "/v1/models") + + +def rate_limited(request: Request) -> Reply: + if is_model_probe(request): + return Reply(body=b'{"object": "list", "data": []}') + body: Final = {"error": {"message": "scripted 429", "type": "rate_limit_error", "code": "rate_limit_exceeded"}} + return Reply(status=429, body=json.dumps(body).encode()) + + +def cooldown_config(directory: Path, cache: OwnedRedis, prefix: str, wire: Wire) -> Path: + base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + assert isinstance(base, dict) + model_list: Final = [ + { + "model_name": f"{prefix}-{name}", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_base": f"{wire.url}/{prefix}-{name}/{lane}/v1", + "api_key": "synthetic-openai-key", + **LANE_PARAMS.get((name, lane), {}), + }, + "model_info": { + "id": f"{prefix}-{name}-{lane}", + **({"enable_tag_filtering": True} if name == TAGGED else {}), + }, + } + for name, lane in product(GROUPS, LANES) + ] + config: Final = { + **base, + "model_list": model_list, + "router_settings": { + "num_retries": 0, + "allowed_fails": 0, + "cooldown_time": COOLDOWN_SECONDS, + "enable_pre_call_checks": True, + "redis_host": cache.host, + "redis_port": cache.port, + }, + } + path: Final = directory / f"cooldown-{prefix}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("router-cooldown-failure-row") + prefix: Final = f"cooldown-{uuid.uuid4().hex[:8]}" + with ExitStack() as stack: + upstream: Final = stack.enter_context(gateway_from_environment()) + cache: Final = stack.enter_context(owned_redis(directory)) + wire: Final = stack.enter_context(wire_server(rate_limited)) + config: Final = cooldown_config(directory, cache, prefix, wire) + owned: Final = stack.enter_context(owned_proxy_process(upstream, directory, {}, config=config, workers=2)) + yield Rig(owned.gateway, cache, prefix, wire) + + +def cooled(cache: OwnedRedis, deployment_id: str) -> bool: + with Redis(host=cache.host, port=cache.port) as client: + return int(client.exists(f"deployment:{deployment_id}:cooldown")) == 1 + + +def request_body(path: str, group: str, stream: bool) -> dict[str, JsonValue]: + content: Final = f"cooldown row {uuid.uuid4().hex}" + match path: + case "/v1/messages": + return { + "model": group, + "max_tokens": 8, + "messages": [{"role": "user", "content": content}], + "stream": stream, + } + case "/v1/responses": + return {"model": group, "input": content, "stream": stream} + case _: + return {"model": group, "messages": [{"role": "user", "content": content}], "stream": stream} + + +def send(gateway: Gateway, path: str, group: str, stream: bool = False) -> Outcome: + response: Final = gateway.request("POST", path, request_body(path, group, stream), headers={"Connection": "close"}) + return Outcome(response.status_code, response.text, response.headers["x-litellm-call-id"]) + + +def send_chat(rig: Rig, name: str, extra: Mapping[str, JsonValue]) -> Outcome: + body: Final = {**request_body(CHAT, rig.group(name), False), **extra} + response: Final = rig.gateway.request("POST", CHAT, body, headers={"Connection": "close"}) + return Outcome(response.status_code, response.text, response.headers["x-litellm-call-id"]) + + +def failure_row(call_id: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows(FAILURE_ROW, (call_id,)), lambda found: len(found) == 1, seconds=ROW_SECONDS + ) + assert rows[0]["status"] == "failure", rows + return rows[0] + + +def until_router_refusal(attempt: Callable[[], Outcome], tries: int = 10) -> tuple[Outcome, ...]: + def walk() -> Iterator[Outcome]: + for _ in range(tries): + outcome: Final = attempt() + yield outcome + if NO_DEPLOYMENTS in outcome.detail: + return + + history: Final = tuple(walk()) + assert NO_DEPLOYMENTS in history[-1].detail, [(o.status, o.detail[:200]) for o in history] + return history + + +def cool_group(rig: Rig, name: str) -> tuple[Outcome, ...]: + history: Final = until_router_refusal(lambda: send(rig.gateway, CHAT, rig.group(name))) + assert all(cooled(rig.cache, identity) for identity in rig.deployments(name)), history + return history + + +def expected_cooldown(deployments: Mapping[str, str]) -> list[JsonValue]: + return [{"model_id": identity, "api_base": deployments[identity]} for identity in sorted(deployments)] + + +def cooldown_entries(error: Mapping[str, JsonValue]) -> list[JsonValue] | None: + entries: Final = error.get("error_cooldown_deployments") + if not isinstance(entries, list): + return None + return sorted(entries, key=lambda entry: str(object_value(entry)["model_id"])) + + +def assert_refusal_row(call_id: str, deployments: Mapping[str, str]) -> Mapping[str, JsonValue]: + row: Final = failure_row(call_id) + error: Final = object_value(row["error"]) + assert (error["error_class"], error["error_code"]) == ("RouterRateLimitError", "429"), row + assert cooldown_entries(error) == expected_cooldown(deployments), error.get("error_message") + assert (row["model_id"], row["api_base"]) == ("", ""), row + return error + + +def provider_row(call_id: str) -> tuple[str, str, str, JsonValue]: + row: Final = failure_row(call_id) + error: Final = object_value(row["error"]) + return str(row["model_id"]), str(row["api_base"]), str(error["error_code"]), error.get("error_cooldown_deployments") + + +def assert_provider_rows(history: tuple[Outcome, ...], deployments: Mapping[str, str]) -> None: + provider: Final = history[:-1] + assert all(outcome.status == 429 and NO_DEPLOYMENTS not in outcome.detail for outcome in provider), [ + (o.status, o.detail[:200]) for o in provider + ] + rows: Final = tuple(provider_row(outcome.call_id) for outcome in provider) + assert {identity for identity, _, _, _ in rows} == set(deployments), rows + assert all(base.startswith(deployments[identity]) for identity, base, _, _ in rows), rows + assert all((code, cooldown) == ("429", None) for _, _, code, cooldown in rows), rows + + +@pytest.mark.parametrize(("path", "stream"), ENDPOINT_CELLS, ids=ENDPOINT_GROUPS) +def test_all_cooled_refusal_row_records_429_and_every_cooled_deployment(rig: Rig, path: str, stream: bool) -> None: + name: Final = ENDPOINT_GROUPS[ENDPOINT_CELLS.index((path, stream))] + history: Final = until_router_refusal(lambda: send(rig.gateway, path, rig.group(name), stream)) + refusal: Final = history[-1] + assert refusal.status == 429, (refusal.status, refusal.detail[:300]) + assert_refusal_row(refusal.call_id, rig.deployments(name)) + assert_provider_rows(history, rig.deployments(name)) + + +def openai_chat(rig: Rig, group: str) -> Outcome: + client: Final = openai.OpenAI(base_url=f"{rig.gateway.client.base_url}/v1", api_key=rig.gateway.key, max_retries=0) + with pytest.raises(openai.RateLimitError) as raised: + client.chat.completions.create(model=group, messages=[{"role": "user", "content": uuid.uuid4().hex}]) + return Outcome(429, raised.value.message, raised.value.response.headers["x-litellm-call-id"]) + + +def openai_async_stream(rig: Rig, group: str) -> Outcome: + async def call() -> Outcome: + client: Final = openai.AsyncOpenAI( + base_url=f"{rig.gateway.client.base_url}/v1", api_key=rig.gateway.key, max_retries=0 + ) + try: + with pytest.raises(openai.RateLimitError) as raised: + await client.chat.completions.create( + model=group, messages=[{"role": "user", "content": uuid.uuid4().hex}], stream=True + ) + return Outcome(429, raised.value.message, raised.value.response.headers["x-litellm-call-id"]) + finally: + await client.close() + + return asyncio.run(call()) + + +def anthropic_message(rig: Rig, group: str) -> Outcome: + client: Final = anthropic.Anthropic( + base_url=str(rig.gateway.client.base_url), api_key=rig.gateway.key, max_retries=0 + ) + with pytest.raises(anthropic.RateLimitError) as raised: + client.messages.create(model=group, max_tokens=8, messages=[{"role": "user", "content": uuid.uuid4().hex}]) + return Outcome(429, raised.value.message, raised.value.response.headers["x-litellm-call-id"]) + + +SDK_CLIENTS: Final = { + "openai-sync": openai_chat, + "openai-async-stream": openai_async_stream, + "anthropic-sync": anthropic_message, +} + + +@pytest.mark.parametrize("client", SDK_CELLS) +def test_sdk_clients_get_429_and_the_row_names_the_cooled_deployments(rig: Rig, client: str) -> None: + history: Final = until_router_refusal(lambda: SDK_CLIENTS[client](rig, rig.group(client))) + assert_refusal_row(history[-1].call_id, rig.deployments(client)) + assert_provider_rows(history, rig.deployments(client)) + + +def test_the_row_lists_only_the_requested_groups_deployments_while_another_group_cools(rig: Rig) -> None: + cool_group(rig, "neighbour") + cool_group(rig, "scoped") + refusal: Final = eventually( + lambda: send(rig.gateway, CHAT, rig.group("scoped")), + lambda outcome: ( + NO_DEPLOYMENTS in outcome.detail + and all(identity in outcome.detail for identity in rig.deployments("neighbour")) + ), + ) + assert_refusal_row(refusal.call_id, rig.deployments("scoped")) + + +def test_a_concurrent_burst_on_a_cooled_group_writes_one_complete_row_per_refusal(rig: Rig) -> None: + cool_group(rig, "burst") + rig.wire.drain() + calls: Final = tuple(ENDPOINT_CELLS[index % len(ENDPOINT_CELLS)] for index in range(30)) + with ( + httpx.Client( + base_url=rig.gateway.client.base_url, timeout=60, trust_env=False, limits=GATEWAY_LIMITS + ) as client, + ThreadPoolExecutor(max_workers=30) as pool, + ): + patient: Final = Gateway(client, rig.gateway.key, rig.gateway.upstream_url) + outcomes: Final = tuple(pool.map(lambda call: send(patient, call[0], rig.group("burst"), call[1]), calls)) + assert all(outcome.status == 429 for outcome in outcomes), [(o.status, o.detail[:200]) for o in outcomes] + assert len({outcome.call_id for outcome in outcomes}) == len(outcomes), outcomes + refusals: Final = tuple(outcome for outcome in outcomes if NO_DEPLOYMENTS in outcome.detail) + reached: Final = tuple(outcome for outcome in outcomes if NO_DEPLOYMENTS not in outcome.detail) + assert refusals, [(o.status, o.detail[:200]) for o in outcomes] + assert len(tuple(request for request in rig.wire.drain() if not is_model_probe(request))) == len(reached) + for outcome in refusals: + assert_refusal_row(outcome.call_id, rig.deployments("burst")) + for outcome in reached: + identity, api_base, code, cooldown = provider_row(outcome.call_id) + assert api_base.startswith(rig.deployments("burst")[identity]), (identity, api_base) + assert (code, cooldown) == ("429", None), outcome + + +def cool_lane(rig: Rig, name: str, lane: str, extra: Mapping[str, JsonValue]) -> None: + identity: Final = f"{rig.group(name)}-{lane}" + outcomes: Final = eventually(lambda: send_chat(rig, name, extra), lambda _: cooled(rig.cache, identity)) + assert NO_DEPLOYMENTS not in outcomes.detail, outcomes.detail[:300] + + +def router_refusal(rig: Rig, name: str, extra: Mapping[str, JsonValue]) -> Outcome: + refusal: Final = eventually(lambda: send_chat(rig, name, extra), lambda outcome: NO_DEPLOYMENTS in outcome.detail) + assert refusal.status == 429, (refusal.status, refusal.detail[:300]) + return refusal + + +def test_a_tag_routed_refusal_row_names_only_the_cooled_deployments_its_tags_reach(rig: Rig) -> None: + deployments: Final = rig.deployments(TAGGED) + tenant: Final = f"{rig.group(TAGGED)}-a" + cool_lane(rig, TAGGED, "a", TENANT_A) + cool_lane(rig, TAGGED, "b", UNTAGGED) + assert_refusal_row(router_refusal(rig, TAGGED, TENANT_A).call_id, {tenant: deployments[tenant]}) + assert_refusal_row(router_refusal(rig, TAGGED, UNTAGGED).call_id, deployments) + + +def test_a_region_restricted_refusal_row_names_only_the_cooled_deployments_in_its_region(rig: Rig) -> None: + deployments: Final = rig.deployments(REGIONAL) + european: Final = f"{rig.group(REGIONAL)}-a" + customer: Final = f"{rig.prefix}-eu-customer" + with ExitStack() as cleanups: + rig.gateway.post("/customer/new", {"user_id": customer, "allowed_model_region": "eu"}) + cleanups.callback(rig.gateway.post, "/customer/delete", {"user_ids": [customer]}) + cool_group(rig, REGIONAL) + assert_refusal_row(router_refusal(rig, REGIONAL, {"user": customer}).call_id, {european: deployments[european]}) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index d1258e43154..37a985264b7 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -4569,6 +4569,50 @@ def test_get_error_information_falls_back_to_str_when_no_message_attr(): assert result["error_class"] == "ValueError" +def test_get_error_information_records_the_cooled_deployments_of_a_router_refusal(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.types.router import RouterRateLimitError + from litellm.types.utils import CooldownDeploymentInfo + + cooled: Final = ( + CooldownDeploymentInfo(model_id="refused-1", api_base="https://first.example/v1"), + CooldownDeploymentInfo(model_id="refused-2", api_base=None), + ) + refusal: Final = RouterRateLimitError( + model="refused", + cooldown_time=60, + enable_pre_call_checks=False, + cooldown_list=["refused-1", "refused-2", "neighbour-1"], + model_ids=("refused-1", "refused-2"), + cooldown_deployments=cooled, + ) + + result: Final = StandardLoggingPayloadSetup.get_error_information(refusal) + assert (result["error_class"], result["error_code"], result["error_cooldown_deployments"]) == ( + "RouterRateLimitError", + "429", + cooled, + ), result + provider_429: Final = litellm.RateLimitError("scripted 429", "openai", "gpt-5.4") + assert StandardLoggingPayloadSetup.get_error_information(provider_429)["error_cooldown_deployments"] is None + + +def test_get_error_information_handles_a_router_refusal_subclass_that_skips_its_init(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.types.router import RouterRateLimitError + + class PluginRefusal(RouterRateLimitError): + def __init__(self, message: str) -> None: + ValueError.__init__(self, message) + + result: Final = StandardLoggingPayloadSetup.get_error_information(PluginRefusal("plugin refused")) + assert (result["error_class"], result["error_code"], result["error_cooldown_deployments"]) == ( + "PluginRefusal", + "429", + None, + ), result + + # ────────────────────────────────────────────────────────────────────── # Tests for _get_assembled_streaming_response non-streaming early return # ────────────────────────────────────────────────────────────────────── diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 2376e0f4d0c..0c9d7ab9f85 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -1,5 +1,7 @@ import asyncio +import copy import importlib +import itertools import random import time from collections.abc import Mapping @@ -15,6 +17,7 @@ from litellm import Router from litellm._internal_context import current_service_target from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials from litellm.router_utils.cooldown_cache import CooldownCache, CooldownCacheValue from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback from litellm.router_utils.cooldown_handlers import ( @@ -44,8 +47,9 @@ from litellm.types.router import ( AllowedFailsPolicy, DeploymentTypedDict, LiteLLMParamsTypedDict, + RouterRateLimitError, ) -from litellm.types.utils import CredentialItem +from litellm.types.utils import CooldownDeploymentInfo, CredentialItem from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -3004,3 +3008,358 @@ def test_shared_mode_upstream_429_still_cools_down_the_deployment(single_deploym is True ) assert _should_run_cooldown_logic(single_deployment_router, "dep-1", 429, exc) is True + + +REFUSAL_MESSAGES: Final = [{"role": "user", "content": "cooldown"}] +SECRET_API_BASE: Final = "https://first.example/v1?key=sk-scripted-secret" +FIRST_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-1", api_base=mask_api_base_credentials(SECRET_API_BASE)) +SECOND_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-2", api_base=None) +THIRD_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-3", api_base=None) +OTHER_TEAMS_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-team-b", api_base=None) +FIRST_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-1", api_base="https://wildcard.example/v1") +SECOND_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-2", api_base=None) +ALIAS_TARGET: Final = CooldownDeploymentInfo(model_id="tier-1", api_base="https://tier.example/v1") +PROVIDER_MODEL: Final = CooldownDeploymentInfo(model_id="friendly-1", api_base=None) +EVERY_DEPLOYMENT: Final = ( + "refused-1", + "refused-2", + "refused-3", + "refused-team-b", + "neighbour-1", + "wildcard-1", + "wildcard-2", + "tier-1", + "friendly-1", +) + + +def _cooled_router(routing_strategy: str, cooled: tuple[str, ...]) -> Router: + router: Final = Router( + routing_strategy=routing_strategy, + model_group_alias={"entry": "tier", "tier": "refused"}, + model_list=[ + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": SECRET_API_BASE, "use_in_pass_through": True}, + "model_info": {"id": "refused-1"}, + }, + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "use_in_pass_through": True, "tpm": 1}, + "model_info": {"id": "refused-2"}, + }, + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "tpm": 1}, + "model_info": {"id": "refused-3"}, + }, + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "use_in_pass_through": True}, + "model_info": {"id": "refused-team-b", "team_id": "team-b"}, + }, + { + "model_name": "neighbour", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": "https://neighbour.example/v1"}, + "model_info": {"id": "neighbour-1"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_base": "https://wildcard.example/v1"}, + "model_info": {"id": "wildcard-1"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "wildcard-2"}, + }, + { + "model_name": "tier", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": "https://tier.example/v1"}, + "model_info": {"id": "tier-1"}, + }, + { + "model_name": "friendly", + "litellm_params": {"model": "anthropic/claude-friendly"}, + "model_info": {"id": "friendly-1"}, + }, + ], + ) + for model_id in cooled: + router.cooldown_cache.add_deployment_to_cooldown( + model_id=model_id, + original_exception=litellm.RateLimitError("scripted 429", "openai", "gpt-5.4"), + exception_status=429, + cooldown_time=60, + ) + return router + + +def _team_request(team_id: str | None) -> dict[str, object]: + return {"metadata": {"user_api_key_team_id": team_id} if team_id else {}} + + +def _select(router: Router, entrypoint: str, model: str, request_kwargs: dict[str, object]) -> object: + match entrypoint: + case "async": + return asyncio.run( + router.async_get_available_deployment( + model=model, messages=REFUSAL_MESSAGES, request_kwargs=request_kwargs + ) + ) + case "async-pass-through": + return asyncio.run( + router.async_get_available_deployment_for_pass_through(model=model, request_kwargs=request_kwargs) + ) + case "pass-through": + return router.get_available_deployment_for_pass_through(model=model, request_kwargs=request_kwargs) + case "pass-through-chat": + return router.get_available_deployment_for_pass_through( + model=model, messages=REFUSAL_MESSAGES, request_kwargs=request_kwargs + ) + case _: + return router.get_available_deployment( + model=model, messages=REFUSAL_MESSAGES, request_kwargs=request_kwargs + ) + + +ALL_COOLED: Final = (FIRST_REFUSED, SECOND_REFUSED, THIRD_REFUSED) +STRATEGY_COOLED: Final = ("refused-1", "refused-team-b", "neighbour-1") +WILDCARD_COOLED: Final = (FIRST_WILDCARD, SECOND_WILDCARD) + + +@pytest.mark.parametrize( + ("routing_strategy", "cooled", "entrypoint", "model", "team_id", "expected"), + ( + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "refused", "team-a", ALL_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "refused", "team-a", ALL_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "pass-through", "refused", "team-a", (FIRST_REFUSED, SECOND_REFUSED)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async-pass-through", "refused", "team-a", ALL_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "refused", "team-b", (*ALL_COOLED, OTHER_TEAMS_REFUSED)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "refused", None, ALL_COOLED), + ("cost-based-routing", STRATEGY_COOLED, "sync", "refused", "team-a", None), + ("cost-based-routing", STRATEGY_COOLED, "async", "refused", "team-a", None), + ("cost-based-routing", STRATEGY_COOLED, "pass-through", "refused", "team-a", None), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", (ALIAS_TARGET,)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "entry", "team-a", (ALIAS_TARGET,)), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)), + ), + ids=( + "all-cooled-sync", + "all-cooled-async", + "all-cooled-pass-through", + "all-cooled-async-pass-through", + "all-cooled-owning-team", + "all-cooled-async-no-team", + "strategy-refused-sync", + "strategy-refused-async", + "strategy-refused-pass-through", + "wildcard-sync", + "wildcard-async", + "alias-chain-sync", + "alias-chain-async", + "provider-model-sync", + "provider-model-async", + ), +) +def test_no_deployment_error_names_the_cooled_deployments_the_request_could_route_to( + routing_strategy: str, + cooled: tuple[str, ...], + entrypoint: str, + model: str, + team_id: str | None, + expected: tuple[CooldownDeploymentInfo, ...] | None, +) -> None: + router: Final = _cooled_router(routing_strategy, cooled) + with pytest.raises(RouterRateLimitError) as refusal: + _select(router, entrypoint, model, _team_request(team_id)) + assert refusal.value.cooldown_deployments == expected, refusal.value.cooldown_list + + +@pytest.mark.parametrize("entrypoint", ("sync", "async")) +def test_no_deployment_error_names_only_the_cooled_part_of_a_route_emptied_by_other_filters(entrypoint: str) -> None: + router: Final = _cooled_router("simple-shuffle", STRATEGY_COOLED) + request_kwargs: Final = {**_team_request("team-a"), "_excluded_deployment_ids": ["refused-2", "refused-3"]} + with pytest.raises(RouterRateLimitError) as refusal: + _select(router, entrypoint, "refused", request_kwargs) + assert refusal.value.cooldown_deployments == (FIRST_REFUSED,), refusal.value.cooldown_list + assert "sk-scripted-secret" not in repr(refusal.value.cooldown_deployments) + + +TENANT_A: Final = CooldownDeploymentInfo(model_id="tenant-a-1", api_base="https://tenant-a.example/v1") +TENANT_B: Final = CooldownDeploymentInfo(model_id="tenant-b-1", api_base="https://tenant-b.example/v1") +SHARED: Final = CooldownDeploymentInfo(model_id="shared-1", api_base="https://shared.example/v1") +EVERY_TENANT: Final = (TENANT_A, TENANT_B, SHARED) +TENANT_ROUTES: Final = ((TENANT_A, "tenant-a", "eu"), (TENANT_B, "tenant-b", "us"), (SHARED, "default", "eu")) + + +def _tenant_router(cooled: tuple[CooldownDeploymentInfo, ...], **router_settings: bool) -> Router: + router: Final = Router( + model_list=[ + { + "model_name": "tenants", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_base": deployment["api_base"], + "tags": [tag], + "region_name": region, + "use_in_pass_through": True, + }, + "model_info": {"id": deployment["model_id"]}, + } + for deployment, tag, region in TENANT_ROUTES + ], + **router_settings, + ) + for deployment in cooled: + router.cooldown_cache.add_deployment_to_cooldown( + model_id=deployment["model_id"], + original_exception=litellm.RateLimitError("scripted 429", "openai", "gpt-5.4"), + exception_status=429, + cooldown_time=60, + ) + return router + + +TAGS_ON: Final = {"enable_tag_filtering": True} +REGION_CHECKS_ON: Final = {"enable_pre_call_checks": True} +BOTH_ON: Final = {**TAGS_ON, **REGION_CHECKS_ON} + + +@pytest.mark.parametrize( + ("router_settings", "entrypoint", "request_kwargs", "expected"), + ( + (TAGS_ON, "async", {"metadata": {"tags": ["tenant-a"]}}, (TENANT_A,)), + (TAGS_ON, "async", {"metadata": {"tags": ["tenant-b"]}}, (TENANT_B,)), + (TAGS_ON, "async", {"metadata": {}}, EVERY_TENANT), + (TAGS_ON, "async", {}, EVERY_TENANT), + (TAGS_ON, "async", {"metadata": "unparsed"}, ()), + (TAGS_ON, "async", {"litellm_metadata": {"tags": ["tenant-a"]}}, (TENANT_A,)), + (TAGS_ON, "async", {"metadata": {"tags": ["!tenant-a", "&tenant-b"]}}, (TENANT_B,)), + (TAGS_ON, "async", {"metadata": {"tags": ["!tenant-a"]}}, (TENANT_B, SHARED)), + (TAGS_ON, "async", {"metadata": {"tags": ["tenant-z"]}}, (SHARED,)), + (TAGS_ON, "async-pass-through", {"metadata": {"tags": ["tenant-a"]}}, (TENANT_A,)), + ({}, "async", {"metadata": {"tags": ["tenant-a"]}, "enable_tag_filtering": True}, (TENANT_A,)), + ({}, "async", {"metadata": {"tags": ["tenant-a"]}}, EVERY_TENANT), + (REGION_CHECKS_ON, "async", {"metadata": {}, "allowed_model_region": "eu"}, (TENANT_A, SHARED)), + (REGION_CHECKS_ON, "sync", {"metadata": {}, "allowed_model_region": "eu"}, (TENANT_A, SHARED)), + (REGION_CHECKS_ON, "pass-through-chat", {"metadata": {}, "allowed_model_region": "eu"}, (TENANT_A, SHARED)), + (REGION_CHECKS_ON, "pass-through", {"metadata": {}, "allowed_model_region": "eu"}, EVERY_TENANT), + ({}, "sync", {"metadata": {}, "allowed_model_region": "eu"}, EVERY_TENANT), + (REGION_CHECKS_ON, "sync", {"metadata": {}, "allowed_model_region": 1}, ()), + ( + {**TAGS_ON, **REGION_CHECKS_ON}, + "async", + {"metadata": {"tags": ["tenant-a"]}, "allowed_model_region": "us"}, + (), + ), + ), + ids=( + "tag-a", + "tag-b", + "untagged-falls-back-past-the-default", + "no-metadata", + "metadata-not-a-mapping", + "tag-litellm-metadata", + "tag-constraints", + "tag-exclusion-falls-back-in-tiers", + "unknown-tag-gets-default", + "tag-async-pass-through", + "tag-request-level-switch", + "tags-off", + "region-async", + "region-sync", + "region-pass-through", + "region-pass-through-without-input", + "region-without-pre-call-checks", + "region-not-a-string", + "region-and-tag-allow-none", + ), +) +def test_no_deployment_error_names_only_the_cooled_deployments_tag_and_region_routing_allow( + router_settings: dict[str, bool], + entrypoint: str, + request_kwargs: dict[str, object], + expected: tuple[CooldownDeploymentInfo, ...], +) -> None: + router: Final = _tenant_router(EVERY_TENANT, **router_settings) + with pytest.raises(RouterRateLimitError) as refusal: + _select(router, entrypoint, "tenants", request_kwargs) + assert refusal.value.cooldown_deployments == expected, refusal.value.cooldown_list + + +def test_naming_tag_routed_deployments_leaves_the_request_metadata_untouched() -> None: + metadata: Final[dict[str, object]] = {"tags": ["tenant-a"]} + with pytest.raises(RouterRateLimitError) as refusal: + _select(_tenant_router(EVERY_TENANT, **TAGS_ON), "async", "tenants", {"metadata": metadata}) + assert refusal.value.cooldown_deployments == (TENANT_A,) + assert metadata == {"tags": ["tenant-a"]} + + +@pytest.mark.parametrize( + ("has_request_input", "expected"), ((True, (TENANT_A, SHARED)), (False, EVERY_TENANT)), ids=("chat", "no-input") +) +def test_region_routable_deployments_follow_the_pre_call_region_check( + has_request_input: bool, expected: tuple[CooldownDeploymentInfo, ...] +) -> None: + router: Final = _tenant_router((), **REGION_CHECKS_ON) + kept: Final = router._region_routable_deployments( + router.get_model_list(model_name="tenants") or [], {"allowed_model_region": "eu"}, has_request_input + ) + assert router._deployment_ids(kept) == frozenset(deployment["model_id"] for deployment in expected) + + +def test_routable_cooled_deployment_ids_skip_healthy_deployments_and_peel_tag_fallbacks() -> None: + router: Final = _tenant_router((TENANT_A, SHARED), **TAGS_ON) + routable: Final = asyncio.run( + router._async_routable_cooled_deployment_ids( + model="tenants", + request_deployments=router.get_model_list(model_name="tenants") or [], + cooldown_deployments=(TENANT_A["model_id"], SHARED["model_id"]), + request_kwargs={"metadata": {}}, + has_request_input=True, + ) + ) + assert routable == frozenset((TENANT_A["model_id"], SHARED["model_id"])) + + +REACHABILITY_REQUESTS: Final = ( + {"metadata": {"tags": ["tenant-a"]}}, + {"metadata": {}}, + {"metadata": {"tags": ["!tenant-a"]}}, + {"metadata": {"tags": ["tenant-z"]}}, + {"metadata": {"tags": ["!tenant-a", "&tenant-b"]}}, + {"metadata": {}, "allowed_model_region": "eu"}, + {"metadata": {"tags": ["tenant-a"]}, "allowed_model_region": "us"}, +) +HEALTH_STATES: Final = tuple( + itertools.chain.from_iterable( + itertools.combinations(EVERY_TENANT, size) for size in range(1, len(EVERY_TENANT) + 1) + ) +) + + +def _routable_while_up(up: tuple[CooldownDeploymentInfo, ...], request_kwargs: dict[str, object]) -> set[str]: + router: Final = _tenant_router(tuple(deployment for deployment in EVERY_TENANT if deployment not in up), **BOTH_ON) + try: + pool: Final = asyncio.run( + router.async_get_healthy_deployments( + model="tenants", request_kwargs=copy.deepcopy(request_kwargs), messages=REFUSAL_MESSAGES + ) + ) + except ValueError: + return set() + return {str(deployment["model_info"]["id"]) for deployment in pool} + + +@pytest.mark.parametrize("request_kwargs", REACHABILITY_REQUESTS) +def test_the_refusal_names_each_cooled_deployment_some_cooldown_state_routes_the_request_to( + request_kwargs: dict[str, object], +) -> None: + reachable: Final = set().union(*(_routable_while_up(up, request_kwargs) for up in HEALTH_STATES)) + with pytest.raises(RouterRateLimitError) as refusal: + _select(_tenant_router(EVERY_TENANT, **BOTH_ON), "async", "tenants", copy.deepcopy(request_kwargs)) + assert {deployment["model_id"] for deployment in refusal.value.cooldown_deployments or ()} == reachable From 89a194bbd0fd25c57df5456ccdc223978e4ac084 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 13:20:50 -0700 Subject: [PATCH 41/50] test(integration): hold the upstream until the disconnect is settled before reading its spend row (#47303) test_client_disconnect_before_any_content_sends_no_response_scan released the gated upstream right after the client closed, so the upstream's next chunk raced the disconnect into the proxy. When the chunk won, the proxy had " tail" on record for the client and the cut-short scan recorded a post_call success on the spend row, which the test asserts is absent (CircleCI integration-extensions job 2263560 on 68db022d5704: `AssertionError: (('success',),)`). The test now reads the spend row first, which is written once the proxy has settled the disconnect, and releases the upstream after it. The proxy's behavior is unchanged; only the test's ordering is. Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- tests/integration/observability/test_guardrail_effects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index b3481057b3b..086baac6ee5 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -2836,9 +2836,9 @@ def test_client_disconnect_before_any_content_sends_no_response_scan(gateway: Ga with _disconnect_rig(gateway, tmp_path, shape="empty") as rig: try: _stream_and_close(rig, "/v1/chat/completions", _chat_body(rig, 0), "data: ") + rows: Final = _post_call_statuses(rig, rig.model) finally: rig.gate.set() - rows: Final = _post_call_statuses(rig, rig.model) assert rig.scans.response_scans(rig.secret()) == (), rig.scans.seen assert len(rows) == 1 and "success" not in rows[0], rows From 6e148255d921de26381aed7fa199af0e68bd5bb6 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:30:25 -0700 Subject: [PATCH 42/50] chore(pricing): add meta sam-3.1 image segmentation price (#47395) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 17 +++++++++++++++++ model_prices_and_context_window.json | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6e573402427..9d5e0aa8021 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76135,5 +76135,22 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "meta/sam-3.1": { + "input_cost_per_image": 0.0025, + "litellm_provider": "meta", + "mode": "responses", + "source": "https://dev.meta.ai/docs/pricing-rate-limits.md", + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ] } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6e573402427..9d5e0aa8021 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76135,5 +76135,22 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "meta/sam-3.1": { + "input_cost_per_image": 0.0025, + "litellm_provider": "meta", + "mode": "responses", + "source": "https://dev.meta.ai/docs/pricing-rate-limits.md", + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ] } } From 9bafafc2df3c8871bb30277f47d9fb6600e707e4 Mon Sep 17 00:00:00 2001 From: litellm-stampbot Date: Mon, 19 Oct 2026 13:35:04 -0700 Subject: [PATCH 43/50] feat(search): add DeepSeek as a /v1/search provider (#47401) * feat(search): add DeepSeek as a /v1/search provider Adds DeepSeekSearchConfig for POST https://api.deepseek.com/v1/tools/search, registers it under search_provider=deepseek, prices deepseek/search at 0.00075 per query and marks search support in provider_endpoints_support.json * fix(search): reject DeepSeek queries over the 128 character limit before the call * test(search): escape regex metacharacters in DeepSeek query limit match --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/deepseek/search/__init__.py | 7 + .../llms/deepseek/search/transformation.py | 172 ++++++++++++++++++ ...odel_prices_and_context_window_backup.json | 5 + litellm/types/utils.py | 1 + litellm/utils.py | 2 + model_prices_and_context_window.json | 5 + provider_endpoints_support.json | 3 +- tests/unit/llms/deepseek/search/__init__.py | 0 .../test_deepseek_search_transformation.py | 143 +++++++++++++++ 9 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 litellm/llms/deepseek/search/__init__.py create mode 100644 litellm/llms/deepseek/search/transformation.py create mode 100644 tests/unit/llms/deepseek/search/__init__.py create mode 100644 tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py diff --git a/litellm/llms/deepseek/search/__init__.py b/litellm/llms/deepseek/search/__init__.py new file mode 100644 index 00000000000..528dceb223f --- /dev/null +++ b/litellm/llms/deepseek/search/__init__.py @@ -0,0 +1,7 @@ +""" +DeepSeek Search API module. +""" + +from litellm.llms.deepseek.search.transformation import DeepSeekSearchConfig + +__all__ = ["DeepSeekSearchConfig"] diff --git a/litellm/llms/deepseek/search/transformation.py b/litellm/llms/deepseek/search/transformation.py new file mode 100644 index 00000000000..b6bb1c343d8 --- /dev/null +++ b/litellm/llms/deepseek/search/transformation.py @@ -0,0 +1,172 @@ +""" +DeepSeek Search API. +Endpoint: POST https://api.deepseek.com/v1/tools/search +Docs: https://api-docs.deepseek.com/guides/search/ +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Final + +import httpx +from pydantic import ConfigDict, TypeAdapter, ValidationError + +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_JSON_OBJECTS: Final = TypeAdapter(Sequence[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_STR_LIST: Final = TypeAdapter(list[str]) + +DEEPSEEK_SEARCH_MAX_COUNT: Final = 50 +DEEPSEEK_SEARCH_MAX_QUERY_CHARS: Final = 128 +_DEEPSEEK_SEARCH_PATH: Final = "/v1/tools/search" +# DeepSeek chat defaults DEEPSEEK_API_BASE to the /beta prefix; the search route lives at the root. +_CHAT_ONLY_BASE_SUFFIXES: Final = ("/beta", "/v1") + + +class DeepSeekSearchConfig(BaseSearchConfig): + DEEPSEEK_API_BASE = "https://api.deepseek.com" + + @staticmethod + def ui_friendly_name() -> str: + return "DeepSeek" + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseSearchConfig.validate_environment signature + api_key: str | None = None, + api_base: str | None = None, + **kwargs: object, # kwargs-ok: BaseSearchConfig.validate_environment signature + ) -> dict[str, str]: # mutable-ok: the http handler passes this straight to httpx as headers + resolved_key: Final = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("DEEPSEEK_API_KEY",), + base_env_var="DEEPSEEK_API_BASE", + default_api_base=self.DEEPSEEK_API_BASE, + ) + if not resolved_key: + raise ValueError("DEEPSEEK_API_KEY is not set. Set `DEEPSEEK_API_KEY` environment variable.") + return {**headers, "Authorization": f"Bearer {resolved_key}", "Content-Type": "application/json"} + + def get_complete_url( + self, + api_base: str | None, + optional_params: dict[str, object], # mutable-ok: BaseSearchConfig.get_complete_url signature + data: dict[str, object] | list[dict[str, object]] | None = None, # mutable-ok: base signature + **kwargs: object, # kwargs-ok: BaseSearchConfig.get_complete_url signature + ) -> str: + resolved_base: Final = api_base or get_secret_str("DEEPSEEK_API_BASE") or self.DEEPSEEK_API_BASE + clean_base: Final = resolved_base.rstrip("/") + if clean_base.endswith(_DEEPSEEK_SEARCH_PATH): + return clean_base + return f"{_strip_chat_suffix(clean_base)}{_DEEPSEEK_SEARCH_PATH}" + + def transform_search_request( + self, + query: str | list[str], # mutable-ok: BaseSearchConfig.transform_search_request signature + optional_params: dict[str, object], # mutable-ok: base signature + **kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_request signature + ) -> dict[str, object]: # mutable-ok: the http handler sends this as the JSON body + """ + Unified params -> DeepSeek body: ``query`` (list joined with spaces, ``search_domain_filter`` + folded in as ``site:`` operators, the whole string capped at 128 characters by DeepSeek), + ``max_results`` -> ``count`` clamped to [1, 50]. + ``country`` and ``max_tokens_per_page`` have no DeepSeek equivalent and are dropped. + Everything outside the unified spec (``need_content``, ``ori_question``, ``localization``) + is forwarded as-is. + """ + joined_query: Final = " ".join(query) if isinstance(query, list) else query + domains: Final = _domain_filter(optional_params.get("search_domain_filter")) + count: Final = _count(optional_params.get("max_results")) + unified: Final = self.get_supported_perplexity_optional_params() + passthrough: Final = {k: v for k, v in optional_params.items() if k not in unified} + return { + **passthrough, + "query": _checked_query(_with_site_filters(joined_query, domains), domains), + **({"count": count} if count is not None else {}), + } + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_response signature + ) -> SearchResponse: + response_json: Final = _JSON_OBJECT.validate_python(raw_response.json()) + raw_results: Final = _JSON_OBJECTS.validate_python(response_json.get("results", ())) + return SearchResponse( + results=[_to_search_result(item) for item in raw_results], + object="search", + ) + + +def _strip_chat_suffix(base: str) -> str: + return next( + (base.removesuffix(suffix) for suffix in _CHAT_ONLY_BASE_SUFFIXES if base.endswith(suffix)), + base, + ) + + +def _domain_filter(raw: object) -> tuple[str, ...]: + try: + return tuple(_STR_LIST.validate_python(raw)) + except ValidationError: + return () + + +def _with_site_filters(query: str, domains: tuple[str, ...]) -> str: + if not domains: + return query + site_clause: Final = " OR ".join(f"site:{domain}" for domain in domains) + return f"{query} {site_clause}" + + +def _checked_query(query: str, domains: tuple[str, ...]) -> str: + if len(query) <= DEEPSEEK_SEARCH_MAX_QUERY_CHARS: + return query + filter_note: Final = ( + f" ({len(domains)} search_domain_filter entries are sent as site: operators inside the query)" + if domains + else "" + ) + raise BadRequestError( + message=( + f"DeepSeek Search accepts a query of at most {DEEPSEEK_SEARCH_MAX_QUERY_CHARS} characters, " + f"got {len(query)}{filter_note}. Shorten the query or the domain filter." + ), + model="deepseek/search", + llm_provider="deepseek", + ) + + +def _count(raw: object) -> int | None: + if isinstance(raw, bool) or not isinstance(raw, int): + return None + return max(1, min(raw, DEEPSEEK_SEARCH_MAX_COUNT)) + + +def _to_search_result(item: Mapping[str, object]) -> SearchResult: + date: Final = item.get("date") + return SearchResult.model_validate( + { + **item, + "title": _as_str(item.get("title")), + "url": _as_str(item.get("url")), + "snippet": _as_str(item.get("snippet")), + "date": date if isinstance(date, str) else None, + "last_updated": None, + } + ) + + +def _as_str(value: object) -> str: + return value if isinstance(value, str) else "" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9d5e0aa8021..f5eec206e04 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -24366,6 +24366,11 @@ "notes": "Grounding with Bing Search (G1 SKU): $14 per 1,000 transactions. Tokens for the Foundry model deployment that runs the grounded search are billed separately on that deployment." } }, + "deepseek/search": { + "input_cost_per_query": 0.00075, + "litellm_provider": "deepseek", + "mode": "search" + }, "tinyfish/search": { "input_cost_per_query": 0.0, "litellm_provider": "tinyfish", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2e05107f82d..41581d702f3 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4469,6 +4469,7 @@ class SearchProviders(str, Enum): AGENTCORE = "agentcore" NIMBLE = "nimble" BING_GROUNDING = "bing_grounding" + DEEPSEEK = "deepseek" # Create a set of all search provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index ff2c9012ec8..cf3b8756e2f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10297,6 +10297,7 @@ class ProviderConfigManager: from litellm.llms.bedrock.search.transformation import AgentCoreSearchConfig from litellm.llms.brave.search.transformation import BraveSearchConfig from litellm.llms.dataforseo.search.transformation import DataForSEOSearchConfig + from litellm.llms.deepseek.search.transformation import DeepSeekSearchConfig from litellm.llms.duckduckgo.search.transformation import DuckDuckGoSearchConfig from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig @@ -10336,6 +10337,7 @@ class ProviderConfigManager: SearchProviders.AGENTCORE: AgentCoreSearchConfig, SearchProviders.NIMBLE: NimbleSearchConfig, SearchProviders.BING_GROUNDING: BingGroundingSearchConfig, + SearchProviders.DEEPSEEK: DeepSeekSearchConfig, } config_class: Final = PROVIDER_TO_CONFIG_MAP.get(provider, None) if config_class is None: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9d5e0aa8021..f5eec206e04 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24366,6 +24366,11 @@ "notes": "Grounding with Bing Search (G1 SKU): $14 per 1,000 transactions. Tokens for the Foundry model deployment that runs the grounded search are billed separately on that deployment." } }, + "deepseek/search": { + "input_cost_per_query": 0.00075, + "litellm_provider": "deepseek", + "mode": "search" + }, "tinyfish/search": { "input_cost_per_query": 0.0, "litellm_provider": "tinyfish", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index c2cec2c5490..d65da3f41fd 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -923,7 +923,8 @@ "batches": false, "rerank": false, "a2a": true, - "interactions": true + "interactions": true, + "search": true } }, "edenai": { diff --git a/tests/unit/llms/deepseek/search/__init__.py b/tests/unit/llms/deepseek/search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py b/tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py new file mode 100644 index 00000000000..7539876bd37 --- /dev/null +++ b/tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py @@ -0,0 +1,143 @@ +""" +Tests for the DeepSeek Search provider (POST https://api.deepseek.com/v1/tools/search). +""" + +from typing import Final +import re +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import litellm +from litellm.llms.deepseek.search.transformation import DeepSeekSearchConfig + + +@pytest.mark.asyncio +async def test_asearch_sends_deepseek_body_and_maps_results(monkeypatch): + monkeypatch.setenv("DEEPSEEK_API_KEY", "test-deepseek-key") + monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False) + mock_response: Final = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "results": [ + { + "rerank_score": 2.4, + "title": "DeepSeek V4 model released", + "url": "https://example.com/deepseek-v4", + "snippet": "short snippet", + "summary": "long summary for the model", + "content": "full page content", + "site_name": None, + "date": "2026-06-01T00:00:00+00:00", + "authority": 3, + }, + {"title": None, "url": "https://example.com/degraded", "snippet": None, "date": None}, + ] + } + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_post: + response: Final = await litellm.asearch( + query=["DeepSeek V4", "release"], + search_provider="deepseek", + max_results=5, + search_domain_filter=["github.com", "huggingface.co"], + need_content=True, + ) + + assert mock_post.call_count == 1 + call: Final = mock_post.call_args + assert call.kwargs["url"] == "https://api.deepseek.com/v1/tools/search" + assert call.kwargs["headers"]["Authorization"] == "Bearer test-deepseek-key" + assert call.kwargs["headers"]["Content-Type"] == "application/json" + assert call.kwargs["json"] == { + "query": "DeepSeek V4 release site:github.com OR site:huggingface.co", + "count": 5, + "need_content": True, + } + + assert response.object == "search" + assert len(response.results) == 2 + first: Final = response.results[0] + assert (first.title, first.url, first.snippet) == ( + "DeepSeek V4 model released", + "https://example.com/deepseek-v4", + "short snippet", + ) + assert first.date == "2026-06-01T00:00:00+00:00" + assert first.last_updated is None + assert first.model_extra["summary"] == "long summary for the model" + assert first.model_extra["content"] == "full page content" + assert first.model_extra["rerank_score"] == 2.4 + degraded: Final = response.results[1] + assert (degraded.title, degraded.snippet, degraded.date) == ("", "", None) + + +@pytest.mark.parametrize( + ("max_results", "expected_count"), + [(0, 1), (3, 3), (50, 50), (500, 50)], +) +def test_max_results_clamps_to_deepseek_count_range(max_results, expected_count): + body: Final = DeepSeekSearchConfig().transform_search_request( + query="litellm", optional_params={"max_results": max_results} + ) + assert body == {"query": "litellm", "count": expected_count} + + +def test_query_over_deepseek_limit_fails_before_the_call(): + with pytest.raises( + litellm.BadRequestError, match=re.escape("at most 128 characters, got 144 (3 search_domain_filter") + ): + DeepSeekSearchConfig().transform_search_request( + query="litellm proxy search endpoint configuration guide for enterprise deployments with", + optional_params={"search_domain_filter": ["docs.litellm.ai", "github.com", "huggingface.co"]}, + ) + + +def test_unified_params_without_deepseek_equivalent_are_not_sent(): + body: Final = DeepSeekSearchConfig().transform_search_request( + query="litellm", + optional_params={"country": "US", "max_tokens_per_page": 1024, "ori_question": "what is litellm"}, + ) + assert body == {"query": "litellm", "ori_question": "what is litellm"} + + +@pytest.mark.parametrize( + "api_base", + [ + None, + "https://api.deepseek.com", + "https://api.deepseek.com/", + "https://api.deepseek.com/beta", + "https://api.deepseek.com/v1", + "https://api.deepseek.com/v1/tools/search", + ], +) +def test_complete_url_resolves_search_route_from_chat_style_api_base(api_base, monkeypatch): + monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False) + url: Final = DeepSeekSearchConfig().get_complete_url(api_base=api_base, optional_params={}) + assert url == "https://api.deepseek.com/v1/tools/search" + + +def test_missing_api_key_raises(monkeypatch): + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + with pytest.raises(ValueError, match="DEEPSEEK_API_KEY"): + DeepSeekSearchConfig().validate_environment(headers={}) + + +def test_search_cost_comes_from_cost_map(monkeypatch): + monkeypatch.setenv("DEEPSEEK_API_KEY", "test-deepseek-key") + priced: Final[float] = litellm.model_cost["deepseek/search"]["input_cost_per_query"] + assert priced > 0 + mock_response: Final = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"results": [{"title": "t", "url": "https://e.com", "snippet": "s"}]} + with patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", + return_value=mock_response, + ): + response: Final = litellm.search(query="litellm", search_provider="deepseek") + assert response._hidden_params["response_cost"] == pytest.approx(priced) From c0d88b46c9b5a462405e557782341d3d07663aa9 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:35:27 +0000 Subject: [PATCH 44/50] fix(azure): cache the remaining per-request Entra credentials and keep token refresh off the event loop (#47348) * fix(azure): cache the remaining per-request Entra credentials and keep token refresh off the event loop Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(realtime): await the now async azure health-check auth header helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(azure): await async azure_ad_token_provider results in the threaded wrapper * fix(azure): keep transport params when validating async Azure client params on pydantic 2.11 * test(azure): assert the token and validation thread hops by thread identity instead of sleeps * fix(azure): give the OIDC token exchange its own short-lived HTTP client The exchange now runs from worker threads, so it no longer shares litellm.module_level_client's connection pool with other threads * test(azure): check the OIDC exchange closes its client through an in-process transport * refactor(responses): merge caller extra_headers into a new dict instead of updating validated headers in place --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/azure/common_utils.py | 121 +++++++++++----- litellm/llms/azure/search/transformation.py | 5 +- litellm/llms/custom_httpx/llm_http_handler.py | 71 ++++++---- litellm/realtime_api/main.py | 23 ++- litellm/secret_managers/main.py | 9 +- .../test_health_check_helpers.py | 4 +- ...st_bing_grounding_search_transformation.py | 22 +++ .../llms/azure/test_azure_common_utils.py | 133 +++++++++++++++++- .../custom_httpx/test_llm_http_handler.py | 36 +++++ 9 files changed, 348 insertions(+), 76 deletions(-) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index f9f636244f4..52e3cdf380e 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -3,13 +3,15 @@ import hashlib import json import os import re -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping +from contextlib import closing from functools import lru_cache from types import MappingProxyType from typing import Final, Literal, NamedTuple, cast import httpx from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI +from pydantic import ConfigDict, TypeAdapter, with_config from typing_extensions import ReadOnly, TypedDict import litellm @@ -17,6 +19,7 @@ from litellm._logging import verbose_logger from litellm.caching.caching import DualCache from litellm.constants import DEFAULT_MAX_RETRIES from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient from litellm.secret_managers.get_azure_ad_token_provider import ( get_azure_ad_token_provider, @@ -38,6 +41,16 @@ class _AzureV1ClientParams(TypedDict, total=False, extra_items=object): base_url: ReadOnly[str] +@with_config(ConfigDict(extra="allow")) +class _AsyncAzureClientParams(TypedDict, total=False, extra_items=object): + api_key: ReadOnly[str | None] + azure_ad_token: ReadOnly[str | None] + azure_ad_token_provider: ReadOnly[Callable[[], str | Awaitable[str]] | None] + + +_ASYNC_AZURE_CLIENT_PARAMS_ADAPTER: Final = TypeAdapter(_AsyncAzureClientParams) + + class _AzureGatewayClientParams(TypedDict, total=False, extra_items=object): api_version: ReadOnly[str] base_url: ReadOnly[str] @@ -98,11 +111,42 @@ def _cached_entra_id_token_provider( return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope) +@lru_cache(maxsize=128) +def _cached_username_password_token_provider( + client_id: str, + azure_username: str, + azure_password: str, + scope: str, +) -> Callable[[], str]: + from azure.identity import UsernamePasswordCredential, get_bearer_token_provider + + credential: Final = UsernamePasswordCredential( + client_id=client_id, + username=azure_username, + password=azure_password, + ) + return get_bearer_token_provider(credential, scope) + + @lru_cache(maxsize=128) def _cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]: return get_azure_ad_token_provider(azure_scope=scope) +def cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]: + return _cached_azure_ad_token_refresh_provider(scope) + + +def _async_azure_ad_token_provider( + provider: Callable[[], str | Awaitable[str]], +) -> Callable[[], Awaitable[str]]: + async def async_provider() -> str: + token: Final = await asyncio.to_thread(provider) + return token if isinstance(token, str) else await token + + return async_provider + + def get_azure_ad_token_from_entra_id( tenant_id: str, client_id: str, @@ -177,22 +221,19 @@ def get_azure_ad_token_from_username_password( Returns: callable that returns a bearer token. """ - from azure.identity import UsernamePasswordCredential, get_bearer_token_provider - verbose_logger.debug( "client_id=%s, azure_username=[set=%s], azure_password=[set=%s]", client_id, azure_username is not None, azure_password is not None, ) - credential: Final = UsernamePasswordCredential( + token_provider: Final = _cached_username_password_token_provider( client_id=client_id, - username=azure_username, - password=azure_password, + azure_username=azure_username, + azure_password=azure_password, + scope=scope, ) - token_provider: Final = get_bearer_token_provider(credential, scope) - verbose_logger.debug("token_provider %s", token_provider) return token_provider @@ -248,18 +289,17 @@ def get_azure_ad_token_from_oidc( if azure_ad_token_access_token is not None: return azure_ad_token_access_token - client: Final = litellm.module_level_client - - req_token: Final = client.post( - f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token", - data={ - "client_id": azure_client_id, - "grant_type": "client_credentials", - "scope": scope, - "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", - "client_assertion": oidc_token, - }, - ) + with closing(HTTPHandler(timeout=litellm.request_timeout)) as client: + req_token: Final = client.post( + f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token", + data={ + "client_id": azure_client_id, + "grant_type": "client_credentials", + "scope": scope, + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": oidc_token, + }, + ) if req_token.status_code != 200: raise AzureOpenAIError( @@ -560,22 +600,19 @@ class BaseAzureLLM(BaseOpenAILLM): # on every request (via `_refresh_api_key`), so passing # `azure_ad_token_provider` directly preserves Azure AD token refresh # behavior that the regular AzureOpenAI client provides. - v1_api_key: str | Callable[[], object] | None = ( - azure_client_params.get("api_key") - or azure_client_params.get("azure_ad_token_provider") - or azure_client_params.get("azure_ad_token") + v1_client_params: Final[_AsyncAzureClientParams] = _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER.validate_python( + azure_client_params + ) + v1_token_provider: Final = ( + v1_client_params.get("api_key") + or v1_client_params.get("azure_ad_token_provider") + or v1_client_params.get("azure_ad_token") + ) + v1_api_key: Final[str | Callable[[], str | Awaitable[str]] | None] = ( + _async_azure_ad_token_provider(v1_token_provider) + if _is_async is True and callable(v1_token_provider) + else v1_token_provider ) - if _is_async is True and callable(v1_api_key): - # AsyncOpenAI expects an async provider; wrap the sync provider - # returned by azure-identity. Offload to a thread so a token - # refresh (blocking HTTP call to AAD on cache miss) does not - # stall the event loop. - _sync_provider: Final = v1_api_key - - async def _async_v1_api_key() -> str: - return await asyncio.to_thread(_sync_provider) - - v1_api_key = _async_v1_api_key v1_params: Final[_AzureV1ClientParams] = { "api_key": v1_api_key, @@ -597,7 +634,19 @@ class BaseAzureLLM(BaseOpenAILLM): else: # Traditional Azure API uses AzureOpenAI client if _is_async is True: - openai_client = AsyncAzureOpenAI(**azure_client_params) + async_azure_client_params: Final[_AsyncAzureClientParams] = ( + _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER.validate_python(azure_client_params) + ) + async_token_provider_candidate: Final = async_azure_client_params.get("azure_ad_token_provider") + async_azure_client_params_with_threaded_provider: Final[_AsyncAzureClientParams] = ( + { + **async_azure_client_params, + "azure_ad_token_provider": _async_azure_ad_token_provider(async_token_provider_candidate), + } + if async_token_provider_candidate is not None + else async_azure_client_params + ) + openai_client = AsyncAzureOpenAI(**async_azure_client_params_with_threaded_provider) else: openai_client = AzureOpenAI(**azure_client_params) diff --git a/litellm/llms/azure/search/transformation.py b/litellm/llms/azure/search/transformation.py index 4264dae018a..f2bb2ecc68c 100644 --- a/litellm/llms/azure/search/transformation.py +++ b/litellm/llms/azure/search/transformation.py @@ -35,6 +35,7 @@ from typing import TYPE_CHECKING, Final, Literal import httpx from pydantic import ConfigDict, ValidationError +from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, @@ -263,9 +264,7 @@ def _search_tool(optional_params: Mapping[str, object]) -> _BingGroundingTool | def _default_entra_token_minter() -> str: - from litellm.secret_managers.get_azure_ad_token_provider import get_azure_ad_token_provider - - return get_azure_ad_token_provider(azure_scope=ENTRA_SCOPE)() + return cached_azure_ad_token_refresh_provider(ENTRA_SCOPE)() class BingGroundingSearchConfig(BaseSearchConfig): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index b03b8050299..48858dfa441 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2953,14 +2953,15 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers = responses_api_provider_config.validate_environment( - headers=response_api_optional_request_params.get("extra_headers", {}) or {}, - model=model, - litellm_params=litellm_params, - ) - - if extra_headers: - headers.update(extra_headers) + validated_headers: Final = { + **await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=response_api_optional_request_params.get("extra_headers", {}) or {}, + model=model, + litellm_params=litellm_params, + ), + **(extra_headers or {}), + } # Check if streaming is requested stream = response_api_optional_request_params.get("stream", False) @@ -2976,7 +2977,7 @@ class BaseLLMHTTPHandler: input=input, response_api_optional_request_params=response_api_optional_request_params, litellm_params=litellm_params, - headers=headers, + headers=validated_headers, ) data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data) @@ -3011,7 +3012,7 @@ class BaseLLMHTTPHandler: headers, signed_body = await sign_request_off_loop_if_aws( responses_api_provider_config, responses_api_provider_config.sign_request, - headers=headers, + headers=validated_headers, optional_params=dict(litellm_params), request_data=data, api_base=api_base, @@ -3145,8 +3146,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -3399,8 +3403,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -3563,8 +3570,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -4620,8 +4630,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -4795,12 +4808,15 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model=model, litellm_params=litellm_params - ) - - if extra_headers: - headers.update(extra_headers) + validated_headers: Final = { + **await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model=model, + litellm_params=litellm_params, + ), + **(extra_headers or {}), + } api_base: Final = responses_api_provider_config.get_complete_url( api_base=litellm_params.api_base, @@ -4816,14 +4832,14 @@ class BaseLLMHTTPHandler: response_api_optional_request_params=response_api_optional_request_params, api_base=api_base, litellm_params=litellm_params, - headers=headers, + headers=validated_headers, ) data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data) headers, signed_body = await sign_request_off_loop_if_aws( responses_api_provider_config, responses_api_provider_config.sign_request, - headers=headers, + headers=validated_headers, optional_params=dict(litellm_params), request_data=data, api_base=url, @@ -6991,7 +7007,8 @@ class BaseLLMHTTPHandler: **kwargs, } ) - headers: Final = responses_api_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, headers={}, model=model, litellm_params=litellm_params, diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 96b4b081c05..ea47d744cae 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -562,7 +562,12 @@ async def _arealtime( configured_realtime_protocol, query_params=query_params, websocket=websocket ) resolved_azure_ad_token: Final = ( - None if api_key else get_azure_ad_token(GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token)) + None + if api_key + else await asyncio.to_thread( + get_azure_ad_token, + GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token), + ) ) await azure_realtime.async_realtime( model=model, @@ -743,15 +748,21 @@ def _azure_realtime_health_protocol( return "GA", query_params -def _realtime_health_check_auth_headers( +async def _realtime_health_check_auth_headers( custom_llm_provider: str, api_key: str | None, model_params: Mapping[str, object] ) -> Mapping[str, str]: if custom_llm_provider == "azure": + azure_ad_token: Final = ( + None + if api_key + else await asyncio.to_thread( + get_azure_ad_token, + GenericLiteLLMParams.model_validate(dict(model_params)), + ) + ) return azure_realtime.get_auth_headers( api_key=api_key, - azure_ad_token=( - None if api_key else get_azure_ad_token(GenericLiteLLMParams.model_validate(dict(model_params))) - ), + azure_ad_token=azure_ad_token, ) if api_key is None: return _EMPTY_AUTH_HEADERS @@ -807,7 +818,7 @@ async def realtime_health_check( if custom_llm_provider == LlmProviders.OPENAI.value else resolved_api_key ) - auth_headers: Final = _realtime_health_check_auth_headers( + auth_headers: Final = await _realtime_health_check_auth_headers( custom_llm_provider=custom_llm_provider, api_key=health_check_api_key, model_params=resolved_params, diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index 716407d6def..d12e200772f 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -3,6 +3,8 @@ import base64 import os import time import traceback +from collections.abc import Callable +from functools import lru_cache from typing import Final import httpx @@ -23,6 +25,11 @@ oidc_cache: Final = DualCache() _PARSED_LITERAL: Final = TypeAdapter(object) +@lru_cache(maxsize=128) +def _cached_azure_oidc_token_provider(audience: str) -> Callable[[], str]: + return get_azure_ad_token_provider(azure_scope=audience) + + _OIDC_TOKEN_EXPIRY_MARGIN_SECONDS: Final = 60 @@ -283,7 +290,7 @@ def get_secret( verbose_logger.warning( "AZURE_FEDERATED_TOKEN_FILE not found in environment will use Azure AD token provider" ) - azure_token_provider: Final = get_azure_ad_token_provider(azure_scope=oidc_aud) + azure_token_provider: Final = _cached_azure_oidc_token_provider(oidc_aud) try: oidc_token = azure_token_provider() if oidc_token is None: diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 4f62c6fb096..c23749f7a20 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -534,12 +534,12 @@ async def test_ocr_health_check_sends_the_document_kind_the_provider_config_acce assert document[expected_document_type].startswith(expected_uri_prefix) -def test_realtime_health_check_azure_ad_params_drop_reserved_keys(): +async def test_realtime_health_check_azure_ad_params_drop_reserved_keys(): from litellm.realtime_api import main as realtime_main seen = [] with patch.object(realtime_main, "get_azure_ad_token", lambda params: seen.append(params) or "ad-token"): - headers = realtime_main._realtime_health_check_auth_headers( + headers = await realtime_main._realtime_health_check_auth_headers( "azure", None, MappingProxyType({"api_base": "https://x.openai.azure.com", "self": 1, "params": 2, "__class__": 3}), diff --git a/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py b/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py index f8d042c14fd..dfd1474211f 100644 --- a/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py +++ b/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py @@ -1,5 +1,6 @@ import json, litellm from pathlib import Path +from typing import Final from unittest.mock import AsyncMock, Mock, patch import pytest @@ -110,6 +111,27 @@ def test_validate_environment_entra_minter_failure_names_the_options(): assert "no az login" in message +def test_default_entra_minter_reuses_provider(monkeypatch: pytest.MonkeyPatch) -> None: + import importlib + + from litellm.llms.azure import common_utils + from litellm.llms.azure.search.transformation import _default_entra_token_minter + + provider_module: Final = importlib.import_module("litellm.secret_managers.get_azure_ad_token_provider") + provider_factory: Final = Mock(side_effect=lambda **_: lambda: "entra-token") + common_utils._cached_azure_ad_token_refresh_provider.cache_clear() + monkeypatch.setattr(common_utils, "get_azure_ad_token_provider", provider_factory) + monkeypatch.setattr(provider_module, "get_azure_ad_token_provider", provider_factory) + + try: + first: Final = _default_entra_token_minter() + second: Final = _default_entra_token_minter() + assert first == second == "entra-token" + assert provider_factory.call_count == 1 + finally: + common_utils._cached_azure_ad_token_refresh_provider.cache_clear() + + def test_validate_environment_does_not_mutate_and_is_idempotent(): config = _config() caller_headers = {"X-Custom": "keep-me"} diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 80449909b3f..41ad27d6f40 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -1,9 +1,12 @@ import json import os +import threading import traceback -from typing import Callable, Optional +from typing import Callable, Final, Optional +from urllib.parse import parse_qs from unittest.mock import MagicMock, patch +import httpx import pytest import litellm @@ -13,6 +16,8 @@ from litellm.llms.azure.common_utils import ( _cached_entra_id_token_provider, get_azure_ad_token, get_azure_ad_token_from_entra_id, + get_azure_ad_token_from_oidc, + get_azure_ad_token_from_username_password, ) from litellm.secret_managers.get_azure_ad_token_provider import ( get_azure_ad_token_provider, @@ -2141,6 +2146,132 @@ class TestEntraIdTokenProviderCache: assert mock_credential.call_count == 2 +def test_username_password_token_provider_reuses_same_credentials_and_separates_passwords() -> None: + with ( + patch("azure.identity.UsernamePasswordCredential") as mock_credential, + patch( + "azure.identity.get_bearer_token_provider", + side_effect=lambda credential, scope: lambda: "token", + ), + ): + first: Final = get_azure_ad_token_from_username_password( + client_id="client", + azure_username="user", + azure_password="password", + scope="https://cognitiveservices.azure.com/.default", + ) + second: Final = get_azure_ad_token_from_username_password( + client_id="client", + azure_username="user", + azure_password="password", + scope="https://cognitiveservices.azure.com/.default", + ) + changed_password: Final = get_azure_ad_token_from_username_password( + client_id="client", + azure_username="user", + azure_password="different-password", + scope="https://cognitiveservices.azure.com/.default", + ) + + assert first() == second() == changed_password() == "token" + assert mock_credential.call_count == 2 + + +@pytest.mark.asyncio +async def test_non_v1_azure_token_refresh_runs_off_the_event_loop_thread() -> None: + from openai import AsyncAzureOpenAI + + def token_provider_naming_its_thread() -> str: + return str(threading.get_ident()) + + client: Final = BaseAzureLLM().get_azure_openai_client( + api_key=None, + api_base="https://test.openai.azure.com", + api_version="2024-02-01", + litellm_params={"azure_ad_token_provider": token_provider_naming_its_thread}, + _is_async=True, + ) + + assert isinstance(client, AsyncAzureOpenAI) + try: + token_thread: Final = await client._get_azure_ad_token() + finally: + await client.close() + + assert token_thread is not None + assert token_thread != str(threading.get_ident()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_version", ["2024-02-01", "v1"]) +async def test_async_azure_ad_token_provider_reaches_azure_as_bearer_token( + monkeypatch: pytest.MonkeyPatch, api_version: str +) -> None: + def echo_authorization(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "gpt", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": request.headers["authorization"]}, + } + ], + }, + ) + + async def async_token_provider() -> str: + return "async-token" + + monkeypatch.setattr( + litellm, "aclient_session", httpx.AsyncClient(transport=httpx.MockTransport(echo_authorization)) + ) + client: Final = BaseAzureLLM().get_azure_openai_client( + api_key=None, + api_base=f"https://async-token-provider-{api_version}.openai.azure.com", + api_version=api_version, + litellm_params={"azure_ad_token_provider": async_token_provider}, + _is_async=True, + ) + + response: Final = await client.chat.completions.create(model="gpt", messages=[{"role": "user", "content": "hi"}]) + + assert response.choices[0].message.content == "Bearer async-token" + + +def test_oidc_token_exchange_closes_its_own_http_client(monkeypatch: pytest.MonkeyPatch) -> None: + class TokenEndpoint(httpx.BaseTransport): + def __init__(self) -> None: + self.closed = False + + def handle_request(self, request: httpx.Request) -> httpx.Response: + form: Final = parse_qs(request.read().decode()) + return httpx.Response( + 200, json={"access_token": f"minted-for-{form['client_assertion'][0]}", "expires_in": 3600} + ) + + def close(self) -> None: + self.closed = True + + token_endpoint: Final = TokenEndpoint() + monkeypatch.setattr(litellm, "sync_transport", token_endpoint, raising=False) + monkeypatch.setenv("AZURE_TEST_OIDC_ASSERTION", "workload-jwt") + + token: Final = get_azure_ad_token_from_oidc( + azure_ad_token="os.environ/AZURE_TEST_OIDC_ASSERTION", + azure_client_id="client", + azure_tenant_id="tenant-closes-its-client", + ) + + assert token == "minted-for-workload-jwt" + assert token_endpoint.closed + + def test_evicting_an_azure_client_built_on_the_callers_session_leaves_it_open(monkeypatch): """`initialize_azure_sdk_client` puts `litellm.aclient_session` on the SDK client. diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 23c142b61ab..4e529764229 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -476,6 +476,42 @@ async def test_async_response_api_handler_streams_when_provider_transform_adds_s assert client.post.call_args.kwargs["json"]["stream"] is True +@pytest.mark.asyncio +async def test_async_response_api_handler_validates_environment_off_the_event_loop() -> None: + def validate_environment(**_kwargs: object) -> dict[str, str]: + return {"validated-on-thread": str(threading.get_ident()), "x-override": "provider"} + + config: Final = Mock() + config.validate_environment.side_effect = validate_environment + config.get_complete_url.return_value = "https://responses.example.com/v1/responses" + config.async_transform_responses_api_request = AsyncMock(return_value={"model": "model", "input": "hi"}) + config.sign_request.return_value = ({}, None) + client: Final = AsyncMock(spec=AsyncHTTPHandler) + client.post.return_value = httpx.Response( + 200, + request=httpx.Request("POST", "https://responses.example.com/v1/responses"), + ) + logging_obj: Final = Mock() + logging_obj.dynamic_success_callbacks = None + + await BaseLLMHTTPHandler().async_response_api_handler( + model="model", + input="hi", + responses_api_provider_config=config, + response_api_optional_request_params={}, + custom_llm_provider="test", + litellm_params=GenericLiteLLMParams(), + logging_obj=logging_obj, + client=client, + extra_headers={"x-override": "caller"}, + ) + + validated_headers: Final = config.async_transform_responses_api_request.await_args.kwargs["headers"] + assert validated_headers["validated-on-thread"] != str(threading.get_ident()) + assert validated_headers["x-override"] == "caller" + client.post.assert_awaited_once() + + _CHATGPT_SSE_BODY = ( "event: response.output_item.done\n" 'data: {"type": "response.output_item.done", "output_index": 0, "item": {"type": "message", ' From 1212260ab15fe2c9ff28f0c465ad2d0833da8143 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 13:38:22 -0700 Subject: [PATCH 45/50] fix(logging): report the cost-map entry billing used in model_map_information (#47182) * fix(logging): report the cost-map entry billing used in model_map_information Billing resolved the deployment's own cost-map entry and logging ran a second, name-only lookup, so callbacks and spend logs showed the catalog rates for a custom-priced deployment. completion_cost now stores the entry it billed from on the logging object and the standard logging payload reads it, falling back to the old lookup only when billing never ran. * fix(logging): pass the deployment id into the logging cost-map lookup Drop the logging-object plumbing and keep the lookup where it was: get_standard_logging_object_payload now hands the deployment id to get_model_cost_information as router_model_id, the same id billing uses, so the deployment's own cost-map entry is reported instead of the catalog entry. * test(logging): mirror the proxy metadata in the deployment-price regression test * fix(logging): keep the catalog model_map_key, report only the deployment's own rates --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 15 +++- .../test_litellm_logging.py | 83 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3a1dafa331c..b065d4dd91a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -6471,6 +6471,7 @@ class StandardLoggingPayloadSetup: custom_llm_provider: str | None, init_response_obj: object, api_base: str | None = None, + router_model_id: str | None = None, ) -> StandardLoggingModelInformation: model_cost_name: Final = select_model_name_for_cost_calc( model=base_model if custom_pricing else None, @@ -6478,12 +6479,23 @@ class StandardLoggingPayloadSetup: base_model=base_model, custom_pricing=custom_pricing, ) + pricing_model_name: Final = ( + select_model_name_for_cost_calc( + model=base_model if custom_pricing else None, + completion_response=init_response_obj, + base_model=base_model, + custom_pricing=custom_pricing, + router_model_id=router_model_id, + ) + if router_model_id is not None + else model_cost_name + ) if model_cost_name is None: model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None) else: try: _model_cost_information: Final = litellm.get_model_info( - model=model_cost_name, + model=pricing_model_name or model_cost_name, custom_llm_provider=custom_llm_provider, api_base=api_base, ) @@ -7193,6 +7205,7 @@ def get_standard_logging_object_payload( custom_llm_provider=kwargs.get("custom_llm_provider"), init_response_obj=init_response_obj, api_base=litellm_params.get("api_base"), + router_model_id=logging_obj.get_router_model_id(), ) error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 37a985264b7..1d3bb2d0f34 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -10718,6 +10718,89 @@ def test_uncached_price_leaves_no_pricing_state_on_the_call(): litellm.model_cost.pop(deployment_id, None) +@pytest.mark.parametrize( + ("input_cost_per_token", "output_cost_per_token"), + [(1e-06, 2e-06), (0.0, 0.0)], + ids=["custom_price", "explicit_zero_price"], +) +def test_standard_logging_payload_reports_the_deployment_entry_the_request_was_billed_from( + input_cost_per_token: float, output_cost_per_token: float +): + """A deployment with its own price keeps the catalog model as model_map_key but is logged at its own rates.""" + deployment_id: Final = "lit9660-priced-deployment" + deployment_model_info: Final = { + "id": deployment_id, + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": output_cost_per_token, + } + litellm.register_model( + model_cost={ + deployment_id: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": output_cost_per_token, + } + }, + persist_across_reloads=False, + ) + try: + logging_obj: Final = LitellmLogging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hi"}], + stream=False, + call_type="completion", + start_time=time.time(), + litellm_call_id="lit9660", + function_id="fn", + ) + litellm_params: Final = { + "metadata": { + "model_group": "lit9660-group", + "deployment": "openai/gpt-5.4-nano", + "model_info": deployment_model_info, + } + } + logging_obj.update_environment_variables( + model="gpt-5.4-nano", user="", optional_params={}, litellm_params=litellm_params, custom_llm_provider="openai" + ) + response: Final = ModelResponse( + model="gpt-5.4-nano", + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="hi"))], + usage=litellm.Usage(prompt_tokens=10, completion_tokens=6, total_tokens=16), + ) + + response_cost: Final = logging_obj.response_cost_calculator(result=response) + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": "gpt-5.4-nano", + "custom_llm_provider": "openai", + "litellm_params": litellm_params, + "response_cost": response_cost, + }, + init_response_obj=response, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + logging_obj=logging_obj, + status="success", + ) + + assert response_cost == 10 * input_cost_per_token + 6 * output_cost_per_token + assert payload is not None + assert payload["model_map_information"] == { + "model_map_key": "openai/gpt-5.4-nano", + "model_map_value": litellm.get_model_info(model=f"openai/{deployment_id}", custom_llm_provider="openai"), + } + model_map_value: Final = payload["model_map_information"]["model_map_value"] + assert model_map_value is not None + assert (model_map_value["input_cost_per_token"], model_map_value["output_cost_per_token"]) == ( + input_cost_per_token, + output_cost_per_token, + ) + finally: + litellm.model_cost.pop(deployment_id, None) + + def _assert_zeroed_copy(hit_breakdown: CostBreakdown | None, fresh_breakdown: CostBreakdown) -> None: basis_and_rates: Final = ("service_tier", "data_residency", "vertex_location", "discount_percent", "margin_percent") assert hit_breakdown is not None and hit_breakdown.keys() == fresh_breakdown.keys() From 365d7342c0c1f31533f32b189e60b25aa2cd6fb7 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 13:38:26 -0700 Subject: [PATCH 46/50] fix(prometheus): label deployment failures with the deployment's provider instead of None (#47384) * fix(prometheus): label deployment failures with the deployment's provider instead of None * test(prometheus): drive every deployment failure surface through the provider label Streaming chat, responses and messages, dropped and stalled upstreams, JSON-configured, custom-provider-map, prompt-management and unprefixed deployments, forged request metadata, an unauthenticated request, the OpenAI and Anthropic SDKs sync and async, and a concurrent burst counted exactly once per deployment --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/integrations/prometheus.py | 46 +- ...metheus_deployment_failure_api_provider.py | 527 ++++++++++++++++++ .../integrations/test_prometheus_labels.py | 112 ++++ 3 files changed, 684 insertions(+), 1 deletion(-) create mode 100644 tests/integration/observability/test_prometheus_deployment_failure_api_provider.py diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 43250fd0fd4..073e545a93f 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -47,9 +47,11 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, ) +from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider from litellm.litellm_core_utils.service_tier_utils import ( get_service_tier_from_standard_logging_payload, ) +from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.models.end_user import LiteLLM_EndUserTable from litellm.models.team import LiteLLM_TeamTable from litellm.models.user import LiteLLM_UserTable @@ -60,6 +62,7 @@ from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.table_repositories import EndUserRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository +from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata from litellm.types.guardrails import GuardrailEventHooks from litellm.types.integrations.prometheus import * from litellm.types.integrations.prometheus import ( @@ -260,6 +263,26 @@ _POSITIVE_SERIES_TTL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, F _SERIES_CLEANUP_INTERVAL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(ge=0)]) _DEFAULT_SERIES_CLEANUP_INTERVAL_SECONDS: Final = 60.0 _ADDITIONAL_HEADERS: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) +_LOGGED_FIELDS: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_NO_LOGGED_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _logged_fields(value: object) -> Mapping[str, object]: + return _LOGGED_FIELDS.validate_python(value) if isinstance(value, Mapping) else _NO_LOGGED_FIELDS + + +def _is_llm_provider(name: str) -> bool: + routable: Final = ( + name in litellm.provider_list or JSONProviderRegistry.exists(name) or is_registered_custom_provider(name) + ) + return routable and name not in litellm._known_custom_logger_compatible_callbacks # pyright: ignore[reportPrivateUsage] # the router treats these prefixes as prompt-management deployments + + +def _deployment_model_provider(deployment: object) -> str: + if not isinstance(deployment, str): + return "" + prefix, separator, _ = deployment.partition("/") + return prefix if separator and _is_llm_provider(prefix) else "" def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None: @@ -2846,6 +2869,25 @@ class PrometheusLogger(CustomLogger): ) return None + @staticmethod + def _extract_deployment_api_provider(request_kwargs: Mapping[str, object]) -> str: + """ + Provider of the deployment a failed call was routed to: the resolved + ``custom_llm_provider`` on ``litellm_params``, then the one recorded on + ``standard_logging_object``, then the provider prefix of the deployment + model the router wrote into the request metadata (``azure/my-gpt`` -> + ``azure``) when that prefix is an LLM provider the router accepts. + Returns an empty string otherwise. + """ + litellm_params: Final = _logged_fields(request_kwargs.get("litellm_params")) + provider: Final = litellm_params.get("custom_llm_provider") or _logged_fields( + request_kwargs.get("standard_logging_object") + ).get("custom_llm_provider") + if isinstance(provider, str) and provider: + return provider + deployment_metadata: Final = find_deployment_metadata(litellm_params) + return _deployment_model_provider(deployment_metadata.get("deployment") if deployment_metadata else None) + @staticmethod def _extract_api_provider_from_exception(exception: Exception) -> str | None: if not isinstance(exception, litellm.exceptions.RateLimitError): @@ -3124,7 +3166,9 @@ class PrometheusLogger(CustomLogger): label_litellm_model_name = litellm_model_name label_model_id = model_id label_api_base = api_base - label_api_provider = llm_provider + label_api_provider = self._extract_deployment_api_provider( + _LOGGED_FIELDS.validate_python(request_kwargs) + ) label_requested_model = model_group or litellm_model_name label_model_group = model_group else: diff --git a/tests/integration/observability/test_prometheus_deployment_failure_api_provider.py b/tests/integration/observability/test_prometheus_deployment_failure_api_provider.py new file mode 100644 index 00000000000..190f27f346d --- /dev/null +++ b/tests/integration/observability/test_prometheus_deployment_failure_api_provider.py @@ -0,0 +1,527 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import uuid +from collections import Counter +from collections.abc import Callable, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import anthropic +import httpx +import openai +import pytest +import yaml +from pydantic import JsonValue + +from tests.integration._support.client import Gateway, eventually, gateway_from_environment +from tests.integration._support.process import owned_proxy +from tests.integration._support.prometheus_series import Sample, scrape +from tests.integration._support.wire import Reply, Request, Wire, wire_server + +_FAILURES: Final = "litellm_deployment_failure_responses_total" +_TOTALS: Final = "litellm_deployment_total_requests_total" +_SERVED: Final = "limited" +_UNREACHABLE: Final = "unreachable" +_STALLED: Final = "stalled" +_NEVER_RELEASED: Final = threading.Event() +_IN_HOUSE_HANDLER: Final = """ +from litellm import CustomLLM + + +class InHouse(CustomLLM): + pass + + +handler = InHouse() +""" + + +@dataclass(frozen=True, slots=True) +class _Deployment: + group: str + model: str + provider: str + extra: Mapping[str, str | int] + model_id: str = "" + + +@dataclass(frozen=True, slots=True) +class _Rig: + gateway: Gateway + upstream: Wire + + +def _deployment(group: str, model: str, provider: str, **extra: str | int) -> _Deployment: + return _Deployment(group, model, provider, extra, f"{group}-{uuid.uuid4().hex[:8]}") + + +_CHAT: Final = _deployment("deployment-failure-chat", "openai/refused-chat", "openai") +_EMBEDDING: Final = _deployment("deployment-failure-embedding", "openai/refused-embedding", "openai") +_RESPONSES: Final = _deployment("deployment-failure-responses", "openai/refused-responses", "openai") +_MESSAGES: Final = _deployment("deployment-failure-messages", "anthropic/claude-haiku-4-5", "anthropic") +_UNSUPPORTED: Final = _deployment("deployment-failure-unsupported", "anthropic/claude-haiku-4-5", "anthropic") +_LIMITED: Final = _deployment("deployment-failure-limited", f"openai/{_SERVED}-openai", "openai", rpm=1) +_LIMITED_AZURE: Final = _deployment("gpt-4o-mini", f"azure/{_SERVED}-azure", "azure", rpm=1, api_version="2024-10-21") +_STREAMED_CHAT: Final = _deployment("deployment-failure-streamed-chat", "openai/refused-streamed-chat", "openai") +_STREAMED_RESPONSES: Final = _deployment( + "deployment-failure-streamed-responses", "openai/refused-streamed-responses", "openai" +) +_STREAMED_MESSAGES: Final = _deployment( + "deployment-failure-streamed-messages", "anthropic/claude-haiku-4-5", "anthropic" +) +_UNREACHABLE_EMBEDDING: Final = _deployment( + "deployment-failure-unreachable", f"openai/{_UNREACHABLE}-embedding", "openai" +) +_STALLED_EMBEDDING: Final = _deployment( + "deployment-failure-stalled", f"openai/{_STALLED}-embedding", "openai", timeout=1 +) +_JSON_PROVIDER: Final = _deployment("deployment-failure-json-provider", "abliteration/blocked", "abliteration", rpm=0) +_CUSTOM_PROVIDER: Final = _deployment("deployment-failure-custom-provider", "in-house/blocked", "in-house", rpm=0) +_PROMPT_MANAGEMENT: Final = _deployment( + "deployment-failure-prompt-management", "litellm_agent/anthropic/claude-haiku-4-5", "anthropic" +) +_UNPREFIXED: Final = _deployment("deployment-failure-unprefixed", "gpt-4o-mini", "", rpm=0) +_FORGED_EMBEDDING: Final = _deployment("deployment-failure-forged-embedding", "openai/refused-forged", "openai") +_FORGED_BLOCKED: Final = _deployment("deployment-failure-forged-blocked", "openai/blocked-forged", "openai", rpm=0) +_UNAUTHENTICATED: Final = _deployment("deployment-failure-unauthenticated", "openai/refused-unauthenticated", "openai") +_SDK: Final = { + client: _deployment(f"deployment-failure-{client}", model, provider) + for client, model, provider in ( + ("openai-sync", "openai/refused-sdk-sync", "openai"), + ("openai-async", "openai/refused-sdk-async", "openai"), + ("anthropic-sync", "anthropic/claude-haiku-4-5", "anthropic"), + ("anthropic-async", "anthropic/claude-haiku-4-5", "anthropic"), + ) +} +_BURST_EMBEDDING: Final = _deployment("burst-embedding", "openai/refused-burst-embedding", "openai") +_BURST_RESPONSES: Final = _deployment("burst-responses", "openai/refused-burst-responses", "openai") +_BURST_MESSAGES: Final = _deployment("burst-messages", "anthropic/claude-haiku-4-5", "anthropic") +_BURST_UNSUPPORTED: Final = _deployment("burst-unsupported", "anthropic/claude-haiku-4-5", "anthropic") +_BURST_BLOCKED: Final = _deployment("burst-blocked", "gemini/blocked-burst", "gemini", rpm=0) +_BURST: Final = (_BURST_EMBEDDING, _BURST_RESPONSES, _BURST_MESSAGES, _BURST_UNSUPPORTED, _BURST_BLOCKED) +_DEPLOYMENTS: Final = ( + _CHAT, + _EMBEDDING, + _RESPONSES, + _MESSAGES, + _UNSUPPORTED, + _LIMITED, + _LIMITED_AZURE, + _STREAMED_CHAT, + _STREAMED_RESPONSES, + _STREAMED_MESSAGES, + _UNREACHABLE_EMBEDDING, + _STALLED_EMBEDDING, + _JSON_PROVIDER, + _CUSTOM_PROVIDER, + _PROMPT_MANAGEMENT, + _UNPREFIXED, + _FORGED_EMBEDDING, + _FORGED_BLOCKED, + _UNAUTHENTICATED, + *_SDK.values(), + *_BURST, +) + + +def _respond(request: Request) -> Reply: + model: Final = json.loads(request.body).get("model", "") if request.body else "" + if model.startswith(_UNREACHABLE): + return Reply(drop_connection=True) + if model.startswith(_STALLED): + return Reply(chunks=(b"{}",), gate_before_first=_NEVER_RELEASED) + if _SERVED in request.target or model.startswith(_SERVED): + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "served"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ).encode() + ) + if request.target.endswith("/v1/messages"): + return Reply( + status=400, + body=json.dumps( + {"type": "error", "error": {"type": "invalid_request_error", "message": "refused upstream"}} + ).encode(), + ) + return Reply( + status=400, + body=json.dumps( + {"error": {"message": "refused upstream", "type": "invalid_request_error", "code": "refused"}} + ).encode(), + ) + + +def _config(directory: Path, upstream: Wire) -> Path: + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["model_list"] = [ + *configuration["model_list"], + *( + { + "model_name": deployment.group, + "litellm_params": { + "model": deployment.model, + "api_key": "sk-fixture", + "api_base": upstream.url if deployment.provider != "openai" else f"{upstream.url}/v1", + **deployment.extra, + }, + "model_info": {"id": deployment.model_id}, + } + for deployment in _DEPLOYMENTS + ), + ] + (directory / "in_house_handler.py").write_text(_IN_HOUSE_HANDLER) + configuration["litellm_settings"]["callbacks"] = ["prometheus"] + configuration["litellm_settings"]["custom_provider_map"] = [ + {"provider": "in-house", "custom_handler": "in_house_handler.handler"} + ] + router_settings: Final = configuration.setdefault("router_settings", {}) + router_settings["num_retries"] = 0 + router_settings["optional_pre_call_checks"] = ["enforce_model_rate_limits"] + path: Final = directory / "prometheus-deployment-failure-api-provider.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("prometheus-deployment-failure-api-provider") + samples: Final = directory / "prometheus-multiproc" + samples.mkdir() + with ( + wire_server(_respond) as upstream, + gateway_from_environment() as shared, + owned_proxy( + shared, + directory, + {"PROMETHEUS_MULTIPROC_DIR": str(samples)}, + config=_config(directory, upstream), + workers=2, + ) as owned, + ): + yield _Rig(owned, upstream) + + +def _series(samples: Sequence[Sample], name: str, model_id: str) -> tuple[Sample, ...]: + return tuple(sample for sample in samples if sample.name == name and sample.labels.get("model_id") == model_id) + + +def _providers(rig: _Rig, name: str, model_id: str) -> frozenset[str]: + found: Final = eventually(lambda: _series(scrape(rig.gateway), name, model_id), bool, seconds=30) + return frozenset(sample.labels["api_provider"] for sample in found) + + +def _assert_labeled(rig: _Rig, deployment: _Deployment) -> None: + assert _providers(rig, _FAILURES, deployment.model_id) == {deployment.provider} + assert _providers(rig, _TOTALS, deployment.model_id) == {deployment.provider} + + +def _counted(samples: Sequence[Sample], name: str, model_id: str) -> Mapping[str, float]: + counts: Final[Counter[str]] = Counter() + for sample in _series(samples, name, model_id): + counts[sample.labels["api_provider"]] += sample.value + return counts + + +def _counts(rig: _Rig, name: str, deployment: _Deployment, expected: int) -> Mapping[str, float]: + return eventually( + lambda: _counted(scrape(rig.gateway), name, deployment.model_id), + lambda counts: sum(counts.values()) >= expected, + seconds=60, + ) + + +def _post(rig: _Rig, path: str, body: Mapping[str, JsonValue]) -> httpx.Response: + return rig.gateway.request("POST", path, body) + + +def _prompt() -> str: + return f"deployment failure {uuid.uuid4().hex}" + + +def _chat(deployment: _Deployment, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": deployment.group, "messages": [{"role": "user", "content": _prompt()}], **extra} + + +def _messages(deployment: _Deployment, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": deployment.group, "max_tokens": 16, "messages": [{"role": "user", "content": _prompt()}], **extra} + + +def test_a_refused_chat_completion_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None: + response: Final = _post(rig, "/v1/chat/completions", _chat(_CHAT)) + assert response.status_code == 400, response.text + _assert_labeled(rig, _CHAT) + + +def test_a_refused_embedding_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None: + response: Final = _post(rig, "/v1/embeddings", {"model": _EMBEDDING.group, "input": _prompt()}) + assert response.status_code == 400, response.text + assert any(request.target.endswith("/v1/embeddings") for request in rig.upstream.drain()) + _assert_labeled(rig, _EMBEDDING) + + +def test_a_refused_responses_call_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None: + response: Final = _post(rig, "/v1/responses", {"model": _RESPONSES.group, "input": _prompt()}) + assert response.status_code == 400, response.text + assert any(request.target.endswith("/v1/responses") for request in rig.upstream.drain()) + _assert_labeled(rig, _RESPONSES) + + +def test_a_refused_anthropic_messages_call_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None: + response: Final = _post(rig, "/v1/messages", _messages(_MESSAGES)) + assert response.status_code == 400, response.text + assert any(request.target.endswith("/v1/messages") for request in rig.upstream.drain()) + _assert_labeled(rig, _MESSAGES) + + +def test_a_param_the_deployment_does_not_support_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None: + rig.upstream.drain() + response: Final = _post(rig, "/v1/chat/completions", _chat(_UNSUPPORTED, logit_bias={"50256": -100})) + assert response.status_code == 400, response.text + assert "logit_bias" in response.text + assert rig.upstream.drain() == () + _assert_labeled(rig, _UNSUPPORTED) + + +def _statuses(rig: _Rig, deployment: _Deployment) -> Iterator[int]: + for _ in range(8): + yield _post(rig, "/v1/chat/completions", _chat(deployment)).status_code + + +def _until_rate_limited(rig: _Rig, deployment: _Deployment) -> None: + first_refusal: Final = next((status for status in _statuses(rig, deployment) if status != 200), None) + assert first_refusal == 429, f"expected the rpm limit to refuse a request, first non-200 was {first_refusal}" + + +@pytest.mark.parametrize( + "deployment", [_LIMITED, _LIMITED_AZURE], ids=["alias-group", "group-named-like-another-provider"] +) +def test_a_deployment_over_its_rpm_limit_labels_the_failure_with_the_deployment_provider( + rig: _Rig, deployment: _Deployment +) -> None: + _until_rate_limited(rig, deployment) + _assert_labeled(rig, deployment) + + +@pytest.mark.parametrize( + ("path", "body"), + [ + ("/v1/chat/completions", _chat(_STREAMED_CHAT, stream=True)), + ("/v1/responses", {"model": _STREAMED_RESPONSES.group, "input": _prompt(), "stream": True}), + ("/v1/messages", _messages(_STREAMED_MESSAGES, stream=True)), + ], + ids=["chat", "responses", "messages"], +) +def test_a_refused_streaming_call_labels_the_deployment_failure_with_its_provider( + rig: _Rig, path: str, body: Mapping[str, JsonValue] +) -> None: + deployment: Final = next(deployment for deployment in _DEPLOYMENTS if deployment.group == body["model"]) + response: Final = _post(rig, path, body) + assert response.status_code == 400, response.text + _assert_labeled(rig, deployment) + + +@pytest.mark.parametrize( + ("deployment", "status"), + [(_UNREACHABLE_EMBEDDING, 500), (_STALLED_EMBEDDING, 408)], + ids=["connection-dropped", "read-timeout"], +) +def test_an_upstream_that_never_answers_labels_the_deployment_failure_with_its_provider( + rig: _Rig, deployment: _Deployment, status: int +) -> None: + response: Final = _post(rig, "/v1/embeddings", {"model": deployment.group, "input": _prompt()}) + assert response.status_code == status, response.text + _assert_labeled(rig, deployment) + + +@pytest.mark.parametrize( + "deployment", + [_JSON_PROVIDER, _CUSTOM_PROVIDER, _UNPREFIXED], + ids=["json-configured-provider", "custom-provider-map", "unprefixed-model"], +) +def test_a_call_blocked_before_it_leaves_labels_the_failure_by_the_deployment_model_prefix( + rig: _Rig, deployment: _Deployment +) -> None: + rig.upstream.drain() + response: Final = _post(rig, "/v1/chat/completions", _chat(deployment)) + assert response.status_code == 429, response.text + assert rig.upstream.drain() == () + _assert_labeled(rig, deployment) + + +def test_a_prompt_management_deployment_labels_the_failure_with_the_provider_it_resolved_to(rig: _Rig) -> None: + response: Final = _post(rig, "/v1/chat/completions", _chat(_PROMPT_MANAGEMENT, logit_bias={"50256": -100})) + assert response.status_code == 400, response.text + found: Final = eventually( + lambda: tuple( + sample + for sample in scrape(rig.gateway) + if sample.name == _FAILURES and sample.labels.get("model_group") == _PROMPT_MANAGEMENT.group + ), + bool, + seconds=30, + ) + assert {sample.labels["api_provider"] for sample in found} == {_PROMPT_MANAGEMENT.provider} + + +@pytest.mark.parametrize( + "forged", + ["bedrock/forged", 5, ["bedrock/forged"], "", f"bedrock/{'x' * 5000}"], + ids=["provider-string", "int", "list", "empty", "five-kb"], +) +@pytest.mark.parametrize( + ("path", "deployment", "status"), + [("/v1/embeddings", _FORGED_EMBEDDING, 400), ("/v1/chat/completions", _FORGED_BLOCKED, 429)], + ids=["refused-upstream", "blocked-before-the-call"], +) +def test_a_caller_cannot_pick_the_label_through_request_metadata( + rig: _Rig, path: str, deployment: _Deployment, status: int, forged: JsonValue +) -> None: + bucket: Final[dict[str, JsonValue]] = {"deployment": forged, "model_info": {"id": deployment.model_id}} + body: Final = ( + _chat(deployment, metadata=bucket, litellm_metadata=bucket) + if path == "/v1/chat/completions" + else {"model": deployment.group, "input": _prompt(), "metadata": bucket, "litellm_metadata": bucket} + ) + response: Final = _post(rig, path, body) + assert response.status_code == status, response.text + _assert_labeled(rig, deployment) + every_provider: Final = { + sample.labels.get("api_provider") for sample in scrape(rig.gateway) if sample.name in (_FAILURES, _TOTALS) + } + assert not any(str(provider).startswith("bedrock") for provider in every_provider), every_provider + + +def test_an_unauthenticated_request_leaves_the_deployment_series_alone(rig: _Rig) -> None: + body: Final[dict[str, JsonValue]] = {"model": _UNAUTHENTICATED.group, "input": _prompt()} + rejected: Final = rig.gateway.request("POST", "/v1/embeddings", body, key="sk-not-a-litellm-key") + assert rejected.status_code == 401, rejected.text + refused: Final = _post(rig, "/v1/embeddings", body) + assert refused.status_code == 400, refused.text + assert _counts(rig, _FAILURES, _UNAUTHENTICATED, 1) == {"openai": 1} + assert _counts(rig, _TOTALS, _UNAUTHENTICATED, 1) == {"openai": 1} + + +def _sdk_v1(gateway: Gateway) -> str: + return f"{str(gateway.client.base_url).rstrip('/')}/v1" + + +def _sdk_origin(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +def _openai_sync(gateway: Gateway, deployment: _Deployment) -> int: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + openai.OpenAI(api_key=gateway.key, base_url=_sdk_v1(gateway), max_retries=0, http_client=transport) as client, + ): + with pytest.raises(openai.APIStatusError) as refusal: + client.embeddings.create(model=deployment.group, input=_prompt()) + return refusal.value.status_code + + +def _openai_async(gateway: Gateway, deployment: _Deployment) -> int: + async def run() -> int: + async with ( + httpx.AsyncClient(timeout=15, trust_env=False) as transport, + openai.AsyncOpenAI( + api_key=gateway.key, base_url=_sdk_v1(gateway), max_retries=0, http_client=transport + ) as client, + ): + with pytest.raises(openai.APIStatusError) as refusal: + await client.embeddings.create(model=deployment.group, input=_prompt()) + return refusal.value.status_code + + return asyncio.run(run()) + + +def _anthropic_sync(gateway: Gateway, deployment: _Deployment) -> int: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + anthropic.Anthropic( + api_key=gateway.key, base_url=_sdk_origin(gateway), max_retries=0, http_client=transport + ) as client, + ): + with pytest.raises(anthropic.APIStatusError) as refusal: + client.messages.create( + model=deployment.group, max_tokens=16, messages=[{"role": "user", "content": _prompt()}] + ) + return refusal.value.status_code + + +def _anthropic_async(gateway: Gateway, deployment: _Deployment) -> int: + async def run() -> int: + async with ( + httpx.AsyncClient(timeout=15, trust_env=False) as transport, + anthropic.AsyncAnthropic( + api_key=gateway.key, base_url=_sdk_origin(gateway), max_retries=0, http_client=transport + ) as client, + ): + with pytest.raises(anthropic.APIStatusError) as refusal: + await client.messages.create( + model=deployment.group, max_tokens=16, messages=[{"role": "user", "content": _prompt()}] + ) + return refusal.value.status_code + + return asyncio.run(run()) + + +_SDK_CALLS: Final[Mapping[str, Callable[[Gateway, _Deployment], int]]] = { + "openai-sync": _openai_sync, + "openai-async": _openai_async, + "anthropic-sync": _anthropic_sync, + "anthropic-async": _anthropic_async, +} + + +@pytest.mark.parametrize("client", list(_SDK_CALLS)) +def test_a_refusal_through_the_provider_sdk_labels_the_deployment_failure_with_its_provider( + rig: _Rig, client: str +) -> None: + assert _SDK_CALLS[client](rig.gateway, _SDK[client]) == 400 + _assert_labeled(rig, _SDK[client]) + + +_BURST_SIZE: Final = 10 +_BURST_CALLS: Final[Mapping[str, tuple[str, Callable[[], Mapping[str, JsonValue]], int]]] = { + _BURST_EMBEDDING.model_id: ("/v1/embeddings", lambda: {"model": _BURST_EMBEDDING.group, "input": _prompt()}, 400), + _BURST_RESPONSES.model_id: ("/v1/responses", lambda: {"model": _BURST_RESPONSES.group, "input": _prompt()}, 400), + _BURST_MESSAGES.model_id: ("/v1/messages", lambda: _messages(_BURST_MESSAGES), 400), + _BURST_UNSUPPORTED.model_id: ( + "/v1/chat/completions", + lambda: _chat(_BURST_UNSUPPORTED, logit_bias={"50256": -100}), + 400, + ), + _BURST_BLOCKED.model_id: ("/v1/chat/completions", lambda: _chat(_BURST_BLOCKED), 429), +} + + +def test_a_concurrent_burst_of_failures_counts_each_one_once_under_its_deployment_provider(rig: _Rig) -> None: + calls: Final = [deployment.model_id for deployment in _BURST for _ in range(_BURST_SIZE)] + with ThreadPoolExecutor(max_workers=len(calls)) as pool: + statuses: Final = Counter( + pool.map( + lambda model_id: ( + model_id, + _post(rig, _BURST_CALLS[model_id][0], _BURST_CALLS[model_id][1]()).status_code, + ), + calls, + ) + ) + assert statuses == Counter({(model_id, status): _BURST_SIZE for model_id, (_, _, status) in _BURST_CALLS.items()}) + for deployment in _BURST: + assert _counts(rig, _FAILURES, deployment, _BURST_SIZE) == {deployment.provider: _BURST_SIZE} + assert _counts(rig, _TOTALS, deployment, _BURST_SIZE) == {deployment.provider: _BURST_SIZE} diff --git a/tests/unit/integrations/test_prometheus_labels.py b/tests/unit/integrations/test_prometheus_labels.py index ead0ce8ecff..d49ca57a3de 100644 --- a/tests/unit/integrations/test_prometheus_labels.py +++ b/tests/unit/integrations/test_prometheus_labels.py @@ -4,6 +4,7 @@ Unit tests for prometheus metric labels configuration import pytest +import litellm from litellm.types.integrations.prometheus import ( PrometheusMetricLabels, UserAPIKeyLabelNames, @@ -928,6 +929,117 @@ def test_deployment_failure_metrics_emit_model_group_from_standard_logging_paylo _clear_prometheus_registry() +def _routed(deployment: object) -> dict[str, object]: + return {"deployment": deployment, "model_info": {"id": "model-123"}} + + +@pytest.mark.parametrize( + ("litellm_params", "logged_provider", "expected"), + [ + pytest.param( + {"custom_llm_provider": "bedrock", "metadata": _routed("azure/gpt-4o")}, + "vertex_ai", + "bedrock", + id="resolved-provider-wins", + ), + pytest.param( + {"custom_llm_provider": None, "metadata": _routed("anthropic/claude-haiku-4-5")}, + "openai", + "openai", + id="provider-on-the-payload-wins", + ), + pytest.param({"metadata": _routed("azure/limited-azure")}, None, "azure", id="deployment-prefix"), + pytest.param( + {"litellm_metadata": _routed("anthropic/claude-haiku-4-5"), "metadata": {}}, + None, + "anthropic", + id="litellm-metadata-deployment", + ), + pytest.param( + {"litellm_metadata": _routed("anthropic/claude-haiku-4-5"), "metadata": _routed("bedrock/converse/model")}, + None, + "anthropic", + id="litellm-metadata-deployment-wins", + ), + pytest.param( + {"litellm_metadata": {"deployment": "bedrock/forged"}, "metadata": _routed("anthropic/claude-haiku-4-5")}, + None, + "anthropic", + id="routed-bucket-wins-over-a-bare-deployment", + ), + pytest.param({"metadata": _routed("abliteration/model")}, None, "abliteration", id="json-configured-provider"), + pytest.param({"metadata": _routed("langfuse/my-prompt")}, None, "", id="prompt-management-deployment"), + pytest.param({"metadata": _routed("gpt-4o")}, None, "", id="unprefixed-deployment"), + pytest.param({"metadata": _routed("meta-llama/Llama-3")}, None, "", id="prefix-is-not-a-provider"), + pytest.param({"metadata": _routed(5)}, None, "", id="non-string-deployment"), + pytest.param({"metadata": {}}, None, "", id="nothing-known"), + ], +) +def test_deployment_failure_metrics_label_the_selected_deployment_provider(litellm_params, logged_provider, expected): + """ + A failure on a selected deployment labels api_provider with that + deployment's provider, never the string "None" and never a guess from the + requested model name (gpt-4o-mini here would read openai). + """ + from litellm.integrations.prometheus import PrometheusLogger + + _clear_prometheus_registry() + try: + logger = PrometheusLogger() + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": "gpt-4o-mini", + "litellm_params": litellm_params, + "standard_logging_object": { + "model_group": "gpt-4o-mini", + "model_id": "model-123", + "api_base": "https://example.openai.azure.com", + "custom_llm_provider": logged_provider, + "request_tags": [], + }, + "exception": Exception("boom"), + } + ) + + for metric in ( + logger.litellm_deployment_failure_responses, + logger.litellm_deployment_total_requests, + ): + index = metric._labelnames.index("api_provider") + values = {sample_key[index] for sample_key in metric._metrics} + assert values == {expected}, f"expected api_provider={expected!r} on {metric._name}, got {values}" + finally: + _clear_prometheus_registry() + + +def test_deployment_failure_metrics_label_a_custom_provider_deployment(monkeypatch): + from litellm.integrations.prometheus import PrometheusLogger + + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "in-house", "custom_handler": object()}]) + _clear_prometheus_registry() + try: + logger = PrometheusLogger() + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": _routed("in-house/model")}, + "standard_logging_object": { + "model_group": "gpt-4o-mini", + "model_id": "model-123", + "api_base": "https://in-house.example", + "request_tags": [], + }, + "exception": Exception("boom"), + } + ) + + index = logger.litellm_deployment_failure_responses._labelnames.index("api_provider") + values = {sample_key[index] for sample_key in logger.litellm_deployment_failure_responses._metrics} + assert values == {"in-house"} + finally: + _clear_prometheus_registry() + + def test_deployment_tpm_rpm_limit_metrics_emit_model_group_from_enum_values(): """ End-to-end emit wiring for the tpm/rpm limit gauges. From b2f0f4d54c5dace1c1f6ad7b3c2f3bc68cb5982b Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:01:05 -0700 Subject: [PATCH 47/50] fix(vertex-ai): drop floor deprecation_date from gemini-4-argon (#47404) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 1 - model_prices_and_context_window.json | 1 - 2 files changed, 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f5eec206e04..1cadc93969b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -75503,7 +75503,6 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-4-argon": { - "deprecation_date": "2027-10-13", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f5eec206e04..1cadc93969b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -75503,7 +75503,6 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-4-argon": { - "deprecation_date": "2027-10-13", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, From b51b9b7d43dc7da20b4baa49d431736c50c10600 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:13:46 +0000 Subject: [PATCH 48/50] fix(anthropic)!: answer 400 for a malformed tool_result text on /v1/messages and count_tokens (#47347) * fix(anthropic): answer 400 for a malformed tool_result text on the Responses bridge and count_tokens A tool_result text block whose text is null, a number, or missing made /v1/messages and /v1/messages/count_tokens answer 500 on OpenAI and Azure models. Both routes now answer 400 naming the block, and drop the block when drop_params is on. * fix(anthropic): keep a missing tool_result text as empty text, bound the error, honor body drop_params on count_tokens * fix(anthropic): apply the tool_result text check on the chat/completions bridge and every count route The chat/completions bridge used for Azure, Gemini and other providers now answers the same 400 for a null or numeric tool_result text and drops the block under drop_params, so a model group that mixes OpenAI with another provider answers the same way on every deployment. Gemini and Vertex count_tokens apply the same check, count_tokens turns on drop_params for Claude Code and Codex the way /v1/messages does, and a tool_result without a tool_use_id is checked too * fix(proxy): answer 400 for a malformed tool_result text on count_tokens models without a count API Providers with no token counting API fall back to the local tokenizer, which counted a null or numeric tool_result text as fine while /v1/messages on the same model answers 400. The fallback now runs the same check, unless drop_params applies to the deployment or the request * fix(proxy): honor request drop_params on the local count fallback when no deployment resolves * test(integration): cover non-string tool_result text on both /v1/messages bridges and count routes * fix(anthropic): keep a dropped tool_result on Mistral and reject it before a guardrail rewrite Under drop_params a tool_result whose only text block was null or a number became an empty tool message, and Mistral's empty-message filter removed it, so Mistral answered 400 for a missing tool reply. The filter now only removes empty assistant messages, which is what Mistral rejects. A guardrail that rewrites structured_messages wrote the dropped text back as "", so a null or numeric text was silently blanked even without drop_params. The rewrite path now answers the same 400 as the bridges unless the request or global drop_params is set. * test(integration): cover the Mistral empty tool reply and the guardrail rewrite tool_result text check * fix(anthropic): keep a malformed tool_result text through a guardrail rewrite and keep Mistral's null-message drop A guardrail that rewrites structured_messages now writes the caller's null or numeric tool_result text back in place, so the route's bridge and the deployment's drop_params decide after routing, the same as with no guardrail. The rewrite path no longer answers its own 400 before routing. Mistral drops a null-content system or user message again, as main does, and only a tool reply is exempt from that filter so a dropped tool_result keeps its tool call. * test(mistral): expect empty system and user turns dropped and the empty tool reply kept * perf(anthropic): restore kept tool_result text parts in one linear pass * test(integration): cover the tool_result text restore across tool_results, the chat bridge, and a batch upload behind a rewriting guardrail * fix(anthropic): let guardrails scan malformed tool_result text and honor drop_params on native Anthropic A rewriting guardrail now sees a non-string tool_result text as its JSON text and the write-back restores the caller's value by the sanitized tool_use_id when the guardrail left it unchanged. Native Anthropic drops the bad block under drop_params on /v1/messages and count_tokens, and Mistral keeps an empty tool reply only when it answers a tool call, sent as empty text. * fix(anthropic): compare a kept tool_result text to its scanned form once and align the wire tests with drop_params * test(integration): cover native Anthropic drop_params resolution, guardrail scanning of numeric tool_result text, and the Mistral orphan tool message * fix(count_tokens): honor the router's default drop_params when counting a malformed tool_result text * test(count_tokens): send the tool_result count tests through a real request body instead of a mocked parser * test(count_tokens): send the native Claude count test through a real request body * fix(anthropic): keep each non-string tool_result text when tool_use_ids sanitize to the same id * fix(anthropic): keep non-ASCII characters in the guardrail's view of a non-string tool_result text * test(gemini): mock the count_tokens upstream with respx instead of patching the client factory * fix(mistral): keep an empty tool reply only for an open call, and forward non-object message items on the native drop An empty tool reply re-answering a call the conversation already moved past made Mistral reject the request, and a tool call id that is not a string crashed the filter. Native Anthropic under drop_params assumed every message item was an object. * fix(mistral): find answering tool replies in one pass instead of copying the open call set per reply * fix(mistral): ignore dropped empty placeholders when tracking open tool calls * fix(mistral): drop an empty tool reply when another reply to the same call carries the result * refactor(anthropic): share tool_result text helpers to shrink the diff * fix(anthropic): answer 400 for a non-string tool_result text on the OpenAI Responses bridge --- litellm/litellm_core_utils/core_helpers.py | 5 + .../prompt_templates/factory.py | 16 +- .../chat/guardrail_translation/handler.py | 18 +- .../pass_through/adapters/handler.py | 4 + .../pass_through/adapters/transformation.py | 4 +- .../pass_through/convertible_blocks.py | 255 ++++- .../pass_through/messages/transformation.py | 5 +- .../responses_adapters/handler.py | 6 +- .../responses_adapters/transformation.py | 11 +- litellm/llms/gemini/common_utils.py | 5 +- litellm/llms/mistral/chat/transformation.py | 74 +- .../responses/count_tokens/token_counter.py | 6 +- .../responses/count_tokens/transformation.py | 7 +- litellm/llms/vertex_ai/common_utils.py | 5 +- .../llms/vertex_ai/count_tokens/context.py | 11 + litellm/proxy/proxy_server.py | 53 +- .../transformation.py | 6 +- .../test_tool_result_text_wire.py | 868 ++++++++++++++++++ ...llm_core_utils_prompt_templates_factory.py | 6 +- .../test_anthropic_guardrail_handler.py | 135 ++- ...al_pass_through_adapters_transformation.py | 153 +++ ...al_pass_through_messages_transformation.py | 64 ++ .../test_anthropic_messages_router.py | 85 ++ .../test_responses_adapters_handler.py | 85 ++ .../pass_through/test_convertible_blocks.py | 20 + .../llms/gemini/test_gemini_common_utils.py | 78 ++ .../test_mistral_chat_transformation.py | 205 +++++ .../count_tokens/test_token_counter.py | 87 ++ .../vertex_ai/test_vertex_ai_common_utils.py | 72 ++ tests/unit/proxy/test_proxy_token_counter.py | 200 +++- 30 files changed, 2484 insertions(+), 65 deletions(-) create mode 100644 tests/integration/messages_endpoint/test_tool_result_text_wire.py diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 65d0089304b..b7bfd5eeeb3 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_c import httpx from pydantic import TypeAdapter, ValidationError +import litellm from litellm._logging import verbose_logger from litellm.constants import KUBERNETES_POD_ROUTING_KEY from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id @@ -93,6 +94,10 @@ def normalize_drop_params(value: object) -> bool | None: return None +def drops_unsupported_params(drop_params: object) -> bool: + return litellm.drop_params is True or normalize_drop_params(drop_params) is True + + def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool: normalized: Final = normalize_drop_params(value) if normalized is None and value is not None: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index e2fb2be4ec2..481ec6a5076 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1669,7 +1669,7 @@ def _replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Patte return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK -def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: +def sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: """Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$.""" return _replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS) @@ -1815,7 +1815,7 @@ def convert_to_anthropic_tool_result( tool_message: Final[ChatCompletionToolMessage] = message tool_call_id: str = tool_message["tool_call_id"] # Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$ - sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id) + sanitized_tool_use_id = sanitize_anthropic_tool_use_id(tool_call_id) # We can't determine from openai message format whether it's a successful or # error call result so default to the successful result template @@ -1829,7 +1829,7 @@ def convert_to_anthropic_tool_result( function_message: Final[ChatCompletionFunctionMessage] = message tool_call_id = function_message.get("tool_call_id") or str(uuid.uuid4()) # Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$ - sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id) + sanitized_tool_use_id = sanitize_anthropic_tool_use_id(tool_call_id) anthropic_tool_result = AnthropicMessagesToolResultParam( type="tool_result", tool_use_id=sanitized_tool_use_id, @@ -2037,7 +2037,7 @@ def convert_to_anthropic_tool_invoke( elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids: continue else: - sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id) + sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id) _anthropic_tool_use_param = AnthropicMessagesToolUseParam( type="tool_use", id=sanitized_tool_id, @@ -5900,21 +5900,21 @@ def _optional_tool_name(value: object) -> str | None: def _split_tool_call_ids(calls: Sequence[tuple[str | None, int]]) -> tuple[tuple[str | None, ...], ...]: - taken: Final = frozenset(_sanitize_anthropic_tool_use_id(call_id) for call_id, _ in calls if call_id) + taken: Final = frozenset(sanitize_anthropic_tool_use_id(call_id) for call_id, _ in calls if call_id) def fresh(call_id: str) -> Iterator[str]: return filter( - lambda candidate: _sanitize_anthropic_tool_use_id(candidate) not in taken, + lambda candidate: sanitize_anthropic_tool_use_id(candidate) not in taken, (f"{call_id}__concat_{n}" for n in itertools.count(1)), ) suffixes: Final = MappingProxyType( - {_sanitize_anthropic_tool_use_id(call_id): fresh(call_id) for call_id, count in calls if call_id and count > 1} + {sanitize_anthropic_tool_use_id(call_id): fresh(call_id) for call_id, count in calls if call_id and count > 1} ) return tuple( ( call_id, - *(next(suffixes[_sanitize_anthropic_tool_use_id(call_id)]) for _ in range(count - 1)), + *(next(suffixes[sanitize_anthropic_tool_use_id(call_id)]) for _ in range(count - 1)), ) if call_id else (None,) * count diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 7654b56ad9a..aa2bee1f31d 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -30,6 +30,10 @@ from litellm.llms.anthropic.pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, is_provider_native_tool_dict, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import ( + keep_non_string_tool_result_texts, + scannable_tool_result_request, +) from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details from litellm.llms.anthropic.pass_through.stream_assembly import ( build_complete_streaming_response, @@ -493,8 +497,8 @@ class AnthropicMessagesHandler(BaseTranslation): ``get_streaming_string_so_far``): raw SSE ``bytes`` -- which may bundle several events separated by a blank line -- and an already-parsed event ``dict``.""" - if isinstance(item, dict): - return (_as_str_mapping(item),) + if (event := as_str_mapping(item)) is not None: + return (event,) if not isinstance(item, (bytes, bytearray)): return () return tuple( @@ -510,7 +514,7 @@ class AnthropicMessagesHandler(BaseTranslation): chat_completion_compatible_request, _tool_name_mapping, ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( - anthropic_message_request=cast(AnthropicMessagesRequest, data.copy()), + anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)), preserve_midturn_system=True, ) return chat_completion_compatible_request @@ -832,7 +836,7 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _write_back_structured_messages( - data: dict, # mutable-ok: API message payload + data: dict[str, object], # mutable-ok: API message payload structured_messages: list, # mutable-ok: API message payload hoisted_system_message: object = None, preserve_system_messages: bool = False, @@ -899,7 +903,7 @@ class AnthropicMessagesHandler(BaseTranslation): for block in content: if isinstance(block, dict) and block.get("type") == "thinking": block.pop("cache_control", None) - data["messages"] = converted + data["messages"] = keep_non_string_tool_result_texts(converted, data.get("messages")) @staticmethod def _extract_midturn_system_text( @@ -1601,8 +1605,8 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _rewrite_buffered_item(item: object, rewrite_event: _SSEEventRewriter) -> object: - if isinstance(item, dict): - return _rewritten_event(_as_str_mapping(item), rewrite_event) + if (event := as_str_mapping(item)) is not None: + return _rewritten_event(event, rewrite_event) if isinstance(item, (bytes, bytearray)): return AnthropicMessagesHandler._rewrite_sse_events(bytes(item), rewrite_event) return item diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index a9aa3de69ad..666823d6ec3 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -22,6 +22,7 @@ from litellm.llms.anthropic.pass_through.context_management import ( PolyfillResult, apply_context_management, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import reject_malformed_tool_result_text from litellm.llms.anthropic.pass_through.safeguards import ( SafeguardsEvaluator, build_safeguards_evaluator, @@ -731,6 +732,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs) request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs) + reject_malformed_tool_result_text(messages, model, request_kwargs) sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs) sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs) @@ -853,6 +855,8 @@ class LiteLLMMessagesToCompletionTransformationHandler: **kwargs, ) + reject_malformed_tool_result_text(messages, model, _REQUEST_KWARGS.validate_python(kwargs)) + # Run the context_management polyfill on the sync path too so that # ``litellm.messages.create()`` callers don't silently lose edits like # ``clear_tool_uses_20250919``. The dispatcher is async (so the diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index 0d3e48c49fd..de929acbfb2 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -9,6 +9,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.litellm_core_utils.hidden_params import get_hidden_params +from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_part from litellm.llms.anthropic.pass_through.utils import ( is_reasoning_auto_summary_enabled, json_copy_or_original, @@ -1334,7 +1335,8 @@ class LiteLLMAnthropicMessagesAdapter: block: Final = cast(Mapping[str, object], item) # cast-ok: untrusted client payload match block.get("type"): case "text": - return ChatCompletionTextObject(type="text", text=str(block.get("text") or "")) + text: Final = tool_result_text_part(block) + return None if text is None else ChatCompletionTextObject(type="text", text=text) case "image" | "document": return self._tool_result_image_part(block.get("source")) case "tool_reference": diff --git a/litellm/llms/anthropic/pass_through/convertible_blocks.py b/litellm/llms/anthropic/pass_through/convertible_blocks.py index 543a6efd4c4..fb17910fb97 100644 --- a/litellm/llms/anthropic/pass_through/convertible_blocks.py +++ b/litellm/llms/anthropic/pass_through/convertible_blocks.py @@ -1,11 +1,18 @@ -from collections.abc import Iterator, Mapping, Sequence -from itertools import chain +import json +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from functools import partial +from itertools import chain, groupby +from operator import itemgetter from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import assert_never +import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params +from litellm.litellm_core_utils.prompt_templates.factory import sanitize_anthropic_tool_use_id from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages from litellm.types.utils import TokenCountMessagesFormat @@ -104,3 +111,245 @@ def unconvertible_blocks_error( ) case _: assert_never(messages_format) + + +_OBJECTS: Final = TypeAdapter(tuple[object, ...]) + + +def _list_items(value: object) -> tuple[object, ...]: + return _OBJECTS.validate_python(value) if isinstance(value, list) else () + + +@dataclass(frozen=True, slots=True) +class _KeptPart: + block: object + scanned_text: str + + +_KeptParts: TypeAlias = Mapping[int, _KeptPart] +_NO_KEPT_PARTS: Final[_KeptParts] = MappingProxyType({}) + + +def _scannable_text(text: object) -> str: + return json.dumps(text, default=str, ensure_ascii=False) + + +def _non_str_text(block: object) -> _KeptPart | None: + match as_str_mapping(block): + case {"type": "text", "text": str()}: + return None + case {"type": "text", "text": text}: + return _KeptPart(block=block, scanned_text=_scannable_text(text)) + case _: + return None + + +def tool_result_text_part(block: object) -> str | None: + match block: + case {"type": "text", "text": str(text)}: + return text + case {"type": "text", "text": object()}: + return None + case {"type": "text"}: + return "" + case _: + return None + + +def tool_result_text_parts(tool_result: Mapping[str, object]) -> tuple[str, ...]: + parts: Final = map(tool_result_text_part, _list_items(tool_result.get("content"))) + return tuple(part for part in parts if part is not None) + + +def _user_content_blocks(message: object) -> tuple[object, ...]: + match as_str_mapping(message): + case {"role": "user", "content": content}: + return _list_items(content) + case _: + return () + + +def _with_user_blocks(message: object, transform: Callable[[object], object]) -> object: + match as_str_mapping(message): + case {"role": "user", "content": list()} as user_message: + return {**user_message, "content": [transform(block) for block in _user_content_blocks(user_message)]} + case _: + return message + + +def _non_string_text_errors(label: str, content: object) -> Iterator[str]: + return ( + f"{label} content.{index}.text: Input should be a valid string" + for index, text_block in enumerate(_list_items(content)) + if _non_str_text(text_block) is not None + ) + + +def _tool_result_text_errors(block: object) -> Iterator[str]: + match block: + case {"type": "tool_result", "tool_use_id": object() as tool_use_id, "content": object() as content}: + return _non_string_text_errors(f"tool_result {tool_use_id}", content) + case {"type": "tool_result", "content": object() as content}: + return _non_string_text_errors("tool_result", content) + case _: + return iter(()) + + +def malformed_tool_result_text_error(messages: Iterable[object]) -> str | None: + blocks: Final = chain.from_iterable(map(_user_content_blocks, messages)) + return next(chain.from_iterable(map(_tool_result_text_errors, blocks)), None) + + +def malformed_tool_result_count_error( + messages: Iterable[Mapping[str, object]] | None, + messages_format: TokenCountMessagesFormat, + deployment: Mapping[str, object] | None, +) -> str | None: + if messages_format != "anthropic" or deployment_drops_params(deployment): + return None + return malformed_tool_result_text_error(messages or ()) + + +def _scannable_part(part: object) -> object: + match as_str_mapping(part): + case {"type": "text", "text": str()}: + return part + case {"type": "text", "text": text} as text_block: + return {**text_block, "text": _scannable_text(text)} + case _: + return part + + +def _scannable_block(block: object) -> object: + match as_str_mapping(block): + case {"type": "tool_result", "content": list()} as tool_result: + return { + **tool_result, + "content": [_scannable_part(part) for part in _list_items(tool_result.get("content"))], + } + case _: + return block + + +def _scannable_messages(messages: object) -> object: + items: Final = _list_items(messages) + return [_with_user_blocks(message, _scannable_block) for message in items] if items else messages + + +def scannable_tool_result_request(request: object) -> object: + match as_str_mapping(request): + case {"messages": messages} as request_fields: + return {**request_fields, "messages": _scannable_messages(messages)} + case None: + return request + case request_fields: + return {**request_fields} + + +def _non_string_text_parts(content: object) -> _KeptParts: + return MappingProxyType( + {index: part for index, block in enumerate(_list_items(content)) if (part := _non_str_text(block)) is not None} + ) + + +def _tool_result_non_string_texts(block: object) -> tuple[tuple[str, _KeptParts], ...]: + match as_str_mapping(block): + case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result: + return ((sanitize_anthropic_tool_use_id(tool_use_id), _non_string_text_parts(tool_result.get("content"))),) + case _: + return () + + +def _content_blocks(content: object) -> tuple[object, ...]: + match content: + case str() if content: + return ({"type": "text", "text": content},) + case _: + return _list_items(content) + + +def _restored_part(block: object, kept: _KeptPart | None) -> object: + return kept.block if kept is not None and tool_result_text_part(block) == kept.scanned_text else block + + +def _with_kept_parts(block: object, tool_result: Mapping[str, object], parts: _KeptParts) -> object: + if not parts: + return block + blocks: Final = _content_blocks(tool_result.get("content")) + return {**tool_result, "content": [_restored_part(part, parts.get(index)) for index, part in enumerate(blocks)]} + + +def _restored_block(block: object, kept: Mapping[str, Iterator[_KeptParts]]) -> object: + match as_str_mapping(block): + case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result if ( + sanitize_anthropic_tool_use_id(tool_use_id) in kept + ): + return _with_kept_parts( + block, tool_result, next(kept[sanitize_anthropic_tool_use_id(tool_use_id)], _NO_KEPT_PARTS) + ) + case _: + return block + + +def keep_non_string_tool_result_texts(rewritten: Sequence[object], original: object) -> Sequence[object]: + blocks: Final = chain.from_iterable(map(_user_content_blocks, _list_items(original))) + by_id: Final = tuple(sorted(chain.from_iterable(map(_tool_result_non_string_texts, blocks)), key=itemgetter(0))) + if not any(parts for _, parts in by_id): + return rewritten + kept_in_order: Final = MappingProxyType( + { + tool_use_id: iter(tuple(map(itemgetter(1), group))) + for tool_use_id, group in groupby(by_id, key=itemgetter(0)) + } + ) + return [_with_user_blocks(message, partial(_restored_block, kept=kept_in_order)) for message in rewritten] + + +def _without_non_string_text(block: object) -> object: + match as_str_mapping(block): + case {"type": "tool_result", "content": list()} as tool_result: + parts: Final = _list_items(tool_result.get("content")) + return {**tool_result, "content": [part for part in parts if _non_str_text(part) is None]} + case _: + return block + + +def _without_non_string_texts( + message: dict[str, object], # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] +) -> dict[str, object]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + match message: + case {"role": "user", "content": list()}: + return {**message, "content": [_without_non_string_text(block) for block in _user_content_blocks(message)]} + case _: + return message + + +def drop_non_string_tool_result_texts( + messages: Sequence[dict[str, object]], # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + drop_params: object, +) -> Sequence[dict[str, object]]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + if not drops_unsupported_params(drop_params) or malformed_tool_result_text_error(messages) is None: + return messages + return [_without_non_string_texts(message) for message in messages] + + +def deployment_drops_params(deployment: Mapping[str, object] | None) -> bool: + match deployment: + case {"litellm_params": {"drop_params": object() as drop_params}}: + return drops_unsupported_params(drop_params) + case _: + return drops_unsupported_params(None) + + +def reject_malformed_tool_result_text( + messages: Iterable[Mapping[str, object]], model: str, forwarded_kwargs: Mapping[str, object] +) -> None: + if drops_unsupported_params(forwarded_kwargs.get("drop_params")): + return + error: Final = malformed_tool_result_text_error(messages) + if error is None: + return + provider: Final = forwarded_kwargs.get("custom_llm_provider") + raise litellm.BadRequestError( + message=error, model=model, llm_provider=provider if isinstance(provider, str) else "" + ) diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index 30d45f3cec1..0e8c1b49e8e 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -8,6 +8,7 @@ from litellm._logging import verbose_logger from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, @@ -692,7 +693,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): messages = strip_advisor_blocks_from_messages(messages) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( - messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(messages), + messages=strip_encrypted_reasoning_blocks_from_anthropic_messages( + drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None)) + ), max_tokens=max_tokens, model=model, **anthropic_messages_optional_request_params, diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index 05ea4e4301d..2cbba168ef4 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -14,7 +14,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, supports_openai_prompt_cache_breakpoint, ) -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS @@ -32,6 +32,7 @@ from litellm.types.llms.openai import ResponsesAPIResponse from litellm.utils import ProviderConfigManager from ..adapters.handler import request_safeguards_evaluator +from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper @@ -96,7 +97,7 @@ def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, objec return cache_control_as_prompt_cache_breakpoint if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs): return None - if normalize_drop_params(forwarded_kwargs.get("drop_params")) is True or litellm.drop_params is True: + if drops_unsupported_params(forwarded_kwargs.get("drop_params")): return _without_prompt_cache_breakpoint return None @@ -142,6 +143,7 @@ def _build_responses_kwargs( Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). """ forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) + reject_malformed_tool_result_text(messages, model, forwarded_kwargs) block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs) request_messages: Final[AnthropicRequestMessages] = ( [ diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index fb611b64dda..b14a0dc0756 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_parts from litellm.llms.anthropic.pass_through.messages.utils import ( refusal_stop_details, responses_output_refusal_text, @@ -371,10 +372,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif btype == "tool_result": tool_use_id = block.get("tool_use_id", "") inner = block.get("content") + tool_result_block = cast(Mapping[str, object], block) # cast-ok: TypedDict upcast tool_breakpoint, covered_image_parts, later_image_parts = ( - self._tool_result_images_by_breakpoint( - cast(Mapping[str, object], block) # cast-ok: TypedDict upcast - ) + self._tool_result_images_by_breakpoint(tool_result_block) ) document_candidates = ( tuple( @@ -391,10 +391,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif isinstance(inner, str): output_text = inner elif isinstance(inner, list): - parts = [ - c.get("text", "") for c in inner if isinstance(c, dict) and c.get("type") == "text" - ] - output_text = "\n".join(parts) + output_text = "\n".join(tool_result_text_parts(tool_result_block)) if covered_image_parts or later_image_parts: output_text = ( f"{output_text}\n{TOOL_RESULT_IMAGE_PLACEHOLDER}" diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 4006c89954d..c5ebd84e00f 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -508,6 +508,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter): count_context_error_response, gemini_count_contents, inference_request_view, + malformed_tool_result_text_count_error, provider_error_response, unconvertible_messages_error, vertex_gemini_count_context, @@ -524,7 +525,9 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter): count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools) if isinstance(count_context, VertexGeminiCountContextError): return count_context_error_response(count_context, request_model, model_to_use) - blocks_error: Final = unconvertible_messages_error(request_view.messages, messages_format) + blocks_error: Final = unconvertible_messages_error( + request_view.messages, messages_format + ) or malformed_tool_result_text_count_error(messages, messages_format, deployment) if blocks_error is not None: return count_context_error_response(blocks_error, request_model, model_to_use) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index a40970ca429..b8a6cb1beb9 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -7,6 +7,7 @@ Docs - https://docs.mistral.ai/api/ """ from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping +from itertools import accumulate, count from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload import httpx @@ -53,6 +54,9 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: return accepted +_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) + + class MistralConfig(OpenAIGPTConfig): """ Reference: https://docs.mistral.ai/api/ @@ -273,14 +277,30 @@ class MistralConfig(OpenAIGPTConfig): messages = handle_messages_with_content_list_to_str_conversion(messages) ## 3. Handle name in message - new_messages: Final[list[AllMessageValues]] = [] - for m in messages: - m = MistralConfig._handle_name_in_message(m) - m = MistralConfig._handle_tool_call_message(m) - if MistralConfig._is_empty_assistant_message(m): - continue - m = strip_none_values_from_message(m) # prevents 'extra_forbidden' error - new_messages.append(m) + prepared_messages: Final = tuple( + MistralConfig._handle_tool_call_message(MistralConfig._handle_name_in_message(m)) for m in messages + ) + empty_flags: Final = tuple(map(MistralConfig._is_empty_assistant_message, prepared_messages)) + tool_call_turns: Final = accumulate( + zip(count(), prepared_messages, empty_flags, strict=False), + MistralConfig._tool_call_turn, + initial=_NO_TOOL_CALL_TURN, + ) + tool_reply_keys: Final = tuple(map(MistralConfig._tool_reply_key, prepared_messages, tool_call_turns)) + first_reply_positions: Final = { + key: position for position, key in reversed(tuple(enumerate(tool_reply_keys))) if key is not None + } + keys_answered_with_content: Final = frozenset( + key for key, is_empty in zip(tool_reply_keys, empty_flags, strict=True) if not is_empty + ) + answering_positions: Final = frozenset( + position for key, position in first_reply_positions.items() if key not in keys_answered_with_content + ) + new_messages: Final[list[AllMessageValues]] = [ + strip_none_values_from_message(MistralConfig._with_tool_reply_content(m)) + for position, (m, is_empty) in enumerate(zip(prepared_messages, empty_flags, strict=True)) + if not is_empty or position in answering_positions + ] if is_async: return super()._transform_messages(new_messages, model, True) @@ -469,6 +489,44 @@ class MistralConfig(OpenAIGPTConfig): message["tool_calls"] = mistral_tool_calls return message + @staticmethod + def _tool_call_id(tool_call: object) -> str | None: + match tool_call: + case {"id": str(tool_call_id)}: + return tool_call_id + case _: + return None + + @staticmethod + def _tool_call_turn( + turn: tuple[int, frozenset[str]], message_entry: tuple[int, AllMessageValues, bool] + ) -> tuple[int, frozenset[str]]: + match message_entry: + case (position, {"role": "assistant", "tool_calls": list(tool_calls)}, _): + return position, frozenset(filter(None, map(MistralConfig._tool_call_id, tool_calls))) + case (_, {"role": "tool"}, _) | (_, _, True): + return turn + case _: + return _NO_TOOL_CALL_TURN + + @staticmethod + def _tool_reply_key(message: AllMessageValues, turn: tuple[int, frozenset[str]]) -> tuple[int, str] | None: + match message, turn: + case {"role": "tool", "tool_call_id": str(tool_call_id)}, (position, tool_call_ids) if ( + tool_call_id in tool_call_ids + ): + return position, tool_call_id + case _: + return None + + @staticmethod + def _with_tool_reply_content(message: AllMessageValues) -> AllMessageValues: + match message: + case {"role": "tool"} if not message.get("content"): + return {**message, "content": ""} + case _: + return message + @classmethod def _is_empty_assistant_message(cls, message: AllMessageValues) -> bool: """ diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 633f154c4cd..b12fdb95765 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -11,7 +11,7 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify -from litellm.llms.anthropic.pass_through.convertible_blocks import unconvertible_blocks_error +from litellm.llms.anthropic.pass_through.convertible_blocks import deployment_drops_params, unconvertible_blocks_error from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -110,7 +110,9 @@ class OpenAITokenCounter(BaseTokenCounter): return None malformed_error: Final = ( - OpenAICountTokensConfig.malformed_anthropic_messages_error(messages) + OpenAICountTokensConfig.malformed_anthropic_messages_error( + messages, drop_params=deployment_drops_params(deployment) + ) if messages_format == "anthropic" else None ) diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 90bfa4a8655..438aa05a4e9 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -10,6 +10,7 @@ from typing import Any, Final, Literal, TypeAlias from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict +from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, @@ -295,14 +296,16 @@ class OpenAICountTokensConfig: return input_items, instructions @staticmethod - def malformed_anthropic_messages_error(messages: Sequence[Mapping[str, object]]) -> str | None: + def malformed_anthropic_messages_error( + messages: Sequence[Mapping[str, object]], *, drop_params: bool + ) -> str | None: try: _ANTHROPIC_COUNT_MESSAGES.validate_python(messages) except ValidationError as e: return "; ".join( f"messages.{'.'.join(str(part) for part in error['loc'])}: {error['msg']}" for error in e.errors() ) - return None + return None if drop_params else malformed_tool_result_text_error(messages) @staticmethod def anthropic_messages_to_responses_input( diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 3c9a8429882..8298edd6744 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1375,6 +1375,7 @@ class VertexAITokenCounter(BaseTokenCounter): count_context_error_response, gemini_count_contents, inference_request_view, + malformed_tool_result_text_count_error, provider_error_response, unconvertible_messages_error, vertex_gemini_count_context, @@ -1392,7 +1393,9 @@ class VertexAITokenCounter(BaseTokenCounter): count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools) if isinstance(count_context, VertexGeminiCountContextError): return count_context_error_response(count_context, request_model, model_to_use) - blocks_error: Final = unconvertible_messages_error(request_view.messages, messages_format) + blocks_error: Final = unconvertible_messages_error( + request_view.messages, messages_format + ) or malformed_tool_result_text_count_error(messages, messages_format, deployment) if blocks_error is not None: return count_context_error_response(blocks_error, request_model, model_to_use) diff --git a/litellm/llms/vertex_ai/count_tokens/context.py b/litellm/llms/vertex_ai/count_tokens/context.py index c218b4710fd..b13f72e96fe 100644 --- a/litellm/llms/vertex_ai/count_tokens/context.py +++ b/litellm/llms/vertex_ai/count_tokens/context.py @@ -132,6 +132,17 @@ def unconvertible_messages_error( return None if error_message is None else VertexGeminiCountContextError(error_message=error_message) +def malformed_tool_result_text_count_error( + messages: Sequence[Mapping[str, object]] | None, + messages_format: TokenCountMessagesFormat, + deployment: Mapping[str, object] | None, +) -> VertexGeminiCountContextError | None: + from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_count_error + + error_message: Final = malformed_tool_result_count_error(messages, messages_format, deployment) + return None if error_message is None else VertexGeminiCountContextError(error_message=error_message) + + def count_context_error_response( error: VertexGeminiCountContextError, request_model: str, model_used: str ) -> TokenCountResponse: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 660d7d529cb..c1102cee8d1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -368,6 +368,11 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_keys, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import ( + deployment_drops_params, + drop_non_string_tool_result_texts, + malformed_tool_result_count_error, +) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.openai.live import parse_live_session_start from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS @@ -14716,6 +14721,32 @@ def _get_provider_token_counter( return None, None, None +_DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) + + +def _deployment_with_request_drop_params( + deployment: dict[str, object] | None, # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment + drop_params: object, +) -> dict[str, object] | None: # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment + if drop_params is None: + return deployment + resolved_deployment: Final = deployment or {} + litellm_params: Final = _DEPLOYMENT_LITELLM_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) + return {**resolved_deployment, "litellm_params": {**litellm_params, "drop_params": drop_params}} + + +def _counted_request_messages( + messages: Sequence[dict[str, object]] | None, # mutable-ok: TokenCountRequest.messages is typed list[dict] + messages_format: TokenCountMessagesFormat, + deployment: Mapping[str, object] | None, +) -> list[dict[str, object]] | None: # mutable-ok: BaseTokenCounter.count_tokens takes list messages + if messages is None: + return None + if messages_format != "anthropic": + return list(messages) + return list(drop_non_string_tool_result_texts(messages, deployment_drops_params(deployment))) + + async def _try_provider_token_count( provider_counter: "BaseTokenCounter", custom_llm_provider: str | None, @@ -15082,11 +15113,6 @@ _PROVIDERS_COUNTING_WITHOUT_MESSAGES: Final = frozenset( ) -def _drops_params(deployment: Mapping[str, object] | None) -> bool: - litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) - return litellm.drop_params is True or (litellm_params is not None and litellm_params.get("drop_params") is True) - - _ESTIMABLE_GOOGLE_PART_KEYS: Final = frozenset({"text", "functionCall", "functionResponse", "thoughtSignature"}) @@ -15095,7 +15121,7 @@ def _estimable_contents(contents: Sequence[Mapping[str, object]]) -> bool: return all(set(as_str_mapping(part) or ()) <= _ESTIMABLE_GOOGLE_PART_KEYS for part in parts) -def _router_default_litellm_params(router: Router) -> Mapping[str, object]: +def _router_default_litellm_params(router: Router | None) -> Mapping[str, object]: defaults: Final[Mapping[str, object] | None] = getattr(router, "default_litellm_params", None) if defaults is None: return {} @@ -15216,8 +15242,10 @@ async def count_request_tokens( if _model is not None: model_to_use = _model + sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params} + count_deployment: Final = _deployment_with_request_drop_params(deployment, sent_params.get("drop_params")) conversation_empty: Final = not messages and not contents - if prompt and not conversation_empty and not _drops_params(deployment): + if prompt and not conversation_empty and not deployment_drops_params(count_deployment): raise ProxyException( message="prompt cannot be sent with messages or contents. Set drop_params to count them without the prompt", type="invalid_request_error", @@ -15225,6 +15253,7 @@ async def count_request_tokens( code=400, ) prompt_only: Final = bool(prompt) and conversation_empty + request_messages: Final = _counted_request_messages(messages, messages_format, count_deployment) contents_estimable: Final = not contents or (bool(messages) and _estimable_contents(contents)) prompt_as_message: Final = prompt_only and custom_llm_provider in _PROVIDERS_COUNTING_WITHOUT_MESSAGES ladder: Final = ( @@ -15235,9 +15264,9 @@ async def count_request_tokens( _try_provider_token_count, provider_counter, custom_llm_provider, - messages=[{"role": "user", "content": prompt}] if prompt_as_message else messages, + messages=[{"role": "user", "content": prompt}] if prompt_as_message else request_messages, contents=None if prompt_as_message else contents, - deployment=deployment, + deployment=count_deployment, request_model=request.model, tools=tools, system=system, @@ -15253,6 +15282,10 @@ async def count_request_tokens( if isinstance(ladder, TokenCountResponse): return ladder + local_count_error: Final = malformed_tool_result_count_error(request_messages, messages_format, count_deployment) + if local_count_error is not None: + raise ProxyException(message=local_count_error, type="token_counting_error", param="messages", code=400) + # Check if token counter is disabled before fallback if litellm.disable_token_counter is True: raise ProxyException( @@ -15273,7 +15306,7 @@ async def count_request_tokens( prompt_turn: Final = prompt_only and not (system_message is None and not tools) typed_messages: Final = cast( # cast-ok: request messages are raw chat-shaped dicts that token_counter normalizes Sequence[AllMessageValues] | None, - [{"role": "user", "content": prompt}] if prompt_turn else (None if prompt_only else messages), + [{"role": "user", "content": prompt}] if prompt_turn else (None if prompt_only else request_messages), ) counted_messages: Final = ( typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 842230699b7..a2acbc09784 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -45,7 +45,7 @@ from litellm.constants import ( REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER, ) from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) @@ -326,10 +326,6 @@ def _dropped_cache_control(_: object) -> None: return None -def drops_unsupported_params(drop_params: object) -> bool: - return litellm.drop_params is True or normalize_drop_params(drop_params) is True - - def _without_cache_control_key(item: object) -> object: if not isinstance(item, dict): return item diff --git a/tests/integration/messages_endpoint/test_tool_result_text_wire.py b/tests/integration/messages_endpoint/test_tool_result_text_wire.py new file mode 100644 index 00000000000..f8c6eb6b0ca --- /dev/null +++ b/tests/integration/messages_endpoint/test_tool_result_text_wire.py @@ -0,0 +1,868 @@ +import json +import re +import uuid +from collections.abc import Callable, Mapping, Sequence +from email.parser import BytesParser +from email.policy import HTTP +from types import MappingProxyType +from typing import Final + +import anthropic +import httpx +import pytest +from integration._support.client import Gateway, Scenario, eventually, string_value +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_JSON_OBJECTS: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]) +_TOOL_USE_ID: Final = "toolu_01" +_WEATHER: Final = "sunny, 21C" +_FIELD_ERROR: Final = f"tool_result {_TOOL_USE_ID} content.0.text: Input should be a valid string" +_CLAUDE_CODE: Final = MappingProxyType({"user-agent": "claude-cli/2.1.0 (external, cli)"}) +_OPENAI_COUNT: Final = 4242 +_GOOGLE_COUNT: Final = 777 +_ANTHROPIC_COUNT: Final = 58 +_BEDROCK_COUNT: Final = 1345 +_GEMINI: Final = "gemini-4-argon" +_VERTEX_PROJECT: Final = "scripted-project" +_CITY: Final = "Paris" +_REDACTED_CITY: Final = "[CITY]" +_TOOLS: Final[list[JsonValue]] = [ + { + "name": "get_weather", + "description": "Get the weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } +] + + +def _messages(*blocks: JsonValue) -> list[JsonValue]: + return [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": _TOOL_USE_ID, "name": "get_weather", "input": {"city": "Paris"}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": _TOOL_USE_ID, "content": list(blocks)}]}, + ] + + +_NULL_TEXT: Final = _messages({"type": "text", "text": None}) +_NUMBER_THEN_WEATHER: Final = _messages({"type": "text", "text": 5}, {"type": "text", "text": _WEATHER}) +_WEATHER_ONLY: Final = _messages({"type": "text", "text": _WEATHER}) +_WEATHER_THEN_MISSING: Final = _messages({"type": "text", "text": _WEATHER}, {"type": "text"}) + + +def _body(model: str, messages: list[JsonValue], **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "messages": messages, "tools": _TOOLS, **extra} + + +def _responses_reply() -> bytes: + return json.dumps( + { + "id": f"resp_{uuid.uuid4().hex}", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-5.4-mini", + "output": [ + { + "type": "message", + "id": f"msg_{uuid.uuid4().hex}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "It is sunny.", "annotations": []}], + } + ], + "usage": {"input_tokens": 41, "output_tokens": 5, "total_tokens": 46}, + } + ).encode() + + +def _chat_reply() -> bytes: + return json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "mistral-small-latest", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "It is sunny."}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13}, + } + ).encode() + + +def _anthropic_reply() -> bytes: + return json.dumps( + { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "It is sunny."}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 9, "output_tokens": 4}, + } + ).encode() + + +def _openai_peer(request: Request) -> Reply: + if request.target.endswith("/responses/input_tokens"): + return Reply(body=json.dumps({"input_tokens": _OPENAI_COUNT}).encode()) + if request.target.endswith("/responses"): + return Reply(body=_responses_reply()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _chat_peer(request: Request) -> Reply: + if request.target.endswith("/chat/completions"): + return Reply(body=_chat_reply()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _google_peer(request: Request) -> Reply: + if request.target.endswith(":countTokens"): + return Reply(body=json.dumps({"totalTokens": _GOOGLE_COUNT}).encode()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _anthropic_peer(request: Request) -> Reply: + if request.target.endswith("/v1/messages/count_tokens"): + return Reply(body=json.dumps({"input_tokens": _ANTHROPIC_COUNT}).encode()) + if request.target.endswith("/v1/messages"): + return Reply(body=_anthropic_reply()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _bedrock_peer(request: Request) -> Reply: + if request.target.endswith("/count-tokens"): + return Reply(body=json.dumps({"inputTokens": _BEDROCK_COUNT}).encode()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _openai_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model(model="openai/gpt-5.4-mini", api_base=wire.url, api_key="synthetic-openai-key", **parameters) + + +def _mistral_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model( + model="mistral/mistral-small-latest", api_base=wire.url, api_key="synthetic-mistral-key", **parameters + ) + + +def _anthropic_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model( + model="anthropic/claude-sonnet-4-6", api_base=wire.url, api_key="synthetic-anthropic-key", **parameters + ) + + +def _google_model(gateway: Gateway, scenario: Scenario, wire: Wire, provider: str, **parameters: JsonValue) -> str: + if provider == "gemini": + return scenario.model(model=f"gemini/{_GEMINI}", api_base=wire.url, api_key="scripted-gemini-key", **parameters) + return scenario.model( + model=f"vertex_ai/{_GEMINI}", + api_base=wire.url, + api_key=None, + vertex_project=_VERTEX_PROJECT, + vertex_location="us-east5", + vertex_credentials=service_account_json(_VERTEX_PROJECT, gateway.upstream_url.rstrip("/")), + **parameters, + ) + + +def _json(request: Request) -> dict[str, JsonValue]: + return _JSON_OBJECT.validate_json(request.body) + + +def _only(requests: Sequence[Request], suffix: str) -> dict[str, JsonValue]: + matching: Final = tuple(request for request in requests if request.target.endswith(suffix)) + assert len(matching) == 1, [request.target for request in requests] + return _json(matching[0]) + + +def _function_call_output(body: Mapping[str, JsonValue]) -> JsonValue: + outputs: Final = tuple( + item["output"] + for item in _JSON_OBJECTS.validate_python(body["input"]) + if item.get("type") == "function_call_output" + ) + assert len(outputs) == 1, body + return outputs[0] + + +def _tool_message_text(body: Mapping[str, JsonValue]) -> str: + contents: Final = tuple( + message.get("content") + for message in _JSON_OBJECTS.validate_python(body["messages"]) + if message.get("role") == "tool" + ) + assert len(contents) == 1, body + content: Final = contents[0] + if isinstance(content, str): + return content + return "".join(str(part["text"]) for part in _JSON_OBJECTS.validate_python(content)) + + +def _anthropic_error_message(status: int, text: str, payload: bytes) -> str: + assert status == 400, text + error: Final = _JSON_OBJECT.validate_json(payload) + assert error["type"] == "error", text + detail: Final = _JSON_OBJECT.validate_python(error["error"]) + assert detail["type"] == "invalid_request_error", text + return str(detail["message"]) + + +def _assert_field_rejection(gateway: Gateway, path: str, body: Mapping[str, JsonValue], **headers: str) -> None: + response: Final = gateway.request("POST", path, body, headers=headers) + assert _FIELD_ERROR in _anthropic_error_message(response.status_code, response.text, response.content), ( + response.text + ) + + +def _sdk(gateway: Gateway) -> anthropic.Anthropic: + return anthropic.Anthropic( + api_key=gateway.key, base_url=str(gateway.client.base_url), max_retries=0, http_client=gateway.client + ) + + +def _assert_sdk_bad_request(raised: pytest.ExceptionInfo[anthropic.BadRequestError]) -> None: + error: Final = raised.value + assert error.status_code == 400, error + assert _FIELD_ERROR in _anthropic_error_message(error.status_code, error.message, error.response.content), error + + +def test_openai_rejects_a_non_string_tool_result_text_on_both_routes(gateway: Gateway) -> None: + client: Final = _sdk(gateway) + with wire_server(_openai_peer) as wire, gateway.scenario() as scenario: + model: Final = _openai_model(scenario, wire) + with pytest.raises(anthropic.BadRequestError) as created: + client.messages.create(model=model, max_tokens=64, messages=_NULL_TEXT, tools=_TOOLS) + _assert_sdk_bad_request(created) + with pytest.raises(anthropic.BadRequestError) as streamed: + client.messages.create(model=model, max_tokens=64, messages=_NULL_TEXT, tools=_TOOLS, stream=True) + _assert_sdk_bad_request(streamed) + with pytest.raises(anthropic.BadRequestError) as counted: + client.messages.count_tokens(model=model, messages=_NULL_TEXT, tools=_TOOLS) + _assert_sdk_bad_request(counted) + assert wire.drain() == () + + +_DROP_SOURCES: Final = MappingProxyType( + { + "deployment": (MappingProxyType({"drop_params": True}), MappingProxyType({}), MappingProxyType({})), + "body": (MappingProxyType({}), MappingProxyType({"drop_params": True}), MappingProxyType({})), + "claude_code": (MappingProxyType({}), MappingProxyType({}), _CLAUDE_CODE), + } +) + + +@pytest.mark.parametrize("source", sorted(_DROP_SOURCES)) +def test_openai_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None: + deployment, extra, headers = _DROP_SOURCES[source] + with wire_server(_openai_peer) as wire, gateway.scenario() as scenario: + model: Final = _openai_model(scenario, wire, **deployment) + body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _OPENAI_COUNT, counted.text + requests: Final = wire.drain() + assert _function_call_output(_only(requests, "/responses")) == _WEATHER + assert _function_call_output(_only(requests, "/responses/input_tokens")) == _WEATHER + + +def test_explicit_drop_params_false_wins_on_both_routes(gateway: Gateway) -> None: + with wire_server(_openai_peer) as wire, gateway.scenario() as scenario: + model: Final = _openai_model(scenario, wire, drop_params=True) + body: Final = _body(model, _NULL_TEXT, drop_params=False) + _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64}, **_CLAUDE_CODE) + _assert_field_rejection(gateway, "/v1/messages/count_tokens", body, **_CLAUDE_CODE) + assert wire.drain() == () + + +def test_chat_bridge_and_local_count_reject_a_non_string_tool_result_text(gateway: Gateway) -> None: + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + body: Final = _body(model, _NULL_TEXT) + _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64}) + _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64, "stream": True}) + _assert_field_rejection(gateway, "/v1/messages/count_tokens", body) + assert wire.drain() == () + + +@pytest.mark.parametrize("source", ("deployment", "body")) +def test_chat_bridge_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None: + deployment, extra, _ = _DROP_SOURCES[source] + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire, **deployment) + body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _tool_message_text(_only(wire.drain(), "/chat/completions")) == _WEATHER + + +def test_count_without_a_deployment_rejects_and_honors_body_drop_params(gateway: Gateway) -> None: + model: Final = f"integration-unrouted-{uuid.uuid4().hex}" + _assert_field_rejection(gateway, "/v1/messages/count_tokens", _body(model, _NULL_TEXT)) + dropped: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(model, _NULL_TEXT, drop_params=True)) + assert dropped.status_code == 200, dropped.text + tokens: Final = _JSON_OBJECT.validate_json(dropped.content)["input_tokens"] + assert isinstance(tokens, int) and tokens > 0, dropped.text + + +def _counted_contents(body: Mapping[str, JsonValue]) -> JsonValue: + if "contents" in body: + return body["contents"] + return _JSON_OBJECT.validate_python(body["generateContentRequest"])["contents"] + + +@pytest.mark.parametrize("provider", ("gemini", "vertex_ai")) +def test_google_count_rejects_and_drops_a_non_string_tool_result_text(gateway: Gateway, provider: str) -> None: + with wire_server(_google_peer) as wire, gateway.scenario() as scenario: + strict: Final = _google_model(gateway, scenario, wire, provider) + dropping: Final = _google_model(gateway, scenario, wire, provider, drop_params=True) + _assert_field_rejection(gateway, "/v1/messages/count_tokens", _body(strict, _NULL_TEXT)) + assert tuple(request for request in wire.drain() if request.target.endswith(":countTokens")) == () + dropped: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(dropping, _NUMBER_THEN_WEATHER)) + assert dropped.status_code == 200, dropped.text + dropped_body: Final = _only(wire.drain(), ":countTokens") + clean: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(dropping, _WEATHER_ONLY)) + assert clean.status_code == 200, clean.text + assert _JSON_OBJECT.validate_json(dropped.content)["input_tokens"] == _GOOGLE_COUNT, dropped.text + assert _counted_contents(dropped_body) == _counted_contents(_only(wire.drain(), ":countTokens")) + + +def _forwarded_tool_result(body: Mapping[str, JsonValue]) -> JsonValue: + last: Final = _JSON_OBJECTS.validate_python(body["messages"])[-1] + return _JSON_OBJECTS.validate_python(last["content"])[0]["content"] + + +@pytest.mark.parametrize( + ("headers", "forwarded"), + ((MappingProxyType({}), [{"type": "text", "text": None}]), (_CLAUDE_CODE, [])), + ids=("no-drop-params", "claude-code-drop-params"), +) +def test_native_anthropic_forwards_a_null_tool_result_text_unless_drop_params_drops_it( + gateway: Gateway, headers: Mapping[str, str], forwarded: list[JsonValue] +) -> None: + with wire_server(_anthropic_peer) as wire, gateway.scenario() as scenario: + model: Final = _anthropic_model(scenario, wire) + body: Final = _body(model, _NULL_TEXT) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _ANTHROPIC_COUNT, counted.text + requests: Final = wire.drain() + assert _forwarded_tool_result(_only(requests, "/v1/messages")) == forwarded + assert _forwarded_tool_result(_only(requests, "/v1/messages/count_tokens")) == forwarded + + +def test_bedrock_count_with_a_claude_code_user_agent_still_counts(gateway: Gateway) -> None: + with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="bedrock/anthropic.claude-sonnet-4-6", + api_base=wire.url, + api_key=None, + aws_access_key_id="AKIAINTEGRATIONTOOLRESULT", + aws_secret_access_key="integration-tool-result-secret", + aws_region_name="us-east-1", + ) + counted: Final = gateway.request( + "POST", "/v1/messages/count_tokens", _body(model, _WEATHER_ONLY), headers=_CLAUDE_CODE + ) + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _BEDROCK_COUNT, counted.text + assert len(tuple(request for request in wire.drain() if request.target.endswith("/count-tokens"))) == 1 + + +@pytest.mark.parametrize("provider", ("openai", "mistral")) +def test_valid_tool_result_text_reaches_the_upstream(gateway: Gateway, provider: str) -> None: + peer: Final = _openai_peer if provider == "openai" else _chat_peer + with wire_server(peer) as wire, gateway.scenario() as scenario: + model: Final = (_openai_model if provider == "openai" else _mistral_model)(scenario, wire) + body: Final = _body(model, _WEATHER_THEN_MISSING) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + requests: Final = wire.drain() + if provider == "openai": + assert _function_call_output(_only(requests, "/responses")) == f"{_WEATHER}\n" + assert _function_call_output(_only(requests, "/responses/input_tokens")) == f"{_WEATHER}\n" + else: + assert _tool_message_text(_only(requests, "/chat/completions")) == _WEATHER + + +def _rewritten(content: JsonValue, rewrite: Callable[[str], str]) -> JsonValue: + if isinstance(content, str): + return rewrite(content) + if isinstance(content, list): + return [_rewritten(part, rewrite) for part in content] + if isinstance(content, dict) and isinstance(text := content.get("text"), str): + return {**content, "text": rewrite(text)} + return content + + +def _redacted(content: JsonValue) -> JsonValue: + return _rewritten(content, lambda text: text.replace(_CITY, _REDACTED_CITY)) + + +def _redacted_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return {**row, "content": _redacted(row["content"])} if "content" in row else dict(row) + + +def _digits_masked_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + if "content" not in row: + return dict(row) + return {**row, "content": _rewritten(row["content"], lambda text: re.sub(r"\d", "#", text))} + + +def _joined_tool_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + redacted: Final = _redacted_row(row) + content: Final = redacted.get("content") + if row.get("role") != "tool" or not isinstance(content, list): + return redacted + texts: Final = ( + part["text"] for part in _JSON_OBJECTS.validate_python(content) if isinstance(part.get("text"), str) + ) + return {**redacted, "content": " | ".join(str(text) for text in texts)} + + +_ROW_REWRITES: Final = MappingProxyType({"join": _joined_tool_row, "digits": _digits_masked_row}) + + +def _guardrail_peer(request: Request) -> Reply: + rows: Final = _JSON_OBJECTS.validate_python(_json(request).get("structured_messages") or []) + if request.target.startswith("/none/") or not rows: + return Reply(body=json.dumps({"action": "NONE"}).encode()) + rewrite: Final = _ROW_REWRITES.get(request.target.split("/")[1], _redacted_row) + verdict: Final = {"action": "GUARDRAIL_INTERVENED", "structured_messages": [rewrite(row) for row in rows]} + return Reply(body=json.dumps(verdict).encode()) + + +def _delete_guardrail(gateway: Gateway, identity: str) -> None: + deleted: Final = gateway.request("DELETE", f"/guardrails/{identity}") + assert deleted.status_code == 200, deleted.text + + +def _guardrail(gateway: Gateway, scenario: Scenario, wire: Wire, mode: str) -> str: + name: Final = f"tool-result-{mode}-{uuid.uuid4().hex}" + params: Final[dict[str, JsonValue]] = { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": f"{wire.url}/{mode}", + "api_key": "synthetic-guardrail-key", + } + created: Final = gateway.post("/guardrails", {"guardrail": {"guardrail_name": name, "litellm_params": params}}) + scenario.cleanups.callback(_delete_guardrail, gateway, string_value(created["guardrail_id"])) + return name + + +def _first_turn(messages: JsonValue) -> str: + return json.dumps(_JSON_OBJECTS.validate_python(messages)[0]) + + +def test_rewriting_guardrail_leaves_the_rejection_to_the_openai_route(gateway: Gateway) -> None: + with ( + wire_server(_openai_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _openai_model(scenario, provider) + body: Final = _body(model, _NULL_TEXT, max_tokens=64, guardrails=[_guardrail(gateway, scenario, guard, "rows")]) + _assert_field_rejection(gateway, "/v1/messages", body) + _assert_field_rejection(gateway, "/v1/messages", {**body, "stream": True}) + assert provider.drain() == () + + +@pytest.mark.parametrize("source", sorted(_DROP_SOURCES)) +def test_rewriting_guardrail_then_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None: + deployment, extra, headers = _DROP_SOURCES[source] + with ( + wire_server(_openai_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _openai_model(scenario, provider, **deployment) + guardrail: Final = _guardrail(gateway, scenario, guard, "rows") + body: Final = _body(model, _NUMBER_THEN_WEATHER, max_tokens=64, guardrails=[guardrail], **extra) + created: Final = gateway.request("POST", "/v1/messages", body, headers=headers) + assert created.status_code == 200, created.text + forwarded: Final = _only(provider.drain(), "/responses") + first_turn: Final = _first_turn(forwarded["input"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + assert _function_call_output(forwarded) == _WEATHER, forwarded + + +@pytest.mark.parametrize( + ("deployment", "forwarded"), + ( + (MappingProxyType({}), [{"type": "text", "text": 5}, {"type": "text", "text": _WEATHER}]), + (MappingProxyType({"drop_params": True}), [{"type": "text", "text": _WEATHER}]), + ), + ids=("no-drop-params", "deployment-drop-params"), +) +def test_rewriting_guardrail_forwards_the_malformed_text_to_anthropic_unless_drop_params_drops_it( + gateway: Gateway, deployment: Mapping[str, JsonValue], forwarded: list[JsonValue] +) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider, **deployment) + guardrail: Final = _guardrail(gateway, scenario, guard, "rows") + body: Final = _body(model, _NUMBER_THEN_WEATHER, max_tokens=64, guardrails=[guardrail]) + created: Final = gateway.request("POST", "/v1/messages", body) + assert created.status_code == 200, created.text + forwarded_body: Final = _only(provider.drain(), "/v1/messages") + first_turn: Final = _first_turn(forwarded_body["messages"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + assert _forwarded_tool_result(forwarded_body) == forwarded + + +_MORNING: Final = f"{_CITY} morning: sunny" +_EVENING: Final = f"{_CITY} evening: 18C" +_TOMORROW: Final = f"{_CITY} tomorrow: rain" +_WEEKEND: Final = f"{_CITY} weekend: clear" + + +def _text(value: JsonValue) -> dict[str, JsonValue]: + return {"type": "text", "text": value} + + +def _tool_use(tool_use_id: str) -> dict[str, JsonValue]: + return {"type": "tool_use", "id": tool_use_id, "name": "get_weather", "input": {"city": _CITY}} + + +def _tool_result(tool_use_id: str, *parts: JsonValue) -> dict[str, JsonValue]: + return {"type": "tool_result", "tool_use_id": tool_use_id, "content": [_text(part) for part in parts]} + + +_SEVERAL_TOOL_RESULTS: Final[list[JsonValue]] = [ + {"role": "user", "content": f"What is the weather in {_CITY}?"}, + {"role": "assistant", "content": [_tool_use("toolu_01"), _tool_use("toolu_02"), _tool_use("toolu_03")]}, + { + "role": "user", + "content": [ + _tool_result("toolu_01", None, _MORNING, 5, _EVENING), + _tool_result("toolu_02", _TOMORROW, 7.5), + _tool_result("toolu_03", _WEEKEND), + ], + }, +] + + +def _forwarded_tool_results(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + last: Final = _JSON_OBJECTS.validate_python(body["messages"])[-1] + return { + string_value(block["tool_use_id"]): block["content"] + for block in _JSON_OBJECTS.validate_python(last["content"]) + if block.get("type") == "tool_result" + } + + +def _texts(content: JsonValue) -> tuple[JsonValue, ...]: + if isinstance(content, str): + return (content,) + return tuple(part.get("text") for part in _JSON_OBJECTS.validate_python(content)) + + +_RESTORED_TOOL_RESULTS: Final = MappingProxyType( + { + "rows": ( + [_text(None), _text(_redacted(_MORNING)), _text(5), _text(_redacted(_EVENING))], + [_text(_redacted(_TOMORROW)), _text(7.5)], + ), + "join": ( + [_text(f"null | {_redacted(_MORNING)} | 5 | {_redacted(_EVENING)}")], + [_text(f"{_redacted(_TOMORROW)} | 7.5")], + ), + } +) + + +@pytest.mark.parametrize("mode", sorted(_RESTORED_TOOL_RESULTS)) +def test_rewriting_guardrail_puts_back_each_malformed_text_it_left_unchanged(gateway: Gateway, mode: str) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, mode) + body: Final = _body(model, _SEVERAL_TOOL_RESULTS, max_tokens=64, guardrails=[guardrail]) + created: Final = gateway.request("POST", "/v1/messages", body) + assert created.status_code == 200, created.text + forwarded: Final = _only(provider.drain(), "/v1/messages") + first_turn: Final = _first_turn(forwarded["messages"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + tool_results: Final = _forwarded_tool_results(forwarded) + assert _CITY not in json.dumps(tool_results), tool_results + assert (tool_results["toolu_01"], tool_results["toolu_02"]) == _RESTORED_TOOL_RESULTS[mode], tool_results + assert _texts(tool_results["toolu_03"]) == (_redacted(_WEEKEND),), tool_results + + +def _null_text_answering(tool_use_id: str) -> list[JsonValue]: + return [ + {"role": "user", "content": f"What is the weather in {_CITY}?"}, + {"role": "assistant", "content": [_tool_use(tool_use_id)]}, + {"role": "user", "content": [_tool_result(tool_use_id, None)]}, + ] + + +@pytest.mark.parametrize("tool_use_id", ("toolu_01", "functions.get_weather:0")) +def test_rewriting_guardrail_leaves_the_rejection_to_the_chat_bridge_for_any_tool_use_id( + gateway: Gateway, tool_use_id: str +) -> None: + with ( + wire_server(_chat_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _mistral_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, "rows") + body: Final = _body(model, _null_text_answering(tool_use_id), max_tokens=64, guardrails=[guardrail]) + response: Final = gateway.request("POST", "/v1/messages", body) + message: Final = _anthropic_error_message(response.status_code, response.text, response.content) + assert "content.0.text: Input should be a valid string" in message, message + assert provider.drain() == () + + +def _files_peer(request: Request) -> Reply: + if request.method == "POST" and request.target.endswith("/files"): + uploaded: Final = { + "id": f"file-{uuid.uuid4().hex}", + "object": "file", + "bytes": len(request.body), + "created_at": 1789788253, + "filename": "batch.jsonl", + "purpose": "batch", + "status": "processed", + } + return Reply(body=json.dumps(uploaded).encode()) + if request.target.endswith("/models"): + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _uploaded_records(request: Request) -> tuple[dict[str, JsonValue], ...]: + envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body + parts: Final = BytesParser(policy=HTTP).parsebytes(envelope).iter_parts() + files: Final = tuple(part.get_payload(decode=True) for part in parts if part.get_filename() is not None) + assert len(files) == 1, request.headers + payload: Final = files[0] + assert isinstance(payload, bytes), payload + return tuple(_JSON_OBJECT.validate_json(line) for line in payload.splitlines() if line.strip()) + + +def _listed_models(gateway: Gateway) -> frozenset[str]: + with httpx.Client(base_url=gateway.client.base_url, timeout=15, trust_env=False) as client: + listed: Final = client.get("/v1/models", headers={"Authorization": f"Bearer {gateway.key}"}) + assert listed.status_code == 200, listed.text + entries: Final = _JSON_OBJECTS.validate_python(_JSON_OBJECT.validate_json(listed.content)["data"]) + return frozenset(string_value(entry["id"]) for entry in entries) + + +def _uploaded_batch_body( + gateway: Gateway, scenario: Scenario, guard: Wire, mode: str, messages: list[JsonValue] +) -> dict[str, JsonValue]: + with wire_server(_files_peer) as provider: + model: Final = _openai_model(scenario, provider) + key: Final = scenario.key(metadata={"guardrails": [_guardrail(gateway, scenario, guard, mode)]}) + eventually( + lambda: tuple(_listed_models(gateway) for _ in range(8)), + lambda listings: all(model in listing for listing in listings), + seconds=90, + ) + record: Final = { + "custom_id": f"weather-{uuid.uuid4().hex}", + "method": "POST", + "url": "/v1/messages", + "body": _body(model, messages, max_tokens=64), + } + uploaded: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": model}, + {"file": ("batch.jsonl", json.dumps(record).encode() + b"\n", "application/jsonl")}, + key=key, + ) + assert uploaded.status_code == 200, uploaded.text + uploads: Final = tuple(request for request in provider.drain() if request.target.endswith("/files")) + assert len(uploads) == 1, [request.target for request in uploads] + (forwarded,) = _uploaded_records(uploads[0]) + assert forwarded["custom_id"] == record["custom_id"], forwarded + return _JSON_OBJECT.validate_python(forwarded["body"]) + + +def test_batch_upload_behind_a_rewriting_guardrail_keeps_the_malformed_text(gateway: Gateway) -> None: + with wire_server(_guardrail_peer) as guard, gateway.scenario() as scenario: + forwarded_body: Final = _uploaded_batch_body(gateway, scenario, guard, "rows", _NUMBER_THEN_WEATHER) + first_turn: Final = _first_turn(forwarded_body["messages"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + assert _forwarded_tool_result(forwarded_body) == [_text(5), _text(_WEATHER)], forwarded_body + + +def test_guardrail_that_rewrites_nothing_forwards_a_null_tool_result_text_unchanged(gateway: Gateway) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, "none") + created: Final = gateway.request( + "POST", "/v1/messages", _body(model, _NULL_TEXT, max_tokens=64, guardrails=[guardrail]) + ) + assert created.status_code == 200, created.text + assert len(guard.drain()) == 1 + assert _forwarded_tool_result(_only(provider.drain(), "/v1/messages")) == [{"type": "text", "text": None}] + + +def _forwarded_chat_turns(wire: Wire) -> tuple[dict[str, JsonValue], ...]: + return _JSON_OBJECTS.validate_python(_only(wire.drain(), "/chat/completions")["messages"]) + + +def test_mistral_receives_the_tool_reply_whose_only_text_drop_params_dropped(gateway: Gateway) -> None: + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + created: Final = gateway.request( + "POST", "/v1/messages", _body(model, _NULL_TEXT, max_tokens=64, drop_params=True) + ) + assert created.status_code == 200, created.text + turns: Final = _forwarded_chat_turns(wire) + assert [turn["role"] for turn in turns] == ["user", "assistant", "tool"], turns + calls: Final = _JSON_OBJECTS.validate_python(turns[1]["tool_calls"]) + assert [call["id"] for call in calls] == [turns[2]["tool_call_id"]], turns + assert _tool_message_text({"messages": list(turns)}) == "", turns + + +_WEATHER_CALL: Final[dict[str, JsonValue]] = { + "id": "call_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"city": _CITY})}, +} + + +def test_mistral_chat_completions_keeps_an_empty_tool_reply(gateway: Gateway) -> None: + ask: Final = f"What is the weather in {_CITY}? {uuid.uuid4().hex}" + messages: Final[list[JsonValue]] = [ + {"role": "system", "content": ""}, + {"role": "user", "content": ask}, + {"role": "assistant", "content": ""}, + {"role": "assistant", "content": None, "tool_calls": [_WEATHER_CALL]}, + {"role": "tool", "tool_call_id": "call_weather", "content": ""}, + {"role": "user", "content": ""}, + ] + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + response: Final = gateway.request("POST", "/v1/chat/completions", {"model": model, "messages": messages}) + assert response.status_code == 200, response.text + turns: Final = _forwarded_chat_turns(wire) + assert [(turn["role"], turn.get("content")) for turn in turns] == [ + ("user", ask), + ("assistant", None), + ("tool", ""), + ], turns + assert turns[2]["tool_call_id"] == "call_weather", turns + + +def test_mistral_chat_completions_drops_an_orphan_tool_message_and_sends_a_null_tool_reply_as_empty_text( + gateway: Gateway, +) -> None: + ask: Final = f"What is the weather in {_CITY}? {uuid.uuid4().hex}" + messages: Final[list[JsonValue]] = [ + {"role": "user", "content": ask}, + {"role": "tool", "tool_call_id": "call_orphan", "content": ""}, + {"role": "assistant", "content": None, "tool_calls": [_WEATHER_CALL]}, + {"role": "tool", "tool_call_id": "call_weather", "content": None}, + ] + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + response: Final = gateway.request("POST", "/v1/chat/completions", {"model": model, "messages": messages}) + assert response.status_code == 200, response.text + turns: Final = _forwarded_chat_turns(wire) + assert [(turn["role"], turn.get("tool_call_id"), turn.get("content")) for turn in turns] == [ + ("user", None, ask), + ("assistant", None, None), + ("tool", "call_weather", ""), + ], turns + + +_NATIVE_DROP_PARAMS_CASES: Final = MappingProxyType( + { + "body": (_DROP_SOURCES["body"], [_text(_WEATHER)]), + "deployment": (_DROP_SOURCES["deployment"], [_text(_WEATHER)]), + "explicit-false": ( + (MappingProxyType({"drop_params": True}), MappingProxyType({"drop_params": False}), _CLAUDE_CODE), + [_text(5), _text(_WEATHER)], + ), + } +) + + +@pytest.mark.parametrize("case", sorted(_NATIVE_DROP_PARAMS_CASES)) +def test_native_anthropic_resolves_drop_params_the_same_way_on_both_routes(gateway: Gateway, case: str) -> None: + (deployment, extra, headers), forwarded = _NATIVE_DROP_PARAMS_CASES[case] + with wire_server(_anthropic_peer) as wire, gateway.scenario() as scenario: + model: Final = _anthropic_model(scenario, wire, **deployment) + body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _ANTHROPIC_COUNT, counted.text + requests: Final = wire.drain() + assert _forwarded_tool_result(_only(requests, "/v1/messages")) == forwarded + assert _forwarded_tool_result(_only(requests, "/v1/messages/count_tokens")) == forwarded + + +_CARD_NUMBER: Final = 4111111111111111 +_CARD_NULL_WEATHER: Final = _messages(_text(_CARD_NUMBER), _text(None), _text(_WEATHER)) +_SCANNED_CARD_NULL_WEATHER: Final = ("4111111111111111", "null", _WEATHER) +_MASKED_CARD_NULL_WEATHER: Final = [_text("################"), _text(None), _text("sunny, ##C")] + + +def _scanned_tool_texts(guard: Wire) -> tuple[JsonValue, ...]: + requests: Final = guard.drain() + assert len(requests) == 1, [request.target for request in requests] + rows: Final = _JSON_OBJECTS.validate_python(_json(requests[0])["structured_messages"]) + tool_rows: Final = tuple(row for row in rows if row.get("role") == "tool") + assert len(tool_rows) == 1, rows + return _texts(tool_rows[0]["content"]) + + +def test_masking_guardrail_scans_a_numeric_tool_result_text_before_anthropic_gets_it(gateway: Gateway) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, "digits") + created: Final = gateway.request( + "POST", "/v1/messages", _body(model, _CARD_NULL_WEATHER, max_tokens=64, guardrails=[guardrail]) + ) + assert created.status_code == 200, created.text + scanned: Final = _scanned_tool_texts(guard) + forwarded: Final = _forwarded_tool_result(_only(provider.drain(), "/v1/messages")) + assert scanned == _SCANNED_CARD_NULL_WEATHER, scanned + assert forwarded == _MASKED_CARD_NULL_WEATHER, forwarded + + +def test_masking_guardrail_scans_a_numeric_tool_result_text_in_a_batch_upload(gateway: Gateway) -> None: + with wire_server(_guardrail_peer) as guard, gateway.scenario() as scenario: + forwarded_body: Final = _uploaded_batch_body(gateway, scenario, guard, "digits", _CARD_NULL_WEATHER) + scanned: Final = _scanned_tool_texts(guard) + assert scanned == _SCANNED_CARD_NULL_WEATHER, scanned + assert _forwarded_tool_result(forwarded_body) == _MASKED_CARD_NULL_WEATHER, forwarded_body diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index ec6af7f27c3..046f448046b 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -25,7 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_tools_pt, _rename_duplicate_bedrock_document_names, _convert_to_bedrock_tool_call_invoke, - _sanitize_anthropic_tool_use_id, + sanitize_anthropic_tool_use_id, _convert_to_bedrock_tool_call_result, anthropic_messages_pt, convert_to_anthropic_tool_result, @@ -4503,7 +4503,7 @@ def test_get_tool_calls_from_response_keeps_sanitized_concat_ids_distinct(): ) ids: Final = [call["id"] for call in get_tool_calls_from_response(response)] - sanitized: Final = [_sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] + sanitized: Final = [sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] assert len(sanitized) == len(set(sanitized)) assert ids == ["a:b", "a:b__concat_2", "a_b__concat_1"] @@ -4517,7 +4517,7 @@ def test_get_tool_calls_from_response_bumps_suffix_when_sibling_sanitizes_onto_i ) ids: Final = [call["id"] for call in get_tool_calls_from_response(response)] - sanitized: Final = [_sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] + sanitized: Final = [sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] assert len(ids) == len(sanitized) assert len(sanitized) == len(set(sanitized)) diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 0aca60351bf..58e3b55773b 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -155,6 +155,17 @@ class MockStructuredMaskingGuardrail(CustomGuardrail): return masked +class MockNumberMaskingGuardrail(MockStructuredMaskingGuardrail): + @staticmethod + def _mask(text: str) -> str: + return ( + text.replace("bob@example.com", "") + .replace("5", "") + .replace("null", "") + .replace("山田太郎", "") + ) + + class TestAnthropicMessagesHandlerStreamingRequestData: """Post-call guardrails on streaming /v1/messages receive the response and identity metadata""" @@ -2499,7 +2510,7 @@ class TestStructuredWriteBackKeepsToolResults: """A guardrail rewrite must never leave a tool_use without its tool_result (Claude Code ToolSearch, LIT-6103).""" @staticmethod - def _claude_code_tool_search_turns(tool_result_content): + def _claude_code_tool_search_turns(tool_result_content, tool_use_id="toolu_01"): return [ {"role": "user", "content": "load WebFetch for bob@example.com"}, { @@ -2507,7 +2518,7 @@ class TestStructuredWriteBackKeepsToolResults: "content": [ { "type": "tool_use", - "id": "toolu_01", + "id": tool_use_id, "name": "ToolSearch", "input": {"query": "select:WebFetch"}, } @@ -2516,7 +2527,7 @@ class TestStructuredWriteBackKeepsToolResults: { "role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}, + {"type": "tool_result", "tool_use_id": tool_use_id, "content": tool_result_content}, {"type": "text", "text": "Now fetch the page."}, ], }, @@ -2563,6 +2574,124 @@ class TestStructuredWriteBackKeepsToolResults: later_blocks = [b for m in messages[tool_use_index + 1 :] for b in self._blocks(m)] assert {"type": "text", "text": "Now fetch the page."} in later_blocks + @pytest.mark.parametrize( + ("tool_result_content", "expected_written_back_content"), + [ + ( + [{"type": "text", "text": None}, {"type": "text", "text": "log for bob@example.com"}], + [{"type": "text", "text": None}, {"type": "text", "text": "log for "}], + ), + ( + [{"type": "text", "text": "log for bob@example.com"}, {"type": "text", "text": 73}], + [{"type": "text", "text": "log for "}, {"type": "text", "text": 73}], + ), + ([{"type": "text", "text": None}], [{"type": "text", "text": None}]), + ( + [ + {"type": "text", "text": None}, + {"type": "text", "text": "log for bob@example.com"}, + {"type": "text", "text": 5}, + {"type": "text", "text": "bob@example.com again"}, + ], + [ + {"type": "text", "text": None}, + {"type": "text", "text": "log for "}, + {"type": "text", "text": 5}, + {"type": "text", "text": " again"}, + ], + ), + ], + ids=["null-first", "number-last", "null-only", "interleaved"], + ) + async def test_rewrite_keeps_a_non_string_tool_result_text_in_place( + self, tool_result_content, expected_written_back_content + ): + data = {"model": "claude-fable-5", "messages": self._claude_code_tool_search_turns(tool_result_content)} + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockStructuredMaskingGuardrail() + ) + + assert "bob@example.com" not in json.dumps(data["messages"]) + assert data["messages"][2]["content"][0] == { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": expected_written_back_content, + } + + async def test_rewrite_keeps_a_non_string_tool_result_text_whose_tool_use_id_gets_sanitized(self): + tool_result_content = [{"type": "text", "text": None}, {"type": "text", "text": "log for bob@example.com"}] + data = { + "model": "claude-fable-5", + "messages": self._claude_code_tool_search_turns(tool_result_content, tool_use_id="functions.get_weather:0"), + } + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockStructuredMaskingGuardrail() + ) + + assert data["messages"][2]["content"][0]["content"] == [ + {"type": "text", "text": None}, + {"type": "text", "text": "log for "}, + ] + + async def test_rewrite_keeps_each_non_string_text_when_two_tool_use_ids_sanitize_alike(self): + data = { + "model": "claude-fable-5", + "messages": [ + {"role": "user", "content": "check both logs for bob@example.com"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "read:log", "name": "Read", "input": {"path": "a.log"}}, + {"type": "tool_use", "id": "read_log", "name": "Read", "input": {"path": "b.log"}}, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "read:log", + "content": [{"type": "text", "text": "ok"}, {"type": "text", "text": None}], + }, + { + "type": "tool_result", + "tool_use_id": "read_log", + "content": [{"type": "text", "text": 5}, {"type": "text", "text": "bob@example.com"}], + }, + ], + }, + ], + } + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockStructuredMaskingGuardrail() + ) + + assert [block["content"] for block in data["messages"][2]["content"]] == [ + [{"type": "text", "text": "ok"}, {"type": "text", "text": None}], + [{"type": "text", "text": 5}, {"type": "text", "text": ""}], + ] + + @pytest.mark.parametrize( + ("text", "masked"), + [(5, ""), (None, ""), ({"owner": "山田太郎"}, '{"owner": ""}')], + ids=["number", "null", "non-ascii-object"], + ) + async def test_guardrail_scans_and_masks_a_non_string_tool_result_text(self, text, masked): + tool_result_content = [{"type": "text", "text": text}, {"type": "text", "text": "log for bob@example.com"}] + data = {"model": "claude-fable-5", "messages": self._claude_code_tool_search_turns(tool_result_content)} + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockNumberMaskingGuardrail() + ) + + assert data["messages"][2]["content"][0]["content"] == [ + {"type": "text", "text": masked}, + {"type": "text", "text": "log for "}, + ] + class TestAnthropicMessagesHandlerStreamingScanKey: """get_streaming_scan_key mirrors what process_output_streaming_response would scan""" diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 9e0bd5c60b8..0e918cd49c5 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1,10 +1,13 @@ import base64 import functools import json +from collections.abc import Mapping from types import MappingProxyType from typing import Any, Final, cast +import httpx import pytest +import respx import litellm @@ -5235,3 +5238,153 @@ def test_eager_input_streaming_tool_reaches_bedrock_converse_as_beta(): assert data["additionalModelRequestFields"]["anthropic_beta"] == ["fine-grained-tool-streaming-2025-05-14"] assert data["toolConfig"]["tools"][0]["toolSpec"]["inputSchema"]["json"] == EAGER_INPUT_SCHEMA + + +_CHAT_BRIDGE_MODEL: Final = "azure/gpt-5.6" +_CHAT_BRIDGE_API_BASE: Final = "https://tool-result-text.openai.azure.com" +_CHAT_BRIDGE_URL: Final = r".*tool-result-text\.openai\.azure\.com/openai/deployments/gpt-5\.6/chat/completions.*" +_CHAT_BRIDGE_REPLY: Final = { + "id": "chatcmpl-tool-result-text", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "build passed"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 3, "total_tokens": 15}, +} +_MALFORMED_TOOL_RESULT_TEXTS: Final = (pytest.param(None, id="null"), pytest.param(5, id="number")) +_MALFORMED_TOOL_RESULT_TEXT_ERROR: Final = ( + "litellm.BadRequestError: tool_result toolu_log content.1.text: Input should be a valid string" +) + + +def _tool_result_with_text_blocks(text_block: Mapping[str, object]) -> list[dict[str, object]]: + blocks: Final = [{"type": "text", "text": "build ok"}, text_block, {"type": "text", "text": "done"}, text_block] + return [ + {"role": "user", "content": "run the build"}, + dict(_anthropic_tool_use_turn("toolu_log")), + dict(_anthropic_tool_result_turn({"toolu_log": blocks})), + ] + + +def _chat_bridge_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(url__regex=_CHAT_BRIDGE_URL).mock(return_value=httpx.Response(200, json=_CHAT_BRIDGE_REPLY)) + + +def _sent_tool_messages(route: respx.Route) -> list[object]: + sent: Final = json.loads(route.calls.last.request.read()) + return [message for message in sent["messages"] if message["role"] == "tool"] + + +@pytest.fixture +def _chat_bridge_httpx(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"]) +@pytest.mark.parametrize("text", _MALFORMED_TOOL_RESULT_TEXTS) +async def test_chat_bridge_async_malformed_tool_result_text_answers_400_before_any_provider_call( + respx_mock: respx.MockRouter, text: object, stream: bool +) -> None: + route: Final = _chat_bridge_route(respx_mock) + + with pytest.raises(litellm.BadRequestError) as rejected: + await litellm.anthropic.messages.acreate( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + stream=stream, + messages=_tool_result_with_text_blocks({"type": "text", "text": text}), + ) + + assert rejected.value.status_code == 400 + assert rejected.value.llm_provider == "azure" + assert rejected.value.message == _MALFORMED_TOOL_RESULT_TEXT_ERROR + assert route.call_count == 0 + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"]) +@pytest.mark.parametrize("text", _MALFORMED_TOOL_RESULT_TEXTS) +def test_chat_bridge_sync_malformed_tool_result_text_answers_400_before_any_provider_call( + respx_mock: respx.MockRouter, text: object, stream: bool +) -> None: + route: Final = _chat_bridge_route(respx_mock) + + with pytest.raises(litellm.BadRequestError) as rejected: + litellm.anthropic.messages.create( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + stream=stream, + messages=_tool_result_with_text_blocks({"type": "text", "text": text}), + ) + + assert rejected.value.status_code == 400 + assert rejected.value.llm_provider == "azure" + assert rejected.value.message == _MALFORMED_TOOL_RESULT_TEXT_ERROR + assert route.call_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize("drop_params_source", ["request", "global"]) +@pytest.mark.parametrize("text", _MALFORMED_TOOL_RESULT_TEXTS) +async def test_chat_bridge_drop_params_drops_a_malformed_tool_result_text( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, text: object, drop_params_source: str +) -> None: + route: Final = _chat_bridge_route(respx_mock) + if drop_params_source == "global": + monkeypatch.setattr(litellm, "drop_params", True) + + response: Final = await litellm.anthropic.messages.acreate( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + messages=_tool_result_with_text_blocks({"type": "text", "text": text}), + **({"drop_params": True} if drop_params_source == "request" else {}), + ) + + assert _sent_tool_messages(route) == [ + { + "role": "tool", + "tool_call_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": "done"}], + } + ] + assert isinstance(response, dict) + assert response["content"] == [{"type": "text", "text": "build passed"}] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_chat_bridge_tool_result_text_block_without_text_reaches_the_model_as_empty_text( + respx_mock: respx.MockRouter, +) -> None: + route: Final = _chat_bridge_route(respx_mock) + + await litellm.anthropic.messages.acreate( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + messages=_tool_result_with_text_blocks({"type": "text"}), + ) + + assert _sent_tool_messages(route) == [ + { + "role": "tool", + "tool_call_id": "toolu_log", + "content": [ + {"type": "text", "text": "build ok"}, + {"type": "text", "text": ""}, + {"type": "text", "text": "done"}, + {"type": "text", "text": ""}, + ], + } + ] diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py index c63bfbdd301..7fbec3ba12e 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py @@ -300,3 +300,67 @@ async def test_a_json_provider_gets_safeguards_and_the_beta_together_unless_drop assert sent.headers["anthropic-beta"] == forwarded_beta assert "safeguard_results" not in response assert response["content"][0]["text"] == "hello" + + +@pytest.mark.parametrize( + ("drop_params", "expected_tool_result_content"), + [ + (True, [{"type": "text", "text": "sunny"}]), + (None, [{"type": "text", "text": 5}, {"type": "text", "text": "sunny"}]), + ], + ids=["drop_params", "no_drop_params"], +) +def test_drop_params_drops_a_non_string_tool_result_text_on_the_native_request( + drop_params, expected_tool_result_content +): + messages = [ + {"role": "user", "content": "What is the weather?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": 5}, {"type": "text", "text": "sunny"}], + } + ], + }, + ] + + request = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-5", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + assert request["messages"][2]["content"][0]["content"] == expected_tool_result_content + + +def test_drop_params_forwards_a_non_object_message_item_as_sent_on_the_native_request(): + tool_result_turn = { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": None}, {"type": "text", "text": "sunny"}], + } + ], + } + + request = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "What is the weather?"}, 7, tool_result_turn], + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + + assert request["messages"][1] == 7 + assert request["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "sunny"}] diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py index c55fa476c06..409e64635f7 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py @@ -617,3 +617,88 @@ async def test_tool_search_forwards_deferred_tools_and_beta_header( assert [block for block in response["content"] if block["type"] == "tool_use"] == [ {"type": "tool_use", "id": "toolu_1", "name": tool_name, "input": tool_input} ] + + +_MIXED_GROUP: Final = "gpt-5.6-mixed" +_MIXED_AZURE_API_BASE: Final = "https://mixed-group.openai.azure.com" +_MIXED_AZURE_CHAT_URL: Final = r".*mixed-group\.openai\.azure\.com/openai/deployments/.*/chat/completions.*" +_MIXED_DEPLOYMENT_IDS: Final = ("mixed-openai-responses-bridge", "mixed-azure-chat-bridge") +_MIXED_AZURE_CHAT_BODY: Final = { + "id": "chatcmpl-mixed", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": _JOKE}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, +} + + +def _mixed_openai_azure_router() -> Router: + return Router( + model_list=[ + { + "model_name": _MIXED_GROUP, + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "fake-key"}, + "model_info": {"id": _MIXED_DEPLOYMENT_IDS[0]}, + }, + { + "model_name": _MIXED_GROUP, + "litellm_params": {"model": "azure/gpt-5.6", "api_key": "fake-key", "api_base": _MIXED_AZURE_API_BASE}, + "model_info": {"id": _MIXED_DEPLOYMENT_IDS[1]}, + }, + ], + num_retries=0, + ) + + +async def _null_tool_result_text_rejection(router: Router, deployment_id: str) -> litellm.BadRequestError: + with pytest.raises(litellm.BadRequestError) as rejected: + await router.aanthropic_messages( + model=deployment_id, + max_tokens=64, + messages=[ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_build", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_build", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": None}], + } + ], + }, + ], + ) + return rejected.value + + +@pytest.mark.asyncio +async def test_mixed_openai_azure_group_answers_the_same_400_for_a_null_tool_result_text_on_either_deployment( + respx_mock, +): + openai_route: Final = respx_mock.post(url__regex=_OPENAI_RESPONSES_URL).mock( + return_value=httpx.Response(200, json=_OPENAI_BODY) + ) + azure_route: Final = respx_mock.post(url__regex=_MIXED_AZURE_CHAT_URL).mock( + return_value=httpx.Response(200, json=_MIXED_AZURE_CHAT_BODY) + ) + router: Final = _mixed_openai_azure_router() + + rejections: Final = [ + await _null_tool_result_text_rejection(router, deployment_id) for deployment_id in _MIXED_DEPLOYMENT_IDS + ] + + assert [(rejection.status_code, rejection.llm_provider) for rejection in rejections] == [ + (400, "openai"), + (400, "azure"), + ] + assert {rejection.message.partition("\n\n")[0] for rejection in rejections} == { + "litellm.BadRequestError: tool_result toolu_build content.1.text: Input should be a valid string" + } + assert openai_route.call_count == 0 + assert azure_route.call_count == 0 diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index b6f831d4bef..8fe6bd31082 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -576,6 +576,91 @@ def test_client_cache_control_changes_nothing_for_a_deployment_that_does_not_rea assert marked == unmarked +MALFORMED_TEXT_BLOCKS: Final = [ + pytest.param({"type": "text", "text": None}, id="null"), + pytest.param({"type": "text", "text": 7}, id="number"), +] + + +def _tool_result_turn_with(text_block: Mapping[str, object]) -> list[dict[str, object]]: + return [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, text_block, {"type": "text", "text": "done"}], + } + ], + } + ] + + +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +def test_malformed_tool_result_text_answers_400_naming_the_block(text_block: Mapping[str, object]) -> None: + with pytest.raises(litellm.BadRequestError) as rejected: + _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs=OPENAI_API, + max_tokens=1024, + messages=_tool_result_turn_with(text_block), + ) + + assert rejected.value.status_code == 400 + assert "tool_result toolu_log content.1.text: Input should be a valid string" in rejected.value.message + + +def test_many_malformed_tool_result_texts_answer_one_bounded_error() -> None: + tool_use_id: Final = "toolu_" + "x" * 4096 + messages: Final = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_use_id, + "content": [{"type": "text", "text": None}] * 2048, + } + ], + } + ] + + with pytest.raises(litellm.BadRequestError) as rejected: + _build_responses_kwargs(model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, messages=messages) + + assert rejected.value.status_code == 400 + assert f"tool_result {tool_use_id} content.0.text: Input should be a valid string" in rejected.value.message + assert rejected.value.message.count(tool_use_id) == 1 + + +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +def test_drop_params_drops_a_malformed_tool_result_text(text_block: Mapping[str, object]) -> None: + responses_kwargs: Final = _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs={**OPENAI_API, "drop_params": True}, + max_tokens=1024, + messages=_tool_result_turn_with(text_block), + ) + + assert responses_kwargs["input"] == [ + {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\ndone"}, + ] + + +def test_tool_result_text_block_without_text_still_reaches_the_model_as_empty_text() -> None: + responses_kwargs: Final = _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs=OPENAI_API, + max_tokens=1024, + messages=_tool_result_turn_with({"type": "text"}), + ) + + assert responses_kwargs["input"] == [ + {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"}, + ] + + REASONING_MODEL: Final = "gpt-bridge-reasoning-cost" REASONING_RATES: Final = { "input_cost_per_token": 2e-06, diff --git a/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py b/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py index b5301b413ff..eac75486da4 100644 --- a/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py +++ b/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py @@ -8,6 +8,7 @@ from pydantic import JsonValue, TypeAdapter from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.llms.anthropic.pass_through.convertible_blocks import ( + malformed_tool_result_text_error, unconvertible_anthropic_blocks, unconvertible_blocks_error, ) @@ -190,3 +191,22 @@ def test_an_encrypted_web_search_inference_cannot_send_is_named() -> None: error: Final = unconvertible_blocks_error(_replayed_web_search("opaque"), "anthropic") assert error is not None assert error.endswith("server_tool_use, web_search_tool_result") + + +@pytest.mark.parametrize( + ("tool_result", "error"), + ( + ( + {"type": "tool_result", "tool_use_id": "toolu_01", "content": [{"type": "text", "text": None}]}, + "tool_result toolu_01 content.0.text: Input should be a valid string", + ), + ( + {"type": "tool_result", "content": [{"type": "text", "text": "ok"}, {"type": "text", "text": 5}]}, + "tool_result content.1.text: Input should be a valid string", + ), + ), +) +def test_a_non_string_tool_result_text_is_named_with_or_without_a_tool_use_id( + tool_result: Mapping[str, object], error: str +) -> None: + assert malformed_tool_result_text_error([{"role": "user", "content": [tool_result]}]) == error diff --git a/tests/unit/llms/gemini/test_gemini_common_utils.py b/tests/unit/llms/gemini/test_gemini_common_utils.py index e54023c6596..918a7475eec 100644 --- a/tests/unit/llms/gemini/test_gemini_common_utils.py +++ b/tests/unit/llms/gemini/test_gemini_common_utils.py @@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, patch import httpx import pytest +import respx from pydantic import JsonValue, TypeAdapter import litellm @@ -153,6 +154,83 @@ async def test_google_ai_studio_count_tokens_answers_with_the_status_google_reje assert "Invalid function name" in (result.error_message or "") +def _weather_tool_result_messages(text: JsonValue) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "Paris"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": text}, {"type": "text", "text": "partly cloudy"}], + } + ], + }, + ] + + +_COUNT_TOKENS_URL: Final = "https://count-tokens.test/v1beta/models/gemini-2.5-flash:countTokens" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text", (None, 5)) +async def test_google_ai_studio_count_tokens_rejects_a_non_string_tool_result_text_without_request( + text: JsonValue, respx_mock: respx.MockRouter +) -> None: + count_route: Final = respx_mock.post(_COUNT_TOKENS_URL).respond(200, json={"totalTokens": 11}) + + result: Final = await GoogleAIStudioTokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(text), + contents=None, + deployment={"litellm_params": {"api_key": "test-key", "api_base": "https://count-tokens.test"}}, + request_model="gemini/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is True + assert result.status_code == 400 + assert result.error_message == "tool_result toolu_01 content.0.text: Input should be a valid string" + assert not count_route.called + + +@pytest.mark.asyncio +async def test_google_ai_studio_count_tokens_leaves_out_a_numeric_tool_result_text_under_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + count_route: Final = respx_mock.post(_COUNT_TOKENS_URL).respond(200, json={"totalTokens": 11}) + + result: Final = await GoogleAIStudioTokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(5), + contents=None, + deployment={ + "litellm_params": {"api_key": "test-key", "api_base": "https://count-tokens.test", "drop_params": True} + }, + request_model="gemini/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is False + assert result.total_tokens == 11 + assert _JSON_BODY.validate_json(count_route.calls.last.request.content)["contents"] == [ + {"role": "user", "parts": [{"text": "What is the weather in Paris?"}]}, + {"role": "model", "parts": [{"function_call": {"name": "get_weather", "args": {"city": "Paris"}}}]}, + { + "role": "user", + "parts": [{"function_response": {"name": "get_weather", "response": {"content": "partly cloudy"}}}], + }, + ] + + @pytest.mark.asyncio @pytest.mark.parametrize( ("system_instruction", "tools", "expected_body"), diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index 2e88bccd49b..10f57be33fe 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -920,6 +920,85 @@ class TestMistralStripsOutputOnlyFields: assert "reasoning_content" not in result[-1] +def test_mistral_transform_request_keeps_an_empty_tool_message_after_its_tool_call(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "read the log"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + result = request["messages"] + assert [m.get("role") for m in result] == ["user", "assistant", "tool"] + assert result[2].get("tool_call_id") == "call_1" + + +@pytest.mark.parametrize("reply", [{"content": None}, {}], ids=["null", "missing"]) +def test_mistral_transform_request_sends_a_null_tool_reply_as_empty_text(reply): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "read the log"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", **reply}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"][2] == {"role": "tool", "tool_call_id": "call_1", "content": ""} + + +def test_mistral_transform_request_drops_an_empty_tool_message_no_tool_call_asked_for(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "hi"}, + {"role": "tool", "tool_call_id": "call_9", "content": ""}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"] == [{"role": "user", "content": "hi"}] + + +@pytest.mark.parametrize("role", ["system", "user"]) +def test_mistral_transform_request_drops_a_message_with_null_content(role): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [{"role": role, "content": None}, {"role": "user", "content": "hi"}], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"] == [{"role": "user", "content": "hi"}] + + def test_mistral_transform_request_hoists_tool_message_image(): """Images inside role:"tool" messages must be moved to a following user message (Mistral rejects/ignores non-text tool content), including when @@ -957,3 +1036,129 @@ def test_mistral_transform_request_hoists_tool_message_image(): {"type": "text", "text": TOOL_RESULT_IMAGE_BOUNDARY}, {"type": "image_url", "image_url": {"url": data_uri}}, ] + + +def test_mistral_transform_request_drops_an_empty_tool_message_for_a_call_already_answered_earlier(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + {"role": "assistant", "content": "It is sunny"}, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant", "tool", "assistant", "user"] + + +def test_mistral_transform_request_drops_an_empty_tool_message_when_a_later_reply_carries_the_result(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant", "tool", "user"] + assert request["messages"][2]["content"] == "sunny" + + +def test_mistral_transform_request_keeps_an_empty_tool_message_answering_a_call_in_the_first_message(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["assistant", "tool", "user"] + assert request["messages"][1]["content"] == "" + + +@pytest.mark.parametrize("placeholder_role", ["assistant", "user"]) +def test_mistral_transform_request_keeps_an_empty_tool_message_after_a_dropped_empty_placeholder(placeholder_role): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": placeholder_role, "content": ""}, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant", "tool", "user"] + assert request["messages"][2]["content"] == "" + + +def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_not_a_string(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "read the log"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": {"nested": "call_1"}, "type": "function", "function": {"name": "read", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": {"nested": "call_1"}, "content": ""}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index be394b71ec7..7495d98b42a 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -1,5 +1,6 @@ import json import operator +from collections.abc import Mapping from typing import Final import httpx @@ -395,6 +396,92 @@ async def test_undecodable_count_body_becomes_a_500_error_response_carrying_the_ ) +MALFORMED_TEXT_BLOCKS: Final = [ + pytest.param({"type": "text", "text": None}, id="null"), + pytest.param({"type": "text", "text": 7}, id="number"), +] + + +def _tool_result_turns_with(text_block: Mapping[str, object]) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_log", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, text_block, {"type": "text", "text": "done"}], + } + ], + }, + ] + + +async def _count_tool_result_turns( + monkeypatch: pytest.MonkeyPatch, text_block: Mapping[str, object], litellm_params: dict[str, object] +) -> tuple[TokenCountResponse | None, respx.Route]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9})) + result: Final = await OpenAITokenCounter().count_tokens( + model_to_use="gpt-4o", + messages=_tool_result_turns_with(text_block), + contents=None, + deployment={"litellm_params": litellm_params}, + request_model="openai/gpt-4o", + messages_format="anthropic", + ) + return result, route + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +async def test_anthropic_format_malformed_tool_result_text_answers_400_without_calling_openai( + monkeypatch: pytest.MonkeyPatch, text_block: Mapping[str, object] +) -> None: + result, route = await _count_tool_result_turns(monkeypatch, text_block, {"api_key": "sk-test"}) + + assert route.called is False + assert result is not None + assert (result.error, result.status_code, result.error_message) == ( + True, + 400, + "tool_result toolu_log content.1.text: Input should be a valid string", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +async def test_anthropic_format_drop_params_counts_the_tool_result_without_the_malformed_text( + monkeypatch: pytest.MonkeyPatch, text_block: Mapping[str, object] +) -> None: + result, route = await _count_tool_result_turns( + monkeypatch, text_block, {"api_key": "sk-test", "drop_params": True} + ) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 9) + sent: Final = json.loads(route.calls.last.request.content) + assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\ndone"} in sent["input"] + + +@pytest.mark.asyncio +async def test_anthropic_format_tool_result_text_block_without_text_still_counts_as_empty_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_tool_result_turns(monkeypatch, {"type": "text"}, {"api_key": "sk-test"}) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 9) + sent: Final = json.loads(route.calls.last.request.content) + assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"} in sent["input"] + + @pytest.mark.asyncio async def test_count_uses_the_global_openai_key_chat_would_use(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 20dba332c08..7d380205ca1 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -117,6 +117,7 @@ def _vertex_count_tokens_peer(status_code: int) -> Iterator[_VertexCountTokensPe _VERTEX_DEPLOYMENT: Final = {"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-central1"}} +_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue]) @pytest.mark.asyncio @@ -164,6 +165,77 @@ async def test_vertex_gemini_count_rejects_an_image_url_it_refuses_to_fetch_with assert peer.requests == () +def _weather_tool_result_messages(text: JsonValue) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "Paris"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": text}, {"type": "text", "text": "partly cloudy"}], + } + ], + }, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text", (None, 5)) +async def test_vertex_gemini_count_rejects_a_non_string_tool_result_text_without_request(text: JsonValue) -> None: + with _vertex_count_tokens_peer(200) as peer: + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(text), + contents=None, + deployment=_VERTEX_DEPLOYMENT, + request_model="vertex_ai/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is True + assert result.status_code == 400 + assert result.error_message == "tool_result toolu_01 content.0.text: Input should be a valid string" + assert peer.requests == () + + +@pytest.mark.asyncio +async def test_vertex_gemini_count_leaves_out_a_numeric_tool_result_text_when_the_deployment_drops_params() -> None: + with _vertex_count_tokens_peer(200) as peer: + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(5), + contents=None, + deployment={ + "litellm_params": { + "vertex_project": "test-project", + "vertex_location": "us-central1", + "drop_params": True, + } + }, + request_model="vertex_ai/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is False + assert result.total_tokens == 37 + assert _JSON_BODY.validate_json(peer.requests[0].content)["contents"] == [ + {"role": "user", "parts": [{"text": "What is the weather in Paris?"}]}, + {"role": "model", "parts": [{"function_call": {"name": "get_weather", "args": {"city": "Paris"}}}]}, + { + "role": "user", + "parts": [{"function_response": {"name": "get_weather", "response": {"content": "partly cloudy"}}}], + }, + ] + + _TOOL_OUTPUT: Final = "line of a long file that the agent read earlier. " * 180 diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 720739a730b..18fe5a2ad32 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -21,6 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import anyio import httpx import pytest +import respx import tiktoken from dotenv import load_dotenv from pydantic import BaseModel, ConfigDict, JsonValue @@ -36,6 +37,7 @@ load_dotenv() from fastapi import Request from fastapi.responses import JSONResponse +from starlette.types import Message import litellm from litellm import Router @@ -56,7 +58,7 @@ from litellm.llms.bedrock.count_tokens.kimi_k3 import ( from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import VertexAIPartnerModelsTokenCounter -from litellm.proxy._types import ProxyException, TokenCountRequest +from litellm.proxy._types import ProxyException, TokenCountRequest, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.endpoints import ( count_tokens as anthropic_count_tokens, ) @@ -1275,6 +1277,202 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): proxy_server.count_request_tokens = original_token_counter +def _json_request(body: Mapping[str, object], user_agent: str) -> Request: + payload: Final = json.dumps(body).encode() + + async def receive() -> Message: + return {"type": "http.request", "body": payload, "more_body": False} + + return Request( + { + "type": "http", + "method": "POST", + "path": "/v1/messages/count_tokens", + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"user-agent", user_agent.encode())], + }, + receive, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_drop_params", "user_agent"), + (({"drop_params": True}, "python-httpx/0.28.1"), ({}, "claude-cli/2.1.0 (external, cli)")), +) +async def test_anthropic_count_tokens_drops_a_malformed_tool_result_text_when_drop_params_applies( + monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, bool], user_agent: str +) -> None: + import litellm.proxy.proxy_server as proxy_server + + body: Final = { + "model": "gpt-tool-result", + **request_drop_params, + "messages": [ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_log", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": None}], + } + ], + }, + ], + } + router: Final = Router( + model_list=[ + {"model_name": "gpt-tool-result", "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-test"}} + ] + ) + request: Final = _json_request(body, user_agent) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.post("https://api.openai.com/v1/responses/input_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 9}) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert response["input_tokens"] == 9 + sent: Final = json.loads(route.calls.last.request.content) + assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok"} in sent["input"] + + +def _build_log_messages(log_text: JsonValue) -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_log", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": log_text}], + } + ], + }, + ] + + +async def _count_on_a_model_without_a_count_api( + monkeypatch: pytest.MonkeyPatch, body: Mapping[str, object], router_defaults: dict[str, JsonValue] | None = None +) -> JSONResponse | dict[str, JsonValue]: + import litellm.proxy.proxy_server as proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "mistral-tool-result", + "litellm_params": {"model": "mistral/mistral-small-latest", "api_key": "sk-test"}, + } + ], + default_litellm_params=router_defaults, + ) + request: Final = _json_request(body, "python-httpx/0.28.1") + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + with respx.mock: + return await anthropic_count_tokens(request, UserAPIKeyAuth()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("log_text", (None, 5)) +async def test_count_tokens_without_a_count_api_rejects_a_non_string_tool_result_text( + monkeypatch: pytest.MonkeyPatch, log_text: JsonValue +) -> None: + response: Final = await _count_on_a_model_without_a_count_api( + monkeypatch, {"model": "mistral-tool-result", "messages": _build_log_messages(log_text)} + ) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert json.loads(bytes(response.body))["error"] == { + "type": "invalid_request_error", + "message": "tool_result toolu_log content.1.text: Input should be a valid string", + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("mistral-tool-result", "model-not-in-the-router")) +async def test_count_tokens_without_a_count_api_counts_a_non_string_tool_result_text_under_drop_params( + monkeypatch: pytest.MonkeyPatch, model: str +) -> None: + response: Final = await _count_on_a_model_without_a_count_api( + monkeypatch, {"model": model, "drop_params": True, "messages": _build_log_messages(None)} + ) + + assert isinstance(response, dict) + assert isinstance(response["input_tokens"], int) and response["input_tokens"] > 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("body_drop_params", "expected_status"), (({}, None), ({"drop_params": False}, 400))) +async def test_count_tokens_honors_router_default_drop_params_unless_the_body_overrides_it( + monkeypatch: pytest.MonkeyPatch, body_drop_params: dict[str, JsonValue], expected_status: int | None +) -> None: + response: Final = await _count_on_a_model_without_a_count_api( + monkeypatch, + {"model": "mistral-tool-result", "messages": _build_log_messages(None), **body_drop_params}, + router_defaults={"drop_params": True}, + ) + + match response: + case JSONResponse(): + assert response.status_code == expected_status + case _: + assert expected_status is None + assert isinstance(response["input_tokens"], int) and response["input_tokens"] > 0 + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_tool_result_text_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "claude-native-drop", + "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", + "api_key": "sk-ant-test", + "drop_params": True, + }, + } + ] + ) + request: Final = _json_request( + {"model": "claude-native-drop", "messages": _build_log_messages(5)}, "python-httpx/0.28.1" + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 7}) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert response["input_tokens"] == 7 + sent: Final = json.loads(route.calls.last.request.content) + assert sent["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "build ok"}] + + class _ScriptedCounter(BaseTokenCounter): def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None: self._replies = replies From 53821f665417c0d02be0c4495e188ea9c0c3584e Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 14:19:14 -0700 Subject: [PATCH 49/50] fix(decisions): log every request field the caller sent on /v1/systemone and /v1/decisions (#47183) * fix(decisions): log every request field the caller sent on /v1/systemone and /v1/decisions * fix(decisions): serialize the logged request with model_dump_json so no mutable dict is built * refactor(decisions): drop the state-only messages pick in function_setup, prepare_call logs the request * fix(decisions): return the single logged message so no mutable list type is annotated * fix(decisions): log the caller's raw request fields before validation so a rejected request is still recorded * fix(decisions): serialize the raw request fields with pydantic_core so question objects and mapping proxies log as JSON * fix(decisions): log the caller request at the SDK entry so requests rejected before prepare_call are recorded too * test(decisions): assert the exact BadRequestError for the rejected-questions case --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/decisions/call.py | 27 +++++++ litellm/decisions/main.py | 11 +++ litellm/systemone/main.py | 9 ++- litellm/utils.py | 7 -- tests/unit/decisions/test_call.py | 114 ++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 10 deletions(-) create mode 100644 tests/unit/decisions/test_call.py diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 4943b22e1fb..b31cb1f69db 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -1,3 +1,4 @@ +import json from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from functools import partial @@ -6,6 +7,7 @@ from typing import Final, Literal, TypeAlias import httpx from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError +from pydantic_core import to_jsonable_python from typing_extensions import assert_never import litellm @@ -56,6 +58,7 @@ from litellm.types.decisions import ( OpenAIDecisionResponse, UnsupportedDecisionsRequest, ) +from litellm.types.llms.openai import ChatCompletionUserMessage from litellm.types.utils import LlmProviders, ProviderSpecificHeader from litellm.utils import ProviderConfigManager, get_model_cost_key @@ -415,6 +418,30 @@ def _outbound_headers( return MappingProxyType({**(extra_headers or {}), **forwarded}) +_LOGGED_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def _jsonable_fallback(value: object) -> object: + return dict(_LOGGED_FIELDS_ADAPTER.validate_python(value)) if isinstance(value, Mapping) else str(value) + + +def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage: + caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python( + to_jsonable_python(caller_request, fallback=_jsonable_fallback) + ) + sent_fields: Final = json.dumps( + dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {})) + ) + return ChatCompletionUserMessage(role="user", content=sent_fields) + + +def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> None: + """Record every request field the caller sent as the logged `messages`, before any validation.""" + logging_obj: Final = kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.update_messages([_logged_request_message(caller_request)]) + + def prepare_call( *, model: str, diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 7d2ee03da0e..4e609fd2ab8 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -17,6 +17,7 @@ from litellm.decisions.call import ( SystemOneQuestions, SystemOneRequestFields, asend, + log_caller_request, prepare_call, reject_images_next_to_input, send, @@ -78,6 +79,16 @@ def _prepare_decisions_call( extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], ) -> DecisionsCall: + log_caller_request( + kwargs, + { + "input": decision_input, + "questions": questions, + "audio": audio, + "videos": videos, + **{key: kwargs[key] for key in _SYSTEMONE_FIELDS if key in kwargs}, + }, + ) return prepare_call( model=model, fields=_request_fields(model, decision_input, questions, audio, videos, kwargs), diff --git a/litellm/systemone/main.py b/litellm/systemone/main.py index 01f151ef4d0..27fa84e3f9b 100644 --- a/litellm/systemone/main.py +++ b/litellm/systemone/main.py @@ -14,6 +14,7 @@ from litellm.decisions.call import ( SystemOneQuestions, SystemOneRequestFields, asend, + log_caller_request, prepare_call, reject_other_format, send, @@ -39,12 +40,14 @@ def _prepare_systemone_call( extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], ) -> DecisionsCall: + fields: Final = SystemOneRequestFields( + state=state, questions=questions, images=images, assets=assets, audio=audio, videos=videos + ) + log_caller_request(kwargs, fields) reject_other_format(model=model, kwargs=kwargs, field_name="input", other_function="decisions") return prepare_call( model=model, - fields=SystemOneRequestFields( - state=state, questions=questions, images=images, assets=assets, audio=audio, videos=videos - ), + fields=fields, safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, diff --git a/litellm/utils.py b/litellm/utils.py index cf3b8756e2f..143c2470d33 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -998,10 +998,6 @@ def _ocr_document_summary(document: object) -> str: # Runs once per call to check if the user wants to send their data anywhere - PostHog/Sentry/Slack/etc. -_DECISION_CALL_TYPES: Final[frozenset[str]] = frozenset( - (CallTypes.decisions.value, CallTypes.adecisions.value, CallTypes.systemone.value, CallTypes.asystemone.value) -) - def function_setup( original_function: str, @@ -1249,9 +1245,6 @@ def function_setup( if isinstance(search_query, list) else search_query ) - elif call_type in _DECISION_CALL_TYPES: - decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state") or kwargs.get("input") or "" - messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state, default=str) elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): messages = args[1] if len(args) > 1 else kwargs.get("prompt") elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value): diff --git a/tests/unit/decisions/test_call.py b/tests/unit/decisions/test_call.py new file mode 100644 index 00000000000..03c135369b8 --- /dev/null +++ b/tests/unit/decisions/test_call.py @@ -0,0 +1,114 @@ +import json +from collections.abc import Mapping +from datetime import datetime, timezone +from typing import Final + +import pytest + +import litellm +from litellm.decisions.main import _prepare_decisions_call +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.systemone.main import _prepare_systemone_call + +QUESTION_TEXT: Final = "Is this about a billing problem?" +CONNECTION: Final = { + "model": "hosted_vllm/jev-fake", + "safety_identifier": None, + "api_key": "fake-provider-key", + "api_base": "http://127.0.0.1:4010", + "timeout": None, + "custom_llm_provider": None, + "extra_headers": None, +} + + +def _logging_obj(call_type: str) -> Logging: + return Logging( + model="hosted_vllm/jev-fake", + messages="placeholder", + stream=False, + call_type=call_type, + start_time=datetime.now(timezone.utc), + litellm_call_id="lit-9659", + function_id="lit-9659", + ) + + +def _logged_request(logging_obj: Logging) -> Mapping[str, object]: + messages: Final = logging_obj.model_call_details["messages"] + assert len(messages) == 1 and messages[0]["role"] == "user" + return json.loads(messages[0]["content"]) + + +def test_systemone_logs_every_request_field_the_caller_sent(): + logging_obj: Final = _logging_obj("asystemone") + + _prepare_systemone_call( + **CONNECTION, + state={"message": "My card was charged twice."}, + questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}}, + images=None, + assets=None, + audio=None, + videos=None, + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["state"] == {"message": "My card was charged twice."} + assert logged["questions"]["billing"]["instructions"] == QUESTION_TEXT + assert set(logged) == {"state", "questions"} + + +def test_decisions_logs_every_request_field_the_caller_sent(): + logging_obj: Final = _logging_obj("adecisions") + + _prepare_decisions_call( + **CONNECTION, + decision_input="My card was charged twice.", + questions=[{"type": "predicate", "instructions": QUESTION_TEXT}], + audio=None, + videos=None, + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["input"] == "My card was charged twice." + assert logged["questions"][0]["instructions"] == QUESTION_TEXT + assert set(logged) == {"input", "questions"} + + +def test_request_is_logged_even_when_it_is_rejected_before_the_provider_call(): + logging_obj: Final = _logging_obj("asystemone") + + with pytest.raises(litellm.BadRequestError): + _prepare_systemone_call( + **CONNECTION, + state={"message": "My card was charged twice."}, + questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}}, + images=None, + assets=None, + audio=None, + videos=None, + kwargs={"litellm_logging_obj": logging_obj, "input": "sent to the wrong function"}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["questions"]["billing"]["instructions"] == QUESTION_TEXT + + +def test_decisions_request_is_logged_even_when_its_questions_fail_validation(): + logging_obj: Final = _logging_obj("adecisions") + + with pytest.raises(litellm.BadRequestError): + _prepare_decisions_call( + **CONNECTION, + decision_input="My card was charged twice.", + questions=[{"type": "not-a-question-type", "instructions": QUESTION_TEXT}], + audio=None, + videos=None, + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["questions"][0]["instructions"] == QUESTION_TEXT From 62ce515d15fd61f15985d54120265f5976fbce4d Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 14:27:29 -0700 Subject: [PATCH 50/50] feat(router): let a deployment set a timeout per service_tier (#47406) * feat(router): let a deployment set a timeout per service_tier Adds timeout_by_service_tier and stream_timeout_by_service_tier to a deployment's litellm_params. A request whose service_tier matches a key uses that value in place of the deployment's timeout or stream_timeout. A timeout the client sends still wins and requests without a matching tier are unchanged Co-authored-by: adc77 * style(router): sort the request_timeout_resolver import * fix(router): read the service tier maps without the untyped params get * fix(router): look the service tier timeout up through a typed get --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: adc77 --- .../request_timeout_resolver.py | 23 +++++++- litellm/passthrough/timeout_utils.py | 20 +++++-- litellm/router.py | 3 + litellm/types/litellm_params.py | 2 + litellm/types/router.py | 5 ++ .../test_pass_through_endpoints.py | 26 +++++++++ tests/unit/test_router/test_router_timeout.py | 56 +++++++++++++++++++ tests/unit/types/test_litellm_params.py | 2 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 16 ++++++ 9 files changed, 147 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/request_timeout_resolver.py b/litellm/litellm_core_utils/request_timeout_resolver.py index a32651fedee..0e011dc06a0 100644 --- a/litellm/litellm_core_utils/request_timeout_resolver.py +++ b/litellm/litellm_core_utils/request_timeout_resolver.py @@ -12,10 +12,15 @@ tell "user asked for this" from "nobody set it". This resolver answers that: from __future__ import annotations -from typing import Final +from collections.abc import Mapping +from typing import Final, Literal, Protocol + +from pydantic import TypeAdapter from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +_TIMEOUTS_BY_SERVICE_TIER: Final[TypeAdapter[Mapping[str, float] | None]] = TypeAdapter(Mapping[str, float] | None) + def get_configured_request_timeout() -> float | None: """Return the explicitly-configured ``litellm.request_timeout``, else ``None``.""" @@ -27,3 +32,19 @@ def get_configured_request_timeout() -> float | None: if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS): return timeout return None + + +class _DeploymentParams(Protocol): + def get(self, key: str, /) -> object: ... + + +def get_service_tier_timeout( + service_tier: object, + litellm_params: _DeploymentParams, + key: Literal["timeout_by_service_tier", "stream_timeout_by_service_tier"], +) -> float | None: + """Return the deployment's ``key`` timeout for the request's ``service_tier``, else ``None``.""" + if not isinstance(service_tier, str): + return None + timeouts: Final = _TIMEOUTS_BY_SERVICE_TIER.validate_python(litellm_params.get(key)) + return None if timeouts is None else timeouts.get(service_tier) diff --git a/litellm/passthrough/timeout_utils.py b/litellm/passthrough/timeout_utils.py index f600c7817f2..ce13552ff49 100644 --- a/litellm/passthrough/timeout_utils.py +++ b/litellm/passthrough/timeout_utils.py @@ -5,7 +5,10 @@ from typing import Final from pydantic import TypeAdapter -from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout +from litellm.litellm_core_utils.request_timeout_resolver import ( + get_configured_request_timeout, + get_service_tier_timeout, +) DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0 @@ -50,21 +53,27 @@ def resolve_llm_passthrough_timeout( Anthropic /v1/messages). Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params - timeout/request_timeout -> router_timeout -> litellm.request_timeout (litellm_settings.request_timeout, + timeout_by_service_tier[kwargs service_tier] -> litellm_params timeout/request_timeout -> router_timeout -> litellm.request_timeout (litellm_settings.request_timeout, when explicitly set) -> general_settings.pass_through_request_timeout -> 600s default. Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before any generic timeout, matching ``Router._get_stream_timeout`` on the completion route: - kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the - non-streaming chain above. + kwargs stream_timeout -> litellm_params stream_timeout_by_service_tier[kwargs service_tier] -> + litellm_params stream_timeout -> router_stream_timeout, then the non-streaming chain above. Only the first set value is validated as seconds, so a value in a lower-precedence field never fails the call. """ request: Final = kwargs if kwargs is not None else _NO_PARAMS deployment: Final = litellm_params if litellm_params is not None else _NO_PARAMS + service_tier: Final = request.get("service_tier") stream_candidates: Final = ( - (request.get("stream_timeout"), deployment.get("stream_timeout"), router_stream_timeout) + ( + request.get("stream_timeout"), + get_service_tier_timeout(service_tier, deployment, "stream_timeout_by_service_tier"), + deployment.get("stream_timeout"), + router_stream_timeout, + ) if request.get("stream") else () ) @@ -72,6 +81,7 @@ def resolve_llm_passthrough_timeout( *stream_candidates, request.get("timeout"), request.get("request_timeout"), + get_service_tier_timeout(service_tier, deployment, "timeout_by_service_tier"), deployment.get("timeout"), deployment.get("request_timeout"), router_timeout, diff --git a/litellm/router.py b/litellm/router.py index af79fa763c4..5bf46c68b20 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -118,6 +118,7 @@ from litellm.litellm_core_utils.ptu_pricing import ( ) from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, + get_service_tier_timeout, ) from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.sensitive_data_masker import ( @@ -4773,6 +4774,7 @@ class Router: """Helper to get stream timeout from kwargs or deployment params""" return ( kwargs.get("stream_timeout", None) # the params dynamically set by user + or get_service_tier_timeout(kwargs.get("service_tier"), data, "stream_timeout_by_service_tier") or data.get("stream_timeout", None) # timeout set on litellm_params for this deployment or self.stream_timeout # timeout set on router or self.request_timeout # litellm_settings.request_timeout (per-attempt) @@ -4786,6 +4788,7 @@ class Router: return ( kwargs.get("timeout", None) # the params dynamically set by user or kwargs.get("request_timeout", None) # the params dynamically set by user + or get_service_tier_timeout(kwargs.get("service_tier"), data, "timeout_by_service_tier") or data.get("timeout", None) # timeout set on litellm_params for this deployment or data.get("request_timeout", None) # timeout set on litellm_params for this deployment or self.request_timeout # litellm_settings.request_timeout (per-attempt) diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index b05d25462aa..978eb9c4557 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -77,6 +77,8 @@ class ProviderConnection: request_timeout: float | None = None force_timeout: float | None = None stream_timeout: float | str | None = None + timeout_by_service_tier: Mapping[str, float] | None = None + stream_timeout_by_service_tier: Mapping[str, float] | None = None max_retries: int | None = None tenant_id: str | None = None client_id: str | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index 46617e1b753..f83127806db 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -505,6 +505,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): timeout: float | str | httpx.Timeout | SDKTimeout | None = None # if str, pass in as os.environ/ # read timeout of a live stream, a faked stream takes the longer of this and timeout. If str, pass in as os.environ/ stream_timeout: float | str | None = None + # per service_tier overrides of timeout / stream_timeout, used when the request's service_tier matches a key + timeout_by_service_tier: Mapping[str, float] | None = None + stream_timeout_by_service_tier: Mapping[str, float] | None = None max_retries: int | None = None drop_params: bool | str | None = None organization: str | None = None # for openai orgs @@ -709,6 +712,8 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): responses_api_version: ReadOnly[str | None] timeout: float | str | httpx.Timeout | SDKTimeout | None # writable-ok: preserve timeout updates stream_timeout: float | str | None + timeout_by_service_tier: ReadOnly[Mapping[str, float] | None] + stream_timeout_by_service_tier: ReadOnly[Mapping[str, float] | None] max_retries: int | None organization: list | str | None # for openai orgs provider_affinity_header: ReadOnly[str | None] diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 5412ea56ac0..4c3c7ae0f0a 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1857,6 +1857,32 @@ def test_resolve_llm_passthrough_timeout_validates_only_the_winning_value( assert resolve_llm_passthrough_timeout(kwargs=kwargs, litellm_params=litellm_params) == expected +def test_resolve_llm_passthrough_timeout_uses_the_deployment_timeout_for_the_requests_service_tier(): + litellm_params = { + "timeout": 610, + "stream_timeout": 30, + "timeout_by_service_tier": {"flex": 900}, + "stream_timeout_by_service_tier": {"flex": 950}, + } + assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex"}, litellm_params=litellm_params) == 900.0 + assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "default"}, litellm_params=litellm_params) == 610.0 + assert ( + resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "timeout": 20}, litellm_params=litellm_params) + == 20.0 + ) + assert ( + resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "stream": True}, litellm_params=litellm_params) + == 950.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"service_tier": "flex", "stream": True}, + litellm_params={"timeout": 610, "stream_timeout": 30, "timeout_by_service_tier": {"flex": 900}}, + ) + == 30.0 + ) + + def test_resolve_llm_passthrough_timeout_rejects_a_non_numeric_winner(): with pytest.raises(ValidationError): resolve_llm_passthrough_timeout(kwargs={"timeout": httpx.Timeout(30.0)}) diff --git a/tests/unit/test_router/test_router_timeout.py b/tests/unit/test_router/test_router_timeout.py index 3265179a513..8fc32620f71 100644 --- a/tests/unit/test_router/test_router_timeout.py +++ b/tests/unit/test_router/test_router_timeout.py @@ -153,6 +153,62 @@ def test_unit_test_streaming_timeout(stream): assert stream_timeout_val == normal_timeout +@pytest.mark.parametrize( + "request_kwargs, expected_timeout", + [ + ({"service_tier": "flex"}, 900.0), + ({"service_tier": "default"}, 610.0), + ({}, 610.0), + ({"service_tier": "flex", "timeout": 20}, 20), + ({"service_tier": "flex", "stream": True}, 950.0), + ({"service_tier": "default", "stream": True}, 610.0), + ({"service_tier": "flex", "stream": True, "stream_timeout": 30}, 30), + ], +) +def test_the_deployment_timeout_for_the_requests_service_tier_wins_over_its_general_timeout( + request_kwargs: dict[str, object], expected_timeout: float +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "openai-gpt-5.1", + "litellm_params": { + "model": "openai/gpt-5.1", + "api_key": "sk-test", + "timeout": 610, + "stream_timeout": 610, + "timeout_by_service_tier": {"flex": 900}, + "stream_timeout_by_service_tier": {"flex": 950}, + }, + } + ] + ) + deployment_params: Final = router.model_list[0]["litellm_params"] + + assert router._get_timeout(kwargs=request_kwargs, data=deployment_params) == expected_timeout + + +def test_a_stream_keeps_the_deployment_stream_timeout_when_only_the_request_timeout_map_names_its_tier() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "openai-gpt-5.1", + "litellm_params": { + "model": "openai/gpt-5.1", + "api_key": "sk-test", + "timeout": 610, + "stream_timeout": 30, + "timeout_by_service_tier": {"flex": 900}, + }, + } + ] + ) + deployment_params: Final = router.model_list[0]["litellm_params"] + + assert router._get_timeout(kwargs={"service_tier": "flex", "stream": True}, data=deployment_params) == 30.0 + assert router._get_timeout(kwargs={"service_tier": "flex"}, data=deployment_params) == 900.0 + + async def _drain_responses_stream(router: Router) -> None: response: Final = await router.aresponses(model="gpt", input="hi", stream=True) _ = [event async for event in response] diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index ce763378656..e88d757519c 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -75,6 +75,8 @@ CONNECTION_NAMES: Final = ( "request_timeout", "force_timeout", "stream_timeout", + "timeout_by_service_tier", + "stream_timeout_by_service_tier", "max_retries", "tenant_id", "client_id", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index daffc3c0c41..733a093bcac 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35424,6 +35424,10 @@ export interface components { } | null; /** Stream Timeout */ stream_timeout?: number | string | null; + /** Stream Timeout By Service Tier */ + stream_timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Tag Regex */ tag_regex?: string[] | null; /** Tags */ @@ -35436,6 +35440,10 @@ export interface components { }[] | null; /** Timeout */ timeout?: number | string | null; + /** Timeout By Service Tier */ + timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Token Exchange Audience */ token_exchange_audience?: string | null; /** Token Exchange Endpoint */ @@ -50666,6 +50674,10 @@ export interface components { } | null; /** Stream Timeout */ stream_timeout?: number | string | null; + /** Stream Timeout By Service Tier */ + stream_timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Tag Regex */ tag_regex?: string[] | null; /** Tags */ @@ -50678,6 +50690,10 @@ export interface components { }[] | null; /** Timeout */ timeout?: number | string | null; + /** Timeout By Service Tier */ + timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Token Exchange Audience */ token_exchange_audience?: string | null; /** Token Exchange Endpoint */