diff --git a/tests/test_litellm/batches/test_responses_batch_cost.py b/tests/test_litellm/batches/test_responses_batch_cost.py new file mode 100644 index 00000000000..7ce026bd103 --- /dev/null +++ b/tests/test_litellm/batches/test_responses_batch_cost.py @@ -0,0 +1,119 @@ +"""Token and spend reconciliation for /v1/responses batches. + +Regression guard for https://github.com/BerriAI/litellm/issues/35363: a batch +output line built by the Responses API reports ``input_tokens`` / +``output_tokens`` where a chat line reports ``prompt_tokens`` / +``completion_tokens``. The usage object was constructed straight from the raw +dict, which accepts the unrecognized names without raising and yields zeros, so +a completed Responses batch reconciled to 0 tokens and $0.00 spend with no +error, and per-key budgets were never charged for it. + +Line shape decides the parse, not the batch's declared endpoint, so an output +file mixing Responses-shaped and chat-shaped lines sums across both. +""" + +from typing import Literal, get_args, get_type_hints + +import pytest + +import litellm +import litellm.batches.batch_utils as bu +from litellm.types.llms.openai import CreateBatchRequest + +MODEL = "gpt-5.6" + + +def _responses_line(input_tokens: int, output_tokens: int) -> dict: + return { + "response": { + "status_code": 200, + "body": { + "model": MODEL, + "usage": { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + }, + }, + } + } + + +def _chat_line(prompt_tokens: int, completion_tokens: int) -> dict: + return { + "response": { + "status_code": 200, + "body": { + "model": MODEL, + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + }, + }, + } + } + + +def test_responses_shaped_usage_maps_onto_prompt_and_completion_tokens(): + """The Responses names land on the chat-shaped counters instead of being + dropped for unrecognized keys.""" + usage = bu._get_batch_job_usage_from_response_body( + {"usage": {"input_tokens": 100, "output_tokens": 50, "total_tokens": 150}} + ) + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (100, 50, 150) + + +async def test_responses_batch_reconciles_to_real_tokens_and_spend(local_model_cost_map): + """A completed Responses batch records the provider's token counts and a + non-zero spend at the model's batch rates.""" + model_info = litellm.get_model_info(model=MODEL, custom_llm_provider="openai") + input_tokens = 33 + output_tokens = 57 + + cost, usage, models = await bu.calculate_batch_cost_and_usage( + file_content_dictionary=[_responses_line(input_tokens, output_tokens)], + custom_llm_provider="openai", + model_name=MODEL, + model_info=model_info, + ) + + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == ( + input_tokens, + output_tokens, + input_tokens + output_tokens, + ) + assert models == [MODEL] + assert cost == pytest.approx( + input_tokens * model_info["input_cost_per_token_batches"] + + output_tokens * model_info["output_cost_per_token_batches"] + ) + assert cost > 0.0 + + +async def test_mixed_shape_batch_output_sums_across_both_line_shapes(local_model_cost_map): + """An output file carrying both line shapes sums both. A fix keyed off the + batch's declared endpoint rather than each line's shape would miss this.""" + model_info = litellm.get_model_info(model=MODEL, custom_llm_provider="openai") + + cost, usage, _ = await bu.calculate_batch_cost_and_usage( + file_content_dictionary=[_responses_line(100, 50), _chat_line(33, 57)], + custom_llm_provider="openai", + model_name=MODEL, + model_info=model_info, + ) + + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (133, 107, 240) + assert cost == pytest.approx( + 133 * model_info["input_cost_per_token_batches"] + 107 * model_info["output_cost_per_token_batches"] + ) + + +def test_create_batch_endpoint_accepts_v1_responses(): + """A type-checked caller can pass endpoint="/v1/responses", which the runtime + already forwarded correctly.""" + endpoint_annotation = get_type_hints(CreateBatchRequest)["endpoint"] + assert "/v1/responses" in get_args(endpoint_annotation) + + for create_fn in (litellm.create_batch, litellm.acreate_batch): + assert "/v1/responses" in get_args(get_type_hints(create_fn)["endpoint"]) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py new file mode 100644 index 00000000000..92cb417f96c --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py @@ -0,0 +1,136 @@ +"""Cache-write pricing for OpenAI models, on both the chat and Responses paths. + +Regression guard for https://github.com/BerriAI/litellm/issues/33772: OpenAI +reports cache-write tokens under ``prompt_tokens_details.cache_write_tokens`` +(chat) / ``input_tokens_details.cache_write_tokens`` (responses), where +Anthropic reports ``cache_creation_tokens``. The cost path read only the +Anthropic name, so cache-write tokens were billed at the plain input rate, +the Responses transform dropped the split before cost ran, and the tiered +``cache_creation_input_token_cost_{priority,flex,above_272k_tokens}`` keys were +discarded by ``get_model_info``. +""" + +import pytest + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.types.utils import Usage + +MODEL = "gpt-5.6" + + +def _openai_chat_usage(prompt_tokens: int, cache_write_tokens: int, completion_tokens: int) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details={ + "cached_tokens": 0, + "cache_write_tokens": cache_write_tokens, + }, + ) + + +def test_openai_cache_write_tokens_billed_at_the_cache_creation_rate(local_model_cost_map): + """A cache-write request costs the cache-creation rate on the written tokens, + not the plain input rate.""" + rates = litellm.model_cost[MODEL] + input_rate = rates["input_cost_per_token"] + cache_write_rate = rates["cache_creation_input_token_cost"] + output_rate = rates["output_cost_per_token"] + assert cache_write_rate == pytest.approx(input_rate * 1.25) + + prompt_tokens = 12317 + cache_write_tokens = 12314 + fresh_tokens = prompt_tokens - cache_write_tokens + completion_tokens = 5 + + prompt_cost, completion_cost = generic_cost_per_token( + model=MODEL, + usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens), + custom_llm_provider="openai", + ) + + assert prompt_cost == pytest.approx(fresh_tokens * input_rate + cache_write_tokens * cache_write_rate) + assert completion_cost == pytest.approx(completion_tokens * output_rate) + assert prompt_cost > prompt_tokens * input_rate + + +def test_responses_api_cache_write_costs_the_same_as_chat(local_model_cost_map): + """The Responses usage transform carries the cache-write split through, so the + same request costs the same whichever route it took.""" + prompt_tokens = 12317 + cache_write_tokens = 12314 + completion_tokens = 5 + + responses_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + { + "input_tokens": prompt_tokens, + "output_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + "input_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": cache_write_tokens, + }, + } + ) + assert responses_usage.prompt_tokens_details.cache_write_tokens == cache_write_tokens + + responses_prompt_cost, responses_completion_cost = generic_cost_per_token( + model=MODEL, + usage=responses_usage, + custom_llm_provider="openai", + ) + chat_prompt_cost, chat_completion_cost = generic_cost_per_token( + model=MODEL, + usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens), + custom_llm_provider="openai", + ) + + rates = litellm.model_cost[MODEL] + fresh_tokens = prompt_tokens - cache_write_tokens + assert responses_prompt_cost == pytest.approx( + fresh_tokens * rates["input_cost_per_token"] + + cache_write_tokens * rates["cache_creation_input_token_cost"] + ) + assert responses_prompt_cost == pytest.approx(chat_prompt_cost) + assert responses_completion_cost == pytest.approx(chat_completion_cost) + + +@pytest.mark.parametrize( + "service_tier,prompt_tokens,rate_key", + [ + (None, 100000, "cache_creation_input_token_cost"), + ("priority", 100000, "cache_creation_input_token_cost_priority"), + ("flex", 100000, "cache_creation_input_token_cost_flex"), + (None, 300000, "cache_creation_input_token_cost_above_272k_tokens"), + ], +) +def test_tiered_cache_creation_rates_are_registered_and_billed( + local_model_cost_map, service_tier, prompt_tokens, rate_key +): + """The tiered cache-creation keys survive ``get_model_info`` and are the rate the + cost path actually charges for priority, flex, and >272k requests.""" + model_info = litellm.get_model_info(model=MODEL, custom_llm_provider="openai") + tiered_rate = litellm.model_cost[MODEL][rate_key] + assert model_info.get(rate_key) == tiered_rate + + cache_write_tokens = prompt_tokens - 1000 + fresh_tokens = 1000 + input_rate_key = { + "cache_creation_input_token_cost": "input_cost_per_token", + "cache_creation_input_token_cost_priority": "input_cost_per_token_priority", + "cache_creation_input_token_cost_flex": "input_cost_per_token_flex", + "cache_creation_input_token_cost_above_272k_tokens": "input_cost_per_token_above_272k_tokens", + }[rate_key] + input_rate = litellm.model_cost[MODEL][input_rate_key] + + prompt_cost, _ = generic_cost_per_token( + model=MODEL, + usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens=100), + custom_llm_provider="openai", + service_tier=service_tier, + ) + + assert prompt_cost == pytest.approx(fresh_tokens * input_rate + cache_write_tokens * tiered_rate) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_responses_cache_cost_breakdown.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_responses_cache_cost_breakdown.py new file mode 100644 index 00000000000..0d387233be7 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_responses_cache_cost_breakdown.py @@ -0,0 +1,113 @@ +"""Itemized cache costs in the logged cost breakdown for the OpenAI Responses API. + +Regression guard for https://github.com/BerriAI/litellm/issues/34309: the +breakdown block derived its cache token counts only from the Anthropic-style +top-level ``cache_read_input_tokens`` / ``cache_creation_input_tokens``. OpenAI's +Responses API reports them under ``input_tokens_details.{cached_tokens, +cache_write_tokens}`` instead, so ``cost_breakdown.cache_read_cost`` and +``cache_creation_cost`` serialized as null for every OpenAI request while the +grand total stayed correct. + +``input_cost`` is the full prompt-side cost and the two cache fields are additive +break-outs that overlap it, matching the Anthropic path. A request with no cache +activity leaves both fields unset, which is the documented behavior. +""" + +from datetime import datetime + +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.types.utils import Choices, Message, ModelResponse + +MODEL = "gpt-5.6" + + +def _logging_obj() -> Logging: + return Logging( + model=MODEL, + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="responses-cache-breakdown", + function_id="f", + ) + + +def _responses_completion(cached_tokens: int, cache_write_tokens: int, fresh_tokens: int, output_tokens: int): + input_tokens = cached_tokens + cache_write_tokens + fresh_tokens + usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + "input_tokens_details": { + "cached_tokens": cached_tokens, + "cache_write_tokens": cache_write_tokens, + }, + } + ) + return ModelResponse( + id="x", + created=1, + model=MODEL, + object="chat.completion", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=usage, + ) + + +def test_responses_api_cache_costs_are_itemized_in_the_breakdown(local_model_cost_map): + """Cache-read and cache-write dollars are broken out for an OpenAI Responses + request, at the model's own cache rates.""" + rates = litellm.model_cost[MODEL] + cached_tokens = 4012 + cache_write_tokens = 5000 + fresh_tokens = 1000 + output_tokens = 200 + + logging_obj = _logging_obj() + total = litellm.completion_cost( + completion_response=_responses_completion(cached_tokens, cache_write_tokens, fresh_tokens, output_tokens), + model=MODEL, + custom_llm_provider="openai", + litellm_logging_obj=logging_obj, + ) + + breakdown = logging_obj.cost_breakdown + assert breakdown is not None + + expected_cache_read = cached_tokens * rates["cache_read_input_token_cost"] + expected_cache_creation = cache_write_tokens * rates["cache_creation_input_token_cost"] + expected_input = fresh_tokens * rates["input_cost_per_token"] + expected_cache_read + expected_cache_creation + expected_output = output_tokens * rates["output_cost_per_token"] + + assert breakdown["cache_read_cost"] == pytest.approx(expected_cache_read) + assert breakdown["cache_creation_cost"] == pytest.approx(expected_cache_creation) + assert breakdown["input_cost"] == pytest.approx(expected_input) + assert breakdown["output_cost"] == pytest.approx(expected_output) + assert breakdown["total_cost"] == pytest.approx(expected_input + expected_output) + assert total == pytest.approx(breakdown["total_cost"]) + + +def test_cache_fields_stay_unset_when_there_was_no_cache_activity(local_model_cost_map): + """A request that neither read nor wrote the cache leaves both break-out fields + off the breakdown, so they serialize as null rather than a misleading $0.""" + logging_obj = _logging_obj() + litellm.completion_cost( + completion_response=_responses_completion( + cached_tokens=0, cache_write_tokens=0, fresh_tokens=1000, output_tokens=200 + ), + model=MODEL, + custom_llm_provider="openai", + litellm_logging_obj=logging_obj, + ) + + breakdown = logging_obj.cost_breakdown + assert breakdown is not None + assert "cache_read_cost" not in breakdown + assert "cache_creation_cost" not in breakdown + assert breakdown["input_cost"] == pytest.approx(1000 * litellm.model_cost[MODEL]["input_cost_per_token"]) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 570ce152714..9e58ded81bd 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1,5 +1,8 @@ +import asyncio import json import os +import uuid +from typing import Any, Dict, List import httpx import pytest @@ -10,6 +13,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import litellm from litellm.anthropic_interface import messages +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.utils import Delta, ModelResponse, StreamingChoices @@ -1103,3 +1108,181 @@ def test_messages_sync_streaming_reports_provider_local_model(): first_event = next(iter(stream)) assert json.loads(first_event.decode().split("data: ", 1)[1])["message"]["model"] == "perplexity/kimi-k3" + + +_RESPONSES_COMPLETED_BODY: Dict[str, Any] = { + "id": "resp-1", + "object": "response", + "created_at": 0, + "model": "gpt-4o-mini", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg-1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "hello there"}], + } + ], + "usage": {"input_tokens": 11, "output_tokens": 7, "total_tokens": 18}, +} + +_RESPONSES_SSE_EVENTS: List[Dict[str, Any]] = [ + { + "type": "response.created", + "response": { + **_RESPONSES_COMPLETED_BODY, + "status": "in_progress", + "output": [], + "usage": None, + }, + }, + { + "type": "response.output_text.delta", + "item_id": "msg-1", + "output_index": 0, + "content_index": 0, + "delta": "hello there", + }, + {"type": "response.completed", "response": _RESPONSES_COMPLETED_BODY}, +] + + +def _sse_body(events: List[Dict[str, Any]]) -> bytes: + return b"".join(f"data: {json.dumps(event)}\n\n".encode() for event in events) + + +class _SuccessPayloadCapture(CustomLogger): + def __init__(self, tracking_id: str): + super().__init__() + self.tracking_id = tracking_id + self.payloads: List[Dict[str, Any]] = [] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + if kwargs.get("litellm_call_id") == self.tracking_id: + self.payloads.append(kwargs.get("standard_logging_object") or {}) + + +@pytest.fixture +def capture_success_payloads(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + capture = _SuccessPayloadCapture(tracking_id=f"messages-stream-{uuid.uuid4()}") + monkeypatch.setattr(litellm, "callbacks", [capture]) + return capture + + +async def _drain(sse_stream) -> List[bytes]: + return [chunk async for chunk in sse_stream] + + +def _bind_logging_worker_to_running_loop() -> None: + """The worker's queue keeps the loop it was built on, so one left over from an earlier + test makes ``flush`` raise "bound to a different event loop". ``start`` runs + ``_ensure_queue``, which rebinds the queue when the running loop has changed.""" + GLOBAL_LOGGING_WORKER.start() + + +async def _flush_logging_worker(capture: "_SuccessPayloadCapture") -> None: + await asyncio.sleep(0) + try: + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + except (asyncio.TimeoutError, RuntimeError): + pass + deadline = asyncio.get_running_loop().time() + 10.0 + while not capture.payloads and asyncio.get_running_loop().time() < deadline: + await asyncio.sleep(0.05) + + +def _assert_anthropic_sse(chunks: List[bytes]) -> None: + body = b"".join(chunks).decode() + assert "message_start" in body + assert "message_stop" in body + + +class TestMessagesStreamingSuccessLogging: + """A streamed /v1/messages call routed to an ``openai/`` backend must still reach + success logging once the SSE stream is drained. The client gets a correct Anthropic + SSE body and the provider bills the call, but no StandardLoggingPayload is produced, + so the request is invisible to spend tracking and every success-logging integration. + + Logging is fired by the inner CustomStreamWrapper / ResponsesAPIStreamingIterator the + Anthropic wrappers drain, not by the wrappers themselves, so these assert on the + callback that actually reaches an integration rather than on a wrapper attribute. + """ + + MESSAGES = [{"role": "user", "content": "hi"}] + + @pytest.mark.asyncio + async def test_responses_bridge_streaming_emits_success_logging(self, capture_success_payloads): + """The Responses bridge, which is the default for openai/ deployments.""" + from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler import ( + LiteLLMMessagesToResponsesAPIHandler, + ) + + _bind_logging_worker_to_running_loop() + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = httpx.Response( + 200, + content=_sse_body(_RESPONSES_SSE_EVENTS), + headers={"content-type": "text/event-stream"}, + ) + sse_stream = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=100, + messages=self.MESSAGES, + model="openai/gpt-4o-mini", + stream=True, + custom_llm_provider="openai", + litellm_call_id=capture_success_payloads.tracking_id, + ) + chunks = await _drain(sse_stream) + + await _flush_logging_worker(capture_success_payloads) + + _assert_anthropic_sse(chunks) + assert len(capture_success_payloads.payloads) == 1 + payload = capture_success_payloads.payloads[0] + assert payload["call_type"] == "aresponses" + assert payload["prompt_tokens"] == 11 + assert payload["completion_tokens"] == 7 + assert payload["total_tokens"] == 18 + assert payload["response_cost"] > 0 + + @pytest.mark.asyncio + async def test_chat_completions_bridge_streaming_emits_success_logging(self, capture_success_payloads): + """The chat-completions bridge, reached via + litellm.use_chat_completions_url_for_anthropic_messages. Its router lookup is + stubbed to what an SDK caller with no proxy running already resolves to.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + _bind_logging_worker_to_running_loop() + + with patch( + "litellm.llms.anthropic.experimental_pass_through.adapters.handler._proxy_router_fallback", + return_value=None, + ): + sse_stream = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( + max_tokens=100, + messages=self.MESSAGES, + model="openai/gpt-4o-mini", + stream=True, + custom_llm_provider="openai", + mock_response="hello there", + litellm_call_id=capture_success_payloads.tracking_id, + ) + chunks = await _drain(sse_stream) + + await _flush_logging_worker(capture_success_payloads) + + _assert_anthropic_sse(chunks) + assert len(capture_success_payloads.payloads) == 1 + payload = capture_success_payloads.payloads[0] + assert payload["call_type"] == "acompletion" + assert payload["total_tokens"] > 0 + assert payload["response_cost"] > 0 diff --git a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py index b548b0b3135..2f6a5a3b0e0 100644 --- a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py @@ -29,6 +29,7 @@ added to this layer raises instead of silently passing - the inventory of seams cannot drift without a test failure. """ +import json from contextlib import ExitStack from dataclasses import dataclass from typing import Any, Dict, Optional @@ -50,7 +51,7 @@ from litellm.router import Router from litellm.types.llms.openai import BatchJobStatus from litellm.types.utils import CredentialItem, LiteLLMBatch -from fastapi import Response +from fastapi import Request, Response # --------------------------------------------------------------------------- # # Fixtures: distinguishable credentials per model so a wrong/hardcoded model_id @@ -851,6 +852,48 @@ async def test_create__missing_required_param_is_400(harness, body, missing_para harness.router_acreate.assert_not_called() +def _raw_batches_request(body: Dict[str, Any]) -> MagicMock: + """A request that reaches the real pre-call logic, which the `harness` fixture + mocks out. Metadata validation lives there, so it cannot be seen through the seam.""" + request = MagicMock(spec=Request) + request.url = MagicMock() + request.url.__str__.return_value = "http://localhost/v1/batches" + request.url.path = "/v1/batches" + request.method = "POST" + request.query_params = {} + request.headers = {"Content-Type": "application/json"} + request.client = MagicMock() + request.client.host = "127.0.0.1" + request.body = AsyncMock(return_value=json.dumps(body).encode()) + return request + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["metadata", "litellm_metadata"]) +async def test_create__non_object_metadata_is_400(field): + """A non-object metadata field is rejected with a 400 naming it (#37147), rather + than being coerced to None and silently dropped, or crashing behind a 500 with + "'str' object has no attribute 'update'".""" + body = { + "input_file_id": "file-abc", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + field: "abc", + } + + with pytest.raises(ProxyException) as exc_info: + await endpoints.create_batch( + request=_raw_batches_request(body), + fastapi_response=Response(), + provider=None, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + + assert exc_info.value.code == "400" + assert exc_info.value.param == field + assert "has no attribute 'update'" not in exc_info.value.message + + # =========================================================================== # # Team-level batch expiry enforcement (independent of routing). # =========================================================================== # diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index 1b16e036ad7..23552f2fa31 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -1,6 +1,6 @@ import json -from typing import Final, List -from unittest.mock import ANY, AsyncMock +from typing import Final, List, Optional +from unittest.mock import ANY, AsyncMock, MagicMock import pytest import respx @@ -4302,3 +4302,177 @@ def test_batch_upload_closes_the_spools_it_opened(monkeypatch, llm_router: Route finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) ProxyLogging._callback_capabilities_cache.clear() + + +def _managed_file_row( + unified_file_id: str, + stored_file_id: Optional[str] = None, + created_by: str = "test-user", + team_id: Optional[str] = None, +) -> MagicMock: + file_object = OpenAIFileObject( + id=stored_file_id or unified_file_id, + bytes=100, + created_at=1700000000, + filename="batch.jsonl", + object="file", + purpose="batch", + status="processed", + ) + return MagicMock( + unified_file_id=unified_file_id, + file_object=file_object.model_dump(), + created_by=created_by, + team_id=team_id, + ) + + +def _row_matches_where(row, where) -> bool: + for field, expected in where.items(): + if field == "OR": + if not any(_row_matches_where(row, clause) for clause in expected): + return False + elif getattr(row, field) != expected: + return False + return True + + +class _ManagedFileTableOverRows: + """Enough of the Prisma table for the real hook's owner-scoped keyset query.""" + + def __init__(self, rows): + self.rows = list(rows) + + def _owned_rows(self, where): + return [row for row in self.rows if _row_matches_where(row, where)] + + async def find_first(self, where): + return next(iter(self._owned_rows(where)), None) + + async def find_many(self, where, take=None, order=None, cursor=None, skip=0): + rows = self._owned_rows(where) + if cursor is not None: + start = next( + index + for index, row in enumerate(rows) + if row.unified_file_id == cursor["unified_file_id"] + ) + rows = rows[start + skip :] + return rows if take is None else rows[:take] + + +def _setup_unscoped_list_files_route_over_real_hook( + mocker, monkeypatch, llm_router: Router, rows +): + """Wire GET /v1/files to a real managed-files hook over `rows`, with no provider + credentials in the process. The neighbouring setup stubs the hook with a + MagicMock, so it cannot see anything past the route's argument plumbing.""" + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + from litellm_enterprise.proxy.hooks.managed_files import ( + _PROXY_LiteLLMManagedFiles, + ) + + for env_var in ("OPENAI_API_KEY", "OPENAI_ADMIN_KEY", "OPENAI_ORGANIZATION"): + monkeypatch.delenv(env_var, raising=False) + monkeypatch.setattr(litellm, "api_key", None, raising=False) + monkeypatch.setattr(litellm, "openai_key", None, raising=False) + + managed_files = _PROXY_LiteLLMManagedFiles( + internal_usage_cache=MagicMock(), prisma_client=MagicMock() + ) + managed_files.prisma_client.db.litellm_managedfiletable = _ManagedFileTableOverRows(rows) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, llm_router) + proxy_logging_obj.proxy_hook_mapping["managed_files"] = managed_files + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=None) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", llm_router) + + provider_list = mocker.patch.object(litellm, "afile_list", new=mocker.AsyncMock()) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + ) + return proxy_logging_obj, provider_list + + +def test_unscoped_list_files_reads_the_store_without_any_provider_key( + mocker: MockerFixture, monkeypatch, llm_router: Router +): + """The exact shape `client.files.list()` produces, with no target_model_names, no + provider and no OPENAI_API_KEY in the process, must return the caller's managed + files instead of falling through to a keyless provider client (#35362).""" + proxy_logging_obj, provider_list = _setup_unscoped_list_files_route_over_real_hook( + mocker, monkeypatch, llm_router, [_managed_file_row("unified-file-1")] + ) + + response = _get_unscoped_list_files("") + + assert response.status_code == 200, response.text + assert [file["id"] for file in response.json()["data"]] == ["unified-file-1"] + provider_list.assert_not_awaited() + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_unscoped_list_files_returns_unified_ids_for_rows_storing_a_raw_provider_id( + mocker: MockerFixture, monkeypatch, llm_router: Router +): + """Batch output rows store the provider's file object, whose id is a raw `file-` + the caller cannot act on. The listing hands back the row's unified id so + files.retrieve and files.content work on what it returned.""" + _setup_unscoped_list_files_route_over_real_hook( + mocker, + monkeypatch, + llm_router, + [_managed_file_row("unified-batch-output", stored_file_id="file-raw-provider-123")], + ) + + response = _get_unscoped_list_files("") + + assert response.status_code == 200, response.text + assert [file["id"] for file in response.json()["data"]] == ["unified-batch-output"] + assert response.json()["data"][0]["filename"] == "batch.jsonl" + + +def test_unscoped_list_files_does_not_leak_another_callers_files( + mocker: MockerFixture, monkeypatch, llm_router: Router +): + """Listing without a model pinned is still owner-scoped.""" + _setup_unscoped_list_files_route_over_real_hook( + mocker, + monkeypatch, + llm_router, + [ + _managed_file_row("unified-mine"), + _managed_file_row("unified-theirs", created_by="other-user"), + ], + ) + + response = _get_unscoped_list_files("") + + assert response.status_code == 200, response.text + assert [file["id"] for file in response.json()["data"]] == ["unified-mine"] + + +def test_scoped_list_files_still_resolves_deployment_credentials( + mocker: MockerFixture, monkeypatch, llm_router: Router +): + """target_model_names keeps routing to the provider on deployment credentials, so + the unscoped path does not swallow that route.""" + _, provider_list = _setup_unscoped_list_files_route_over_real_hook( + mocker, monkeypatch, llm_router, [_managed_file_row("unified-file-1")] + ) + provider_list.return_value = [] + + response = _get_unscoped_list_files("?target_model_names=gpt-3.5-turbo") + + assert response.status_code == 200, response.text + provider_list.assert_awaited_once() + assert provider_list.await_args.kwargs["custom_llm_provider"] == "openai" + assert provider_list.await_args.kwargs["api_key"] == "openai_api_key" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index f0b2feeb377..6f9142c85df 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -7,6 +7,7 @@ import httpx import pytest +import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler import ( OpenAIPassthroughLoggingHandler, @@ -14,6 +15,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrou from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, ) @@ -1809,5 +1811,229 @@ class TestOpenAIPassthroughIntegration: assert mock_logging_obj.model_call_details["custom_llm_provider"] == "openai" +class TestOpenAIPassthroughResponsesStreamingSpendLog: + """A streamed OpenAI-passthrough `/v1/responses` call must write a priced spend + log row (#36523). + + `_handle_logging_openai_collected_chunks` received `url_route` and ignored it, so + a Responses SSE stream was reassembled by the chat-completions chunk parser. The + assembled object carried a synthetic `chatcmpl-` id, `model=None` and zero usage, + and the spend row landed at zero tokens and zero spend while the identical + buffered call priced exactly. + """ + + RESPONSE_ID = "resp_0c72ddebf05f8751" + MODEL_MAP_KEY = "gpt-4o-mini-2024-07-18" + INPUT_TOKENS = 14 + OUTPUT_TOKENS = 2 + + def setup_method(self): + self.start_time = datetime.now() + self.end_time = datetime.now() + rates = litellm.model_cost[self.MODEL_MAP_KEY] + self.expected_spend = ( + self.INPUT_TOKENS * rates["input_cost_per_token"] + + self.OUTPUT_TOKENS * rates["output_cost_per_token"] + ) + + def _responses_stream_chunks(self) -> List[str]: + """Usage arrives only on the terminal `response.completed` event, nested under + `response` as `input_tokens` / `output_tokens`. No event carries a top-level + `id`, `model` or `usage`.""" + response_body = { + "id": self.RESPONSE_ID, + "object": "response", + "created_at": 1786374786, + "model": self.MODEL_MAP_KEY, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": {}, + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + } + created_event = { + "type": "response.created", + "sequence_number": 0, + "response": {**response_body, "status": "in_progress", "output": [], "usage": None}, + } + delta_event = { + "type": "response.output_text.delta", + "sequence_number": 3, + "item_id": "msg_abc", + "output_index": 0, + "content_index": 0, + "delta": "OK", + } + completed_event = { + "type": "response.completed", + "sequence_number": 8, + "response": { + **response_body, + "status": "completed", + "output": [ + { + "id": "msg_abc", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK", "annotations": []}], + } + ], + "usage": { + "input_tokens": self.INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": self.OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": self.INPUT_TOKENS + self.OUTPUT_TOKENS, + }, + }, + } + return [ + f"data: {json.dumps(created_event)}", + f"data: {json.dumps(delta_event)}", + f"data: {json.dumps(completed_event)}", + "data: [DONE]", + ] + + def _logging_obj(self) -> LiteLLMLoggingObj: + logging_obj = LiteLLMLoggingObj( + model="gpt-4o-mini", + messages=[], + stream=True, + call_type="pass_through_endpoint", + start_time=self.start_time, + litellm_call_id="323dfe4f-2741-4473-b6e2-000000000000", + function_id="1234", + ) + logging_obj.model_call_details["custom_llm_provider"] = "openai" + return logging_obj + + def test_streamed_responses_passthrough_spend_log_is_priced(self): + """The spend row books the same tokens, spend and `resp_` id as the buffered call.""" + result = OpenAIPassthroughLoggingHandler._handle_logging_openai_collected_chunks( + litellm_logging_obj=self._logging_obj(), + passthrough_success_handler_obj=None, + url_route="https://api.openai.com/v1/responses", + request_body={ + "model": "gpt-4o-mini", + "input": "Say hi in one word.", + "max_output_tokens": 16, + "stream": True, + }, + endpoint_type=None, + start_time=self.start_time, + all_chunks=self._responses_stream_chunks(), + end_time=self.end_time, + ) + + spend_log_row = get_logging_payload( + kwargs=result["kwargs"], + response_obj=result["result"], + start_time=self.start_time, + end_time=self.end_time, + ) + + assert spend_log_row["prompt_tokens"] == self.INPUT_TOKENS + assert spend_log_row["completion_tokens"] == self.OUTPUT_TOKENS + assert spend_log_row["total_tokens"] == self.INPUT_TOKENS + self.OUTPUT_TOKENS + assert spend_log_row["spend"] == self.expected_spend + assert spend_log_row["request_id"] == self.RESPONSE_ID + assert spend_log_row["model"] == "gpt-4o-mini" + + row_metadata = json.loads(spend_log_row["metadata"]) + assert row_metadata["model_map_information"]["model_map_key"] == self.MODEL_MAP_KEY + + +class TestOpenAIPassthroughEmbeddingsSpendLog: + """An OpenAI-passthrough `/v1/embeddings` call must write a priced spend log row + (#36646). + + `_is_supported_openai_endpoint` ORed four route predicates, none of which matched + `/v1/embeddings`, so the dispatcher never entered the OpenAI handler and billable + embedding tokens produced no spend row at all, under-enforcing key budgets. + """ + + EMBEDDINGS_URL = "https://api.openai.com/v1/embeddings" + MODEL = "text-embedding-3-small" + PROMPT_TOKENS = 14 + CALL_ID = "11024cc3-b143-4a63-954a-ec06081df768" + + def setup_method(self): + self.start_time = datetime.now() + self.end_time = datetime.now() + self.expected_spend = self.PROMPT_TOKENS * litellm.model_cost[self.MODEL]["input_cost_per_token"] + self.response_body = { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.0, 1.0]}], + "model": self.MODEL, + "usage": {"prompt_tokens": self.PROMPT_TOKENS, "total_tokens": self.PROMPT_TOKENS}, + } + self.request_body = {"model": self.MODEL, "input": "hello"} + + def _create_mock_httpx_response(self) -> httpx.Response: + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.text = json.dumps(self.response_body) + mock_response.json.return_value = self.response_body + mock_response.headers = {"content-type": "application/json"} + return mock_response + + def _logging_obj(self) -> LiteLLMLoggingObj: + logging_obj = LiteLLMLoggingObj( + model=self.MODEL, + messages=[], + stream=False, + call_type="pass_through_endpoint", + start_time=self.start_time, + litellm_call_id=self.CALL_ID, + function_id="1234", + ) + logging_obj.model_call_details["passthrough_logging_payload"] = PassthroughStandardLoggingPayload( + url=self.EMBEDDINGS_URL, + request_body=self.request_body, + request_method="POST", + ) + return logging_obj + + def test_embeddings_passthrough_spend_log_is_priced(self): + """The dispatched call books prompt tokens and cost onto the spend row.""" + dispatched = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=self._create_mock_httpx_response(), + response_body=self.response_body, + request_body=self.request_body, + logging_obj=self._logging_obj(), + url_route=self.EMBEDDINGS_URL, + result="", + start_time=self.start_time, + end_time=self.end_time, + cache_hit=False, + custom_llm_provider="openai", + litellm_call_id=self.CALL_ID, + litellm_params={}, + call_type="pass_through_endpoint", + ) + + assert dispatched["standard_logging_response_object"] is not None + assert dispatched["kwargs"]["response_cost"] == self.expected_spend + + spend_log_row = get_logging_payload( + kwargs=dispatched["kwargs"], + response_obj=dispatched["standard_logging_response_object"], + start_time=self.start_time, + end_time=self.end_time, + ) + + assert spend_log_row["prompt_tokens"] == self.PROMPT_TOKENS + assert spend_log_row["total_tokens"] == self.PROMPT_TOKENS + assert spend_log_row["spend"] == self.expected_spend + assert spend_log_row["model"] == self.MODEL + assert spend_log_row["custom_llm_provider"] == "openai" + assert spend_log_row["request_id"] == self.CALL_ID + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index e65d79a83f4..cbaac4d89a7 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1805,6 +1805,51 @@ class TestLLMClassifier: call_kwargs = mock_router_instance.acompletion.call_args.kwargs assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"} + @pytest.mark.asyncio + @pytest.mark.parametrize( + "request_kwargs", + [ + pytest.param({"metadata": {"user_api_key": "sk-abc"}}, id="metadata-bucket"), + pytest.param( + {"litellm_metadata": {"user_api_key": "sk-abc"}}, id="litellm-metadata-bucket" + ), + pytest.param({}, id="no-caller-context"), + pytest.param(None, id="no-request-kwargs"), + ], + ) + async def test_aclassify_reaches_the_llm_for_every_caller_metadata_shape( + self, llm_classifier_config, request_kwargs + ): + """Whatever the caller's metadata bucket looks like, the configured classifier must + actually run. The forwarded metadata reaches litellm's own metadata handling, which + raises "'NoneType' object has no attribute 'update'" on a shape it does not expect; + aclassify catches that and silently degrades to heuristic scoring, so the tier is + decided by word counting while the config says otherwise. A real Router is used here + because a mocked acompletion accepts any shape and never reaches that handling. + """ + real_router = Router( + model_list=[ + { + "model_name": "haiku-classifier", + "litellm_params": { + "model": "openai/haiku-classifier", + "api_key": "sk-classifier", + "mock_response": '{"tier": "COMPLEX"}', + }, + } + ] + ) + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=real_router, + complexity_router_config=llm_classifier_config, + ) + + outcome = await router.aclassify("hi", request_kwargs=request_kwargs) + + assert outcome.cause == "llm_classifier" + assert outcome.tier == ComplexityTier.COMPLEX + @pytest.mark.asyncio async def test_aclassify_captures_request_body_in_proxy_server_request( self, llm_complexity_router, mock_router_instance diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index c05f25430c2..99b1cc826aa 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -850,6 +850,36 @@ def test_responses_api_bridge_check_gpt_5_4_tools_with_default_reasoning_routes_ assert model_info.get("mode") == "responses" +@pytest.mark.parametrize("model_name", ["gpt-5.6-sol", "gpt-5.6-luna", "gpt-5.6-terra"]) +def test_responses_api_bridge_check_gpt_5_6_tools_with_default_reasoning_routes_to_responses( + monkeypatch, model_name +): + """ + The whole gpt-5.6 family must bridge on function tools alone. The bridge used to + require an explicit reasoning_effort, so a gpt-5.6 call carrying tools and no effort + was rejected with "Function tools with reasoning_effort are not supported for + gpt-5.6-sol in /v1/chat/completions". + """ + import litellm + from litellm.main import responses_api_bridge_check + + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model=model_name, + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=None, + ) + + assert model == model_name + assert model_info.get("mode") == "responses" + + def test_responses_api_bridge_check_gpt_5_4_tools_with_reasoning_none_stays_chat(): """ Explicit reasoning_effort "none" is OpenAI's documented escape hatch that keeps @@ -2833,9 +2863,17 @@ def _priced_at(prompt_tokens, completion_tokens): @pytest.fixture def local_cost_map(monkeypatch): """The prices these tests assert are the checked-in ones. Setting the environment - variable alone does not reload the map, so pin the map itself.""" + variable alone does not reload the map, so pin the map itself. + + ``get_model_info`` is lru_cached, so pinning ``model_cost`` is not enough on its + own: a cached entry warmed against the network-fetched map keeps its old prices + and ``completion_cost`` bills at those while the assertions read the pinned map. + Clear on the way in and out so entries never leak across tests in either direction.""" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() def test_a_streamed_response_bills_the_usage_the_provider_reported(local_cost_map): diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 56e00ecdad6..d00fbf589e3 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -6,6 +6,7 @@ import os import threading from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest @@ -7304,6 +7305,78 @@ async def test_acreate_batch_surfaces_owning_provider_error_without_disable_fall assert attempted_models == ["owning-model"] +@pytest.mark.asyncio +async def test_acreate_batch_still_falls_back_within_the_owning_model_group(): + """Holding a batch inside the model group that owns its input file must not + disable fallbacks outright (#35359): the owning group's second deployment is + still tried in `order`, and only the cross-group target is skipped.""" + completion_window_error = "Invalid value: '5m'. Supported values are: '24h'." + attempted_models = [] + + async def _acreate_batch(**kwargs): + model = kwargs["model"] + attempted_models.append(model) + if model.startswith("azure/"): + raise litellm.BadRequestError( + message="Error code: 400 - {'error': {'code': 'quotaExceeded'}}", + model=model, + llm_provider="azure", + ) + raise litellm.BadRequestError( + message=completion_window_error, + model=model, + llm_provider="openai", + ) + + router = litellm.Router( + model_list=[ + { + "model_name": "my-gpt", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-owning", + "order": 1, + }, + "model_info": {"id": "my-gpt-1"}, + }, + { + "model_name": "my-gpt", + "litellm_params": { + "model": "openai/gpt-4o-mini-backup", + "api_key": "sk-owning", + "order": 2, + }, + "model_info": {"id": "my-gpt-2"}, + }, + { + "model_name": "my-azure-gpt", + "litellm_params": { + "model": "azure/gpt-4o-mini", + "api_key": "sk-fallback", + "api_base": "https://fallback.openai.azure.com", + "api_version": "2024-08-01-preview", + }, + "model_info": {"id": "my-azure-gpt-1"}, + }, + ], + fallbacks=[{"my-gpt": ["my-azure-gpt"]}], + num_retries=0, + ) + + with patch.object(litellm, "acreate_batch", new=_acreate_batch): + with pytest.raises(litellm.BadRequestError) as raised: + await router.acreate_batch( + model="my-gpt", + input_file_id="file-owned-by-my-gpt", + endpoint="/v1/chat/completions", + completion_window="5m", + ) + + assert "24h" in str(raised.value) + assert "quotaExceeded" not in str(raised.value) + assert attempted_models == ["openai/gpt-4o-mini", "openai/gpt-4o-mini-backup"] + + @pytest.mark.asyncio async def test_acreate_batch_request_bedrock_tags_override_deployment_tags(): import httpx @@ -8264,6 +8337,94 @@ class TestTaggedAutoRouterOnSharedModelName: assert marker_only._model_name_has_plain_deployments("gpt4o") is False +class TestAutoRouterSharedModelNameConnectionParams: + """A plain deployment sharing its model_name with an `auto_router/` marker must not have + its api_base and api_key grafted onto the routed tier's outbound call (#36619).""" + + PLAIN_API_BASE = "https://plain-sibling.openai.example/v1" + PLAIN_API_KEY = "sk-plain-sibling-secret" + + class _FixedRouteLayer: + def __call__(self, text: str): + from semantic_router.schema import RouteChoice + + return RouteChoice(name="gemini-flash") + + @classmethod + def _router(cls, plain_entry_first: bool) -> "litellm.Router": + pytest.importorskip("semantic_router", reason="auto-router needs the semantic-router extra") + plain = { + "model_name": "gpt4o", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": cls.PLAIN_API_KEY, + "api_base": cls.PLAIN_API_BASE, + }, + } + marker = { + "model_name": "gpt4o", + "litellm_params": { + "model": "auto_router/gpt4o-router", + "auto_router_config": json.dumps( + {"routes": [{"name": "gemini-flash", "utterances": ["capital city questions"]}]} + ), + "auto_router_default_model": "gemini-flash", + "auto_router_embedding_model": "text-embedding-3-small", + "drop_params": True, + }, + } + tier = { + "model_name": "gemini-flash", + "litellm_params": {"model": "gemini/gemini-3.6-flash", "api_key": "sk-tier-key"}, + } + shared_name_entries = [plain, marker] if plain_entry_first else [marker, plain] + router = litellm.Router(model_list=[*shared_name_entries, tier]) + router.auto_routers["gpt4o"][0].strategy.routelayer = cls._FixedRouteLayer() + return router + + @staticmethod + def _gemini_response() -> httpx.Response: + return httpx.Response( + status_code=200, + json={ + "candidates": [ + {"content": {"parts": [{"text": "Paris"}], "role": "model"}, "finishReason": "STOP"} + ], + "usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 1, "totalTokenCount": 6}, + "modelVersion": "gemini-3.6-flash", + }, + request=httpx.Request("POST", "https://generativelanguage.googleapis.com"), + ) + + @pytest.mark.parametrize( + "plain_entry_first", [True, False], ids=["plain_entry_first", "marker_entry_first"] + ) + async def test_routed_tier_call_goes_out_on_its_own_endpoint_and_credentials(self, plain_entry_first): + """The outbound provider request for the routed tier hits the tier's own Gemini host + with the tier's own key, never the plain sibling's api_base or api_key.""" + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + router = self._router(plain_entry_first) + + with patch.object( + AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=self._gemini_response() + ) as mock_post: + await router.acompletion( + model="gpt4o", + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + + call = mock_post.call_args + outbound_url = str(call.kwargs["url"] if "url" in call.kwargs else call.args[0]) + outbound_headers = dict(call.kwargs.get("headers") or {}) + + assert "generativelanguage.googleapis.com" in outbound_url + assert "gemini-3.6-flash" in outbound_url + assert self.PLAIN_API_BASE not in outbound_url + assert self.PLAIN_API_KEY not in outbound_url + assert self.PLAIN_API_KEY not in json.dumps(outbound_headers) + + class TestGetAllowedFailsFromPolicy: def _make_router(self, **policy_kwargs) -> litellm.Router: from litellm.types.router import AllowedFailsPolicy