mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
test: add regression coverage for twelve closed issues (#37974)
* test: add regression coverage for twelve closed issues Adds targeted regression tests for behavior that was fixed but left ungated, so the fixes cannot silently regress: - #33772 openai cache_write_tokens cost - #34309 Responses API cache cost_breakdown - #35363 /v1/responses batch spend - #36619 auto-router api_base/api_key leak on a shared model name - #35359 batch fallbacks within the owning model group - #36523 passthrough streamed Responses spend log - #36646 passthrough embeddings spend log - #37147 non-object metadata on create_batch is a 400 - #35362 unscoped list files reads the managed-file store - #33221 gpt-5.6 bridges to Responses on function tools alone - #34487 LLM complexity classifier runs for every caller metadata shape - #35124 streamed /v1/messages emits success logging on both bridges Cost assertions read rates from litellm.model_cost rather than hardcoding dollar amounts, so they do not drift on repricing. * fix: stop the new regression tests polluting and tripping over shared global state Two shard failures, both from global state the new tests share with their neighbours rather than from the behaviour under test. test_main.py's local_cost_map pinned litellm.model_cost but left the get_model_info lru_cache warm, so completion_cost billed at whatever prices were cached earlier in the process while the assertions read the pinned map. Clear the cache on both sides of the fixture, matching the local_model_cost_map fixture in tests/test_litellm/conftest.py. The anthropic messages streaming tests called GLOBAL_LOGGING_WORKER.flush() on whatever queue happened to be around. A queue left non-empty by an earlier test is still bound to that test's loop, so join() either hangs or raises "bound to a different event loop". Rebind to the running loop before the call and wait for the captured payload instead of a fixed sleep.
This commit is contained in:
Mateo Wang
GitHub
parent
ba07340964
commit
75613bf22f
10 changed files with 1242 additions and 4 deletions
119
tests/test_litellm/batches/test_responses_batch_cost.py
Normal file
119
tests/test_litellm/batches/test_responses_batch_cost.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
"""Token and spend reconciliation for /v1/responses batches.
|
||||
|
||||
Regression guard for https://github.com/BerriAI/litellm/issues/35363: a batch
|
||||
output line built by the Responses API reports ``input_tokens`` /
|
||||
``output_tokens`` where a chat line reports ``prompt_tokens`` /
|
||||
``completion_tokens``. The usage object was constructed straight from the raw
|
||||
dict, which accepts the unrecognized names without raising and yields zeros, so
|
||||
a completed Responses batch reconciled to 0 tokens and $0.00 spend with no
|
||||
error, and per-key budgets were never charged for it.
|
||||
|
||||
Line shape decides the parse, not the batch's declared endpoint, so an output
|
||||
file mixing Responses-shaped and chat-shaped lines sums across both.
|
||||
"""
|
||||
|
||||
from typing import Literal, get_args, get_type_hints
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
import litellm.batches.batch_utils as bu
|
||||
from litellm.types.llms.openai import CreateBatchRequest
|
||||
|
||||
MODEL = "gpt-5.6"
|
||||
|
||||
|
||||
def _responses_line(input_tokens: int, output_tokens: int) -> dict:
|
||||
return {
|
||||
"response": {
|
||||
"status_code": 200,
|
||||
"body": {
|
||||
"model": MODEL,
|
||||
"usage": {
|
||||
"input_tokens": input_tokens,
|
||||
"output_tokens": output_tokens,
|
||||
"total_tokens": input_tokens + output_tokens,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _chat_line(prompt_tokens: int, completion_tokens: int) -> dict:
|
||||
return {
|
||||
"response": {
|
||||
"status_code": 200,
|
||||
"body": {
|
||||
"model": MODEL,
|
||||
"usage": {
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": completion_tokens,
|
||||
"total_tokens": prompt_tokens + completion_tokens,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_responses_shaped_usage_maps_onto_prompt_and_completion_tokens():
|
||||
"""The Responses names land on the chat-shaped counters instead of being
|
||||
dropped for unrecognized keys."""
|
||||
usage = bu._get_batch_job_usage_from_response_body(
|
||||
{"usage": {"input_tokens": 100, "output_tokens": 50, "total_tokens": 150}}
|
||||
)
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (100, 50, 150)
|
||||
|
||||
|
||||
async def test_responses_batch_reconciles_to_real_tokens_and_spend(local_model_cost_map):
|
||||
"""A completed Responses batch records the provider's token counts and a
|
||||
non-zero spend at the model's batch rates."""
|
||||
model_info = litellm.get_model_info(model=MODEL, custom_llm_provider="openai")
|
||||
input_tokens = 33
|
||||
output_tokens = 57
|
||||
|
||||
cost, usage, models = await bu.calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=[_responses_line(input_tokens, output_tokens)],
|
||||
custom_llm_provider="openai",
|
||||
model_name=MODEL,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
input_tokens + output_tokens,
|
||||
)
|
||||
assert models == [MODEL]
|
||||
assert cost == pytest.approx(
|
||||
input_tokens * model_info["input_cost_per_token_batches"]
|
||||
+ output_tokens * model_info["output_cost_per_token_batches"]
|
||||
)
|
||||
assert cost > 0.0
|
||||
|
||||
|
||||
async def test_mixed_shape_batch_output_sums_across_both_line_shapes(local_model_cost_map):
|
||||
"""An output file carrying both line shapes sums both. A fix keyed off the
|
||||
batch's declared endpoint rather than each line's shape would miss this."""
|
||||
model_info = litellm.get_model_info(model=MODEL, custom_llm_provider="openai")
|
||||
|
||||
cost, usage, _ = await bu.calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=[_responses_line(100, 50), _chat_line(33, 57)],
|
||||
custom_llm_provider="openai",
|
||||
model_name=MODEL,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (133, 107, 240)
|
||||
assert cost == pytest.approx(
|
||||
133 * model_info["input_cost_per_token_batches"] + 107 * model_info["output_cost_per_token_batches"]
|
||||
)
|
||||
|
||||
|
||||
def test_create_batch_endpoint_accepts_v1_responses():
|
||||
"""A type-checked caller can pass endpoint="/v1/responses", which the runtime
|
||||
already forwarded correctly."""
|
||||
endpoint_annotation = get_type_hints(CreateBatchRequest)["endpoint"]
|
||||
assert "/v1/responses" in get_args(endpoint_annotation)
|
||||
|
||||
for create_fn in (litellm.create_batch, litellm.acreate_batch):
|
||||
assert "/v1/responses" in get_args(get_type_hints(create_fn)["endpoint"])
|
||||
|
|
@ -0,0 +1,136 @@
|
|||
"""Cache-write pricing for OpenAI models, on both the chat and Responses paths.
|
||||
|
||||
Regression guard for https://github.com/BerriAI/litellm/issues/33772: OpenAI
|
||||
reports cache-write tokens under ``prompt_tokens_details.cache_write_tokens``
|
||||
(chat) / ``input_tokens_details.cache_write_tokens`` (responses), where
|
||||
Anthropic reports ``cache_creation_tokens``. The cost path read only the
|
||||
Anthropic name, so cache-write tokens were billed at the plain input rate,
|
||||
the Responses transform dropped the split before cost ran, and the tiered
|
||||
``cache_creation_input_token_cost_{priority,flex,above_272k_tokens}`` keys were
|
||||
discarded by ``get_model_info``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
MODEL = "gpt-5.6"
|
||||
|
||||
|
||||
def _openai_chat_usage(prompt_tokens: int, cache_write_tokens: int, completion_tokens: int) -> Usage:
|
||||
return Usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
prompt_tokens_details={
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": cache_write_tokens,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_openai_cache_write_tokens_billed_at_the_cache_creation_rate(local_model_cost_map):
|
||||
"""A cache-write request costs the cache-creation rate on the written tokens,
|
||||
not the plain input rate."""
|
||||
rates = litellm.model_cost[MODEL]
|
||||
input_rate = rates["input_cost_per_token"]
|
||||
cache_write_rate = rates["cache_creation_input_token_cost"]
|
||||
output_rate = rates["output_cost_per_token"]
|
||||
assert cache_write_rate == pytest.approx(input_rate * 1.25)
|
||||
|
||||
prompt_tokens = 12317
|
||||
cache_write_tokens = 12314
|
||||
fresh_tokens = prompt_tokens - cache_write_tokens
|
||||
completion_tokens = 5
|
||||
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model=MODEL,
|
||||
usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens),
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(fresh_tokens * input_rate + cache_write_tokens * cache_write_rate)
|
||||
assert completion_cost == pytest.approx(completion_tokens * output_rate)
|
||||
assert prompt_cost > prompt_tokens * input_rate
|
||||
|
||||
|
||||
def test_responses_api_cache_write_costs_the_same_as_chat(local_model_cost_map):
|
||||
"""The Responses usage transform carries the cache-write split through, so the
|
||||
same request costs the same whichever route it took."""
|
||||
prompt_tokens = 12317
|
||||
cache_write_tokens = 12314
|
||||
completion_tokens = 5
|
||||
|
||||
responses_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
{
|
||||
"input_tokens": prompt_tokens,
|
||||
"output_tokens": completion_tokens,
|
||||
"total_tokens": prompt_tokens + completion_tokens,
|
||||
"input_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": cache_write_tokens,
|
||||
},
|
||||
}
|
||||
)
|
||||
assert responses_usage.prompt_tokens_details.cache_write_tokens == cache_write_tokens
|
||||
|
||||
responses_prompt_cost, responses_completion_cost = generic_cost_per_token(
|
||||
model=MODEL,
|
||||
usage=responses_usage,
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
chat_prompt_cost, chat_completion_cost = generic_cost_per_token(
|
||||
model=MODEL,
|
||||
usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens),
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
rates = litellm.model_cost[MODEL]
|
||||
fresh_tokens = prompt_tokens - cache_write_tokens
|
||||
assert responses_prompt_cost == pytest.approx(
|
||||
fresh_tokens * rates["input_cost_per_token"]
|
||||
+ cache_write_tokens * rates["cache_creation_input_token_cost"]
|
||||
)
|
||||
assert responses_prompt_cost == pytest.approx(chat_prompt_cost)
|
||||
assert responses_completion_cost == pytest.approx(chat_completion_cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"service_tier,prompt_tokens,rate_key",
|
||||
[
|
||||
(None, 100000, "cache_creation_input_token_cost"),
|
||||
("priority", 100000, "cache_creation_input_token_cost_priority"),
|
||||
("flex", 100000, "cache_creation_input_token_cost_flex"),
|
||||
(None, 300000, "cache_creation_input_token_cost_above_272k_tokens"),
|
||||
],
|
||||
)
|
||||
def test_tiered_cache_creation_rates_are_registered_and_billed(
|
||||
local_model_cost_map, service_tier, prompt_tokens, rate_key
|
||||
):
|
||||
"""The tiered cache-creation keys survive ``get_model_info`` and are the rate the
|
||||
cost path actually charges for priority, flex, and >272k requests."""
|
||||
model_info = litellm.get_model_info(model=MODEL, custom_llm_provider="openai")
|
||||
tiered_rate = litellm.model_cost[MODEL][rate_key]
|
||||
assert model_info.get(rate_key) == tiered_rate
|
||||
|
||||
cache_write_tokens = prompt_tokens - 1000
|
||||
fresh_tokens = 1000
|
||||
input_rate_key = {
|
||||
"cache_creation_input_token_cost": "input_cost_per_token",
|
||||
"cache_creation_input_token_cost_priority": "input_cost_per_token_priority",
|
||||
"cache_creation_input_token_cost_flex": "input_cost_per_token_flex",
|
||||
"cache_creation_input_token_cost_above_272k_tokens": "input_cost_per_token_above_272k_tokens",
|
||||
}[rate_key]
|
||||
input_rate = litellm.model_cost[MODEL][input_rate_key]
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model=MODEL,
|
||||
usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens=100),
|
||||
custom_llm_provider="openai",
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(fresh_tokens * input_rate + cache_write_tokens * tiered_rate)
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
"""Itemized cache costs in the logged cost breakdown for the OpenAI Responses API.
|
||||
|
||||
Regression guard for https://github.com/BerriAI/litellm/issues/34309: the
|
||||
breakdown block derived its cache token counts only from the Anthropic-style
|
||||
top-level ``cache_read_input_tokens`` / ``cache_creation_input_tokens``. OpenAI's
|
||||
Responses API reports them under ``input_tokens_details.{cached_tokens,
|
||||
cache_write_tokens}`` instead, so ``cost_breakdown.cache_read_cost`` and
|
||||
``cache_creation_cost`` serialized as null for every OpenAI request while the
|
||||
grand total stayed correct.
|
||||
|
||||
``input_cost`` is the full prompt-side cost and the two cache fields are additive
|
||||
break-outs that overlap it, matching the Anthropic path. A request with no cache
|
||||
activity leaves both fields unset, which is the documented behavior.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
MODEL = "gpt-5.6"
|
||||
|
||||
|
||||
def _logging_obj() -> Logging:
|
||||
return Logging(
|
||||
model=MODEL,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="responses-cache-breakdown",
|
||||
function_id="f",
|
||||
)
|
||||
|
||||
|
||||
def _responses_completion(cached_tokens: int, cache_write_tokens: int, fresh_tokens: int, output_tokens: int):
|
||||
input_tokens = cached_tokens + cache_write_tokens + fresh_tokens
|
||||
usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
{
|
||||
"input_tokens": input_tokens,
|
||||
"output_tokens": output_tokens,
|
||||
"total_tokens": input_tokens + output_tokens,
|
||||
"input_tokens_details": {
|
||||
"cached_tokens": cached_tokens,
|
||||
"cache_write_tokens": cache_write_tokens,
|
||||
},
|
||||
}
|
||||
)
|
||||
return ModelResponse(
|
||||
id="x",
|
||||
created=1,
|
||||
model=MODEL,
|
||||
object="chat.completion",
|
||||
choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
|
||||
def test_responses_api_cache_costs_are_itemized_in_the_breakdown(local_model_cost_map):
|
||||
"""Cache-read and cache-write dollars are broken out for an OpenAI Responses
|
||||
request, at the model's own cache rates."""
|
||||
rates = litellm.model_cost[MODEL]
|
||||
cached_tokens = 4012
|
||||
cache_write_tokens = 5000
|
||||
fresh_tokens = 1000
|
||||
output_tokens = 200
|
||||
|
||||
logging_obj = _logging_obj()
|
||||
total = litellm.completion_cost(
|
||||
completion_response=_responses_completion(cached_tokens, cache_write_tokens, fresh_tokens, output_tokens),
|
||||
model=MODEL,
|
||||
custom_llm_provider="openai",
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
breakdown = logging_obj.cost_breakdown
|
||||
assert breakdown is not None
|
||||
|
||||
expected_cache_read = cached_tokens * rates["cache_read_input_token_cost"]
|
||||
expected_cache_creation = cache_write_tokens * rates["cache_creation_input_token_cost"]
|
||||
expected_input = fresh_tokens * rates["input_cost_per_token"] + expected_cache_read + expected_cache_creation
|
||||
expected_output = output_tokens * rates["output_cost_per_token"]
|
||||
|
||||
assert breakdown["cache_read_cost"] == pytest.approx(expected_cache_read)
|
||||
assert breakdown["cache_creation_cost"] == pytest.approx(expected_cache_creation)
|
||||
assert breakdown["input_cost"] == pytest.approx(expected_input)
|
||||
assert breakdown["output_cost"] == pytest.approx(expected_output)
|
||||
assert breakdown["total_cost"] == pytest.approx(expected_input + expected_output)
|
||||
assert total == pytest.approx(breakdown["total_cost"])
|
||||
|
||||
|
||||
def test_cache_fields_stay_unset_when_there_was_no_cache_activity(local_model_cost_map):
|
||||
"""A request that neither read nor wrote the cache leaves both break-out fields
|
||||
off the breakdown, so they serialize as null rather than a misleading $0."""
|
||||
logging_obj = _logging_obj()
|
||||
litellm.completion_cost(
|
||||
completion_response=_responses_completion(
|
||||
cached_tokens=0, cache_write_tokens=0, fresh_tokens=1000, output_tokens=200
|
||||
),
|
||||
model=MODEL,
|
||||
custom_llm_provider="openai",
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
breakdown = logging_obj.cost_breakdown
|
||||
assert breakdown is not None
|
||||
assert "cache_read_cost" not in breakdown
|
||||
assert "cache_creation_cost" not in breakdown
|
||||
assert breakdown["input_cost"] == pytest.approx(1000 * litellm.model_cost[MODEL]["input_cost_per_token"])
|
||||
|
|
@ -1,5 +1,8 @@
|
|||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -10,6 +13,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
|
||||
import litellm
|
||||
from litellm.anthropic_interface import messages
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.types.utils import Delta, ModelResponse, StreamingChoices
|
||||
|
||||
|
|
@ -1103,3 +1108,181 @@ def test_messages_sync_streaming_reports_provider_local_model():
|
|||
first_event = next(iter(stream))
|
||||
|
||||
assert json.loads(first_event.decode().split("data: ", 1)[1])["message"]["model"] == "perplexity/kimi-k3"
|
||||
|
||||
|
||||
_RESPONSES_COMPLETED_BODY: Dict[str, Any] = {
|
||||
"id": "resp-1",
|
||||
"object": "response",
|
||||
"created_at": 0,
|
||||
"model": "gpt-4o-mini",
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg-1",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "hello there"}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 11, "output_tokens": 7, "total_tokens": 18},
|
||||
}
|
||||
|
||||
_RESPONSES_SSE_EVENTS: List[Dict[str, Any]] = [
|
||||
{
|
||||
"type": "response.created",
|
||||
"response": {
|
||||
**_RESPONSES_COMPLETED_BODY,
|
||||
"status": "in_progress",
|
||||
"output": [],
|
||||
"usage": None,
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"item_id": "msg-1",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": "hello there",
|
||||
},
|
||||
{"type": "response.completed", "response": _RESPONSES_COMPLETED_BODY},
|
||||
]
|
||||
|
||||
|
||||
def _sse_body(events: List[Dict[str, Any]]) -> bytes:
|
||||
return b"".join(f"data: {json.dumps(event)}\n\n".encode() for event in events)
|
||||
|
||||
|
||||
class _SuccessPayloadCapture(CustomLogger):
|
||||
def __init__(self, tracking_id: str):
|
||||
super().__init__()
|
||||
self.tracking_id = tracking_id
|
||||
self.payloads: List[Dict[str, Any]] = []
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
if kwargs.get("litellm_call_id") == self.tracking_id:
|
||||
self.payloads.append(kwargs.get("standard_logging_object") or {})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def capture_success_payloads(monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
|
||||
capture = _SuccessPayloadCapture(tracking_id=f"messages-stream-{uuid.uuid4()}")
|
||||
monkeypatch.setattr(litellm, "callbacks", [capture])
|
||||
return capture
|
||||
|
||||
|
||||
async def _drain(sse_stream) -> List[bytes]:
|
||||
return [chunk async for chunk in sse_stream]
|
||||
|
||||
|
||||
def _bind_logging_worker_to_running_loop() -> None:
|
||||
"""The worker's queue keeps the loop it was built on, so one left over from an earlier
|
||||
test makes ``flush`` raise "bound to a different event loop". ``start`` runs
|
||||
``_ensure_queue``, which rebinds the queue when the running loop has changed."""
|
||||
GLOBAL_LOGGING_WORKER.start()
|
||||
|
||||
|
||||
async def _flush_logging_worker(capture: "_SuccessPayloadCapture") -> None:
|
||||
await asyncio.sleep(0)
|
||||
try:
|
||||
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0)
|
||||
except (asyncio.TimeoutError, RuntimeError):
|
||||
pass
|
||||
deadline = asyncio.get_running_loop().time() + 10.0
|
||||
while not capture.payloads and asyncio.get_running_loop().time() < deadline:
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
|
||||
def _assert_anthropic_sse(chunks: List[bytes]) -> None:
|
||||
body = b"".join(chunks).decode()
|
||||
assert "message_start" in body
|
||||
assert "message_stop" in body
|
||||
|
||||
|
||||
class TestMessagesStreamingSuccessLogging:
|
||||
"""A streamed /v1/messages call routed to an ``openai/`` backend must still reach
|
||||
success logging once the SSE stream is drained. The client gets a correct Anthropic
|
||||
SSE body and the provider bills the call, but no StandardLoggingPayload is produced,
|
||||
so the request is invisible to spend tracking and every success-logging integration.
|
||||
|
||||
Logging is fired by the inner CustomStreamWrapper / ResponsesAPIStreamingIterator the
|
||||
Anthropic wrappers drain, not by the wrappers themselves, so these assert on the
|
||||
callback that actually reaches an integration rather than on a wrapper attribute.
|
||||
"""
|
||||
|
||||
MESSAGES = [{"role": "user", "content": "hi"}]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_bridge_streaming_emits_success_logging(self, capture_success_payloads):
|
||||
"""The Responses bridge, which is the default for openai/ deployments."""
|
||||
from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler import (
|
||||
LiteLLMMessagesToResponsesAPIHandler,
|
||||
)
|
||||
|
||||
_bind_logging_worker_to_running_loop()
|
||||
|
||||
with patch(
|
||||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_post:
|
||||
mock_post.return_value = httpx.Response(
|
||||
200,
|
||||
content=_sse_body(_RESPONSES_SSE_EVENTS),
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
sse_stream = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
|
||||
max_tokens=100,
|
||||
messages=self.MESSAGES,
|
||||
model="openai/gpt-4o-mini",
|
||||
stream=True,
|
||||
custom_llm_provider="openai",
|
||||
litellm_call_id=capture_success_payloads.tracking_id,
|
||||
)
|
||||
chunks = await _drain(sse_stream)
|
||||
|
||||
await _flush_logging_worker(capture_success_payloads)
|
||||
|
||||
_assert_anthropic_sse(chunks)
|
||||
assert len(capture_success_payloads.payloads) == 1
|
||||
payload = capture_success_payloads.payloads[0]
|
||||
assert payload["call_type"] == "aresponses"
|
||||
assert payload["prompt_tokens"] == 11
|
||||
assert payload["completion_tokens"] == 7
|
||||
assert payload["total_tokens"] == 18
|
||||
assert payload["response_cost"] > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completions_bridge_streaming_emits_success_logging(self, capture_success_payloads):
|
||||
"""The chat-completions bridge, reached via
|
||||
litellm.use_chat_completions_url_for_anthropic_messages. Its router lookup is
|
||||
stubbed to what an SDK caller with no proxy running already resolves to."""
|
||||
from litellm.llms.anthropic.experimental_pass_through.adapters.handler import (
|
||||
LiteLLMMessagesToCompletionTransformationHandler,
|
||||
)
|
||||
|
||||
_bind_logging_worker_to_running_loop()
|
||||
|
||||
with patch(
|
||||
"litellm.llms.anthropic.experimental_pass_through.adapters.handler._proxy_router_fallback",
|
||||
return_value=None,
|
||||
):
|
||||
sse_stream = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler(
|
||||
max_tokens=100,
|
||||
messages=self.MESSAGES,
|
||||
model="openai/gpt-4o-mini",
|
||||
stream=True,
|
||||
custom_llm_provider="openai",
|
||||
mock_response="hello there",
|
||||
litellm_call_id=capture_success_payloads.tracking_id,
|
||||
)
|
||||
chunks = await _drain(sse_stream)
|
||||
|
||||
await _flush_logging_worker(capture_success_payloads)
|
||||
|
||||
_assert_anthropic_sse(chunks)
|
||||
assert len(capture_success_payloads.payloads) == 1
|
||||
payload = capture_success_payloads.payloads[0]
|
||||
assert payload["call_type"] == "acompletion"
|
||||
assert payload["total_tokens"] > 0
|
||||
assert payload["response_cost"] > 0
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ added to this layer raises instead of silently passing - the inventory of seams
|
|||
cannot drift without a test failure.
|
||||
"""
|
||||
|
||||
import json
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional
|
||||
|
|
@ -50,7 +51,7 @@ from litellm.router import Router
|
|||
from litellm.types.llms.openai import BatchJobStatus
|
||||
from litellm.types.utils import CredentialItem, LiteLLMBatch
|
||||
|
||||
from fastapi import Response
|
||||
from fastapi import Request, Response
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fixtures: distinguishable credentials per model so a wrong/hardcoded model_id
|
||||
|
|
@ -851,6 +852,48 @@ async def test_create__missing_required_param_is_400(harness, body, missing_para
|
|||
harness.router_acreate.assert_not_called()
|
||||
|
||||
|
||||
def _raw_batches_request(body: Dict[str, Any]) -> MagicMock:
|
||||
"""A request that reaches the real pre-call logic, which the `harness` fixture
|
||||
mocks out. Metadata validation lives there, so it cannot be seen through the seam."""
|
||||
request = MagicMock(spec=Request)
|
||||
request.url = MagicMock()
|
||||
request.url.__str__.return_value = "http://localhost/v1/batches"
|
||||
request.url.path = "/v1/batches"
|
||||
request.method = "POST"
|
||||
request.query_params = {}
|
||||
request.headers = {"Content-Type": "application/json"}
|
||||
request.client = MagicMock()
|
||||
request.client.host = "127.0.0.1"
|
||||
request.body = AsyncMock(return_value=json.dumps(body).encode())
|
||||
return request
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("field", ["metadata", "litellm_metadata"])
|
||||
async def test_create__non_object_metadata_is_400(field):
|
||||
"""A non-object metadata field is rejected with a 400 naming it (#37147), rather
|
||||
than being coerced to None and silently dropped, or crashing behind a 500 with
|
||||
"'str' object has no attribute 'update'"."""
|
||||
body = {
|
||||
"input_file_id": "file-abc",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
field: "abc",
|
||||
}
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await endpoints.create_batch(
|
||||
request=_raw_batches_request(body),
|
||||
fastapi_response=Response(),
|
||||
provider=None,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "400"
|
||||
assert exc_info.value.param == field
|
||||
assert "has no attribute 'update'" not in exc_info.value.message
|
||||
|
||||
|
||||
# =========================================================================== #
|
||||
# Team-level batch expiry enforcement (independent of routing).
|
||||
# =========================================================================== #
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import json
|
||||
from typing import Final, List
|
||||
from unittest.mock import ANY, AsyncMock
|
||||
from typing import Final, List, Optional
|
||||
from unittest.mock import ANY, AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
|
|
@ -4302,3 +4302,177 @@ def test_batch_upload_closes_the_spools_it_opened(monkeypatch, llm_router: Route
|
|||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
|
||||
|
||||
def _managed_file_row(
|
||||
unified_file_id: str,
|
||||
stored_file_id: Optional[str] = None,
|
||||
created_by: str = "test-user",
|
||||
team_id: Optional[str] = None,
|
||||
) -> MagicMock:
|
||||
file_object = OpenAIFileObject(
|
||||
id=stored_file_id or unified_file_id,
|
||||
bytes=100,
|
||||
created_at=1700000000,
|
||||
filename="batch.jsonl",
|
||||
object="file",
|
||||
purpose="batch",
|
||||
status="processed",
|
||||
)
|
||||
return MagicMock(
|
||||
unified_file_id=unified_file_id,
|
||||
file_object=file_object.model_dump(),
|
||||
created_by=created_by,
|
||||
team_id=team_id,
|
||||
)
|
||||
|
||||
|
||||
def _row_matches_where(row, where) -> bool:
|
||||
for field, expected in where.items():
|
||||
if field == "OR":
|
||||
if not any(_row_matches_where(row, clause) for clause in expected):
|
||||
return False
|
||||
elif getattr(row, field) != expected:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class _ManagedFileTableOverRows:
|
||||
"""Enough of the Prisma table for the real hook's owner-scoped keyset query."""
|
||||
|
||||
def __init__(self, rows):
|
||||
self.rows = list(rows)
|
||||
|
||||
def _owned_rows(self, where):
|
||||
return [row for row in self.rows if _row_matches_where(row, where)]
|
||||
|
||||
async def find_first(self, where):
|
||||
return next(iter(self._owned_rows(where)), None)
|
||||
|
||||
async def find_many(self, where, take=None, order=None, cursor=None, skip=0):
|
||||
rows = self._owned_rows(where)
|
||||
if cursor is not None:
|
||||
start = next(
|
||||
index
|
||||
for index, row in enumerate(rows)
|
||||
if row.unified_file_id == cursor["unified_file_id"]
|
||||
)
|
||||
rows = rows[start + skip :]
|
||||
return rows if take is None else rows[:take]
|
||||
|
||||
|
||||
def _setup_unscoped_list_files_route_over_real_hook(
|
||||
mocker, monkeypatch, llm_router: Router, rows
|
||||
):
|
||||
"""Wire GET /v1/files to a real managed-files hook over `rows`, with no provider
|
||||
credentials in the process. The neighbouring setup stubs the hook with a
|
||||
MagicMock, so it cannot see anything past the route's argument plumbing."""
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
from litellm_enterprise.proxy.hooks.managed_files import (
|
||||
_PROXY_LiteLLMManagedFiles,
|
||||
)
|
||||
|
||||
for env_var in ("OPENAI_API_KEY", "OPENAI_ADMIN_KEY", "OPENAI_ORGANIZATION"):
|
||||
monkeypatch.delenv(env_var, raising=False)
|
||||
monkeypatch.setattr(litellm, "api_key", None, raising=False)
|
||||
monkeypatch.setattr(litellm, "openai_key", None, raising=False)
|
||||
|
||||
managed_files = _PROXY_LiteLLMManagedFiles(
|
||||
internal_usage_cache=MagicMock(), prisma_client=MagicMock()
|
||||
)
|
||||
managed_files.prisma_client.db.litellm_managedfiletable = _ManagedFileTableOverRows(rows)
|
||||
|
||||
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, llm_router)
|
||||
proxy_logging_obj.proxy_hook_mapping["managed_files"] = managed_files
|
||||
proxy_logging_obj.update_request_status = mocker.AsyncMock()
|
||||
proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=None)
|
||||
proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock()
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", llm_router)
|
||||
|
||||
provider_list = mocker.patch.object(litellm, "afile_list", new=mocker.AsyncMock())
|
||||
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
api_key="test-key",
|
||||
user_role=LitellmUserRoles.INTERNAL_USER,
|
||||
user_id="test-user",
|
||||
)
|
||||
return proxy_logging_obj, provider_list
|
||||
|
||||
|
||||
def test_unscoped_list_files_reads_the_store_without_any_provider_key(
|
||||
mocker: MockerFixture, monkeypatch, llm_router: Router
|
||||
):
|
||||
"""The exact shape `client.files.list()` produces, with no target_model_names, no
|
||||
provider and no OPENAI_API_KEY in the process, must return the caller's managed
|
||||
files instead of falling through to a keyless provider client (#35362)."""
|
||||
proxy_logging_obj, provider_list = _setup_unscoped_list_files_route_over_real_hook(
|
||||
mocker, monkeypatch, llm_router, [_managed_file_row("unified-file-1")]
|
||||
)
|
||||
|
||||
response = _get_unscoped_list_files("")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert [file["id"] for file in response.json()["data"]] == ["unified-file-1"]
|
||||
provider_list.assert_not_awaited()
|
||||
proxy_logging_obj.post_call_failure_hook.assert_not_called()
|
||||
|
||||
|
||||
def test_unscoped_list_files_returns_unified_ids_for_rows_storing_a_raw_provider_id(
|
||||
mocker: MockerFixture, monkeypatch, llm_router: Router
|
||||
):
|
||||
"""Batch output rows store the provider's file object, whose id is a raw `file-`
|
||||
the caller cannot act on. The listing hands back the row's unified id so
|
||||
files.retrieve and files.content work on what it returned."""
|
||||
_setup_unscoped_list_files_route_over_real_hook(
|
||||
mocker,
|
||||
monkeypatch,
|
||||
llm_router,
|
||||
[_managed_file_row("unified-batch-output", stored_file_id="file-raw-provider-123")],
|
||||
)
|
||||
|
||||
response = _get_unscoped_list_files("")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert [file["id"] for file in response.json()["data"]] == ["unified-batch-output"]
|
||||
assert response.json()["data"][0]["filename"] == "batch.jsonl"
|
||||
|
||||
|
||||
def test_unscoped_list_files_does_not_leak_another_callers_files(
|
||||
mocker: MockerFixture, monkeypatch, llm_router: Router
|
||||
):
|
||||
"""Listing without a model pinned is still owner-scoped."""
|
||||
_setup_unscoped_list_files_route_over_real_hook(
|
||||
mocker,
|
||||
monkeypatch,
|
||||
llm_router,
|
||||
[
|
||||
_managed_file_row("unified-mine"),
|
||||
_managed_file_row("unified-theirs", created_by="other-user"),
|
||||
],
|
||||
)
|
||||
|
||||
response = _get_unscoped_list_files("")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert [file["id"] for file in response.json()["data"]] == ["unified-mine"]
|
||||
|
||||
|
||||
def test_scoped_list_files_still_resolves_deployment_credentials(
|
||||
mocker: MockerFixture, monkeypatch, llm_router: Router
|
||||
):
|
||||
"""target_model_names keeps routing to the provider on deployment credentials, so
|
||||
the unscoped path does not swallow that route."""
|
||||
_, provider_list = _setup_unscoped_list_files_route_over_real_hook(
|
||||
mocker, monkeypatch, llm_router, [_managed_file_row("unified-file-1")]
|
||||
)
|
||||
provider_list.return_value = []
|
||||
|
||||
response = _get_unscoped_list_files("?target_model_names=gpt-3.5-turbo")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
provider_list.assert_awaited_once()
|
||||
assert provider_list.await_args.kwargs["custom_llm_provider"] == "openai"
|
||||
assert provider_list.await_args.kwargs["api_key"] == "openai_api_key"
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import httpx
|
|||
import pytest
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler import (
|
||||
OpenAIPassthroughLoggingHandler,
|
||||
|
|
@ -14,6 +15,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrou
|
|||
from litellm.proxy.pass_through_endpoints.success_handler import (
|
||||
PassThroughEndpointLogging,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
PassthroughStandardLoggingPayload,
|
||||
)
|
||||
|
|
@ -1809,5 +1811,229 @@ class TestOpenAIPassthroughIntegration:
|
|||
assert mock_logging_obj.model_call_details["custom_llm_provider"] == "openai"
|
||||
|
||||
|
||||
class TestOpenAIPassthroughResponsesStreamingSpendLog:
|
||||
"""A streamed OpenAI-passthrough `/v1/responses` call must write a priced spend
|
||||
log row (#36523).
|
||||
|
||||
`_handle_logging_openai_collected_chunks` received `url_route` and ignored it, so
|
||||
a Responses SSE stream was reassembled by the chat-completions chunk parser. The
|
||||
assembled object carried a synthetic `chatcmpl-` id, `model=None` and zero usage,
|
||||
and the spend row landed at zero tokens and zero spend while the identical
|
||||
buffered call priced exactly.
|
||||
"""
|
||||
|
||||
RESPONSE_ID = "resp_0c72ddebf05f8751"
|
||||
MODEL_MAP_KEY = "gpt-4o-mini-2024-07-18"
|
||||
INPUT_TOKENS = 14
|
||||
OUTPUT_TOKENS = 2
|
||||
|
||||
def setup_method(self):
|
||||
self.start_time = datetime.now()
|
||||
self.end_time = datetime.now()
|
||||
rates = litellm.model_cost[self.MODEL_MAP_KEY]
|
||||
self.expected_spend = (
|
||||
self.INPUT_TOKENS * rates["input_cost_per_token"]
|
||||
+ self.OUTPUT_TOKENS * rates["output_cost_per_token"]
|
||||
)
|
||||
|
||||
def _responses_stream_chunks(self) -> List[str]:
|
||||
"""Usage arrives only on the terminal `response.completed` event, nested under
|
||||
`response` as `input_tokens` / `output_tokens`. No event carries a top-level
|
||||
`id`, `model` or `usage`."""
|
||||
response_body = {
|
||||
"id": self.RESPONSE_ID,
|
||||
"object": "response",
|
||||
"created_at": 1786374786,
|
||||
"model": self.MODEL_MAP_KEY,
|
||||
"error": None,
|
||||
"incomplete_details": None,
|
||||
"instructions": None,
|
||||
"metadata": {},
|
||||
"parallel_tool_calls": True,
|
||||
"temperature": 1.0,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"top_p": 1.0,
|
||||
}
|
||||
created_event = {
|
||||
"type": "response.created",
|
||||
"sequence_number": 0,
|
||||
"response": {**response_body, "status": "in_progress", "output": [], "usage": None},
|
||||
}
|
||||
delta_event = {
|
||||
"type": "response.output_text.delta",
|
||||
"sequence_number": 3,
|
||||
"item_id": "msg_abc",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": "OK",
|
||||
}
|
||||
completed_event = {
|
||||
"type": "response.completed",
|
||||
"sequence_number": 8,
|
||||
"response": {
|
||||
**response_body,
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_abc",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "OK", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": self.INPUT_TOKENS,
|
||||
"input_tokens_details": {"cached_tokens": 0},
|
||||
"output_tokens": self.OUTPUT_TOKENS,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": self.INPUT_TOKENS + self.OUTPUT_TOKENS,
|
||||
},
|
||||
},
|
||||
}
|
||||
return [
|
||||
f"data: {json.dumps(created_event)}",
|
||||
f"data: {json.dumps(delta_event)}",
|
||||
f"data: {json.dumps(completed_event)}",
|
||||
"data: [DONE]",
|
||||
]
|
||||
|
||||
def _logging_obj(self) -> LiteLLMLoggingObj:
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="gpt-4o-mini",
|
||||
messages=[],
|
||||
stream=True,
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=self.start_time,
|
||||
litellm_call_id="323dfe4f-2741-4473-b6e2-000000000000",
|
||||
function_id="1234",
|
||||
)
|
||||
logging_obj.model_call_details["custom_llm_provider"] = "openai"
|
||||
return logging_obj
|
||||
|
||||
def test_streamed_responses_passthrough_spend_log_is_priced(self):
|
||||
"""The spend row books the same tokens, spend and `resp_` id as the buffered call."""
|
||||
result = OpenAIPassthroughLoggingHandler._handle_logging_openai_collected_chunks(
|
||||
litellm_logging_obj=self._logging_obj(),
|
||||
passthrough_success_handler_obj=None,
|
||||
url_route="https://api.openai.com/v1/responses",
|
||||
request_body={
|
||||
"model": "gpt-4o-mini",
|
||||
"input": "Say hi in one word.",
|
||||
"max_output_tokens": 16,
|
||||
"stream": True,
|
||||
},
|
||||
endpoint_type=None,
|
||||
start_time=self.start_time,
|
||||
all_chunks=self._responses_stream_chunks(),
|
||||
end_time=self.end_time,
|
||||
)
|
||||
|
||||
spend_log_row = get_logging_payload(
|
||||
kwargs=result["kwargs"],
|
||||
response_obj=result["result"],
|
||||
start_time=self.start_time,
|
||||
end_time=self.end_time,
|
||||
)
|
||||
|
||||
assert spend_log_row["prompt_tokens"] == self.INPUT_TOKENS
|
||||
assert spend_log_row["completion_tokens"] == self.OUTPUT_TOKENS
|
||||
assert spend_log_row["total_tokens"] == self.INPUT_TOKENS + self.OUTPUT_TOKENS
|
||||
assert spend_log_row["spend"] == self.expected_spend
|
||||
assert spend_log_row["request_id"] == self.RESPONSE_ID
|
||||
assert spend_log_row["model"] == "gpt-4o-mini"
|
||||
|
||||
row_metadata = json.loads(spend_log_row["metadata"])
|
||||
assert row_metadata["model_map_information"]["model_map_key"] == self.MODEL_MAP_KEY
|
||||
|
||||
|
||||
class TestOpenAIPassthroughEmbeddingsSpendLog:
|
||||
"""An OpenAI-passthrough `/v1/embeddings` call must write a priced spend log row
|
||||
(#36646).
|
||||
|
||||
`_is_supported_openai_endpoint` ORed four route predicates, none of which matched
|
||||
`/v1/embeddings`, so the dispatcher never entered the OpenAI handler and billable
|
||||
embedding tokens produced no spend row at all, under-enforcing key budgets.
|
||||
"""
|
||||
|
||||
EMBEDDINGS_URL = "https://api.openai.com/v1/embeddings"
|
||||
MODEL = "text-embedding-3-small"
|
||||
PROMPT_TOKENS = 14
|
||||
CALL_ID = "11024cc3-b143-4a63-954a-ec06081df768"
|
||||
|
||||
def setup_method(self):
|
||||
self.start_time = datetime.now()
|
||||
self.end_time = datetime.now()
|
||||
self.expected_spend = self.PROMPT_TOKENS * litellm.model_cost[self.MODEL]["input_cost_per_token"]
|
||||
self.response_body = {
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.0, 1.0]}],
|
||||
"model": self.MODEL,
|
||||
"usage": {"prompt_tokens": self.PROMPT_TOKENS, "total_tokens": self.PROMPT_TOKENS},
|
||||
}
|
||||
self.request_body = {"model": self.MODEL, "input": "hello"}
|
||||
|
||||
def _create_mock_httpx_response(self) -> httpx.Response:
|
||||
mock_response = MagicMock(spec=httpx.Response)
|
||||
mock_response.status_code = 200
|
||||
mock_response.text = json.dumps(self.response_body)
|
||||
mock_response.json.return_value = self.response_body
|
||||
mock_response.headers = {"content-type": "application/json"}
|
||||
return mock_response
|
||||
|
||||
def _logging_obj(self) -> LiteLLMLoggingObj:
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model=self.MODEL,
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=self.start_time,
|
||||
litellm_call_id=self.CALL_ID,
|
||||
function_id="1234",
|
||||
)
|
||||
logging_obj.model_call_details["passthrough_logging_payload"] = PassthroughStandardLoggingPayload(
|
||||
url=self.EMBEDDINGS_URL,
|
||||
request_body=self.request_body,
|
||||
request_method="POST",
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
def test_embeddings_passthrough_spend_log_is_priced(self):
|
||||
"""The dispatched call books prompt tokens and cost onto the spend row."""
|
||||
dispatched = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
|
||||
httpx_response=self._create_mock_httpx_response(),
|
||||
response_body=self.response_body,
|
||||
request_body=self.request_body,
|
||||
logging_obj=self._logging_obj(),
|
||||
url_route=self.EMBEDDINGS_URL,
|
||||
result="",
|
||||
start_time=self.start_time,
|
||||
end_time=self.end_time,
|
||||
cache_hit=False,
|
||||
custom_llm_provider="openai",
|
||||
litellm_call_id=self.CALL_ID,
|
||||
litellm_params={},
|
||||
call_type="pass_through_endpoint",
|
||||
)
|
||||
|
||||
assert dispatched["standard_logging_response_object"] is not None
|
||||
assert dispatched["kwargs"]["response_cost"] == self.expected_spend
|
||||
|
||||
spend_log_row = get_logging_payload(
|
||||
kwargs=dispatched["kwargs"],
|
||||
response_obj=dispatched["standard_logging_response_object"],
|
||||
start_time=self.start_time,
|
||||
end_time=self.end_time,
|
||||
)
|
||||
|
||||
assert spend_log_row["prompt_tokens"] == self.PROMPT_TOKENS
|
||||
assert spend_log_row["total_tokens"] == self.PROMPT_TOKENS
|
||||
assert spend_log_row["spend"] == self.expected_spend
|
||||
assert spend_log_row["model"] == self.MODEL
|
||||
assert spend_log_row["custom_llm_provider"] == "openai"
|
||||
assert spend_log_row["request_id"] == self.CALL_ID
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__])
|
||||
|
|
|
|||
|
|
@ -1805,6 +1805,51 @@ class TestLLMClassifier:
|
|||
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
|
||||
assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"request_kwargs",
|
||||
[
|
||||
pytest.param({"metadata": {"user_api_key": "sk-abc"}}, id="metadata-bucket"),
|
||||
pytest.param(
|
||||
{"litellm_metadata": {"user_api_key": "sk-abc"}}, id="litellm-metadata-bucket"
|
||||
),
|
||||
pytest.param({}, id="no-caller-context"),
|
||||
pytest.param(None, id="no-request-kwargs"),
|
||||
],
|
||||
)
|
||||
async def test_aclassify_reaches_the_llm_for_every_caller_metadata_shape(
|
||||
self, llm_classifier_config, request_kwargs
|
||||
):
|
||||
"""Whatever the caller's metadata bucket looks like, the configured classifier must
|
||||
actually run. The forwarded metadata reaches litellm's own metadata handling, which
|
||||
raises "'NoneType' object has no attribute 'update'" on a shape it does not expect;
|
||||
aclassify catches that and silently degrades to heuristic scoring, so the tier is
|
||||
decided by word counting while the config says otherwise. A real Router is used here
|
||||
because a mocked acompletion accepts any shape and never reaches that handling.
|
||||
"""
|
||||
real_router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "haiku-classifier",
|
||||
"litellm_params": {
|
||||
"model": "openai/haiku-classifier",
|
||||
"api_key": "sk-classifier",
|
||||
"mock_response": '{"tier": "COMPLEX"}',
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
router = ComplexityRouter(
|
||||
model_name="test-complexity-router",
|
||||
litellm_router_instance=real_router,
|
||||
complexity_router_config=llm_classifier_config,
|
||||
)
|
||||
|
||||
outcome = await router.aclassify("hi", request_kwargs=request_kwargs)
|
||||
|
||||
assert outcome.cause == "llm_classifier"
|
||||
assert outcome.tier == ComplexityTier.COMPLEX
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aclassify_captures_request_body_in_proxy_server_request(
|
||||
self, llm_complexity_router, mock_router_instance
|
||||
|
|
|
|||
|
|
@ -850,6 +850,36 @@ def test_responses_api_bridge_check_gpt_5_4_tools_with_default_reasoning_routes_
|
|||
assert model_info.get("mode") == "responses"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", ["gpt-5.6-sol", "gpt-5.6-luna", "gpt-5.6-terra"])
|
||||
def test_responses_api_bridge_check_gpt_5_6_tools_with_default_reasoning_routes_to_responses(
|
||||
monkeypatch, model_name
|
||||
):
|
||||
"""
|
||||
The whole gpt-5.6 family must bridge on function tools alone. The bridge used to
|
||||
require an explicit reasoning_effort, so a gpt-5.6 call carrying tools and no effort
|
||||
was rejected with "Function tools with reasoning_effort are not supported for
|
||||
gpt-5.6-sol in /v1/chat/completions".
|
||||
"""
|
||||
import litellm
|
||||
from litellm.main import responses_api_bridge_check
|
||||
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
|
||||
with patch("litellm.main._get_model_info_helper") as mock_get_model_info:
|
||||
mock_get_model_info.return_value = {"max_tokens": 128000}
|
||||
model_info, model = responses_api_bridge_check(
|
||||
model=model_name,
|
||||
custom_llm_provider="openai",
|
||||
tools=[{"type": "function", "function": {"name": "get_capital"}}],
|
||||
reasoning_effort=None,
|
||||
)
|
||||
|
||||
assert model == model_name
|
||||
assert model_info.get("mode") == "responses"
|
||||
|
||||
|
||||
def test_responses_api_bridge_check_gpt_5_4_tools_with_reasoning_none_stays_chat():
|
||||
"""
|
||||
Explicit reasoning_effort "none" is OpenAI's documented escape hatch that keeps
|
||||
|
|
@ -2833,9 +2863,17 @@ def _priced_at(prompt_tokens, completion_tokens):
|
|||
@pytest.fixture
|
||||
def local_cost_map(monkeypatch):
|
||||
"""The prices these tests assert are the checked-in ones. Setting the environment
|
||||
variable alone does not reload the map, so pin the map itself."""
|
||||
variable alone does not reload the map, so pin the map itself.
|
||||
|
||||
``get_model_info`` is lru_cached, so pinning ``model_cost`` is not enough on its
|
||||
own: a cached entry warmed against the network-fetched map keeps its old prices
|
||||
and ``completion_cost`` bills at those while the assertions read the pinned map.
|
||||
Clear on the way in and out so entries never leak across tests in either direction."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def test_a_streamed_response_bills_the_usage_the_provider_reported(local_cost_map):
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import os
|
|||
import threading
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
|
||||
|
|
@ -7304,6 +7305,78 @@ async def test_acreate_batch_surfaces_owning_provider_error_without_disable_fall
|
|||
assert attempted_models == ["owning-model"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acreate_batch_still_falls_back_within_the_owning_model_group():
|
||||
"""Holding a batch inside the model group that owns its input file must not
|
||||
disable fallbacks outright (#35359): the owning group's second deployment is
|
||||
still tried in `order`, and only the cross-group target is skipped."""
|
||||
completion_window_error = "Invalid value: '5m'. Supported values are: '24h'."
|
||||
attempted_models = []
|
||||
|
||||
async def _acreate_batch(**kwargs):
|
||||
model = kwargs["model"]
|
||||
attempted_models.append(model)
|
||||
if model.startswith("azure/"):
|
||||
raise litellm.BadRequestError(
|
||||
message="Error code: 400 - {'error': {'code': 'quotaExceeded'}}",
|
||||
model=model,
|
||||
llm_provider="azure",
|
||||
)
|
||||
raise litellm.BadRequestError(
|
||||
message=completion_window_error,
|
||||
model=model,
|
||||
llm_provider="openai",
|
||||
)
|
||||
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "my-gpt",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "sk-owning",
|
||||
"order": 1,
|
||||
},
|
||||
"model_info": {"id": "my-gpt-1"},
|
||||
},
|
||||
{
|
||||
"model_name": "my-gpt",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini-backup",
|
||||
"api_key": "sk-owning",
|
||||
"order": 2,
|
||||
},
|
||||
"model_info": {"id": "my-gpt-2"},
|
||||
},
|
||||
{
|
||||
"model_name": "my-azure-gpt",
|
||||
"litellm_params": {
|
||||
"model": "azure/gpt-4o-mini",
|
||||
"api_key": "sk-fallback",
|
||||
"api_base": "https://fallback.openai.azure.com",
|
||||
"api_version": "2024-08-01-preview",
|
||||
},
|
||||
"model_info": {"id": "my-azure-gpt-1"},
|
||||
},
|
||||
],
|
||||
fallbacks=[{"my-gpt": ["my-azure-gpt"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
with patch.object(litellm, "acreate_batch", new=_acreate_batch):
|
||||
with pytest.raises(litellm.BadRequestError) as raised:
|
||||
await router.acreate_batch(
|
||||
model="my-gpt",
|
||||
input_file_id="file-owned-by-my-gpt",
|
||||
endpoint="/v1/chat/completions",
|
||||
completion_window="5m",
|
||||
)
|
||||
|
||||
assert "24h" in str(raised.value)
|
||||
assert "quotaExceeded" not in str(raised.value)
|
||||
assert attempted_models == ["openai/gpt-4o-mini", "openai/gpt-4o-mini-backup"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acreate_batch_request_bedrock_tags_override_deployment_tags():
|
||||
import httpx
|
||||
|
|
@ -8264,6 +8337,94 @@ class TestTaggedAutoRouterOnSharedModelName:
|
|||
assert marker_only._model_name_has_plain_deployments("gpt4o") is False
|
||||
|
||||
|
||||
class TestAutoRouterSharedModelNameConnectionParams:
|
||||
"""A plain deployment sharing its model_name with an `auto_router/` marker must not have
|
||||
its api_base and api_key grafted onto the routed tier's outbound call (#36619)."""
|
||||
|
||||
PLAIN_API_BASE = "https://plain-sibling.openai.example/v1"
|
||||
PLAIN_API_KEY = "sk-plain-sibling-secret"
|
||||
|
||||
class _FixedRouteLayer:
|
||||
def __call__(self, text: str):
|
||||
from semantic_router.schema import RouteChoice
|
||||
|
||||
return RouteChoice(name="gemini-flash")
|
||||
|
||||
@classmethod
|
||||
def _router(cls, plain_entry_first: bool) -> "litellm.Router":
|
||||
pytest.importorskip("semantic_router", reason="auto-router needs the semantic-router extra")
|
||||
plain = {
|
||||
"model_name": "gpt4o",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": cls.PLAIN_API_KEY,
|
||||
"api_base": cls.PLAIN_API_BASE,
|
||||
},
|
||||
}
|
||||
marker = {
|
||||
"model_name": "gpt4o",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/gpt4o-router",
|
||||
"auto_router_config": json.dumps(
|
||||
{"routes": [{"name": "gemini-flash", "utterances": ["capital city questions"]}]}
|
||||
),
|
||||
"auto_router_default_model": "gemini-flash",
|
||||
"auto_router_embedding_model": "text-embedding-3-small",
|
||||
"drop_params": True,
|
||||
},
|
||||
}
|
||||
tier = {
|
||||
"model_name": "gemini-flash",
|
||||
"litellm_params": {"model": "gemini/gemini-3.6-flash", "api_key": "sk-tier-key"},
|
||||
}
|
||||
shared_name_entries = [plain, marker] if plain_entry_first else [marker, plain]
|
||||
router = litellm.Router(model_list=[*shared_name_entries, tier])
|
||||
router.auto_routers["gpt4o"][0].strategy.routelayer = cls._FixedRouteLayer()
|
||||
return router
|
||||
|
||||
@staticmethod
|
||||
def _gemini_response() -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"candidates": [
|
||||
{"content": {"parts": [{"text": "Paris"}], "role": "model"}, "finishReason": "STOP"}
|
||||
],
|
||||
"usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 1, "totalTokenCount": 6},
|
||||
"modelVersion": "gemini-3.6-flash",
|
||||
},
|
||||
request=httpx.Request("POST", "https://generativelanguage.googleapis.com"),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plain_entry_first", [True, False], ids=["plain_entry_first", "marker_entry_first"]
|
||||
)
|
||||
async def test_routed_tier_call_goes_out_on_its_own_endpoint_and_credentials(self, plain_entry_first):
|
||||
"""The outbound provider request for the routed tier hits the tier's own Gemini host
|
||||
with the tier's own key, never the plain sibling's api_base or api_key."""
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
||||
router = self._router(plain_entry_first)
|
||||
|
||||
with patch.object(
|
||||
AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=self._gemini_response()
|
||||
) as mock_post:
|
||||
await router.acompletion(
|
||||
model="gpt4o",
|
||||
messages=[{"role": "user", "content": "What is the capital of France?"}],
|
||||
)
|
||||
|
||||
call = mock_post.call_args
|
||||
outbound_url = str(call.kwargs["url"] if "url" in call.kwargs else call.args[0])
|
||||
outbound_headers = dict(call.kwargs.get("headers") or {})
|
||||
|
||||
assert "generativelanguage.googleapis.com" in outbound_url
|
||||
assert "gemini-3.6-flash" in outbound_url
|
||||
assert self.PLAIN_API_BASE not in outbound_url
|
||||
assert self.PLAIN_API_KEY not in outbound_url
|
||||
assert self.PLAIN_API_KEY not in json.dumps(outbound_headers)
|
||||
|
||||
|
||||
class TestGetAllowedFailsFromPolicy:
|
||||
def _make_router(self, **policy_kwargs) -> litellm.Router:
|
||||
from litellm.types.router import AllowedFailsPolicy
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue