mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-15 23:31:29 +00:00
test(e2e): source the cheap Anthropic model from e2e_config across suites
This commit is contained in:
parent
36dff0f63a
commit
e44aa07a8a
16 changed files with 38 additions and 36 deletions
|
|
@ -13,7 +13,7 @@ from __future__ import annotations
|
|||
|
||||
import pytest
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import unwrap
|
||||
from models import ChatBody, ChatMessage
|
||||
from passthrough_client import PassthroughClient
|
||||
|
|
@ -22,7 +22,7 @@ pytestmark = pytest.mark.e2e
|
|||
|
||||
CHAT_MODELS: tuple[tuple[str, str], ...] = (
|
||||
("gpt-5.5", "openai"),
|
||||
("claude-haiku-4-5", "anthropic"),
|
||||
(CHEAP_ANTHROPIC_MODEL, "anthropic"),
|
||||
("gemini-2.5-flash", "gemini"),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ A passthrough call returning non-2xx fails hard (never a skip); once it returns
|
|||
|
||||
import pytest
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import StreamingResponse, require_successful_call
|
||||
from models import SpendLogRow
|
||||
from passthrough_client import (
|
||||
|
|
@ -112,7 +112,7 @@ def test_gemini_passthrough_tool_call_logs_cost(
|
|||
def test_anthropic_passthrough_nonstreaming_logs_cost(
|
||||
client: PassthroughClient, scoped_key: str
|
||||
) -> None:
|
||||
result = client.anthropic_message(scoped_key, "claude-haiku-4-5", "Say hello")
|
||||
result = client.anthropic_message(scoped_key, CHEAP_ANTHROPIC_MODEL, "Say hello")
|
||||
require_successful_call(result)
|
||||
|
||||
row = _fetch_cost_breakdown(client, result)
|
||||
|
|
@ -124,7 +124,7 @@ def test_anthropic_passthrough_streaming_logs_cost(
|
|||
client: PassthroughClient, scoped_key: str
|
||||
) -> None:
|
||||
result = client.anthropic_message(
|
||||
scoped_key, "claude-haiku-4-5", "Count to five", stream=True
|
||||
scoped_key, CHEAP_ANTHROPIC_MODEL, "Count to five", stream=True
|
||||
)
|
||||
require_successful_call(result)
|
||||
assert result.chunks > 0, "streaming passthrough produced no events"
|
||||
|
|
@ -138,7 +138,7 @@ def test_anthropic_passthrough_tool_call_logs_cost(
|
|||
) -> None:
|
||||
result = client.anthropic_message(
|
||||
scoped_key,
|
||||
"claude-haiku-4-5",
|
||||
CHEAP_ANTHROPIC_MODEL,
|
||||
"What is the weather in Paris? Use the get_weather tool.",
|
||||
tools=[
|
||||
AnthropicTool(
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from typing import Callable, List, Type
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from lifecycle import run_case
|
||||
|
||||
|
|
@ -31,7 +31,7 @@ def _assert_budget_blocks(client: BudgetClient, key: str, *, user: str = "") ->
|
|||
for _ in range(40):
|
||||
result = client.chat(
|
||||
key,
|
||||
"claude-haiku-4-5",
|
||||
CHEAP_ANTHROPIC_MODEL,
|
||||
f"spend {unique_marker()}",
|
||||
max_tokens=16,
|
||||
user=user or None,
|
||||
|
|
@ -88,7 +88,7 @@ class EndUserBudgetCase(_BudgetCase):
|
|||
customer = f"e2e-budget-cust-{unique_marker()}"
|
||||
self.client.create_customer(customer, max_budget=3e-6)
|
||||
self._undo.append(lambda: self.client.delete_customers([customer]))
|
||||
self.key = self.client.generate_key(models=["claude-haiku-4-5"])
|
||||
self.key = self.client.generate_key(models=[CHEAP_ANTHROPIC_MODEL])
|
||||
self._undo.append(lambda: self.client.delete_key(self.key))
|
||||
self._customer = customer
|
||||
|
||||
|
|
|
|||
|
|
@ -9,13 +9,13 @@ import time
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient, model_budget
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from lifecycle import ResourceManager
|
||||
from models import AnthropicMessagesResponse
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
PRIMARY_MODEL = "claude-haiku-4-5"
|
||||
PRIMARY_MODEL = CHEAP_ANTHROPIC_MODEL
|
||||
FALLBACK_MODEL = "gpt-5.5"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from datetime import datetime
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from lifecycle import ResourceManager
|
||||
from models import BudgetWindow
|
||||
|
|
@ -32,7 +32,7 @@ TINY_CAP = 3e-6
|
|||
|
||||
|
||||
def _call(client: BudgetClient, key: str):
|
||||
return client.chat(key, "claude-haiku-4-5", f"advance {unique_marker()}", max_tokens=16)
|
||||
return client.chat(key, CHEAP_ANTHROPIC_MODEL, f"advance {unique_marker()}", max_tokens=16)
|
||||
|
||||
|
||||
def _as_datetime(value: str) -> datetime:
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import time
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
|
|
@ -22,7 +22,7 @@ pytestmark = pytest.mark.e2e
|
|||
|
||||
def _call(client: BudgetClient, key: str):
|
||||
return client.chat(
|
||||
key, "claude-haiku-4-5", f"reset {unique_marker()}", max_tokens=16
|
||||
key, CHEAP_ANTHROPIC_MODEL, f"reset {unique_marker()}", max_tokens=16
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,13 +11,13 @@ import time
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient, is_budget_block, model_budget
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
CAPPED_MODEL = "claude-haiku-4-5"
|
||||
CAPPED_MODEL = CHEAP_ANTHROPIC_MODEL
|
||||
FREE_MODEL = "gemini-2.5-flash"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import time
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from lifecycle import ResourceManager
|
||||
from models import BudgetWindow
|
||||
|
|
@ -25,7 +25,7 @@ WINDOW_SECONDS = 30 # the tight window; calls succeed again only after it elaps
|
|||
|
||||
def _call(client: BudgetClient, key: str):
|
||||
return client.chat(
|
||||
key, "claude-haiku-4-5", f"window {unique_marker()}", max_tokens=16
|
||||
key, CHEAP_ANTHROPIC_MODEL, f"window {unique_marker()}", max_tokens=16
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ load-bearing behavior: soft != block.
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
|
|
@ -27,7 +27,7 @@ def test_soft_budget_does_not_block(
|
|||
|
||||
for _ in range(3):
|
||||
result = client.chat(
|
||||
key, "claude-haiku-4-5", f"hi {unique_marker()}", max_tokens=16
|
||||
key, CHEAP_ANTHROPIC_MODEL, f"hi {unique_marker()}", max_tokens=16
|
||||
)
|
||||
assert not is_budget_block(result), (
|
||||
"soft_budget blocked a request; it must alert only, not block "
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ import pytest
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from budget_client import BudgetClient
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import StreamingResponse
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
|
|
@ -36,7 +36,7 @@ if TYPE_CHECKING:
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
MODEL = "claude-haiku-4-5"
|
||||
MODEL = CHEAP_ANTHROPIC_MODEL
|
||||
ACCUMULATE_CALLS = 24
|
||||
BURST = 6
|
||||
# proxy_batch_write_at (60s) flushes the spend to the DB and default_redis_ttl (20s)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import time
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
|
|
@ -23,7 +23,7 @@ TINY_BUDGET = 1e-6
|
|||
def _tagged_call(client: BudgetClient, key: str, tag: str):
|
||||
result = client.chat(
|
||||
key,
|
||||
"claude-haiku-4-5",
|
||||
CHEAP_ANTHROPIC_MODEL,
|
||||
f"hi {unique_marker()}",
|
||||
tags=[tag],
|
||||
max_tokens=64,
|
||||
|
|
|
|||
|
|
@ -19,14 +19,14 @@ from dataclasses import dataclass
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import Success, require_successful_call
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
MODEL = "claude-haiku-4-5"
|
||||
MODEL = CHEAP_ANTHROPIC_MODEL
|
||||
TEAM_BUDGET = 100.0
|
||||
MEMBER_BUDGET = 3e-6
|
||||
BURST = 6
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from datetime import datetime
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
|
|
@ -34,7 +34,7 @@ def test_team_member_budget_reset_keeps_advancing(client: BudgetClient, resource
|
|||
# the member can spend within the team while the window is live
|
||||
key = client.generate_key(team_id=team_id, user_id=user_id)
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
require_successful_call(client.chat(key, "claude-haiku-4-5", f"reset {unique_marker()}", max_tokens=16))
|
||||
require_successful_call(client.chat(key, CHEAP_ANTHROPIC_MODEL, f"reset {unique_marker()}", max_tokens=16))
|
||||
|
||||
# once the window elapses the reset job must move budget_reset_at forward; a job
|
||||
# that skips the member's budget row (the #25109 regression) leaves it pinned at
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import time
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from lifecycle import ResourceManager
|
||||
from models import BudgetWindow
|
||||
|
|
@ -29,7 +29,7 @@ WINDOW_SECONDS = 30
|
|||
|
||||
|
||||
def _call(client: BudgetClient, key: str):
|
||||
return client.chat(key, "claude-haiku-4-5", f"team-window {unique_marker()}", max_tokens=16)
|
||||
return client.chat(key, CHEAP_ANTHROPIC_MODEL, f"team-window {unique_marker()}", max_tokens=16)
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.team_multi_window.blocks_then_resets")
|
||||
|
|
@ -42,7 +42,7 @@ def test_team_short_window_blocks_then_resets(client: BudgetClient, resources: R
|
|||
],
|
||||
)
|
||||
resources.defer(lambda: client.delete_team(team_id))
|
||||
key = client.generate_key(team_id=team_id, models=["claude-haiku-4-5"])
|
||||
key = client.generate_key(team_id=team_id, models=[CHEAP_ANTHROPIC_MODEL])
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
||||
# 1. exhaust the tight window -> litellm returns budget_exceeded
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from typing import Iterator
|
|||
|
||||
import pytest
|
||||
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL
|
||||
from models import LiteLLMParamsBody
|
||||
from spend_e2e_client import SpendClient, build_client
|
||||
|
||||
|
|
@ -32,7 +33,7 @@ def _driver_params(provider_model: str, env_var: str) -> LiteLLMParamsBody:
|
|||
|
||||
DRIVER_MODELS: tuple[tuple[str, str, str], ...] = (
|
||||
("gemini-2.5-flash", "gemini/gemini-2.5-flash", "GEMINI_API_KEY"),
|
||||
("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY"),
|
||||
(CHEAP_ANTHROPIC_MODEL, "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY"),
|
||||
("openai-text-embedding-3-small", "openai/text-embedding-3-small", "OPENAI_API_KEY"),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from concurrent.futures import ThreadPoolExecutor
|
|||
|
||||
import pytest
|
||||
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL
|
||||
from e2e_http import Result, Success
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatResponse, SpendLogs, SpendLogsParams
|
||||
|
|
@ -397,14 +398,14 @@ def test_each_model_on_a_shared_key_gets_its_own_row(
|
|||
)
|
||||
claude = unwrap(
|
||||
client.chat(
|
||||
scoped_key, "claude-haiku-4-5", f"one word {unique_marker()}", max_tokens=16
|
||||
scoped_key, CHEAP_ANTHROPIC_MODEL, f"one word {unique_marker()}", max_tokens=16
|
||||
)
|
||||
)
|
||||
|
||||
def both_models_costed(rows: list[SpendLogRow]) -> bool:
|
||||
costed = [r.model or "" for r in rows if (r.spend or 0) > 0]
|
||||
return any("gemini-2.5-flash" in m for m in costed) and any(
|
||||
"claude-haiku-4-5" in m for m in costed
|
||||
CHEAP_ANTHROPIC_MODEL in m for m in costed
|
||||
)
|
||||
|
||||
rows = client.poll_logs_for_key(scoped_key, min_rows=2, predicate=both_models_costed)
|
||||
|
|
@ -412,7 +413,7 @@ def test_each_model_on_a_shared_key_gets_its_own_row(
|
|||
rows, lambda r: "gemini-2.5-flash" in (r.model or ""), "for the gemini call"
|
||||
)
|
||||
claude_row = _require_row(
|
||||
rows, lambda r: "claude-haiku-4-5" in (r.model or ""), "for the claude call"
|
||||
rows, lambda r: CHEAP_ANTHROPIC_MODEL in (r.model or ""), "for the claude call"
|
||||
)
|
||||
|
||||
assert (gemini_row.spend or 0) > 0, f"gemini row should cost > 0: {_summarize(rows)}"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue