test(e2e): source the cheap Anthropic model from e2e_config across suites

This commit is contained in:
mateo-berri 2026-07-11 17:04:27 -07:00
parent 36dff0f63a
commit e44aa07a8a
16 changed files with 38 additions and 36 deletions

View file

@ -13,7 +13,7 @@ from __future__ import annotations
import pytest
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import unwrap
from models import ChatBody, ChatMessage
from passthrough_client import PassthroughClient
@ -22,7 +22,7 @@ pytestmark = pytest.mark.e2e
CHAT_MODELS: tuple[tuple[str, str], ...] = (
("gpt-5.5", "openai"),
("claude-haiku-4-5", "anthropic"),
(CHEAP_ANTHROPIC_MODEL, "anthropic"),
("gemini-2.5-flash", "gemini"),
)

View file

@ -13,7 +13,7 @@ A passthrough call returning non-2xx fails hard (never a skip); once it returns
import pytest
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import StreamingResponse, require_successful_call
from models import SpendLogRow
from passthrough_client import (
@ -112,7 +112,7 @@ def test_gemini_passthrough_tool_call_logs_cost(
def test_anthropic_passthrough_nonstreaming_logs_cost(
client: PassthroughClient, scoped_key: str
) -> None:
result = client.anthropic_message(scoped_key, "claude-haiku-4-5", "Say hello")
result = client.anthropic_message(scoped_key, CHEAP_ANTHROPIC_MODEL, "Say hello")
require_successful_call(result)
row = _fetch_cost_breakdown(client, result)
@ -124,7 +124,7 @@ def test_anthropic_passthrough_streaming_logs_cost(
client: PassthroughClient, scoped_key: str
) -> None:
result = client.anthropic_message(
scoped_key, "claude-haiku-4-5", "Count to five", stream=True
scoped_key, CHEAP_ANTHROPIC_MODEL, "Count to five", stream=True
)
require_successful_call(result)
assert result.chunks > 0, "streaming passthrough produced no events"
@ -138,7 +138,7 @@ def test_anthropic_passthrough_tool_call_logs_cost(
) -> None:
result = client.anthropic_message(
scoped_key,
"claude-haiku-4-5",
CHEAP_ANTHROPIC_MODEL,
"What is the weather in Paris? Use the get_weather tool.",
tools=[
AnthropicTool(

View file

@ -17,7 +17,7 @@ from typing import Callable, List, Type
import pytest
from budget_client import BudgetClient, is_budget_block
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import require_successful_call
from lifecycle import run_case
@ -31,7 +31,7 @@ def _assert_budget_blocks(client: BudgetClient, key: str, *, user: str = "") ->
for _ in range(40):
result = client.chat(
key,
"claude-haiku-4-5",
CHEAP_ANTHROPIC_MODEL,
f"spend {unique_marker()}",
max_tokens=16,
user=user or None,
@ -88,7 +88,7 @@ class EndUserBudgetCase(_BudgetCase):
customer = f"e2e-budget-cust-{unique_marker()}"
self.client.create_customer(customer, max_budget=3e-6)
self._undo.append(lambda: self.client.delete_customers([customer]))
self.key = self.client.generate_key(models=["claude-haiku-4-5"])
self.key = self.client.generate_key(models=[CHEAP_ANTHROPIC_MODEL])
self._undo.append(lambda: self.client.delete_key(self.key))
self._customer = customer

View file

@ -9,13 +9,13 @@ import time
import pytest
from budget_client import BudgetClient, model_budget
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from lifecycle import ResourceManager
from models import AnthropicMessagesResponse
pytestmark = pytest.mark.e2e
PRIMARY_MODEL = "claude-haiku-4-5"
PRIMARY_MODEL = CHEAP_ANTHROPIC_MODEL
FALLBACK_MODEL = "gpt-5.5"

View file

@ -19,7 +19,7 @@ from datetime import datetime
import pytest
from budget_client import BudgetClient, is_budget_block
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import require_successful_call
from lifecycle import ResourceManager
from models import BudgetWindow
@ -32,7 +32,7 @@ TINY_CAP = 3e-6
def _call(client: BudgetClient, key: str):
return client.chat(key, "claude-haiku-4-5", f"advance {unique_marker()}", max_tokens=16)
return client.chat(key, CHEAP_ANTHROPIC_MODEL, f"advance {unique_marker()}", max_tokens=16)
def _as_datetime(value: str) -> datetime:

View file

@ -13,7 +13,7 @@ import time
import pytest
from budget_client import BudgetClient, is_budget_block
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import require_successful_call
from lifecycle import ResourceManager
@ -22,7 +22,7 @@ pytestmark = pytest.mark.e2e
def _call(client: BudgetClient, key: str):
return client.chat(
key, "claude-haiku-4-5", f"reset {unique_marker()}", max_tokens=16
key, CHEAP_ANTHROPIC_MODEL, f"reset {unique_marker()}", max_tokens=16
)

View file

@ -11,13 +11,13 @@ import time
import pytest
from budget_client import BudgetClient, is_budget_block, model_budget
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import require_successful_call
from lifecycle import ResourceManager
pytestmark = pytest.mark.e2e
CAPPED_MODEL = "claude-haiku-4-5"
CAPPED_MODEL = CHEAP_ANTHROPIC_MODEL
FREE_MODEL = "gemini-2.5-flash"

View file

@ -13,7 +13,7 @@ import time
import pytest
from budget_client import BudgetClient, is_budget_block
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import require_successful_call
from lifecycle import ResourceManager
from models import BudgetWindow
@ -25,7 +25,7 @@ WINDOW_SECONDS = 30 # the tight window; calls succeed again only after it elaps
def _call(client: BudgetClient, key: str):
return client.chat(
key, "claude-haiku-4-5", f"window {unique_marker()}", max_tokens=16
key, CHEAP_ANTHROPIC_MODEL, f"window {unique_marker()}", max_tokens=16
)

View file

@ -10,7 +10,7 @@ load-bearing behavior: soft != block.
import pytest
from budget_client import BudgetClient, is_budget_block
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import require_successful_call
from lifecycle import ResourceManager
@ -27,7 +27,7 @@ def test_soft_budget_does_not_block(
for _ in range(3):
result = client.chat(
key, "claude-haiku-4-5", f"hi {unique_marker()}", max_tokens=16
key, CHEAP_ANTHROPIC_MODEL, f"hi {unique_marker()}", max_tokens=16
)
assert not is_budget_block(result), (
"soft_budget blocked a request; it must alert only, not block "

View file

@ -26,7 +26,7 @@ import pytest
from pydantic import TypeAdapter, ValidationError
from budget_client import BudgetClient
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import StreamingResponse
from lifecycle import ResourceManager
@ -36,7 +36,7 @@ if TYPE_CHECKING:
pytestmark = pytest.mark.e2e
MODEL = "claude-haiku-4-5"
MODEL = CHEAP_ANTHROPIC_MODEL
ACCUMULATE_CALLS = 24
BURST = 6
# proxy_batch_write_at (60s) flushes the spend to the DB and default_redis_ttl (20s)

View file

@ -11,7 +11,7 @@ import time
import pytest
from budget_client import BudgetClient, is_budget_block
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import require_successful_call
from lifecycle import ResourceManager
@ -23,7 +23,7 @@ TINY_BUDGET = 1e-6
def _tagged_call(client: BudgetClient, key: str, tag: str):
result = client.chat(
key,
"claude-haiku-4-5",
CHEAP_ANTHROPIC_MODEL,
f"hi {unique_marker()}",
tags=[tag],
max_tokens=64,

View file

@ -19,14 +19,14 @@ from dataclasses import dataclass
import pytest
from budget_client import BudgetClient, is_budget_block
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import Success, require_successful_call
from lifecycle import ResourceManager
from models import ChatBody, ChatMessage
pytestmark = pytest.mark.e2e
MODEL = "claude-haiku-4-5"
MODEL = CHEAP_ANTHROPIC_MODEL
TEAM_BUDGET = 100.0
MEMBER_BUDGET = 3e-6
BURST = 6

View file

@ -4,7 +4,7 @@ from datetime import datetime
import pytest
from budget_client import BudgetClient
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import require_successful_call
from lifecycle import ResourceManager
@ -34,7 +34,7 @@ def test_team_member_budget_reset_keeps_advancing(client: BudgetClient, resource
# the member can spend within the team while the window is live
key = client.generate_key(team_id=team_id, user_id=user_id)
resources.defer(lambda: client.delete_key(key))
require_successful_call(client.chat(key, "claude-haiku-4-5", f"reset {unique_marker()}", max_tokens=16))
require_successful_call(client.chat(key, CHEAP_ANTHROPIC_MODEL, f"reset {unique_marker()}", max_tokens=16))
# once the window elapses the reset job must move budget_reset_at forward; a job
# that skips the member's budget row (the #25109 regression) leaves it pinned at

View file

@ -18,7 +18,7 @@ import time
import pytest
from budget_client import BudgetClient, is_budget_block
from e2e_config import unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import require_successful_call
from lifecycle import ResourceManager
from models import BudgetWindow
@ -29,7 +29,7 @@ WINDOW_SECONDS = 30
def _call(client: BudgetClient, key: str):
return client.chat(key, "claude-haiku-4-5", f"team-window {unique_marker()}", max_tokens=16)
return client.chat(key, CHEAP_ANTHROPIC_MODEL, f"team-window {unique_marker()}", max_tokens=16)
@pytest.mark.covers("quota_management.budget.team_multi_window.blocks_then_resets")
@ -42,7 +42,7 @@ def test_team_short_window_blocks_then_resets(client: BudgetClient, resources: R
],
)
resources.defer(lambda: client.delete_team(team_id))
key = client.generate_key(team_id=team_id, models=["claude-haiku-4-5"])
key = client.generate_key(team_id=team_id, models=[CHEAP_ANTHROPIC_MODEL])
resources.defer(lambda: client.delete_key(key))
# 1. exhaust the tight window -> litellm returns budget_exceeded

View file

@ -19,6 +19,7 @@ from typing import Iterator
import pytest
from e2e_config import CHEAP_ANTHROPIC_MODEL
from models import LiteLLMParamsBody
from spend_e2e_client import SpendClient, build_client
@ -32,7 +33,7 @@ def _driver_params(provider_model: str, env_var: str) -> LiteLLMParamsBody:
DRIVER_MODELS: tuple[tuple[str, str, str], ...] = (
("gemini-2.5-flash", "gemini/gemini-2.5-flash", "GEMINI_API_KEY"),
("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY"),
(CHEAP_ANTHROPIC_MODEL, "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY"),
("openai-text-embedding-3-small", "openai/text-embedding-3-small", "OPENAI_API_KEY"),
)

View file

@ -21,6 +21,7 @@ from concurrent.futures import ThreadPoolExecutor
import pytest
from e2e_config import CHEAP_ANTHROPIC_MODEL
from e2e_http import Result, Success
from lifecycle import ResourceManager
from models import ChatResponse, SpendLogs, SpendLogsParams
@ -397,14 +398,14 @@ def test_each_model_on_a_shared_key_gets_its_own_row(
)
claude = unwrap(
client.chat(
scoped_key, "claude-haiku-4-5", f"one word {unique_marker()}", max_tokens=16
scoped_key, CHEAP_ANTHROPIC_MODEL, f"one word {unique_marker()}", max_tokens=16
)
)
def both_models_costed(rows: list[SpendLogRow]) -> bool:
costed = [r.model or "" for r in rows if (r.spend or 0) > 0]
return any("gemini-2.5-flash" in m for m in costed) and any(
"claude-haiku-4-5" in m for m in costed
CHEAP_ANTHROPIC_MODEL in m for m in costed
)
rows = client.poll_logs_for_key(scoped_key, min_rows=2, predicate=both_models_costed)
@ -412,7 +413,7 @@ def test_each_model_on_a_shared_key_gets_its_own_row(
rows, lambda r: "gemini-2.5-flash" in (r.model or ""), "for the gemini call"
)
claude_row = _require_row(
rows, lambda r: "claude-haiku-4-5" in (r.model or ""), "for the claude call"
rows, lambda r: CHEAP_ANTHROPIC_MODEL in (r.model or ""), "for the claude call"
)
assert (gemini_row.spend or 0) > 0, f"gemini row should cost > 0: {_summarize(rows)}"