mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
Give e2e tests typed metadata for what they drive
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
`@pytest.mark.covers("cell.id")` is a registry key, not a description: it cannot answer "which tests drive /v1/responses on Anthropic". This adds the declared half of the per-test metadata, on top of the recorded step log
`@meta(Subject(...))` from tests/e2e/e2e_metadata.py takes one frozen dataclass whose fields are closed enums (domain, route, providers, capabilities, mode) plus free-string models, so a typo is a basedpyright error at the call site rather than a property that silently never appears. providers, models and capabilities are tuples because one test node often drives several (the claude_code matrix runs haiku, sonnet and opus in one body), with no positional pairing between them. Each is deduped and sorted at declaration so committed run artifacts diff cleanly, and anything but a tuple is refused at import, so models=("gpt-5.5") is a collection error naming the file instead of one model per character
Serialization is one pass over dataclasses.asdict: each scalar is one <property> under its field name, each plural value a repeated property under its singular name (provider, model, capability). Empty fields emit nothing, and the fixed package/covers/source prefix stays byte-identical, with the declared fields appended behind it. `covers` is untouched: the marker is separate because a dataclass passed to covers would be dropped silently by dedupe_covers and hard-fail collection in tests/integration/conftest.py, and @meta goes below @covers so every source deep link keeps its line
Provider mirrors litellm's LlmProviders values instead of importing them, because tests/e2e is shipped to the runner image on its own and a module-scope `from litellm...` would make the package a collection-time dependency. TestProviderMirrorsLitellm fails on drift wherever litellm is importable
tests/e2e/quota_management/ (29 files) is annotated as the pilot, and tests that drive more than one provider or model declare all of them. Every field is optional until the backfill of the rest of the suite lands
Its harness tests sit beside the step log's in tests/code_coverage_tests/test_e2e_metadata.py and test_e2e_junit_report.py, since tests/e2e holds only tests that drive a live proxy
This commit is contained in:
parent
a49940ccf3
commit
11dbe70092
36 changed files with 1439 additions and 17 deletions
|
|
@ -37,7 +37,7 @@ from collections.abc import Iterator
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from e2e_metadata import step
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta, step
|
||||
|
||||
FIRST_ATTEMPT_MADE = Path(__file__).with_name("first-attempt-made")
|
||||
|
||||
|
|
@ -99,6 +99,20 @@ def test_passes_on_the_rerun(key: None) -> None:
|
|||
FIRST_ATTEMPT_MADE.touch()
|
||||
chat(ok=not first_attempt)
|
||||
poll_spend_logs()
|
||||
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.MESSAGES,
|
||||
providers=(Provider.BEDROCK, Provider.ANTHROPIC),
|
||||
models=("claude-sonnet-4-5", "claude-opus-4-7", "claude-haiku-4-5"),
|
||||
capabilities=(Capability.VISION, Capability.FUNCTION_CALLING),
|
||||
mode=Mode.STREAM,
|
||||
)
|
||||
)
|
||||
def test_declares_two_providers_and_three_models() -> None:
|
||||
assert Provider.BEDROCK.value == "bedrock"
|
||||
"""
|
||||
|
||||
WIDE_FINALIZER_SUITE: Final = """
|
||||
|
|
@ -147,6 +161,15 @@ def test_dies_in_a_module_scoped_fixture(identity: None) -> None:
|
|||
assert identity is None
|
||||
"""
|
||||
|
||||
BARE_STR_SUITE: Final = """
|
||||
from e2e_metadata import Subject, meta
|
||||
|
||||
|
||||
@meta(Subject(models=("gpt-5.5")))
|
||||
def test_never_collected() -> None:
|
||||
assert Subject is not None
|
||||
"""
|
||||
|
||||
Properties = tuple[tuple[str, str], ...]
|
||||
|
||||
|
||||
|
|
@ -238,7 +261,7 @@ def report(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFact
|
|||
assert xml.exists(), f"the child run wrote no JUnit report:\n{child.stdout}\n{child.stderr}"
|
||||
testsuite: Final = next(ElementTree.parse(xml).getroot().iter("testsuite"))
|
||||
outcomes: Final = {name: testsuite.get(name) for name in ("tests", "failures", "errors", "skipped")}
|
||||
assert outcomes == {"tests": "6", "failures": "1", "errors": "2", "skipped": "0"}, child.stdout
|
||||
assert outcomes == {"tests": "7", "failures": "1", "errors": "2", "skipped": "0"}, child.stdout
|
||||
return properties_by_test(testsuite)
|
||||
|
||||
|
||||
|
|
@ -286,3 +309,38 @@ class TestStepsReachTheReport:
|
|||
already read, on every outcome including a setup error."""
|
||||
for name in ("test_passes", "test_fails", "test_errors_in_setup"):
|
||||
assert tuple(prop for prop, _ in report[name])[:4] == ("package", "covers", "source", "step"), name
|
||||
|
||||
|
||||
class TestDeclaredPropertiesReachTheReport:
|
||||
def test_repeated_provider_model_and_capability_round_trip(self, report: Mapping[str, Properties]) -> None:
|
||||
"""One <property> per member under the SINGULAR name, deduped and sorted,
|
||||
with no pairing between the two providers and the three models."""
|
||||
declared: Final = tuple(
|
||||
(prop, value)
|
||||
for prop, value in report["test_declares_two_providers_and_three_models"]
|
||||
if prop not in {"package", "covers", "source"}
|
||||
)
|
||||
assert declared == (
|
||||
("domain", "llm-translation"),
|
||||
("route", "messages"),
|
||||
("provider", "anthropic"),
|
||||
("provider", "bedrock"),
|
||||
("model", "claude-haiku-4-5"),
|
||||
("model", "claude-opus-4-7"),
|
||||
("model", "claude-sonnet-4-5"),
|
||||
("capability", "function_calling"),
|
||||
("capability", "vision"),
|
||||
("mode", "stream"),
|
||||
)
|
||||
|
||||
|
||||
class TestBareStrIsACollectionError:
|
||||
def test_a_str_where_a_tuple_belongs_fails_collection_and_names_the_fix(self, tmp_path: Path) -> None:
|
||||
"""`models=("gpt-5.5")` raises where the decorator runs, which is import, so
|
||||
pytest stops at collection and points at the file. Nothing is run and no
|
||||
one-letter `model` properties are ever shipped."""
|
||||
write_suite(tmp_path, {"test_bare_str.py": BARE_STR_SUITE})
|
||||
child: Final = run_child_pytest(tmp_path)
|
||||
assert child.returncode == pytest.ExitCode.INTERRUPTED, child.stdout
|
||||
assert "Subject.models must be a tuple, got str: 'gpt-5.5'" in child.stdout
|
||||
assert "models=(x,), not models=(x)" in child.stdout
|
||||
|
|
|
|||
|
|
@ -13,11 +13,32 @@ import threading
|
|||
import warnings
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import fields, replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from e2e_metadata import MAX_STEPS, STEP_FRAMES, STEPS, step, step_properties
|
||||
from junit_properties import attach_result_properties, attach_step_properties
|
||||
from e2e_metadata import (
|
||||
MAX_STEPS,
|
||||
STEP_FRAMES,
|
||||
STEPS,
|
||||
Capability,
|
||||
Domain,
|
||||
Mode,
|
||||
Provider,
|
||||
Route,
|
||||
Subject,
|
||||
meta,
|
||||
step,
|
||||
step_properties,
|
||||
subject_properties,
|
||||
)
|
||||
from junit_properties import (
|
||||
attach_result_properties,
|
||||
attach_step_properties,
|
||||
package_from_nodeid,
|
||||
result_properties,
|
||||
source_from_item,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -34,6 +55,216 @@ def collected_item(request: pytest.FixtureRequest, name: str) -> pytest.Item:
|
|||
return next(item for item in request.session.items if item.path == request.path and item.name == name)
|
||||
|
||||
|
||||
def fixed_prefix(item: pytest.Item, covers: str) -> tuple[tuple[str, str], ...]:
|
||||
"""The three-tuple every testcase in the suite has carried since before the
|
||||
typed marker existed. Its shape and order are spelled out rather than taken
|
||||
from `result_properties`, so a change to either fails a test instead of
|
||||
agreeing with itself. `package` and `source` are read off the item, since
|
||||
this file sits outside the suite root; tests/e2e/test_junit_properties.py
|
||||
pins their values."""
|
||||
return (
|
||||
("package", package_from_nodeid(item.nodeid)),
|
||||
("covers", covers),
|
||||
("source", source_from_item(item)),
|
||||
)
|
||||
|
||||
|
||||
class TestSubjectProperties:
|
||||
"""The declared half: `@meta(Subject(...))` -> `<property>` pairs.
|
||||
|
||||
Markers are applied at run time via `request.applymarker`, the idiom the
|
||||
`covers` tests above already use, so the coverage registry's collect-only pass
|
||||
never sees a marker that exists only to be serialized.
|
||||
"""
|
||||
|
||||
def test_every_declared_field_becomes_a_property_in_field_order(self, request: pytest.FixtureRequest) -> None:
|
||||
"""One pass over `dataclasses.asdict`: declaration order is emission order,
|
||||
a (str, Enum) member is written as its `.value` and never `str(member)`,
|
||||
and every plural field emits a repeated SINGULAR name (`provider`, `model`,
|
||||
`capability`), one <property> per member and never a delimiter-joined value."""
|
||||
test = type(self).test_every_declared_field_becomes_a_property_in_field_order
|
||||
request.applymarker(
|
||||
meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI, Provider.ANTHROPIC),
|
||||
models=("gemini-2.5-flash", "claude-haiku-4-5"),
|
||||
capabilities=(Capability.VISION, Capability.FUNCTION_CALLING, Capability.VISION),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
)
|
||||
assert subject_properties(collected_item(request, test.__name__)) == (
|
||||
("domain", "spend-budgets"),
|
||||
("route", "chat_completions"),
|
||||
("provider", "anthropic"),
|
||||
("provider", "gemini"),
|
||||
("model", "claude-haiku-4-5"),
|
||||
("model", "gemini-2.5-flash"),
|
||||
("capability", "function_calling"),
|
||||
("capability", "vision"),
|
||||
("mode", "nonstream"),
|
||||
)
|
||||
|
||||
def test_one_provider_with_three_models_pairs_nothing(self, request: pytest.FixtureRequest) -> None:
|
||||
"""The claude_code matrix shape: one test node drives haiku, sonnet and opus
|
||||
through a single provider. The two lists are independent sets, so their
|
||||
lengths need not agree and no model is tied to a provider by position."""
|
||||
test = type(self).test_one_provider_with_three_models_pairs_nothing
|
||||
request.applymarker(
|
||||
meta(
|
||||
Subject(
|
||||
providers=(Provider.BEDROCK,),
|
||||
models=("claude-sonnet-4-5", "claude-opus-4-7", "claude-haiku-4-5"),
|
||||
)
|
||||
)
|
||||
)
|
||||
assert subject_properties(collected_item(request, test.__name__)) == (
|
||||
("provider", "bedrock"),
|
||||
("model", "claude-haiku-4-5"),
|
||||
("model", "claude-opus-4-7"),
|
||||
("model", "claude-sonnet-4-5"),
|
||||
)
|
||||
|
||||
def test_an_empty_plural_field_emits_nothing(self, request: pytest.FixtureRequest) -> None:
|
||||
"""No `provider`, `model` or `capability` property at all, rather than one
|
||||
with an empty value: the emitter is what turns absence into `[]`."""
|
||||
test = type(self).test_an_empty_plural_field_emits_nothing
|
||||
request.applymarker(meta(Subject(domain=Domain.MANAGEMENT)))
|
||||
assert subject_properties(collected_item(request, test.__name__)) == (("domain", "management"),)
|
||||
|
||||
def test_scalar_property_names_are_the_dataclass_field_names(self, request: pytest.FixtureRequest) -> None:
|
||||
"""The mapping is `asdict`, not a hand-written table: a scalar field added
|
||||
to `Subject` later serializes under its own name with no edit to the
|
||||
serializer. Proven by reading the field list back off the dataclass."""
|
||||
test = type(self).test_scalar_property_names_are_the_dataclass_field_names
|
||||
request.applymarker(meta(Subject(domain=Domain.UNKNOWN, route=Route.HEALTH, mode=Mode.STREAM)))
|
||||
declared = tuple(field.name for field in fields(Subject))
|
||||
emitted = tuple(name for name, _ in subject_properties(collected_item(request, test.__name__)))
|
||||
assert emitted == tuple(name for name in declared if name in {"domain", "route", "mode"})
|
||||
|
||||
def test_every_plural_field_is_deduped_and_sorted_at_declaration(self) -> None:
|
||||
"""Canonicalized in `__post_init__`, so two tests that spelled the same set
|
||||
in different orders produce byte-identical properties and the committed run
|
||||
files diff cleanly. Sorted by the value that is serialized, which for an
|
||||
enum is its `.value` and not its member name."""
|
||||
subject = Subject(
|
||||
providers=(Provider.OPENAI, Provider.ANTHROPIC, Provider.OPENAI),
|
||||
models=("gpt-5.5", "claude-haiku-4-5", "gpt-5.5"),
|
||||
capabilities=(Capability.VISION, Capability.REASONING, Capability.VISION),
|
||||
)
|
||||
assert subject.providers == (Provider.ANTHROPIC, Provider.OPENAI)
|
||||
assert subject.models == ("claude-haiku-4-5", "gpt-5.5")
|
||||
assert subject.capabilities == (Capability.REASONING, Capability.VISION)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value"),
|
||||
[
|
||||
("models", "gpt-5.5"),
|
||||
("models", ["gpt-5.5"]),
|
||||
("providers", Provider.OPENAI),
|
||||
("providers", [Provider.OPENAI]),
|
||||
("capabilities", Capability.VISION),
|
||||
("capabilities", frozenset({Capability.VISION})),
|
||||
],
|
||||
)
|
||||
def test_a_plural_field_refuses_anything_but_a_tuple(self, field: str, value: object) -> None:
|
||||
"""`models=("gpt-5.5")` is a str, not a one-member tuple: the parentheses
|
||||
do nothing without the trailing comma, and iterating the str would declare
|
||||
one model per character. basedpyright flags it at the call site; this is
|
||||
the runtime half, raised where the decorator runs, so it lands as a
|
||||
collection error naming the file. `replace` is the untyped way in, since
|
||||
the typed constructor would not let the test spell the mistake."""
|
||||
with pytest.raises(TypeError, match=rf"Subject\.{field} must be a tuple"):
|
||||
_ = replace(Subject(), **{field: value})
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "member_type"),
|
||||
[
|
||||
("providers", ("openai",), "Provider"),
|
||||
("capabilities", ("vision",), "Capability"),
|
||||
("models", (5,), "str"),
|
||||
],
|
||||
)
|
||||
def test_a_plural_field_refuses_a_member_of_the_wrong_type(
|
||||
self, field: str, value: object, member_type: str
|
||||
) -> None:
|
||||
"""A bare "openai" where `Provider.OPENAI` belongs would serialize fine
|
||||
today and stop joining the day the enum value is renamed."""
|
||||
with pytest.raises(TypeError, match=rf"Subject\.{field} takes {member_type} members"):
|
||||
_ = replace(Subject(), **{field: value})
|
||||
|
||||
def test_a_blank_model_is_dropped_rather_than_refused(self) -> None:
|
||||
"""`models` is fed from env-overridable constants. A blank override is the
|
||||
operator's mistake, and it must cost one missing property, not the
|
||||
collection of the whole module."""
|
||||
assert Subject(models=("", "gpt-5.5")).models == ("gpt-5.5",)
|
||||
|
||||
def test_the_typed_marker_only_ever_appends_to_the_fixed_prefix(self, request: pytest.FixtureRequest) -> None:
|
||||
"""Loki, Grafana and the status page read `package`/`covers`/`source`; the
|
||||
typed fields ride behind them and must not disturb them."""
|
||||
test = type(self).test_the_typed_marker_only_ever_appends_to_the_fixed_prefix
|
||||
request.applymarker(pytest.mark.covers("quota_management.budget.key.blocks_over_limit"))
|
||||
request.applymarker(meta(Subject(route=Route.SPEND_REPORTING)))
|
||||
item = collected_item(request, test.__name__)
|
||||
assert result_properties(item) == fixed_prefix(item, "quota_management.budget.key.blocks_over_limit") + (
|
||||
("route", "spend_reporting"),
|
||||
)
|
||||
|
||||
def test_a_test_with_only_the_old_string_covers_is_unchanged(self, request: pytest.FixtureRequest) -> None:
|
||||
"""The existing `@pytest.mark.covers("cell.id")` call sites keep emitting
|
||||
exactly what they emitted before the typed marker existed."""
|
||||
test = type(self).test_a_test_with_only_the_old_string_covers_is_unchanged
|
||||
request.applymarker(pytest.mark.covers("llm.responses.openai.tool_use.nonstream.works"))
|
||||
item = collected_item(request, test.__name__)
|
||||
assert result_properties(item) == fixed_prefix(item, "llm.responses.openai.tool_use.nonstream.works")
|
||||
|
||||
def test_a_test_with_neither_marker_carries_only_the_prefix(self, request: pytest.FixtureRequest) -> None:
|
||||
"""Which is every test in the suite until the backfill lands: an empty
|
||||
`covers` and no typed properties at all, never five empty ones."""
|
||||
test = type(self).test_a_test_with_neither_marker_carries_only_the_prefix
|
||||
item = collected_item(request, test.__name__)
|
||||
assert subject_properties(item) == ()
|
||||
assert result_properties(item) == fixed_prefix(item, "")
|
||||
|
||||
def test_a_marker_carrying_something_other_than_a_subject_emits_nothing(
|
||||
self, request: pytest.FixtureRequest
|
||||
) -> None:
|
||||
"""`@meta` is typed, but `pytest.mark.meta` is not, and a bare
|
||||
`@pytest.mark.meta` carries no args at all. Neither may produce a property
|
||||
whose value is a repr."""
|
||||
test = type(self).test_a_marker_carrying_something_other_than_a_subject_emits_nothing
|
||||
request.applymarker(pytest.mark.meta("spend-budgets"))
|
||||
assert subject_properties(collected_item(request, test.__name__)) == ()
|
||||
|
||||
|
||||
class TestProviderMirrorsLitellm:
|
||||
"""`Provider` copies litellm's `LlmProviders` values rather than importing
|
||||
them, so collecting tests/e2e never needs the litellm package -- the suite is
|
||||
shipped to the e2e runner image on its own, and a `from litellm...` at module
|
||||
scope in a test file would turn a missing package into a collection error for
|
||||
every test rather than a slow import.
|
||||
|
||||
A copy can drift, so it is checked here, wherever litellm IS importable (a dev
|
||||
checkout, this repo's own CI). Where it is not, the whole point is that
|
||||
nothing fails, so the check skips.
|
||||
"""
|
||||
|
||||
def test_every_provider_value_is_a_real_litellm_provider(self) -> None:
|
||||
"""One direction only. litellm ships 155 providers and the e2e suite names
|
||||
a handful; a value missing from `Provider` is a line to add when a test
|
||||
needs it, but a value that is not a provider at all would ship a property
|
||||
no consumer can join on."""
|
||||
try:
|
||||
from litellm.types.utils import LlmProviders
|
||||
except ImportError: # pragma: no cover - the runner image's shape
|
||||
pytest.skip("litellm is not importable here, which is the property under test")
|
||||
known = {str(member.value) for member in LlmProviders}
|
||||
unknown = sorted(member.value for member in Provider if member.value not in known)
|
||||
assert not unknown, f"not LlmProviders values: {unknown}"
|
||||
|
||||
|
||||
class TestStepRecording:
|
||||
"""`@step`-decorated harness helpers append to the running test's story as
|
||||
they execute.
|
||||
|
|
|
|||
|
|
@ -130,6 +130,28 @@ Current limits: Bedrock cannot be mounted in record or replay (SigV4 signs the H
|
|||
|
||||
The harness is fully typed with no error budget: `make lint-e2e-basedpyright` must report zero basedpyright errors, and CI enforces that on any PR touching `tests/e2e/**/*.py`. When a response field is untyped, model it in `models.py` (just the fields you read) and let pydantic validate it, rather than threading a `dict` or `Any` through the test
|
||||
|
||||
## Typed test metadata
|
||||
|
||||
Separate from the coverage registry and additive to it: `@meta(Subject(...))` from `e2e_metadata.py` says what a test DRIVES, as closed enums rather than a string id. `@pytest.mark.covers("cell.id")` is untouched and keeps working exactly as before; the two markers coexist on the same test, and `@meta` always goes BELOW `@covers` so `Item.location` still anchors at the first decorator and every `source` deep link stays put
|
||||
|
||||
```python
|
||||
@pytest.mark.covers("quota_management.budget.key.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(CHEAP_ANTHROPIC_MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_bare_key_blocks_over_its_own_budget(...) -> None: ...
|
||||
```
|
||||
|
||||
Every field is optional today (the backfill of the rest of the suite is a later PR) and every field is a closed enum, so a typo is a basedpyright error at the call site rather than a property that silently never appears. `providers`, `models` and `capabilities` are tuples even with one member, because one test node routinely drives several: the claude_code matrix runs haiku, sonnet and opus in a single body, and a spend test calls two providers on one key. Declare every provider and every model the test drives, fallbacks included. The three are independent sets with no positional pairing between them (one provider x three models is the common case), and each is deduped and sorted at declaration so the committed run artifacts diff cleanly. `models=("gpt-5.5")` is a str and not a tuple, so anything but a tuple raises a `TypeError` where the decorator runs and shows up as a collection error naming the file. `Subject` is serialized with `dataclasses.asdict`, so a new scalar field needs no serializer edit; empty fields emit no `<property>` at all. A declared model names the constant the test drives (`CHEAP_ANTHROPIC_MODEL`, the file's own `BACKEND`), never a copy of its value, so the property cannot claim one model while an env override runs another. `e2e_metadata` is stdlib-only and so are its call sites: `Provider` mirrors litellm's `LlmProviders` values instead of importing them, because tests/e2e is shipped to the runner image on its own and a `from litellm...` at module scope would make the litellm package a hard dependency of COLLECTING the suite. `TestProviderMirrorsLitellm` in `tests/code_coverage_tests/test_e2e_metadata.py` fails on drift wherever litellm is importable and skips where it is not, so adding a provider is one line in `e2e_metadata`
|
||||
|
||||
Declared fields ride out as JUnit `<property>` entries behind the fixed prefix, the same way steps do: each scalar under its field name, and each plural value as a repeated property under its SINGULAR name (`provider`, `model`, `capability`). The results JSON downstream regroups them under the plural key, so `providers`, `models` and `capabilities` are arrays there, `[]` when empty
|
||||
|
||||
## Recorded test steps
|
||||
|
||||
`@step("POST /chat/completions")` from `e2e_metadata.py` goes on HARNESS helpers - client methods, `ResourceManager.key`, poll loops - never on a test, and appends its label to the running test's `user_properties` in call order. The list IS the test's user story, and because the label is recorded BEFORE the wrapped call, a failing test's LAST step is where it died. Nothing about steps is hand-written: the call sequence cannot drift from what the test actually did. A new public harness method that performs an action (an HTTP call, a poll, a login, a CLI run) gets a `@step`; pure builders, parsers and `_private` helpers do not. Labels are static, lowercase, one beat of the story: a plain-English action ("create team with a budget") or, for a raw route call, the route itself ("POST /v1/messages")
|
||||
|
|
|
|||
|
|
@ -110,6 +110,11 @@ def pytest_configure(config: pytest.Config) -> None:
|
|||
"markers",
|
||||
"covers(cell_id, *, exercised_on=()): coverage-registry cell(s) this test covers",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"meta(subject): typed e2e_metadata.Subject describing what this test drives"
|
||||
" (domain/route/providers/models/capabilities/mode); attach it with @meta(Subject(...))",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"replayable: edge-wired test whose provider traffic replays from a fixture bundle, so it makes "
|
||||
|
|
|
|||
|
|
@ -1,13 +1,28 @@
|
|||
"""Per-test metadata for the e2e suite: the step log each test records as it runs.
|
||||
"""Typed per-test metadata for the e2e suite: what a test drives, and what it did.
|
||||
|
||||
`steps` is appended at runtime by `@step`-decorated harness helpers, in call
|
||||
order, so the list IS the test's user story and its last element is where a
|
||||
failing test died. Nothing about it is hand-written, so it cannot drift from
|
||||
what the test actually did.
|
||||
Two halves, deliberately separated.
|
||||
|
||||
Stdlib-only on purpose. tests/e2e is a black-box HTTP suite that imports litellm
|
||||
in zero files and is shipped to the runner image as tests/e2e alone, and every
|
||||
harness module imports this one.
|
||||
The DECLARED half is `Subject`: one frozen dataclass passed as the single
|
||||
positional argument of `@meta(...)`. Every field is a closed enum (or free
|
||||
strings for `models`), so a typo is a basedpyright error at the call site rather
|
||||
than a silently dropped property. `dataclasses.asdict()` turns the whole thing
|
||||
into <property> pairs with no per-field plumbing -- adding a scalar field later
|
||||
needs zero serializer changes.
|
||||
|
||||
The RECORDED half is `steps`, and it is NOT a field of `Subject`. Steps are
|
||||
appended at runtime by `@step`-decorated harness helpers, in call order, so the
|
||||
list IS the test's user story and its last element is where a failing test died.
|
||||
Putting it on the declarable dataclass would invite hand-writing it, which is
|
||||
exactly what it replaces.
|
||||
|
||||
Stdlib-only on purpose, and that includes the call sites. tests/e2e is a
|
||||
black-box HTTP suite that imports litellm in zero files and is shipped to the
|
||||
runner image as tests/e2e alone; a `from litellm...` at the top of a test module
|
||||
would make the litellm package a COLLECTION-time dependency of the whole suite,
|
||||
so an image without it would fail collection rather than run tests. `Provider`
|
||||
below therefore mirrors litellm's `LlmProviders` values here instead of
|
||||
importing them, and `TestProviderMirrorsLitellm` in test_e2e_metadata.py
|
||||
fails wherever litellm IS importable if the two ever drift.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -17,10 +32,281 @@ import threading
|
|||
from collections import deque
|
||||
from collections.abc import Callable, Generator
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from dataclasses import asdict, dataclass
|
||||
from enum import Enum
|
||||
from functools import wraps
|
||||
from types import TracebackType
|
||||
from typing import Final, ParamSpec, TypeVar, cast
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class Domain(str, Enum):
|
||||
"""The OSS issue-label taxonomy, verbatim.
|
||||
|
||||
Shared with GitHub issue labels so an issue and a test join on one string.
|
||||
Exactly one per test; `UNKNOWN` is the honest answer, not an omission.
|
||||
"""
|
||||
|
||||
LLM_TRANSLATION = "llm-translation"
|
||||
SPEND_BUDGETS = "spend-budgets"
|
||||
UI = "ui"
|
||||
MCP = "mcp"
|
||||
OBSERVABILITY = "observability"
|
||||
ROUTING = "routing"
|
||||
DEPLOY_OPS = "deploy-ops"
|
||||
COST_MAP = "cost-map"
|
||||
PROXY_AUTH = "proxy-auth"
|
||||
GUARDRAILS = "guardrails"
|
||||
MANAGEMENT = "management"
|
||||
SDK = "sdk"
|
||||
PASSTHROUGH = "passthrough"
|
||||
DB = "db"
|
||||
CACHING = "caching"
|
||||
DOCS = "docs"
|
||||
AGENTS_API = "agents-api"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class Route(str, Enum):
|
||||
"""The customer-facing HTTP surface the test DRIVES.
|
||||
|
||||
Orthogonal to the suite directory: a reliability test in router/ and a
|
||||
logging test in logging/ both drive `CHAT_COMPLETIONS`, which is why this is
|
||||
per-test and not per-module.
|
||||
|
||||
"route" here means endpoint, matching litellm's own `LiteLLMRoutes`
|
||||
(litellm/proxy/_types.py). The coverage registry's `LlmCell.route` uses the
|
||||
same word for PROVIDER; that is a different namespace and is left alone.
|
||||
|
||||
Deliberately collapsed against the registry's `LlmEndpoint`:
|
||||
images_generations + images_edits -> IMAGES, audio_speech +
|
||||
audio_transcriptions -> AUDIO, bedrock_native + google_native ->
|
||||
PASSTHROUGH. Those splits are wire detail, not a customer-facing surface,
|
||||
and `models` + `capabilities` already carry them.
|
||||
|
||||
The last four are ops surfaces: logging/, load/, other/ and ui/ have no LLM
|
||||
route of their own and would otherwise have to lie.
|
||||
"""
|
||||
|
||||
CHAT_COMPLETIONS = "chat_completions"
|
||||
MESSAGES = "messages"
|
||||
RESPONSES = "responses"
|
||||
EMBEDDINGS = "embeddings"
|
||||
COMPLETIONS = "completions"
|
||||
FILES = "files"
|
||||
BATCHES = "batches"
|
||||
PASSTHROUGH = "passthrough"
|
||||
MCP = "mcp"
|
||||
GUARDRAILS = "guardrails"
|
||||
KEY_MANAGEMENT = "key_management"
|
||||
TEAM_MANAGEMENT = "team_management"
|
||||
SPEND_REPORTING = "spend_reporting"
|
||||
MODEL_MANAGEMENT = "model_management"
|
||||
IMAGES = "images"
|
||||
AUDIO = "audio"
|
||||
MODERATIONS = "moderations"
|
||||
RERANK = "rerank"
|
||||
OCR = "ocr"
|
||||
VECTOR_STORES = "vector_stores"
|
||||
REALTIME = "realtime"
|
||||
A2A = "a2a"
|
||||
USER_MANAGEMENT = "user_management"
|
||||
BUDGET_MANAGEMENT = "budget_management"
|
||||
HEALTH = "health"
|
||||
METRICS = "metrics"
|
||||
PROXY_CONFIG = "proxy_config"
|
||||
ADMIN_UI = "admin_ui"
|
||||
|
||||
|
||||
class Provider(str, Enum):
|
||||
"""The upstream LLM provider the test drives, spelled exactly as litellm's
|
||||
own `LlmProviders` (litellm/types/utils.py) spells it.
|
||||
|
||||
A deliberate mirror, not an import. Importing `LlmProviders` at the top of a
|
||||
test module pulls `litellm/__init__` (measured: 1.44s, 2474 modules) and,
|
||||
worse, makes the litellm package a hard dependency of COLLECTING tests/e2e --
|
||||
which is shipped to the e2e runner image on its own, so a missing package
|
||||
would not slow the suite down, it would error every test out at collection.
|
||||
The suite has zero runtime litellm imports and this keeps it that way.
|
||||
|
||||
The mirror cannot drift silently: `TestProviderMirrorsLitellm` in
|
||||
test_e2e_metadata.py asserts every value here is a real `LlmProviders`
|
||||
value, and runs wherever litellm is importable (dev checkouts, the repo's own
|
||||
CI) while skipping where it is not. Adding a provider is one line here.
|
||||
"""
|
||||
|
||||
OPENAI = "openai"
|
||||
OPENAI_LIKE = "openai_like"
|
||||
CUSTOM_OPENAI = "custom_openai"
|
||||
AZURE = "azure"
|
||||
AZURE_AI = "azure_ai"
|
||||
ANTHROPIC = "anthropic"
|
||||
GEMINI = "gemini"
|
||||
VERTEX_AI = "vertex_ai"
|
||||
BEDROCK = "bedrock"
|
||||
SAGEMAKER = "sagemaker"
|
||||
XAI = "xai"
|
||||
GROQ = "groq"
|
||||
DEEPSEEK = "deepseek"
|
||||
MISTRAL = "mistral"
|
||||
COHERE = "cohere"
|
||||
PERPLEXITY = "perplexity"
|
||||
OPENROUTER = "openrouter"
|
||||
TOGETHER_AI = "together_ai"
|
||||
FIREWORKS_AI = "fireworks_ai"
|
||||
CEREBRAS = "cerebras"
|
||||
SAMBANOVA = "sambanova"
|
||||
NVIDIA_NIM = "nvidia_nim"
|
||||
DATABRICKS = "databricks"
|
||||
WATSONX = "watsonx"
|
||||
OLLAMA = "ollama"
|
||||
VLLM = "vllm"
|
||||
HOSTED_VLLM = "hosted_vllm"
|
||||
VOYAGE = "voyage"
|
||||
JINA_AI = "jina_ai"
|
||||
DEEPGRAM = "deepgram"
|
||||
ELEVENLABS = "elevenlabs"
|
||||
ASSEMBLYAI = "assemblyai"
|
||||
LITELLM_PROXY = "litellm_proxy"
|
||||
|
||||
|
||||
class Capability(str, Enum):
|
||||
"""A MODEL feature the test depends on, anchored 1:1 to a `supports_*` key
|
||||
in model_prices_and_context_window.json.
|
||||
|
||||
Not to be confused with the coverage registry's `LlmCell.capability`, which
|
||||
means the endpoint feature under test (`basic`, `multi_turn`, ...) and half
|
||||
of whose values have no `supports_*` key at all.
|
||||
|
||||
Plural by necessity, never a single enum: `thinking_with_tool_use` and
|
||||
`tool_search_history` only exist in the registry because a single-enum field
|
||||
had nowhere to put a conjunction. Here they are
|
||||
(REASONING, FUNCTION_CALLING) and (TOOL_SEARCH,).
|
||||
|
||||
`audio_output` is deliberately absent: litellm/utils.py reads
|
||||
`supports_audio_input` for it, so the value would silently alias
|
||||
AUDIO_INPUT. Add it once that bug is fixed upstream.
|
||||
"""
|
||||
|
||||
FUNCTION_CALLING = "function_calling"
|
||||
PARALLEL_FUNCTION_CALLING = "parallel_function_calling"
|
||||
TOOL_CHOICE = "tool_choice"
|
||||
TOOL_SEARCH = "tool_search"
|
||||
VISION = "vision"
|
||||
PDF_INPUT = "pdf_input"
|
||||
AUDIO_INPUT = "audio_input"
|
||||
REASONING = "reasoning"
|
||||
WEB_SEARCH = "web_search"
|
||||
PROMPT_CACHING = "prompt_caching"
|
||||
RESPONSE_SCHEMA = "response_schema"
|
||||
MID_CONVERSATION_SYSTEM = "mid_conversation_system"
|
||||
|
||||
|
||||
class Mode(str, Enum):
|
||||
"""How the route was driven. The registry's cell ids already carry this
|
||||
axis as a segment (221 `.nonstream.`, 37 `.stream.`)."""
|
||||
|
||||
NONSTREAM = "nonstream"
|
||||
STREAM = "stream"
|
||||
BATCH = "batch"
|
||||
WEBSOCKET = "websocket"
|
||||
|
||||
|
||||
_M = TypeVar("_M")
|
||||
|
||||
|
||||
def _scalar(value: object) -> str:
|
||||
"""`str(member)` on a (str, Enum) gives 'Route.RESPONSES', not 'responses'
|
||||
-- StrEnum would not, but it is 3.11+ and this repo floors at 3.10. So the
|
||||
value is read explicitly, once, for every enum field."""
|
||||
if isinstance(value, Enum):
|
||||
return str(value.value) # pyright: ignore[reportAny] # Enum.value is Any for every enum
|
||||
return str(value)
|
||||
|
||||
|
||||
def _members(value: object) -> tuple[object, ...] | None:
|
||||
"""The elements of a plural field, or None for anything that is not a tuple.
|
||||
|
||||
Both callers hold the value as a plain object: `_canonical` because a call
|
||||
site can pass anything at runtime, the serializer because `asdict` hands the
|
||||
tuple back inside an untyped dict. The elements are re-declared as plain
|
||||
objects here and converted by `_scalar` like any other value.
|
||||
"""
|
||||
return cast("tuple[object, ...]", value) if isinstance(value, tuple) else None
|
||||
|
||||
|
||||
def _canonical(name: str, value: object, member_type: type[_M]) -> tuple[_M, ...]:
|
||||
"""A plural field's members: validated, deduped, and sorted by the value
|
||||
they serialize to.
|
||||
|
||||
`models=("gpt-5.5")` is a str, not a tuple, and iterating it would declare
|
||||
one model per character. Anything that is not a tuple is refused here, which
|
||||
runs where the decorator does: at import, so pytest reports a collection
|
||||
error naming the file instead of shipping garbage properties. An empty
|
||||
string member is dropped rather than refused, because `models` is fed from
|
||||
env-overridable constants and a blank override must not break collection.
|
||||
"""
|
||||
members = _members(value)
|
||||
if members is None:
|
||||
raise TypeError(
|
||||
f"Subject.{name} must be a tuple, got {type(value).__name__}: {value!r}."
|
||||
f" A one-member tuple needs its trailing comma: {name}=(x,), not {name}=(x)"
|
||||
)
|
||||
typed = tuple(member for member in members if isinstance(member, member_type))
|
||||
if len(typed) != len(members):
|
||||
raise TypeError(f"Subject.{name} takes {member_type.__name__} members, got {value!r}")
|
||||
return tuple(sorted(frozenset(member for member in typed if _scalar(member)), key=_scalar))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Subject:
|
||||
"""What a test is about.
|
||||
|
||||
Every field is optional in this first phase -- nothing is enforced, and the
|
||||
backfill of the existing ~908 tests comes later. Named `Subject` rather than
|
||||
`TestMeta` because pytest tries to collect any imported class named `Test*`
|
||||
and would warn in every one of the ~570 modules that import it.
|
||||
|
||||
`providers`, `models` and `capabilities` are plural because one test node
|
||||
routinely drives several: the claude_code matrix runs haiku, sonnet and opus
|
||||
in a single body, and a spend test calls two providers on one key. Each is
|
||||
an independent set. No positional pairing is implied between `providers` and
|
||||
`models` (one provider x three models is the common case), and none could
|
||||
survive anyway, since each tuple is deduped and sorted on its own.
|
||||
"""
|
||||
|
||||
domain: Domain | None = None
|
||||
route: Route | None = None
|
||||
providers: tuple[Provider, ...] = ()
|
||||
models: tuple[str, ...] = ()
|
||||
capabilities: tuple[Capability, ...] = ()
|
||||
mode: Mode | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Canonicalize every plural field at declaration, so the committed run
|
||||
files diff cleanly however a test spelled the tuple, and the serializer
|
||||
stays field-agnostic."""
|
||||
object.__setattr__(self, "providers", _canonical("providers", self.providers, Provider))
|
||||
object.__setattr__(self, "models", _canonical("models", self.models, str))
|
||||
object.__setattr__(self, "capabilities", _canonical("capabilities", self.capabilities, Capability))
|
||||
|
||||
|
||||
def meta(subject: Subject) -> pytest.MarkDecorator:
|
||||
"""Attach a `Subject` to a test: `@meta(Subject(route=Route.RESPONSES, ...))`.
|
||||
|
||||
A typed wrapper around `pytest.mark.meta` (registered in conftest.py's
|
||||
`pytest_configure`, like `covers`) so passing the wrong thing is a type
|
||||
error rather than a property that silently never appears.
|
||||
|
||||
Separate from `@pytest.mark.covers` on purpose: `covers` args are flattened
|
||||
by `dedupe_covers`, which drops non-strings silently, and by
|
||||
tests/integration/conftest.py, which has no such filter and would hard-fail
|
||||
collection. `covers` is untouched by this change.
|
||||
"""
|
||||
return pytest.mark.meta(subject)
|
||||
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
_R = TypeVar("_R")
|
||||
_Y = TypeVar("_Y")
|
||||
|
|
@ -185,6 +471,46 @@ def step(label: str) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]:
|
|||
return decorate
|
||||
|
||||
|
||||
_REPEATED: Final[dict[str, str]] = {"providers": "provider", "models": "model", "capabilities": "capability"}
|
||||
|
||||
|
||||
def _declared_subject(args: tuple[object, ...]) -> Subject | None:
|
||||
"""The `Subject` a `meta` marker carries, or None for anything else.
|
||||
|
||||
`Mark.args` is `tuple[Any, ...]`; taking it as `tuple[object, ...]` is what
|
||||
keeps the Any from leaking past this line. A bare `@pytest.mark.meta` (no
|
||||
args) and a `@pytest.mark.meta("spend-budgets")` (wrong type) both land here,
|
||||
and neither may produce a property whose value is a repr.
|
||||
"""
|
||||
first = args[0] if args else None
|
||||
return first if isinstance(first, Subject) else None
|
||||
|
||||
|
||||
def subject_properties(item: pytest.Item) -> tuple[tuple[str, str], ...]:
|
||||
"""The declared half, in dataclass field order.
|
||||
|
||||
`_REPEATED` is the only field-specific knowledge here: which fields are
|
||||
plural, and the SINGULAR name their repeated <property> goes out under. A new
|
||||
scalar field needs no edit. Empty fields emit nothing; the emitter is what
|
||||
guarantees every key exists in the JSON, with `providers` and `models` as
|
||||
`[]` when nothing was declared."""
|
||||
marker = item.get_closest_marker("meta")
|
||||
if marker is None:
|
||||
return ()
|
||||
subject = _declared_subject(marker.args)
|
||||
if subject is None:
|
||||
return ()
|
||||
fields: dict[str, object] = asdict(subject)
|
||||
pairs: list[tuple[str, str]] = []
|
||||
for name, value in fields.items():
|
||||
repeated = _REPEATED.get(name)
|
||||
if repeated is not None:
|
||||
pairs.extend((repeated, _scalar(member)) for member in _members(value) or ())
|
||||
elif value is not None and value != "":
|
||||
pairs.append((name, _scalar(value)))
|
||||
return tuple(pairs)
|
||||
|
||||
|
||||
def step_properties() -> tuple[tuple[str, str], ...]:
|
||||
"""The step log as repeated `step` properties. Appended after the setup and
|
||||
call phases, never at collection."""
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from collections.abc import Iterable
|
|||
|
||||
import pytest
|
||||
from coverage_registry.management_cases import case_properties
|
||||
from e2e_metadata import step_properties
|
||||
from e2e_metadata import step_properties, subject_properties
|
||||
|
||||
# Hardcoded because the runner image copies tests/e2e/ to /app/e2e, so nothing
|
||||
# at runtime names this suite's place in the repo. test_junit_properties.py
|
||||
|
|
@ -90,13 +90,22 @@ def covers_from_item(item: pytest.Item) -> tuple[str, ...]:
|
|||
|
||||
def result_properties(item: pytest.Item) -> tuple[tuple[str, str], ...]:
|
||||
"""The custom signals a standard reporter cannot derive: the normalized suite
|
||||
package, the comma-joined coverage-registry cell ids this test covers, and the
|
||||
repo-relative `path:line` its source sits at."""
|
||||
return (
|
||||
package, the comma-joined coverage-registry cell ids this test covers, the
|
||||
repo-relative `path:line` its source sits at, and the typed `@meta(Subject(...))`
|
||||
fields.
|
||||
|
||||
The fixed three-tuple prefix is load-bearing and stays byte-identical: Loki,
|
||||
Grafana, the status page and tests/integration/conftest.py all read
|
||||
`package`/`covers`/`source` today. `subject_properties` only ever appends, and
|
||||
appends nothing at all for a test with no `meta` marker -- which is every test
|
||||
in the suite until the backfill lands.
|
||||
"""
|
||||
fixed = (
|
||||
("package", package_from_nodeid(item.nodeid)),
|
||||
("covers", ",".join(covers_from_item(item))),
|
||||
("source", source_from_item(item)),
|
||||
) + case_properties(item.nodeid)
|
||||
)
|
||||
return fixed + case_properties(item.nodeid) + subject_properties(item)
|
||||
|
||||
|
||||
def attach_result_properties(item: pytest.Item) -> None:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
addopts = --strict-markers --strict-config --reruns 1 --only-rerun "kind='network'" --only-rerun "status_code=5[0-9][0-9]"
|
||||
markers =
|
||||
e2e: live test that requires a running proxy and real provider keys
|
||||
meta: typed e2e_metadata.Subject describing what this test drives (domain/route/providers/models/capabilities/mode); attach it with @meta(Subject(...)), never as a bare pytest.mark
|
||||
replayable: edge-wired test whose provider traffic replays from a fixture bundle, so it makes zero provider calls in replay mode; the record/replay CI lane selects it with -m replayable
|
||||
load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites
|
||||
weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set
|
||||
|
|
|
|||
|
|
@ -10,12 +10,19 @@ from datetime import datetime, timezone
|
|||
import pytest
|
||||
|
||||
from budget_client import BudgetClient
|
||||
from e2e_metadata import Domain, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
||||
@pytest.mark.covers("mgmt.budget.new.persists")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.BUDGET_MANAGEMENT,
|
||||
)
|
||||
)
|
||||
def test_budget_crud_roundtrip(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
budget_id = client.create_budget(max_budget=12.5, soft_budget=10.0, budget_duration="30d")
|
||||
resources.defer(lambda: client.delete_budget(budget_id))
|
||||
|
|
@ -38,6 +45,12 @@ def test_budget_crud_roundtrip(client: BudgetClient, resources: ResourceManager)
|
|||
|
||||
|
||||
@pytest.mark.covers("mgmt.budget.delete.persists")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.BUDGET_MANAGEMENT,
|
||||
)
|
||||
)
|
||||
def test_budget_delete_removes_it(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
budget_id = client.create_budget(max_budget=1.0)
|
||||
resources.defer(lambda: client.delete_budget(budget_id))
|
||||
|
|
@ -45,6 +58,12 @@ def test_budget_delete_removes_it(client: BudgetClient, resources: ResourceManag
|
|||
assert not client.budget_info(budget_id), "budget still present after delete"
|
||||
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.KEY_MANAGEMENT,
|
||||
)
|
||||
)
|
||||
def test_budget_duration_schedules_reset_on_key(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
key = client.generate_key(max_budget=10.0, budget_duration="30d")
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import StreamingResponse, require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
|
@ -56,6 +57,15 @@ def _assert_blocked_422(client: BudgetClient, key: str) -> StreamingResponse:
|
|||
|
||||
class TestBudgetBlocksPerLevel:
|
||||
@pytest.mark.covers("quota_management.budget.key.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_bare_key_blocks_over_its_own_budget(self, client: BudgetClient, resources: ResourceManager) -> None:
|
||||
key = client.generate_key(max_budget=TINY_CAP)
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
|
@ -63,6 +73,15 @@ class TestBudgetBlocksPerLevel:
|
|||
_assert_blocked_422(client, key)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.team.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_team_budget_blocks_every_team_key(self, client: BudgetClient, resources: ResourceManager) -> None:
|
||||
team_id = client.create_team(alias=f"e2e-budget-team-{unique_marker()}", max_budget=TINY_CAP)
|
||||
resources.defer(lambda: client.delete_team(team_id))
|
||||
|
|
@ -79,6 +98,15 @@ class TestBudgetBlocksPerLevel:
|
|||
)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.internal_user.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_user_budget_enforced_across_their_personal_keys(
|
||||
self, client: BudgetClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
@ -113,6 +141,15 @@ class TestBudgetBlocksPerLevel:
|
|||
require_successful_call(team_result)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.end_user.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_end_user_budget_blocks_attributed_calls(
|
||||
self, client: BudgetClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
@ -125,6 +162,15 @@ class TestBudgetBlocksPerLevel:
|
|||
_assert_budget_blocks(client, key, user=customer)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.organization.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_org_budget_blocks_keys_under_it(self, client: BudgetClient, resources: ResourceManager) -> None:
|
||||
org_id = client.create_org(max_budget=TINY_CAP, alias=f"e2e-budget-org-{unique_marker()}")
|
||||
resources.defer(lambda: client.delete_org(org_id))
|
||||
|
|
@ -139,6 +185,15 @@ class TestBudgetBlocksPerLevel:
|
|||
)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.team_member.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_member_budget_blocks_without_touching_teammates(
|
||||
self, client: BudgetClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
@ -166,6 +221,15 @@ class TestKeyBudgetBlocksAcrossKeyKinds:
|
|||
the capped key is refused, proving nothing around the key was the blocker."""
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.key.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_personal_key_blocks_over_its_own_budget(
|
||||
self, client: BudgetClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
@ -180,6 +244,15 @@ class TestKeyBudgetBlocksAcrossKeyKinds:
|
|||
require_successful_call(_chat(client, control_key))
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.key.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_team_key_blocks_over_its_own_budget(self, client: BudgetClient, resources: ResourceManager) -> None:
|
||||
team_id = client.create_team(alias=f"e2e-key-cap-team-{unique_marker()}", max_budget=ROOMY_CAP)
|
||||
resources.defer(lambda: client.delete_team(team_id))
|
||||
|
|
@ -192,6 +265,15 @@ class TestKeyBudgetBlocksAcrossKeyKinds:
|
|||
require_successful_call(_chat(client, control_key))
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.key.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_team_member_key_blocks_over_its_own_budget(
|
||||
self, client: BudgetClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import pytest
|
|||
|
||||
from budget_client import BudgetClient, model_budget
|
||||
from e2e_config import unique_marker
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import AnthropicMessagesResponse
|
||||
|
||||
|
|
@ -20,6 +21,15 @@ FALLBACK_MODEL = "gpt-5.5"
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.fallback.routes_to_fallback")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.MESSAGES,
|
||||
providers=(Provider.ANTHROPIC, Provider.OPENAI),
|
||||
models=(PRIMARY_MODEL, FALLBACK_MODEL),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_budget_fallback_reroutes_anthropic_messages_to_openai(
|
||||
client: BudgetClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import BudgetWindow
|
||||
|
||||
|
|
@ -70,6 +71,12 @@ def _drive_to_block(client: BudgetClient, key: str) -> None:
|
|||
# ---- Rung 1: scheduling exists at creation -----------------------------------
|
||||
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.KEY_MANAGEMENT,
|
||||
)
|
||||
)
|
||||
def test_key_with_budget_duration_schedules_reset_at_creation(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
"""Baseline: a key created with a budget_duration has budget_reset_at populated
|
||||
immediately. The reset job can only advance a timestamp that was scheduled in
|
||||
|
|
@ -86,6 +93,15 @@ def test_key_with_budget_duration_schedules_reset_at_creation(client: BudgetClie
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.key.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_key_spend_blocks_at_cap(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
"""Sanity that the tiny cap is enforced before we test that it resets: spend
|
||||
accrues across calls and eventually returns budget_exceeded, never a 5xx."""
|
||||
|
|
@ -103,6 +119,15 @@ def test_key_spend_blocks_at_cap(client: BudgetClient, resources: ResourceManage
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.key.resets_after_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_key_budget_reset_at_advances_after_window(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
"""The core #25109 guard: after the window elapses the reset job must move
|
||||
budget_reset_at strictly forward AND zero key.spend. The broken nullable-JSON
|
||||
|
|
@ -139,6 +164,15 @@ def test_key_budget_reset_at_advances_after_window(client: BudgetClient, resourc
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.key_multi_window.resets_windows_independently")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_multi_window_key_resets_each_window_independently(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
"""The JSON-backed path #25109 specifically touched. A tight 30s window and a
|
||||
roomy 1m window: the tight window must reset on its own boundary while the roomy
|
||||
|
|
@ -183,6 +217,15 @@ def test_multi_window_key_resets_each_window_independently(client: BudgetClient,
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.team_member.resets_after_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_team_member_budget_reset_at_advances(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
"""Per-team member windows are also JSON-backed. member_budget_reset_at must
|
||||
advance after the window; the explicit before<after assertion is the #25109
|
||||
|
|
@ -216,6 +259,15 @@ def test_team_member_budget_reset_at_advances(client: BudgetClient, resources: R
|
|||
# ---- Rung 6: error-path edge - resets surface as blocks, never 5xx -----------
|
||||
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_reset_wait_never_yields_non_budget_error(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
"""The other #25109 failure mode: a reset job that ERRORS on the nullable-JSON
|
||||
column surfaces to the caller as a non-budget 5xx. Across the whole reset wait
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
|
@ -49,6 +50,15 @@ def _poll_until_serves_again(client: BudgetClient, key: str) -> None:
|
|||
|
||||
class TestBudgetResetPerLevel:
|
||||
@pytest.mark.covers("quota_management.budget.key.resets_after_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_bare_key_budget_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None:
|
||||
key = client.generate_key(max_budget=TINY_CAP, budget_duration=WINDOW)
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
|
@ -57,6 +67,15 @@ class TestBudgetResetPerLevel:
|
|||
_poll_until_serves_again(client, key)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.team.resets_after_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_team_budget_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None:
|
||||
team_id = client.create_team(
|
||||
alias=f"e2e-team-reset-{unique_marker()}", max_budget=TINY_CAP, budget_duration=WINDOW
|
||||
|
|
@ -69,6 +88,15 @@ class TestBudgetResetPerLevel:
|
|||
_poll_until_serves_again(client, key)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.organization.resets_after_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_org_budget_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None:
|
||||
org_id = client.create_org(
|
||||
max_budget=TINY_CAP, alias=f"e2e-org-reset-{unique_marker()}", budget_duration=WINDOW
|
||||
|
|
@ -91,6 +119,15 @@ class TestBudgetResetPerLevel:
|
|||
_poll_until_serves_again(client, key)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.internal_user.resets_after_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_personal_key_user_budget_resets_after_window(
|
||||
self, client: BudgetClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
@ -109,6 +146,15 @@ class TestKeyBudgetResetAcrossKeyKinds:
|
|||
the only thing that can block and the only thing that has to reset."""
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.key.resets_after_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_personal_key_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None:
|
||||
user_id = client.create_user(max_budget=ROOMY_CAP)
|
||||
resources.defer(lambda: client.delete_user(user_id))
|
||||
|
|
@ -119,6 +165,15 @@ class TestKeyBudgetResetAcrossKeyKinds:
|
|||
_poll_until_serves_again(client, key)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.key.resets_after_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_team_key_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None:
|
||||
team_id = client.create_team(alias=f"e2e-key-reset-team-{unique_marker()}", max_budget=ROOMY_CAP)
|
||||
resources.defer(lambda: client.delete_team(team_id))
|
||||
|
|
@ -129,6 +184,15 @@ class TestKeyBudgetResetAcrossKeyKinds:
|
|||
_poll_until_serves_again(client, key)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.key.resets_after_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_team_member_key_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None:
|
||||
team_id = client.create_team(alias=f"e2e-key-reset-team-{unique_marker()}", max_budget=ROOMY_CAP)
|
||||
resources.defer(lambda: client.delete_team(team_id))
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import StreamingResponse, require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import KeyGenerateBody, LiteLLMParamsBody, ModelInfoBody, ModelNewBody
|
||||
|
||||
|
|
@ -102,6 +103,15 @@ def drained(client: BudgetClient) -> Iterator[DrainedPool]:
|
|||
|
||||
class TestModelAccessGroupBudget:
|
||||
@pytest.mark.covers("quota_management.budget.model_access_group.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_the_key_that_drained_the_pool_stays_blocked(
|
||||
self, client: BudgetClient, drained: DrainedPool
|
||||
) -> None:
|
||||
|
|
@ -114,6 +124,15 @@ class TestModelAccessGroupBudget:
|
|||
)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.model_access_group.enforced_across_keys")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_a_key_that_spent_nothing_is_blocked_by_the_shared_pool(
|
||||
self, client: BudgetClient, resources: ResourceManager, drained: DrainedPool
|
||||
) -> None:
|
||||
|
|
@ -127,6 +146,15 @@ class TestModelAccessGroupBudget:
|
|||
)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.model_access_group.isolates_per_group")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_a_drained_group_does_not_block_a_different_group(
|
||||
self, client: BudgetClient, resources: ResourceManager, drained: DrainedPool
|
||||
) -> None:
|
||||
|
|
@ -141,6 +169,14 @@ class TestModelAccessGroupBudget:
|
|||
require_successful_call(result)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.model_access_group.reports_spend")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.BUDGET_MANAGEMENT,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(BACKEND,),
|
||||
)
|
||||
)
|
||||
def test_the_budget_read_reports_the_spend_drawn_against_the_pool(
|
||||
self, client: BudgetClient, drained: DrainedPool
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block, model_budget
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import ModelBudgetEntry
|
||||
|
||||
|
|
@ -30,6 +31,15 @@ def _call(client: BudgetClient, key: str, model: str):
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.model_max.isolates_per_model")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC, Provider.GEMINI),
|
||||
models=(CAPPED_MODEL, FREE_MODEL),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_model_max_budget_isolates_per_model(
|
||||
client: BudgetClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
@ -61,6 +71,15 @@ def test_model_max_budget_isolates_per_model(
|
|||
|
||||
@pytest.mark.skip(reason="stage red: product gap, end-user model_max_budget rpm_limit is stored but never enforced")
|
||||
@pytest.mark.covers("quota_management.budget.end_user_model_max.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(FREE_MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_end_user_model_max_budget_enforces_per_model_rpm(
|
||||
client: BudgetClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block, window_reset_at
|
||||
from e2e_http import StreamingResponse, require_successful_call
|
||||
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import BudgetWindow
|
||||
|
||||
|
|
@ -57,6 +58,15 @@ def _drive_to_block(client: BudgetClient, key: str) -> StreamingResponse:
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.key_multi_window.blocks_then_resets")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(CHEAP_OPENAI_MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_short_window_blocks_then_resets(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
key = client.generate_key(
|
||||
models=[MODEL],
|
||||
|
|
@ -90,6 +100,15 @@ def test_short_window_blocks_then_resets(client: BudgetClient, resources: Resour
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.key_multi_window.blocks_then_resets")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(CHEAP_OPENAI_MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_long_window_blocks_after_short_window_resets(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
key = client.generate_key(
|
||||
models=[MODEL],
|
||||
|
|
|
|||
|
|
@ -12,12 +12,22 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.soft.alerts_without_blocking")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_soft_budget_does_not_block(
|
||||
client: BudgetClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
from budget_client import BudgetClient
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import StreamingResponse
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -144,6 +145,15 @@ def _accumulate(client: BudgetClient, key: str, count: int) -> None:
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.spend_counter.reseed_matches_db")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_cold_counter_reseed_keeps_counter_equal_to_db_spend(
|
||||
client: BudgetClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
|
@ -34,6 +35,15 @@ def _tagged_call(client: BudgetClient, key: str, tag: str):
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.tag.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_tag_budget_blocks_tagged_requests(
|
||||
client: BudgetClient, scoped_key: str, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import Success, require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage
|
||||
|
||||
|
|
@ -79,6 +80,15 @@ def _send(client: BudgetClient, key: str) -> str | None:
|
|||
|
||||
|
||||
class TestTeamMemberBudget:
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_member_spend_attributed_to_team_and_user(self, client: BudgetClient, member: _Member) -> None:
|
||||
sent = frozenset(rid for rid in (_send(client, member.key) for _ in range(BURST)) if rid)
|
||||
assert sent, "no member call went through; cannot check attribution"
|
||||
|
|
@ -98,6 +108,15 @@ class TestTeamMemberBudget:
|
|||
)
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.team_member.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_member_spend_over_budget_is_blocked(self, client: BudgetClient, member: _Member) -> None:
|
||||
for _ in range(40):
|
||||
result = client.chat(member.key, MODEL, f"spend {unique_marker()}", max_tokens=16)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import Success, require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage
|
||||
|
||||
|
|
@ -88,6 +89,15 @@ def _roomy_send(client: BudgetClient, key: str) -> str:
|
|||
|
||||
class TestTeamMemberBudgetIsolation:
|
||||
@pytest.mark.covers("quota_management.budget.team_member.isolates_per_member")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_blocked_member_does_not_block_peer(self, client: BudgetClient, pair: _Pair) -> None:
|
||||
blocked = False
|
||||
for _ in range(40):
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import pytest
|
|||
from budget_client import BudgetClient
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
|
@ -17,6 +18,15 @@ def _as_datetime(value: str) -> datetime:
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.team_member.resets_after_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_team_member_budget_reset_keeps_advancing(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
team_id = client.create_team(alias=f"e2e-member-reset-{unique_marker()}", max_budget=100.0)
|
||||
resources.defer(lambda: client.delete_team(team_id))
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block, window_reset_at
|
||||
from e2e_http import StreamingResponse, require_successful_call
|
||||
from e2e_config import unique_marker
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import BudgetWindow
|
||||
|
||||
|
|
@ -52,6 +53,15 @@ def _drive_to_block(client: BudgetClient, key: str) -> StreamingResponse:
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.team_multi_window.blocks_then_resets")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_team_short_window_blocks_then_resets(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
team_id = client.create_team(
|
||||
alias=f"e2e-team-window-{unique_marker()}",
|
||||
|
|
@ -85,6 +95,15 @@ def test_team_short_window_blocks_then_resets(client: BudgetClient, resources: R
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.budget.team_multi_window.blocks_then_resets")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=("claude-haiku-4-5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_team_long_window_blocks_after_short_window_resets(client: BudgetClient, resources: ResourceManager) -> None:
|
||||
|
||||
# 0. key with a short budget window and a long budget window
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import pytest
|
|||
from budget_client import BudgetClient, is_budget_block
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import StreamingResponse, require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
|
@ -58,6 +59,15 @@ def _expect_prompt_block(client: BudgetClient, key: str, subject: str) -> None:
|
|||
|
||||
class TestUserBudgetAcrossKeys:
|
||||
@pytest.mark.covers("quota_management.budget.internal_user.enforced_across_keys")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_user_budget_blocks_a_second_key(self, client: BudgetClient, resources: ResourceManager) -> None:
|
||||
user_id = client.create_user(max_budget=TINY_CAP)
|
||||
resources.defer(lambda: client.delete_user(user_id))
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError
|
|||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import StreamingResponse, require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import KeyGenerateBody, KeyMetadata, LiteLLMParamsBody
|
||||
from quota_client import QuotaClient
|
||||
|
|
@ -157,6 +158,15 @@ class TestDynamicRateLimitPriority:
|
|||
"quota_management.ratelimit.priority_generous.picks_under_tpm",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_generous_mode_lets_priority_borrow_past_reservation(
|
||||
self, client: QuotaClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
@ -199,6 +209,15 @@ class TestDynamicRateLimitPriority:
|
|||
"quota_management.ratelimit.priority_strict.picks_under_tpm",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_strict_mode_blocks_saturated_priority_but_serves_the_other(
|
||||
self, client: QuotaClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError
|
|||
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import StreamingResponse, require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import KeyGenerateBody
|
||||
from quota_client import QuotaClient
|
||||
|
|
@ -176,6 +177,15 @@ def _assert_rate_limited(outcome: StreamingResponse, limit_type: str) -> None:
|
|||
|
||||
class TestKeyRateLimits:
|
||||
@pytest.mark.covers("quota_management.ratelimit.rpm.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(CHEAP_ANTHROPIC_MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_rpm_limit_blocks_over_limit(self, client: QuotaClient, resources: ResourceManager) -> None:
|
||||
key = _limited_key(client, resources, rpm_limit=3)
|
||||
info = client.proxy.key_info(key)
|
||||
|
|
@ -188,6 +198,15 @@ class TestKeyRateLimits:
|
|||
_assert_rate_limited(_chat(client, key), "requests")
|
||||
|
||||
@pytest.mark.covers("quota_management.ratelimit.tpm.blocks_over_limit")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(CHEAP_ANTHROPIC_MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_tpm_limit_blocks_over_limit(self, client: QuotaClient, resources: ResourceManager) -> None:
|
||||
key = _limited_key(client, resources, tpm_limit=TPM_LIMIT)
|
||||
info = client.proxy.key_info(key)
|
||||
|
|
@ -207,6 +226,15 @@ class TestKeyRateLimits:
|
|||
_assert_rate_limited(_chat(client, key), "tokens")
|
||||
|
||||
@pytest.mark.covers("quota_management.ratelimit.rpm.resets_after_window")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(CHEAP_ANTHROPIC_MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_rpm_limit_resets_after_window(self, client: QuotaClient, resources: ResourceManager) -> None:
|
||||
key = _limited_key(client, resources, rpm_limit=1)
|
||||
|
||||
|
|
@ -232,6 +260,15 @@ class TestKeyRateLimits:
|
|||
pytest.fail("a blocked key never recovered after the rate-limit window elapsed")
|
||||
|
||||
@pytest.mark.covers("quota_management.ratelimit.rpm.headers_report_remaining")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(CHEAP_ANTHROPIC_MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_headers_report_limit_and_remaining(self, client: QuotaClient, resources: ResourceManager) -> None:
|
||||
key = _limited_key(client, resources, rpm_limit=5, tpm_limit=100000)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import pytest
|
|||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import KeyGenerateBody, LiteLLMParamsBody
|
||||
from quota_client import QuotaClient
|
||||
|
|
@ -40,6 +41,15 @@ class TestRedisBackedRateLimit:
|
|||
"quota_management.ratelimit.redis_backed.blocks_over_limit",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_rpm_limit_one_blocks_second_call(
|
||||
self, client: QuotaClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import pytest
|
|||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import KeyGenerateBody, LiteLLMParamsBody
|
||||
from quota_client import QuotaClient
|
||||
|
|
@ -45,6 +46,15 @@ class TestRedisCircuitBreakerPath:
|
|||
"reliability.circuit_breaker.redis.trips_then_recovers",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_burst_rate_limit_does_not_freeze_fresh_key(
|
||||
self, client: QuotaClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from models import (
|
|||
TextBlock,
|
||||
Usage,
|
||||
)
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
|
||||
from quota_client import QuotaClient
|
||||
|
||||
pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
|
||||
|
|
@ -101,6 +102,16 @@ class TestTpmExcludesCachedTokens:
|
|||
"quota_management.ratelimit.tpm.excludes_cached_tokens",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(ANTHROPIC_MODEL,),
|
||||
capabilities=(Capability.PROMPT_CACHING,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_cache_hit_reduces_tpm_by_non_cached_only(
|
||||
self, client: QuotaClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ from cost_rows import (
|
|||
)
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import unwrap
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import AnthropicMessagesBody, ChatBody, ChatMessage, LiteLLMParamsBody
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -122,6 +123,16 @@ def _assert_cache_read_billed(row: CostRow) -> None:
|
|||
|
||||
class TestCacheCostAccounting:
|
||||
@pytest.mark.covers("quota_management.spend_tracking.cache_write.bills_cache_creation_rate")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(CACHE_WRITE_BACKEND,),
|
||||
capabilities=(Capability.PROMPT_CACHING,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_cache_write_tokens_billed_at_cache_creation_rate(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -152,6 +163,16 @@ class TestCacheCostAccounting:
|
|||
assert_total_is_sum_of_components(row)
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.cost_breakdown.reports_component_costs")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(CACHE_READ_BACKEND,),
|
||||
capabilities=(Capability.PROMPT_CACHING, Capability.REASONING),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_cost_breakdown_reports_component_costs(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -216,6 +237,16 @@ class TestCacheCostAccounting:
|
|||
_assert_cache_read_billed(row)
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(CACHE_READ_BACKEND,),
|
||||
capabilities=(Capability.PROMPT_CACHING,),
|
||||
mode=Mode.STREAM,
|
||||
)
|
||||
)
|
||||
def test_streaming_cache_read_billed_at_cache_read_rate(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -247,6 +278,16 @@ class TestCacheCostAccounting:
|
|||
_assert_cache_read_billed(row)
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.messages_bridge.keeps_cache_tokens")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.MESSAGES,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(BRIDGE_BACKEND,),
|
||||
capabilities=(Capability.PROMPT_CACHING,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_messages_bridge_keeps_cache_tokens(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import pytest
|
|||
from cost_rows import approx_equal, cacheable_prefix, register_priced_model
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import StreamingResponse
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody
|
||||
from spend_e2e_client import SpendClient
|
||||
|
|
@ -60,6 +61,15 @@ def _header_cost(response: StreamingResponse, name: str) -> float:
|
|||
|
||||
class TestCostHeaders:
|
||||
@pytest.mark.covers("quota_management.spend_tracking.cost_headers.additive_components")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_component_cost_headers_sum_to_total(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from datetime import datetime, timedelta, timezone
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from models import KeyGenerateBody
|
||||
from proxy_client import Converged, await_converged
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -61,6 +62,7 @@ EMBED_MODEL: Final = "openai-text-embedding-3-small"
|
|||
BATCH_MODEL: Final = "openai-gpt-4o-mini"
|
||||
BATCH_BACKEND_MODEL: Final = "gpt-4o-mini"
|
||||
BATCH_PROVIDER: Final = "openai"
|
||||
DRIVEN_MODELS: Final = (CHAT_MODEL, MESSAGES_MODEL, RESPONSES_MODEL, EMBED_MODEL, BATCH_MODEL)
|
||||
HEALTH_SERVICE_ACCOUNT: Final = "litellm-internal-health-check"
|
||||
BATCH_TERMINAL_STATUSES: Final = frozenset({"completed", "failed", "cancelled", "expired"})
|
||||
FAILED_BATCH_POLL_SECONDS: Final = 120.0
|
||||
|
|
@ -281,6 +283,14 @@ class TestKeyAttribution:
|
|||
"rust_control_plane",
|
||||
],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.SPEND_REPORTING,
|
||||
providers=(Provider.GEMINI, Provider.ANTHROPIC, Provider.OPENAI),
|
||||
models=DRIVEN_MODELS,
|
||||
)
|
||||
)
|
||||
def test_every_write_path_row_joins_the_key(self, client: SpendClient, driven: DrivenKey) -> None:
|
||||
assert tuple(path.name for path in driven.paths) == WRITE_PATHS
|
||||
found: Final = tuple((path, client.proxy.poll_logs_for_request_id(path.request_id)) for path in driven.paths)
|
||||
|
|
@ -317,6 +327,14 @@ class TestKeyAttribution:
|
|||
"rust_control_plane",
|
||||
],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.SPEND_REPORTING,
|
||||
providers=(Provider.GEMINI, Provider.ANTHROPIC, Provider.OPENAI),
|
||||
models=DRIVEN_MODELS,
|
||||
)
|
||||
)
|
||||
def test_spend_logs_by_key_return_every_row_with_the_alias(self, client: SpendClient, driven: DrivenKey) -> None:
|
||||
expected_ids: Final = frozenset(path.request_id for path in driven.paths)
|
||||
rows: Final = client.poll_logs_for_key(
|
||||
|
|
@ -345,6 +363,14 @@ class TestKeyAttribution:
|
|||
"rust_control_plane",
|
||||
],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.SPEND_REPORTING,
|
||||
providers=(Provider.GEMINI, Provider.ANTHROPIC, Provider.OPENAI),
|
||||
models=DRIVEN_MODELS,
|
||||
)
|
||||
)
|
||||
def test_user_daily_activity_reports_alias_and_email(self, client: SpendClient, driven: DrivenKey) -> None:
|
||||
breakdown: Final[DailyActivityKeyBreakdown | None] = client.poll_daily_activity_for_key(
|
||||
driven.identity.token,
|
||||
|
|
@ -367,6 +393,14 @@ class TestKeyAttribution:
|
|||
"quota_management.spend_tracking.key_attribution.health_rows_keep_service_account",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.HEALTH,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(CHAT_MODEL,),
|
||||
)
|
||||
)
|
||||
def test_health_check_rows_keep_the_service_account_key(self, client: SpendClient) -> None:
|
||||
started_at: Final = datetime.now(timezone.utc)
|
||||
probe: Final = client.health(CHAT_MODEL)
|
||||
|
|
@ -380,6 +414,15 @@ class TestKeyAttribution:
|
|||
"quota_management.spend_tracking.key_attribution.retrieve_batch_cost_joins_retrieving_key",
|
||||
exercised_on=["batches"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.BATCHES,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(BATCH_MODEL,),
|
||||
mode=Mode.BATCH,
|
||||
)
|
||||
)
|
||||
def test_terminal_batch_cost_row_joins_the_retrieving_key(self, client: SpendClient, driven: DrivenKey) -> None:
|
||||
provider_batch_id: Final = _provider_batch_id(_driven_batch_id(driven))
|
||||
fetched: Final = _await_terminal_batch(client, driven.identity.key, provider_batch_id)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ write path are all still under test with zero provider calls.
|
|||
import pytest
|
||||
|
||||
from e2e_config import CHEAP_OPENAI_MODEL, provider_edge_base
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody
|
||||
from spend_e2e_client import SpendClient, unique_marker, unwrap
|
||||
|
|
@ -22,6 +23,15 @@ pytestmark = [pytest.mark.e2e, pytest.mark.replayable]
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.chat_completions.logs_cost")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(f"openai/{CHEAP_OPENAI_MODEL}",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_edge_wired_chat_writes_nonzero_spend_row(
|
||||
client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from cost_rows import (
|
|||
)
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import unwrap
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage, LiteLLMParamsBody
|
||||
from spend_e2e_client import SpendClient
|
||||
|
|
@ -45,6 +46,16 @@ REASONING_EFFORT = "high"
|
|||
|
||||
class TestServiceTierPricing:
|
||||
@pytest.mark.covers("quota_management.spend_tracking.service_tier.bills_tier_rates")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(BACKEND,),
|
||||
capabilities=(Capability.REASONING,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_priority_tier_bills_priority_rates(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from typing import Final
|
|||
import pytest
|
||||
|
||||
from e2e_http import ProbeResult
|
||||
from e2e_metadata import Domain, Route, Subject, meta
|
||||
from models import DateRangeParams
|
||||
from spend_e2e_client import SpendClient
|
||||
|
||||
|
|
@ -102,12 +103,24 @@ def _probe(client: SpendClient, route: str) -> ProbeResult:
|
|||
|
||||
|
||||
@pytest.mark.parametrize("route", SPEND_ROUTES)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.SPEND_REPORTING,
|
||||
)
|
||||
)
|
||||
def test_spend_route_responsive(client: SpendClient, route: str) -> None:
|
||||
result = _probe(client, route)
|
||||
print(f"{route} -> {result.status_code}\n{result.body[:600]}")
|
||||
assert result.healthy, f"{route} -> {result.status_code}\n{result.body[:600]}"
|
||||
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.SPEND_REPORTING,
|
||||
)
|
||||
)
|
||||
def test_schema_listed_spend_routes_are_responsive(client: SpendClient) -> None:
|
||||
"""Probe any spend GET route the schema lists that isn't in SPEND_ROUTES."""
|
||||
schema = client.openapi()
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
from e2e_http import Success
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody, SpendLogs, SpendLogsParams
|
||||
from spend_e2e_client import SpendClient, SpendLogRow, is_ok, unique_marker, unwrap
|
||||
|
|
@ -62,6 +63,15 @@ def _require_row(
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.chat_completions.logs_cost")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=("gemini-2.5-flash",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_chat_completion_writes_nonzero_spend_row(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -97,6 +107,15 @@ def test_chat_completion_writes_nonzero_spend_row(
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.stream.logs_cost")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=("gemini-2.5-flash",),
|
||||
mode=Mode.STREAM,
|
||||
)
|
||||
)
|
||||
def test_streaming_chat_completion_tracks_spend(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -125,6 +144,15 @@ def test_streaming_chat_completion_tracks_spend(
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.messages_bridge.logs_cost")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.MESSAGES,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=("openai-responses-codex",),
|
||||
mode=Mode.STREAM,
|
||||
)
|
||||
)
|
||||
def test_streaming_messages_via_responses_bridge_tracks_spend(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -195,6 +223,15 @@ def test_streaming_messages_via_responses_bridge_tracks_spend(
|
|||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.embeddings.logs_cost")
|
||||
@pytest.mark.covers("llm.embeddings.openai.basic.nonstream.cost_logged")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.EMBEDDINGS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=("openai-text-embedding-3-small",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_embedding_writes_nonzero_spend_row(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -218,6 +255,15 @@ def test_embedding_writes_nonzero_spend_row(
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.cache_hit.zero_cost")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=("gemini-2.5-flash",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_cache_hit_is_zero_cost_and_suffixed(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -254,6 +300,15 @@ def test_cache_hit_is_zero_cost_and_suffixed(
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.key_rollup.matches_sum_of_logs")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=("gemini-2.5-flash",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_key_spend_equals_sum_of_logs(client: SpendClient, scoped_key: str) -> None:
|
||||
for _ in range(2):
|
||||
_ = unwrap(
|
||||
|
|
@ -282,6 +337,15 @@ def test_key_spend_equals_sum_of_logs(client: SpendClient, scoped_key: str) -> N
|
|||
|
||||
@pytest.mark.replayable
|
||||
@pytest.mark.covers("quota_management.spend_tracking.concurrent_burst.loses_no_spend")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=("openai/gpt-5.6-luna",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_burst_of_concurrent_calls_loses_no_spend(
|
||||
client: SpendClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
@ -299,6 +363,15 @@ def test_burst_of_concurrent_calls_loses_no_spend(
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.pagination.keeps_total")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=("gemini-2.5-flash",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_spend_logs_v2_pagination_caps_pages_and_keeps_total(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -352,6 +425,15 @@ def test_spend_logs_v2_pagination_caps_pages_and_keeps_total(
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.tags.attributes_spend")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=("gemini-2.5-flash",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_request_tags_round_trip(client: SpendClient, scoped_key: str) -> None:
|
||||
tag = f"e2e-spend-{unique_marker()}"
|
||||
_ = unwrap(
|
||||
|
|
@ -369,6 +451,15 @@ def test_request_tags_round_trip(client: SpendClient, scoped_key: str) -> None:
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.tags.attributes_spend")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=("gemini-2.5-flash",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_tag_spend_matches_sum_of_tagged_logs(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -407,6 +498,15 @@ def test_tag_spend_matches_sum_of_tagged_logs(
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.end_user.attributes_spend")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=("gemini-2.5-flash",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_end_user_spend_attributed_on_row(
|
||||
client: SpendClient, scoped_key: str, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
@ -425,6 +525,15 @@ def test_end_user_spend_attributed_on_row(
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.per_model.writes_own_rows")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI, Provider.ANTHROPIC),
|
||||
models=("gemini-2.5-flash", "claude-haiku-4-5"),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_each_model_on_a_shared_key_gets_its_own_row(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -474,6 +583,15 @@ def test_each_model_on_a_shared_key_gets_its_own_row(
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.failure.writes_failure_row")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=("openai/gpt-5.5",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_failure_call_writes_failure_status_row(
|
||||
client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
|
|
@ -499,6 +617,14 @@ def test_failure_call_writes_failure_status_row(
|
|||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.spend_calculate.returns_cost")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.SPEND_REPORTING,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=("gemini-2.5-flash",),
|
||||
)
|
||||
)
|
||||
def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
|
||||
cost = client.calculate_spend(
|
||||
"gemini-2.5-flash", "estimate the cost of this request"
|
||||
|
|
@ -509,6 +635,15 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
|
|||
)
|
||||
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=("gemini-2.5-flash",),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_spend_logs_endpoint_returns_spend(
|
||||
client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
from e2e_http import ProbeResult
|
||||
from e2e_metadata import Domain, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from proxy_client import Converged, await_converged
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -82,6 +83,14 @@ def _probe(client: SpendClient, params: BaseModel) -> ProbeResult:
|
|||
class TestTeamDailyActivity:
|
||||
@pytest.mark.replayable
|
||||
@pytest.mark.covers("mgmt.team.daily_activity.happy_path")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.SPEND_REPORTING,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=("openai/gpt-5.6-luna",),
|
||||
)
|
||||
)
|
||||
def test_valid_date_range_returns_results_and_metadata(
|
||||
self, client: SpendClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
@ -199,6 +208,12 @@ class TestTeamDailyActivity:
|
|||
assert empty.metadata.total_failed_requests == 0
|
||||
|
||||
@pytest.mark.covers("mgmt.team.daily_activity.missing_start_date_rejected")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.SPEND_REPORTING,
|
||||
)
|
||||
)
|
||||
def test_missing_start_date_is_rejected(self, client: SpendClient) -> None:
|
||||
end = datetime.now(timezone.utc).date().isoformat()
|
||||
result = _probe(client, TeamDailyActivityParams(end_date=end, page=1))
|
||||
|
|
@ -207,6 +222,12 @@ class TestTeamDailyActivity:
|
|||
)
|
||||
|
||||
@pytest.mark.covers("mgmt.team.daily_activity.missing_end_date_rejected")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.SPEND_REPORTING,
|
||||
)
|
||||
)
|
||||
def test_missing_end_date_is_rejected(self, client: SpendClient) -> None:
|
||||
start = (datetime.now(timezone.utc).date() - timedelta(days=1)).isoformat()
|
||||
result = _probe(client, TeamDailyActivityParams(start_date=start, page=1))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue