mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata (#44949)
* test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata * docs(e2e): name every markerless harness test file that carries no Subject * test(e2e): keep the step discovery comprehensions to one for clause
This commit is contained in:
ryan-crabbe-berri
GitHub
parent
cb76270bd9
commit
0b0fdedd1e
6 changed files with 233 additions and 68 deletions
|
|
@ -11,17 +11,22 @@ test_e2e_junit_report.py.
|
|||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import importlib
|
||||
import inspect
|
||||
import re
|
||||
import string
|
||||
import sys
|
||||
import threading
|
||||
import warnings
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Generator, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import fields, replace
|
||||
from dataclasses import MISSING, dataclass, fields, is_dataclass, replace
|
||||
from functools import cache, reduce
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from types import UnionType
|
||||
from typing import Final, cast, get_args, get_type_hints
|
||||
from types import ModuleType, UnionType
|
||||
from typing import Final, Union, cast, get_args, get_origin, get_type_hints
|
||||
|
||||
import pytest
|
||||
from e2e_metadata import (
|
||||
|
|
@ -42,9 +47,7 @@ from e2e_metadata import (
|
|||
subject_properties,
|
||||
)
|
||||
from junit_properties import package_from_nodeid, result_properties, source_from_item
|
||||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic.fields import FieldInfo
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -335,48 +338,180 @@ class _DeploymentBody(BaseModel):
|
|||
params: _Params
|
||||
|
||||
|
||||
def _field_type(annotation: object) -> object:
|
||||
"""`X | None` is `X`: a placeholder reads the field when it is set."""
|
||||
present: Final = tuple(arg for arg in get_args(annotation) if arg is not type(None))
|
||||
return present[0] if isinstance(annotation, UnionType) and len(present) == 1 else annotation
|
||||
def _alternatives(annotation: object) -> tuple[object, ...]:
|
||||
if not (isinstance(annotation, UnionType) or get_origin(annotation) is Union):
|
||||
return (annotation,)
|
||||
return tuple(arg for arg in cast("tuple[object, ...]", get_args(annotation)) if arg is not type(None))
|
||||
|
||||
|
||||
def _placeholders(owner: type) -> Iterator[tuple[str, str]]:
|
||||
tree: Final = ast.parse(inspect.getsource(owner))
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.FunctionDef):
|
||||
continue
|
||||
for decorator in node.decorator_list:
|
||||
match decorator:
|
||||
case ast.Call(func=ast.Name(id="step"), args=[ast.Constant(value=str(label))]):
|
||||
for _, field, _, _ in string.Formatter().parse(label):
|
||||
if field is not None:
|
||||
yield node.name, field
|
||||
case _:
|
||||
pass
|
||||
def _evaluated(annotation: object, module: str) -> object:
|
||||
holder: Final = type("Hint", (), {"__annotations__": {"value": annotation}, "__module__": module})
|
||||
hints: Final[Mapping[str, object]] = get_type_hints(holder)
|
||||
return hints["value"]
|
||||
|
||||
|
||||
def _dotted_placeholders(owner: type) -> Iterator[tuple[str, str]]:
|
||||
return ((method, field) for method, field in _placeholders(owner) if "." in field)
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StepHelper:
|
||||
path: Path
|
||||
qualname: str
|
||||
label: str
|
||||
owner: object
|
||||
function: Callable[..., object]
|
||||
|
||||
@property
|
||||
def where(self) -> str:
|
||||
return f"{self.path.relative_to(E2E_DIR)}::{self.qualname}"
|
||||
|
||||
def hint(self, placeholder: str) -> object:
|
||||
root: Final = placeholder.split(".")[0]
|
||||
if root == "self":
|
||||
return self.owner
|
||||
annotation: Final = cast("object", inspect.signature(self.function).parameters[root].annotation)
|
||||
return _evaluated(annotation, self.function.__module__)
|
||||
|
||||
|
||||
def _fields_read(owner: type, method: str, field: str) -> tuple[FieldInfo, ...] | None:
|
||||
"""The model fields a dotted placeholder reads, outermost first, or None if one doesn't exist."""
|
||||
root, *attributes = field.split(".")
|
||||
wrapped: Final = cast("Callable[..., object]", getattr(owner, method))
|
||||
hints: Final[Mapping[str, object]] = get_type_hints(inspect.unwrap(wrapped))
|
||||
current: object = _field_type(hints[root]) # rebind-ok: walks one type per attribute
|
||||
read: tuple[FieldInfo, ...] = () # rebind-ok: grows one field per attribute
|
||||
for attribute in attributes:
|
||||
if not (isinstance(current, type) and issubclass(current, BaseModel) and attribute in current.model_fields):
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FieldRead:
|
||||
always_set: bool
|
||||
annotation: object
|
||||
|
||||
|
||||
def _model_field(owner: type[BaseModel], attribute: str) -> FieldRead | None:
|
||||
info: Final = owner.model_fields.get(attribute)
|
||||
if info is None:
|
||||
return None
|
||||
return FieldRead(info.is_required() or (info.default_factory is None and info.default is not None), info.annotation)
|
||||
|
||||
|
||||
def _dataclass_field(owner: type, attribute: str) -> FieldRead | None:
|
||||
found: Final = next((field for field in fields(owner) if field.name == attribute), None)
|
||||
if found is None:
|
||||
return None
|
||||
always_set: Final = found.default_factory is MISSING and found.default is not None
|
||||
return FieldRead(always_set, _evaluated(found.type, owner.__module__))
|
||||
|
||||
|
||||
def _attribute(owner: object, attribute: str) -> FieldRead | None:
|
||||
if isinstance(owner, type) and issubclass(owner, BaseModel):
|
||||
return _model_field(owner, attribute)
|
||||
if isinstance(owner, type) and is_dataclass(owner):
|
||||
return _dataclass_field(owner, attribute)
|
||||
return None
|
||||
|
||||
|
||||
STEP_DECORATOR: Final = re.compile(r"^[ \t]*@step\(", re.MULTILINE)
|
||||
|
||||
|
||||
def _is_step(decorator: ast.expr) -> bool:
|
||||
match decorator:
|
||||
case ast.Call(func=ast.Name(id="step")):
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _decorated_defs(body: list[ast.stmt], prefix: str = "") -> Iterator[str]:
|
||||
for node in body:
|
||||
match node:
|
||||
case ast.ClassDef(name=name, body=inner):
|
||||
yield from _decorated_defs(inner, f"{prefix}{name}.")
|
||||
case (
|
||||
ast.FunctionDef(name=name, decorator_list=decorators)
|
||||
| ast.AsyncFunctionDef(name=name, decorator_list=decorators)
|
||||
):
|
||||
yield from (f"{prefix}{name}" for decorator in decorators if _is_step(decorator))
|
||||
case _:
|
||||
pass
|
||||
|
||||
|
||||
def _import_root(directory: Path) -> Path:
|
||||
return _import_root(directory.parent) if (directory / "__init__.py").exists() else directory
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _importable_from(directory: Path) -> Generator[None]:
|
||||
sys.path.insert(0, str(directory))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
sys.path.remove(str(directory))
|
||||
|
||||
|
||||
def _imported(path: Path) -> ModuleType:
|
||||
root: Final = _import_root(path.parent)
|
||||
with _importable_from(root):
|
||||
module: Final = importlib.import_module(".".join(path.relative_to(root).with_suffix("").parts))
|
||||
assert module.__file__ is not None and Path(module.__file__).resolve() == path, (
|
||||
f"{path} imports as {module.__name__}, which is {module.__file__}"
|
||||
)
|
||||
return module
|
||||
|
||||
|
||||
def _helper(path: Path, module: ModuleType, qualname: str) -> StepHelper:
|
||||
*scope, name = qualname.split(".")
|
||||
owner: Final = reduce(lambda found, part: cast("object", getattr(found, part)), scope, cast("object", module))
|
||||
wrapper: Final = cast("Callable[..., object]", getattr(owner, name))
|
||||
label: Final = cast("object", inspect.getclosurevars(wrapper).nonlocals.get("label"))
|
||||
assert isinstance(label, str), f"{path}::{qualname} is not wrapped by @step"
|
||||
return StepHelper(path, qualname, label, owner, cast("Callable[..., object]", inspect.unwrap(wrapper)))
|
||||
|
||||
|
||||
def _helpers_in(path: Path) -> tuple[StepHelper, ...]:
|
||||
qualnames: Final = tuple(_decorated_defs(ast.parse(path.read_text()).body))
|
||||
if not qualnames:
|
||||
return ()
|
||||
module: Final = _imported(path)
|
||||
return tuple(_helper(path, module, qualname) for qualname in qualnames)
|
||||
|
||||
|
||||
def _harness_files() -> tuple[Path, ...]:
|
||||
return tuple(sorted(path.resolve() for path in E2E_DIR.rglob("*.py") if "node_modules" not in path.parts))
|
||||
|
||||
|
||||
@cache
|
||||
def step_helpers() -> tuple[StepHelper, ...]:
|
||||
return tuple(chain.from_iterable(_helpers_in(path) for path in _harness_files()))
|
||||
|
||||
|
||||
def _placeholders() -> Iterator[tuple[StepHelper, str]]:
|
||||
for helper in step_helpers():
|
||||
for _, field, _, _ in string.Formatter().parse(helper.label):
|
||||
if field is not None:
|
||||
yield helper, field
|
||||
|
||||
|
||||
def _dotted_placeholders() -> Iterator[tuple[StepHelper, str]]:
|
||||
return ((helper, field) for helper, field in _placeholders() if "." in field)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PlaceholderRead:
|
||||
fields: tuple[FieldRead, ...]
|
||||
printed: tuple[object, ...]
|
||||
|
||||
|
||||
def _read(helper: StepHelper, field: str) -> PlaceholderRead | None:
|
||||
"""The fields a placeholder reads, outermost first, across every member of a union,
|
||||
and the types it ends up printing, or None if one of the fields doesn't exist."""
|
||||
read: PlaceholderRead = PlaceholderRead((), _alternatives(helper.hint(field))) # rebind-ok: one hop per attribute
|
||||
for attribute in field.split(".")[1:]:
|
||||
found = tuple(_attribute(owner, attribute) for owner in read.printed)
|
||||
hop = tuple(entry for entry in found if entry is not None)
|
||||
if len(hop) != len(found):
|
||||
return None
|
||||
read = (*read, current.model_fields[attribute]) # rebind-ok: grows one field per attribute
|
||||
current = _field_type(read[-1].annotation) # rebind-ok: walks one type per attribute
|
||||
printed = tuple(chain.from_iterable(_alternatives(entry.annotation) for entry in hop))
|
||||
read = PlaceholderRead((*read.fields, *hop), printed)
|
||||
return read
|
||||
|
||||
|
||||
def _fields_read(helper: StepHelper, field: str) -> tuple[FieldRead, ...] | None:
|
||||
read: Final = _read(helper, field)
|
||||
return None if read is None else read.fields
|
||||
|
||||
|
||||
SECRET_NAME: Final = re.compile(
|
||||
r"secret|password|api_key|access_key|private_key|credential_values|^token$|(access|auth|bearer|refresh|session)_token$"
|
||||
r"|^key$|credentials$|headers$|_host$|_endpoint$|^api_base$"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -393,15 +528,14 @@ def _models_in(annotation: object, seen: frozenset[type] = frozenset()) -> froze
|
|||
return frozenset[type[BaseModel]]().union(*(_models_in(arg, seen) for arg in args))
|
||||
|
||||
|
||||
def _printed_models(owner: type) -> frozenset[type[BaseModel]]:
|
||||
def hint(method: str, field: str) -> object:
|
||||
wrapped: Final = cast("Callable[..., object]", getattr(owner, method))
|
||||
hints: Final = cast("Mapping[str, object]", get_type_hints(inspect.unwrap(wrapped)))
|
||||
return hints[field.split(".")[0]]
|
||||
def _printed(helper: StepHelper, field: str) -> tuple[object, ...]:
|
||||
read: Final = _read(helper, field)
|
||||
return () if read is None else read.printed
|
||||
|
||||
return frozenset[type[BaseModel]]().union(
|
||||
*(_models_in(hint(method, field)) for method, field in _placeholders(owner))
|
||||
)
|
||||
|
||||
def _printed_models() -> frozenset[type[BaseModel]]:
|
||||
printed: Final = chain.from_iterable(_printed(helper, field) for helper, field in _placeholders())
|
||||
return frozenset[type[BaseModel]]().union(*(_models_in(annotation) for annotation in printed))
|
||||
|
||||
|
||||
class TestLabelTemplates:
|
||||
|
|
@ -460,32 +594,37 @@ class TestLabelTemplates:
|
|||
with pytest.raises(TypeError, match=r"body\.messages\[0\]"):
|
||||
_ = step("Send {body.messages[0]}")(chat)
|
||||
|
||||
@pytest.mark.parametrize("owner", [ProxyClient], ids=["ProxyClient"])
|
||||
def test_every_dotted_placeholder_in_the_harness_names_a_real_field(self, owner: type) -> None:
|
||||
def test_every_step_in_the_harness_is_checked(self) -> None:
|
||||
written: Final = Counter({path: len(STEP_DECORATOR.findall(path.read_text())) for path in _harness_files()})
|
||||
discovered: Final = Counter(helper.path for helper in step_helpers())
|
||||
assert written[E2E_DIR / "proxy_client.py"] > 0
|
||||
assert discovered == +written
|
||||
|
||||
def test_every_dotted_placeholder_in_the_harness_names_a_real_field(self) -> None:
|
||||
"""A dotted placeholder is read on every live call, so one naming a field the
|
||||
request model doesn't have would fail the test calling it, not the label."""
|
||||
placeholders: Final = tuple(_dotted_placeholders(owner))
|
||||
placeholders: Final = tuple(_dotted_placeholders())
|
||||
assert placeholders
|
||||
assert [
|
||||
f"{method}: {field}" for method, field in placeholders if _fields_read(owner, method, field) is None
|
||||
] == []
|
||||
missing: Final = tuple(
|
||||
f"{helper.where}: {field}" for helper, field in placeholders if _fields_read(helper, field) is None
|
||||
)
|
||||
assert missing == ()
|
||||
|
||||
@pytest.mark.parametrize("owner", [ProxyClient], ids=["ProxyClient"])
|
||||
def test_every_dotted_placeholder_in_the_harness_reads_a_field_the_caller_must_set(self, owner: type) -> None:
|
||||
"""A field with a default is usually left unset, and an unset field prints
|
||||
nothing, so the step would read "Save a provider credential for "."""
|
||||
def test_every_dotted_placeholder_in_the_harness_reads_a_field_that_is_always_set(self) -> None:
|
||||
"""A field that defaults to None is usually left unset, and an unset field
|
||||
prints as None, so the step would read "Save a provider credential for None".
|
||||
A required field or one with a real default, like a discriminator, always reads."""
|
||||
unset: Final = tuple(
|
||||
f"{method}: {field}"
|
||||
for method, field in _dotted_placeholders(owner)
|
||||
if not all(info.is_required() for info in _fields_read(owner, method, field) or ())
|
||||
f"{helper.where}: {field}"
|
||||
for helper, field in _dotted_placeholders()
|
||||
if not all(read.always_set for read in _fields_read(helper, field) or ())
|
||||
)
|
||||
assert unset == ()
|
||||
|
||||
@pytest.mark.parametrize("owner", [ProxyClient], ids=["ProxyClient"])
|
||||
def test_every_secret_field_a_label_can_print_is_hidden(self, owner: type) -> None:
|
||||
def test_every_secret_field_a_label_can_print_is_hidden(self) -> None:
|
||||
"""A `{body}` label prints nested models too, so a callback's credentials
|
||||
inside key metadata would land in the public report unless marked `repr=False`."""
|
||||
models: Final = _printed_models(owner)
|
||||
models: Final = _printed_models()
|
||||
assert models
|
||||
exposed: Final = sorted(
|
||||
f"{model.__name__}.{name}"
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ def test_bare_key_blocks_over_its_own_budget(...) -> None: ...
|
|||
|
||||
`route` is the endpoint the test is checking: `TEAM_MANAGEMENT` for a `/team/update` test, `SPEND_REPORTING` for a `/spend/logs` test, `MESSAGES` for a test of spend on `/v1/messages`. A test whose chat call only triggers the behavior under test, like the budget block above, leaves it unset, since its steps already name the call
|
||||
|
||||
Every field is optional today (the backfill of the rest of the suite is a later PR) and every field is a closed enum, so a typo is a basedpyright error at the call site rather than a property that silently never appears. `providers`, `models` and `capabilities` are tuples even with one member, because one test node routinely drives several: the claude_code matrix runs haiku, sonnet and opus in a single body, and a spend test calls two providers on one key. Declare every provider and every model the test drives, fallbacks included. The three are independent sets with no positional pairing between them (one provider x three models is the common case), and each is deduped and sorted at declaration so the committed run artifacts diff cleanly. `models=("gpt-5.5")` is a str and not a tuple, so anything but a tuple raises a `TypeError` where the decorator runs and shows up as a collection error naming the file. `Subject` is serialized with `dataclasses.asdict`, so a new scalar field needs no serializer edit; empty fields emit no `<property>` at all. A declared model names the constant the test drives (`CHEAP_ANTHROPIC_MODEL`, the file's own `BACKEND`), never a copy of its value, so the property cannot claim one model while an env override runs another. `e2e_metadata` and its call sites never import litellm, only the stdlib, pytest and pydantic: `Provider` mirrors litellm's `LlmProviders` values instead of importing them, because tests/e2e is shipped to the runner image on its own and a `from litellm...` at module scope would make the litellm package a hard dependency of COLLECTING the suite. `TestProviderMirrorsLitellm` in `tests/code_coverage_tests/test_e2e_metadata.py` fails on drift wherever litellm is importable and skips where it is not, so adding a provider is one line in `e2e_metadata`
|
||||
Every pytest test in the live suites declares a `Subject` with at least its `domain`. Only the markerless harness tests (the root-level `test_*.py` files, `coverage_registry/`, `batches/test_batch_cleanup.py`, `guardrails/test_guardrails_client.py`, `logging/test_datadog_reader.py`, `logging/test_span_selection.py` and claude_code's `_*_unit_tests/`) and the `load/` suite carry none, since they drive nothing. The fields themselves stay optional, since a test that makes no LLM call has no provider, model or mode to name. Every field is a closed enum, so a typo is a basedpyright error at the call site rather than a property that silently never appears. `providers`, `models` and `capabilities` are tuples even with one member, because one test node routinely drives several: the claude_code matrix runs haiku, sonnet and opus in a single body, and a spend test calls two providers on one key. Declare every provider and every model the test drives, fallbacks included. The three are independent sets with no positional pairing between them (one provider x three models is the common case), and each is deduped and sorted at declaration so the committed run artifacts diff cleanly. `models=("gpt-5.5")` is a str and not a tuple, so anything but a tuple raises a `TypeError` where the decorator runs and shows up as a collection error naming the file. `Subject` is serialized with `dataclasses.asdict`, so a new scalar field needs no serializer edit; empty fields emit no `<property>` at all. A declared model names the constant the test drives (`CHEAP_ANTHROPIC_MODEL`, the file's own `BACKEND`), never a copy of its value, so the property cannot claim one model while an env override runs another. `e2e_metadata` and its call sites never import litellm, only the stdlib, pytest and pydantic: `Provider` mirrors litellm's `LlmProviders` values instead of importing them, because tests/e2e is shipped to the runner image on its own and a `from litellm...` at module scope would make the litellm package a hard dependency of COLLECTING the suite. `TestProviderMirrorsLitellm` in `tests/code_coverage_tests/test_e2e_metadata.py` fails on drift wherever litellm is importable and skips where it is not, so adding a provider is one line in `e2e_metadata`
|
||||
|
||||
Declared fields ride out as JUnit `<property>` entries behind the fixed prefix, the same way steps do: each scalar under its field name, and each plural value as a repeated property under its SINGULAR name (`provider`, `model`, `capability`). The results JSON downstream regroups them under the plural key, so `providers`, `models` and `capabilities` are arrays there, `[]` when empty
|
||||
|
||||
|
|
@ -158,7 +158,7 @@ Declared fields ride out as JUnit `<property>` entries behind the fixed prefix,
|
|||
|
||||
`@step` from `e2e_metadata.py` goes on harness helpers (client methods and poll loops), never on a test. Each call adds one plain-English sentence to the running test's list of steps, in call order, so the list reads as what the test did. The step is recorded before the helper runs, so when a test fails, its last step is where it failed. Nobody writes steps by hand. They come from the calls the test actually made, so they can't drift from what happened
|
||||
|
||||
Steps are being added one harness at a time, and today `ProxyClient` and the rate-limit suite's `QuotaClient` have them. In a harness that has steps, every new public method that does something (an HTTP call, a poll, a login, a CLI run) gets a `@step`. Pure builders, parsers and `_private` helpers don't
|
||||
Every harness has steps: `ProxyClient`, each suite's own client, and the module-level helpers that create resources, poll or drive a CLI. Every new public method or function that does something (an HTTP call, a poll, a login, a CLI run) gets a `@step`. Pure builders, parsers and `_private` helpers don't. `TestLabelTemplates` in `tests/code_coverage_tests/test_e2e_metadata.py` finds every `@step` under tests/e2e on its own, so a new harness is checked without being registered anywhere: each dotted placeholder has to name a real field that is always set, and any request model a label can print has to hide its secret fields
|
||||
|
||||
### Writing a label
|
||||
|
||||
|
|
@ -179,7 +179,7 @@ Generate a virtual key with models: claude-haiku-4-5 and rpm limit: 3
|
|||
Send a /chat/completions request to claude-haiku-4-5 with the prompt "reply with one word d3940a1c4288"
|
||||
```
|
||||
|
||||
A request model prints only the fields the test set, and a dotted placeholder like `{body.litellm_params.model}` prints just one field. A field marked `Field(repr=False)` never prints, so mark every secret field that way, and never put a key, token or credential in a label. As a backstop, the recorder replaces the value of every secret-named environment variable (`*_KEY`, `*_SECRET`, `*_TOKEN`, `*_PASSWORD`, `*_CREDENTIALS`) with `***` wherever it shows up in a label. That only covers secrets the environment holds, so a key the proxy hands back during the test is still never named in a label. A placeholder that isn't one of the helper's parameters fails at import, and a literal brace is written `{{id}}`. A filled-in label is squashed onto one line and cut at 200 characters
|
||||
A request model prints only the fields the test set, and a dotted placeholder like `{body.litellm_params.model}` prints just one field. A field marked `Field(repr=False)` never prints, so mark every secret field that way, along with any field that can name an internal host (`api_base`, `langfuse_host`), and never put a key, token or credential in a label. As a backstop, the recorder replaces the value of every secret-named environment variable (`*_KEY`, `*_SECRET`, `*_TOKEN`, `*_PASSWORD`, `*_CREDENTIALS`) with `***` wherever it shows up in a label. That only covers secrets the environment holds, so a key the proxy hands back during the test is still never named in a label. A placeholder that isn't one of the helper's parameters fails at import, and a literal brace is written `{{id}}`. A filled-in label is squashed onto one line and cut at 200 characters
|
||||
|
||||
### Nesting and the step log
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,10 @@ class Route(str, Enum):
|
|||
METRICS = "metrics"
|
||||
PROXY_CONFIG = "proxy_config"
|
||||
ADMIN_UI = "admin_ui"
|
||||
CONTAINERS = "containers"
|
||||
COUNT_TOKENS = "count_tokens"
|
||||
GOOGLE_GENAI = "google_genai"
|
||||
TAG_MANAGEMENT = "tag_management"
|
||||
|
||||
|
||||
class Provider(str, Enum):
|
||||
|
|
@ -115,6 +119,9 @@ class Provider(str, Enum):
|
|||
ELEVENLABS = "elevenlabs"
|
||||
ASSEMBLYAI = "assemblyai"
|
||||
LITELLM_PROXY = "litellm_proxy"
|
||||
BEDROCK_MANTLE = "bedrock_mantle"
|
||||
AWS_POLLY = "aws_polly"
|
||||
SAIL = "sail"
|
||||
|
||||
|
||||
class Capability(str, Enum):
|
||||
|
|
@ -132,6 +139,7 @@ class Capability(str, Enum):
|
|||
PROMPT_CACHING = "prompt_caching"
|
||||
RESPONSE_SCHEMA = "response_schema"
|
||||
MID_CONVERSATION_SYSTEM = "mid_conversation_system"
|
||||
AUDIO_OUTPUT = "audio_output"
|
||||
|
||||
|
||||
class Mode(str, Enum):
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from e2e_http import (
|
|||
post_json_external,
|
||||
unwrap,
|
||||
)
|
||||
from e2e_metadata import step
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
KEYCLOAK_URL_ENV: Final = "E2E_KEYCLOAK_URL"
|
||||
|
|
@ -171,6 +172,7 @@ class Keycloak:
|
|||
case _:
|
||||
return pytest.fail(f"Keycloak refused {context}: {result}")
|
||||
|
||||
@step("Create the Keycloak group {name}")
|
||||
def create_group(self, name: str) -> str:
|
||||
return created_id(
|
||||
post_json_external(
|
||||
|
|
@ -179,6 +181,7 @@ class Keycloak:
|
|||
f"group {name}",
|
||||
)
|
||||
|
||||
@step("Create the Keycloak user {username}")
|
||||
def create_user(
|
||||
self, *, username: str, email: str, password: str, group: str | None = None, groups: tuple[str, ...] = ()
|
||||
) -> str:
|
||||
|
|
@ -196,12 +199,15 @@ class Keycloak:
|
|||
f"user {username}",
|
||||
)
|
||||
|
||||
@step("Delete the Keycloak user")
|
||||
def delete_user(self, user_id: str) -> None:
|
||||
self._delete(f"/users/{user_id}")
|
||||
|
||||
@step("Delete the Keycloak group")
|
||||
def delete_group(self, group_id: str) -> None:
|
||||
self._delete(f"/groups/{group_id}")
|
||||
|
||||
@step("Check that Keycloak's admin API returns 404 for the deleted entry under /{kind}")
|
||||
def assert_absent(self, kind: Literal["users", "groups", "clients"], resource_id: str) -> None:
|
||||
result: Final = get_external(
|
||||
self._admin_url(f"/{kind}/{resource_id}"),
|
||||
|
|
@ -230,11 +236,13 @@ class Keycloak:
|
|||
stacklevel=2,
|
||||
)
|
||||
|
||||
@step("Create the Keycloak group {group} and the user e2e-jwt-user-{marker} in it")
|
||||
def provision(self, *, marker: str, group: str, defer: Callable[[Callable[[], object]], None]) -> Identity:
|
||||
"""Create `group` and a user in it, credentialed with a password generated
|
||||
for this test alone, and hand back the identity a token can be minted for."""
|
||||
return self.provision_groups(marker=marker, groups=(group,), defer=defer)
|
||||
|
||||
@step("Create each Keycloak group the user e2e-jwt-user-{marker} belongs to, then the user")
|
||||
def provision_groups(
|
||||
self, *, marker: str, groups: tuple[str, ...], defer: Callable[[Callable[[], object]], None]
|
||||
) -> Identity:
|
||||
|
|
@ -246,6 +254,7 @@ class Keycloak:
|
|||
group_ids: Final = tuple(provision_group(group) for group in groups)
|
||||
return self.provision_user(marker=marker, groups=groups, group_ids=group_ids, defer=defer)
|
||||
|
||||
@step("Create the Keycloak user e2e-jwt-user-{marker} with a password of its own")
|
||||
def provision_user(
|
||||
self,
|
||||
*,
|
||||
|
|
@ -262,6 +271,7 @@ class Keycloak:
|
|||
defer(lambda: self.delete_user(user_id))
|
||||
return Identity(user_id=user_id, username=username, password=password, groups=groups, group_ids=group_ids)
|
||||
|
||||
@step("Get a Keycloak access token for {identity.username} from the client {client_id}")
|
||||
def access_token(
|
||||
self, identity: Identity, *, client_id: str = TESTS_CLIENT_ID, issuer_host: str | None = None
|
||||
) -> str:
|
||||
|
|
@ -275,9 +285,11 @@ class Keycloak:
|
|||
)
|
||||
return self._token(result, f"a token for {identity.username}")
|
||||
|
||||
@step("Read Keycloak's OpenID Connect discovery document")
|
||||
def discovery(self) -> Discovery:
|
||||
return unwrap(get_external(f"{self.issuer}/.well-known/openid-configuration", response_type=Discovery))
|
||||
|
||||
@step("Register a browser SSO client in Keycloak")
|
||||
def browser_client(self, *, callback_url: str, defer: Callable[[Callable[[], object]], None]) -> BrowserClient:
|
||||
client: Final = BrowserClient(
|
||||
client_id=f"e2e-browser-{secrets.token_hex(8)}",
|
||||
|
|
@ -309,6 +321,7 @@ class Keycloak:
|
|||
assert configured.attributes.pkce == "S256"
|
||||
return client
|
||||
|
||||
@step("Get a Keycloak access token for {identity.username} through the browser SSO client")
|
||||
def browser_token(self, identity: Identity, client: BrowserClient) -> str:
|
||||
return self._token(
|
||||
post_form_external(
|
||||
|
|
@ -325,6 +338,7 @@ class Keycloak:
|
|||
"browser-profile identity mapping",
|
||||
)
|
||||
|
||||
@step("Read the signed-in user's profile from Keycloak's userinfo endpoint")
|
||||
def userinfo(self, token: str) -> UserInfo:
|
||||
return unwrap(
|
||||
get_external(
|
||||
|
|
@ -435,6 +449,7 @@ def _signal_process_group(process_id: int, signum: int) -> bool:
|
|||
return True
|
||||
|
||||
|
||||
@step("Stop the child process and everything it started")
|
||||
def stop_process_group(child: subprocess.Popen[bytes]) -> None:
|
||||
_signal_process_group(child.pid, signal.SIGTERM)
|
||||
deadline: Final = time.monotonic() + 5
|
||||
|
|
@ -457,6 +472,7 @@ def _process_group_exists(process_id: int) -> bool:
|
|||
return True
|
||||
|
||||
|
||||
@step("Run a command against the proxy with a Keycloak browser SSO client")
|
||||
def run_oidc_profile(proxy_url: str, command: list[str]) -> int:
|
||||
idp: Final = keycloak_from_env().with_strict_cleanup()
|
||||
with ExitStack() as cleanup:
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from builtins import ExceptionGroup
|
|||
from dataclasses import dataclass, field
|
||||
from typing import Callable, Final, List, Protocol, runtime_checkable
|
||||
|
||||
from e2e_metadata import step
|
||||
from proxy_client import ProxyClient
|
||||
from models import KeyGenerateBody
|
||||
|
||||
|
|
@ -81,6 +82,7 @@ class ResourceManager:
|
|||
self.defer(lambda: self.client.delete_customers([customer_id]))
|
||||
return customer_id
|
||||
|
||||
@step("Delete every resource the test created, newest first")
|
||||
def teardown(self) -> None:
|
||||
failures: Final = tuple(
|
||||
failure for cleanup in reversed(self._cleanups) if (failure := _run_cleanup(cleanup)) is not None
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ class BudgetWindowState(BudgetWindow):
|
|||
class KeyLoggingCallbackVars(BaseModel):
|
||||
langfuse_public_key: str | None = Field(default=None, repr=False)
|
||||
langfuse_secret_key: str | None = Field(default=None, repr=False)
|
||||
langfuse_host: str | None = None
|
||||
langfuse_host: str | None = Field(default=None, repr=False)
|
||||
wandb_api_key: str | None = Field(default=None, repr=False)
|
||||
weave_project_id: str | None = None
|
||||
|
||||
|
|
@ -662,7 +662,7 @@ class McpServerCreateBody(BaseModel):
|
|||
authorization_url: str | None = None
|
||||
token_url: str | None = None
|
||||
registration_url: str | None = None
|
||||
credentials: McpOauthCredentials | None = None
|
||||
credentials: McpOauthCredentials | None = Field(default=None, repr=False)
|
||||
server_name: str | None = None
|
||||
description: str | None = None
|
||||
mcp_info: McpInfo | None = None
|
||||
|
|
@ -1253,14 +1253,14 @@ class LiteLLMParamsBody(BaseModel):
|
|||
model: str
|
||||
api_key: str | None = Field(default=None, repr=False)
|
||||
litellm_credential_name: str | None = None
|
||||
api_base: str | None = None
|
||||
api_base: str | None = Field(default=None, repr=False)
|
||||
api_version: str | None = None
|
||||
realtime_protocol: str | None = None
|
||||
allowed_openai_params: list[str] | None = None
|
||||
aws_access_key_id: str | None = Field(default=None, repr=False)
|
||||
aws_secret_access_key: str | None = Field(default=None, repr=False)
|
||||
aws_region_name: str | None = None
|
||||
aws_bedrock_runtime_endpoint: str | None = None
|
||||
aws_bedrock_runtime_endpoint: str | None = Field(default=None, repr=False)
|
||||
vertex_project: str | None = None
|
||||
vertex_location: str | None = None
|
||||
vertex_credentials: str | None = Field(default=None, repr=False)
|
||||
|
|
@ -1287,7 +1287,7 @@ class LiteLLMParamsBody(BaseModel):
|
|||
input_cost_per_token_flex: float | None = None
|
||||
output_cost_per_token_flex: float | None = None
|
||||
cache_read_input_token_cost_flex: float | None = None
|
||||
extra_headers: dict[str, str] | None = None
|
||||
extra_headers: dict[str, str] | None = Field(default=None, repr=False)
|
||||
use_in_pass_through: bool | None = None
|
||||
complexity_router_config: dict[str, object] | None = None
|
||||
auto_router_config: str | None = None
|
||||
|
|
@ -1422,7 +1422,7 @@ class KeyUpdateBody(BaseModel):
|
|||
keeps its stored value, `CLEAR` sends an explicit null that clears it (`budget_duration`
|
||||
clears `budget_reset_at` with it), and `metadata` replaces the stored metadata wholesale."""
|
||||
|
||||
key: str
|
||||
key: str = Field(repr=False)
|
||||
project_id: str | Cleared | None = None
|
||||
models: list[str] | None = None
|
||||
key_alias: str | None = None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue