From 736da28ac15c6ef336cc35f80f56994064803856 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:30:28 -0700 Subject: [PATCH 01/35] fix(tests): match the lowercased bind error in the owned-proxy port-race retry (#45097) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/integration/_support/process.py | 4 +-- .../unit/integration_support/test_process.py | 35 +++++++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 8e9f689ec2d..12780540380 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -119,7 +119,7 @@ def _stop(process: subprocess.Popen[bytes]) -> None: _PORT_ATTEMPTS: Final = 3 -_BIND_COLLISION: Final = os.strerror(errno.EADDRINUSE) +_BIND_COLLISION: Final = os.strerror(errno.EADDRINUSE).lower() def _free_port() -> int: @@ -151,7 +151,7 @@ def _launch(command: tuple[str, ...], root: Path, environment: Mapping[str, str] def _lost_port_race(exit_code: int | None, log: Path) -> bool: - return exit_code is not None and _BIND_COLLISION in log.read_text() + return exit_code is not None and _BIND_COLLISION in log.read_text().lower() def _wait_until_ready(launch: _Launch) -> None: diff --git a/tests/unit/integration_support/test_process.py b/tests/unit/integration_support/test_process.py index 042a2447dff..7896786fce8 100644 --- a/tests/unit/integration_support/test_process.py +++ b/tests/unit/integration_support/test_process.py @@ -1,8 +1,10 @@ from __future__ import annotations +import asyncio import errno import importlib -import os +import socket +from collections.abc import Iterator from pathlib import Path from types import ModuleType from typing import Final @@ -10,7 +12,6 @@ from typing import Final import pytest TESTS_DIR: Final = Path(__file__).resolve().parents[2] -BIND_ERROR_LINE: Final = f"ERROR: {OSError(errno.EADDRINUSE, os.strerror(errno.EADDRINUSE))}\n" UNRELATED_CRASH: Final = "Traceback (most recent call last):\nModuleNotFoundError: No module named 'litellm'\n" @@ -20,19 +21,41 @@ def process_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType: return importlib.import_module("integration._support.process") +async def _refused_bind(port: int) -> OSError | None: + try: + await asyncio.get_running_loop().create_server(asyncio.Protocol, "127.0.0.1", port) + except OSError as refused: + return refused + return None + + +@pytest.fixture +def bind_error_line() -> Iterator[str]: + with socket.socket() as held: + held.bind(("127.0.0.1", 0)) + held.listen() + refused: Final = asyncio.run(_refused_bind(held.getsockname()[1])) + assert refused is not None and refused.errno == errno.EADDRINUSE + yield f"ERROR: {refused}\n" + + def _written_log(directory: Path, text: str) -> Path: log: Final = directory / "owned-proxy.log" log.write_text(text) return log -def test_lost_port_race_matches_the_bind_error_the_server_logs(process_module: ModuleType, tmp_path: Path) -> None: - assert process_module._lost_port_race(1, _written_log(tmp_path, BIND_ERROR_LINE)) +def test_lost_port_race_matches_the_bind_error_the_server_logs( + process_module: ModuleType, bind_error_line: str, tmp_path: Path +) -> None: + assert process_module._lost_port_race(1, _written_log(tmp_path, bind_error_line)) def test_lost_port_race_ignores_an_exit_for_another_reason(process_module: ModuleType, tmp_path: Path) -> None: assert not process_module._lost_port_race(1, _written_log(tmp_path, UNRELATED_CRASH)) -def test_lost_port_race_needs_the_process_to_have_exited(process_module: ModuleType, tmp_path: Path) -> None: - assert not process_module._lost_port_race(None, _written_log(tmp_path, BIND_ERROR_LINE)) +def test_lost_port_race_needs_the_process_to_have_exited( + process_module: ModuleType, bind_error_line: str, tmp_path: Path +) -> None: + assert not process_module._lost_port_race(None, _written_log(tmp_path, bind_error_line)) From ade17902a03458b68d23dbcb40bab9b67e389d8e Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 7 Oct 2026 10:30:37 -0700 Subject: [PATCH 02/35] test(e2e): tag llm_translation tests with Subject metadata and record harness steps (#44950) * test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata * test(e2e): tag llm_translation tests with Subject metadata and record harness steps * docs(e2e): name every markerless harness test file that carries no Subject * test(e2e): keep the step discovery comprehensions to one for clause * test(e2e): declare the realtime param tuples Final --- .../llm_translation/conversational_matrix.py | 66 +++- .../e2e/llm_translation/passthrough_client.py | 18 +- .../realtime/realtime_client.py | 6 +- .../realtime/test_realtime_bedrock_e2e.py | 10 + .../realtime/test_realtime_e2e.py | 55 ++- .../test_realtime_pipecat_audio_e2e.py | 62 ++- .../realtime/test_realtime_pipecat_e2e.py | 17 +- .../llm_translation/test_audio_speech_e2e.py | 70 +++- .../test_audio_transcriptions_e2e.py | 49 ++- .../test_bedrock_native_e2e.py | 90 +++++ .../test_bedrock_provider_matrix_e2e.py | 55 +++ ...test_bedrock_web_search_server_tool_e2e.py | 11 + .../e2e/llm_translation/test_cache_control.py | 41 ++ ..._cache_control_injection_tool_calls_e2e.py | 32 +- .../test_chat_completions_contract_e2e.py | 79 ++++ .../test_chat_completions_regression_e2e.py | 369 +++++++++++++++++- .../test_chat_mid_conversation_system_e2e.py | 41 ++ .../test_chat_stream_contract_e2e.py | 14 +- .../test_chat_tool_round_trip_e2e.py | 51 +++ .../test_completions_endpoint_e2e.py | 16 +- .../llm_translation/test_containers_e2e.py | 26 ++ .../test_credential_messages_e2e.py | 15 +- .../test_custom_pricing_e2e.py | 22 ++ .../test_deepseek_reasoning_e2e.py | 31 ++ .../test_embeddings_endpoint_e2e.py | 108 ++++- .../test_files_batches_contract_e2e.py | 23 ++ .../llm_translation/test_google_native_e2e.py | 19 + .../llm_translation/test_image_edits_e2e.py | 33 +- .../test_image_generation_e2e.py | 61 ++- .../test_messages_azure_foundry_e2e.py | 49 +++ .../test_messages_bedrock_e2e.py | 20 + .../e2e/llm_translation/test_messages_e2e.py | 135 ++++++- ...st_messages_mid_conversation_system_e2e.py | 21 + ...onversation_system_native_providers_e2e.py | 41 ++ .../llm_translation/test_moderations_e2e.py | 32 +- .../e2e/llm_translation/test_ocr_rust_e2e.py | 68 +++- .../llm_translation/test_passthrough_e2e.py | 157 +++++++- .../test_passthrough_headers_e2e.py | 10 + .../test_provider_features_e2e.py | 15 +- .../llm_translation/test_realtime_http_e2e.py | 11 + tests/e2e/llm_translation/test_rerank_e2e.py | 27 +- .../test_responses_bridge_streaming_e2e.py | 31 +- .../e2e/llm_translation/test_responses_e2e.py | 208 +++++++++- .../test_responses_retrieve_e2e.py | 55 ++- tests/e2e/llm_translation/test_sail_e2e.py | 37 ++ .../llm_translation/test_together_ai_e2e.py | 129 ++++++ .../test_token_counter_gemini_contents_e2e.py | 30 +- .../llm_translation/test_vector_stores_e2e.py | 6 + .../test_vertex_passthrough_e2e.py | 10 + 49 files changed, 2489 insertions(+), 93 deletions(-) diff --git a/tests/e2e/llm_translation/conversational_matrix.py b/tests/e2e/llm_translation/conversational_matrix.py index 0d6f6ed3d4e..20dbe236fae 100644 --- a/tests/e2e/llm_translation/conversational_matrix.py +++ b/tests/e2e/llm_translation/conversational_matrix.py @@ -33,6 +33,8 @@ from anthropic.types import ( ToolUseBlockParam, ) from e2e_config import provider_edge_base, unique_marker +from e2e_metadata import Capability as MetaCapability +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta, step from lifecycle import ResourceManager from llm_translation.sdk_clients import NO_PROXY_CACHE, SdkClients, response_header from models import CredentialCreateBody, LiteLLMParamsBody @@ -65,6 +67,10 @@ Streaming = Literal["stream", "nonstream"] Assertion = Literal["works", "cost_logged"] ToolMode = Literal["none", "forced", "offered"] +GPT_4O_MINI_BACKEND: Final = "openai/gpt-4o-mini" +GPT_5_4_MINI_BACKEND: Final = "openai/gpt-5.4-mini" +CLAUDE_HAIKU_BACKEND: Final = "anthropic/claude-haiku-4-5" + SURFACES: Final[tuple[SurfaceName, ...]] = ("chat_completions", "messages", "responses") AUTH_METHODS: Final[tuple[AuthMethod, ...]] = ("env_ref", "stored_credential") @@ -104,12 +110,19 @@ class Deployment: assert key, f"{self.api_key_env} is not set in the test process environment" return key + def provider(self) -> Provider: + match self.route: + case "openai": + return Provider.OPENAI + case "anthropic": + return Provider.ANTHROPIC + DEPLOYMENTS: Final[tuple[Deployment, ...]] = ( Deployment( route="openai", label="gpt-4o-mini", - backend="openai/gpt-4o-mini", + backend=GPT_4O_MINI_BACKEND, api_key_env="OPENAI_API_KEY", edge_mount="openai", edge_suffix="/v1", @@ -117,7 +130,7 @@ DEPLOYMENTS: Final[tuple[Deployment, ...]] = ( Deployment( route="openai", label="gpt-5.4-mini", - backend="openai/gpt-5.4-mini", + backend=GPT_5_4_MINI_BACKEND, api_key_env="OPENAI_API_KEY", edge_mount="openai", edge_suffix="/v1", @@ -125,7 +138,7 @@ DEPLOYMENTS: Final[tuple[Deployment, ...]] = ( Deployment( route="anthropic", label="claude-haiku-4-5", - backend="anthropic/claude-haiku-4-5", + backend=CLAUDE_HAIKU_BACKEND, api_key_env="ANTHROPIC_API_KEY", edge_mount="anthropic", edge_suffix="", @@ -146,6 +159,26 @@ class Cell: def registry_id(self, capability: Capability, streaming: Streaming, assertion: Assertion) -> str: return f"llm.{self.surface}.{self.deployment.route}.{capability}.{streaming}.{assertion}" + def subject(self, capability: Capability, streaming: Streaming, assertion: Assertion) -> Subject: + return Subject( + domain=Domain.SPEND_BUDGETS if assertion == "cost_logged" else Domain.LLM_TRANSLATION, + route=_surface_route(self.surface), + providers=(self.deployment.provider(),), + models=(self.deployment.backend,), + capabilities=() if capability == "basic" else (MetaCapability.FUNCTION_CALLING,), + mode=Mode.STREAM if streaming == "stream" else Mode.NONSTREAM, + ) + + +def _surface_route(surface: SurfaceName) -> Route: + match surface: + case "chat_completions": + return Route.CHAT_COMPLETIONS + case "messages": + return Route.MESSAGES + case "responses": + return Route.RESPONSES + CELLS: Final[tuple[Cell, ...]] = tuple( Cell(surface=surface, deployment=deployment, auth=auth) @@ -158,7 +191,14 @@ CELLS: Final[tuple[Cell, ...]] = tuple( def cells_covering(capability: Capability, streaming: Streaming, assertion: Assertion) -> tuple[ParameterSet, ...]: """Every cell as a pytest param carrying the registry id its test proves.""" return tuple( - pytest.param(cell, id=cell.id, marks=pytest.mark.covers(cell.registry_id(capability, streaming, assertion))) + pytest.param( + cell, + id=cell.id, + marks=( + pytest.mark.covers(cell.registry_id(capability, streaming, assertion)), + meta(cell.subject(capability, streaming, assertion)), + ), + ) for cell in CELLS ) @@ -352,9 +392,14 @@ class ChatCompletionsSurface: cost_header=response_header(raw.headers, "x-litellm-response-cost"), ) + @step( + 'Send a /chat/completions request to {model} with the prompt "{prompt}"' + " and forced weather tool use set to {with_tool}" + ) def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: return self._turn(key, model, _chat_history(prompt), "forced" if with_tool else "none") + @step('Send a streaming /chat/completions request to {model} with the prompt "{prompt}"') def stream(self, key: str, model: str, prompt: str) -> StreamedReply: chunks: Final[tuple[ChatCompletionChunk, ...]] = tuple( self.sdk.openai(key).chat.completions.create( @@ -373,6 +418,7 @@ class ChatCompletionsSurface: event_count=len(chunks), ) + @step("Send the {call.name} tool result back to {model} over /chat/completions") def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: tool_call: Final[ChatCompletionMessageFunctionToolCallParam] = { "id": call.call_id, @@ -421,9 +467,14 @@ class MessagesSurface: cost_header=response_header(raw.headers, "x-litellm-response-cost"), ) + @step( + 'Send a /v1/messages request to {model} with the prompt "{prompt}"' + " and forced weather tool use set to {with_tool}" + ) def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: return self._turn(key, model, ({"role": "user", "content": prompt},), "forced" if with_tool else "none") + @step('Send a streaming /v1/messages request to {model} with the prompt "{prompt}"') def stream(self, key: str, model: str, prompt: str) -> StreamedReply: events: Final[tuple[RawMessageStreamEvent, ...]] = tuple( self.sdk.anthropic(key).messages.create( @@ -446,6 +497,7 @@ class MessagesSurface: event_count=len(events), ) + @step("Send the {call.name} tool result back to {model} over /v1/messages") def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: tool_use: Final[ToolUseBlockParam] = { "type": "tool_use", @@ -496,9 +548,14 @@ class ResponsesSurface: cost_header=response_header(raw.headers, "x-litellm-response-cost"), ) + @step( + 'Send a /v1/responses request to {model} with the prompt "{prompt}"' + " and forced weather tool use set to {with_tool}" + ) def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: return self._turn(key, model, [{"role": "user", "content": prompt}], "forced" if with_tool else "none") + @step('Send a streaming /v1/responses request to {model} with the prompt "{prompt}"') def stream(self, key: str, model: str, prompt: str) -> StreamedReply: events: Final[tuple[ResponseStreamEvent, ...]] = tuple( self.sdk.openai(key).responses.create( @@ -519,6 +576,7 @@ class ResponsesSurface: event_count=len(events), ) + @step("Send the {call.name} tool result back to {model} over /v1/responses") def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: function_call: Final[ResponseFunctionToolCallParam] = { "type": "function_call", diff --git a/tests/e2e/llm_translation/passthrough_client.py b/tests/e2e/llm_translation/passthrough_client.py index a56d3dc077e..478148e45aa 100644 --- a/tests/e2e/llm_translation/passthrough_client.py +++ b/tests/e2e/llm_translation/passthrough_client.py @@ -18,6 +18,7 @@ from websockets.exceptions import InvalidStatus from websockets.sync.client import connect from e2e_config import ws_base_url +from e2e_metadata import step from proxy_client import ProxyClient from e2e_http import FileUploadForm, Headers, NoBody, Result, StreamingResponse from models import ChatMessage @@ -34,7 +35,7 @@ class JsonSchema(BaseModel): class GeminiHeaders(Headers): - x_goog_api_key: str = Field(serialization_alias="x-goog-api-key") + x_goog_api_key: str = Field(serialization_alias="x-goog-api-key", repr=False) content_type: str = Field( default="application/json", serialization_alias="Content-Type" ) @@ -42,7 +43,7 @@ class GeminiHeaders(Headers): class AnthropicHeaders(Headers): - x_api_key: str = Field(serialization_alias="x-api-key") + x_api_key: str = Field(serialization_alias="x-api-key", repr=False) anthropic_version: str = Field( default="2023-06-01", serialization_alias="anthropic-version" ) @@ -56,7 +57,7 @@ class VertexHeaders(Headers): # Only the litellm virtual key; the /vertex_ai passthrough mints the Vertex token # from the proxy's own service account (the deployment marked use_in_pass_through), # so no upstream Authorization bearer is sent from the client. - x_litellm_api_key: str = Field(serialization_alias="x-litellm-api-key") + x_litellm_api_key: str = Field(serialization_alias="x-litellm-api-key", repr=False) content_type: str = Field( default="application/json", serialization_alias="Content-Type" ) @@ -246,6 +247,7 @@ class PassthroughClient: # ---- Gemini native passthrough (/gemini/v1beta/...) ----------------- + @step("Send a Gemini generateContent request to {model} through /gemini") def gemini_generate( self, key: str, @@ -263,6 +265,7 @@ class PassthroughClient: ), ) + @step("Send a Gemini streamGenerateContent request to {model} through /gemini") def gemini_stream( self, key: str, model: str, text: str, *, tags: list[str] | None = None ) -> StreamingResponse: @@ -278,6 +281,7 @@ class PassthroughClient: # ---- Vertex AI native passthrough (/vertex_ai/v1/projects/...) ------- + @step("Send a Vertex AI generateContent request to {model} in {location} through /vertex_ai") def vertex_generate( self, key: str, project: str, location: str, model: str, text: str ) -> StreamingResponse: @@ -295,6 +299,7 @@ class PassthroughClient: # ---- Anthropic native passthrough (/anthropic/v1/messages) ---------- + @step("Send a /v1/messages request to {model} through /anthropic with streaming set to {stream}") def anthropic_message( self, key: str, @@ -324,6 +329,7 @@ class PassthroughClient: # Relayed to OpenAI untouched, which is the whole point of the prefix: the # customer opts out of the gateway's managed-file handling here. + @step("Upload {filename} to /openai_passthrough/v1/files") def openai_passthrough_upload_file( self, key: str, *, content: bytes, filename: str ) -> Result[PassthroughFileObject]: @@ -336,6 +342,7 @@ class PassthroughClient: response_type=PassthroughFileObject, ) + @step("Delete the uploaded file through /openai_passthrough/v1/files") def openai_passthrough_delete_file( self, key: str, file_id: str ) -> Result[PassthroughFileDeleted]: @@ -346,6 +353,7 @@ class PassthroughClient: response_type=PassthroughFileDeleted, ) + @step("List batches from /openai_passthrough/v1/batches") def openai_passthrough_list_batches(self, key: str) -> Result[PassthroughBatchList]: return self.proxy.transport.get( "/openai_passthrough/v1/batches", @@ -360,6 +368,7 @@ class PassthroughClient: # budgets against this traffic, so a 200 that logs no spend is money the # gateway never sees. + @step("Send a /v1/responses request to {model} through /openai_passthrough with streaming set to {stream}") def openai_passthrough_responses( self, key: str, model: str, text: str, *, stream: bool = False ) -> StreamingResponse: @@ -370,6 +379,7 @@ class PassthroughClient: stream=stream, ) + @step('Send a /v1/embeddings request to {model} through /openai_passthrough for "{text}"') def openai_passthrough_embed( self, key: str, model: str, text: str ) -> StreamingResponse: @@ -379,6 +389,7 @@ class PassthroughClient: json=OpenAIEmbeddingBody(model=model, input=text), ) + @step("Send a /v1/chat/completions request to {model} through /openai") def openai_chat( self, key: str, model: str, text: str, *, max_completion_tokens: int = 64 ) -> StreamingResponse: @@ -397,6 +408,7 @@ class PassthroughClient: # The same prefixes over an upgrade instead of a POST, for the provider APIs # that only speak websocket (realtime, responses.connect). + @step("Open a websocket to {path} and wait for its first event") def openai_passthrough_websocket( self, key: str, diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index a280f4bc26b..6eae4fdf54d 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -307,9 +307,11 @@ def as_text(message: str | bytes) -> str: class RealtimeSession: connection: Connection - def send(self, event: BaseModel) -> None: + @step("Send the realtime event {event.type} over the websocket") + def send(self, event: SessionUpdate | ConversationItemCreate | ResponseCreate) -> None: self.connection.send(event.model_dump_json(by_alias=True, exclude_none=True)) + @step("Wait for a {stop_type} event on the realtime websocket") def collect_until( self, stop_type: str, *, timeout: float ) -> tuple[ReceivedEvent, ...]: @@ -381,6 +383,7 @@ class RealtimeSession: class RealtimeClient: proxy: ProxyClient + @step("Add a realtime deployment that calls {provider.litellm_params.model}") def provision(self, provider: RealtimeProvider) -> tuple[str, str]: """Register this provider's realtime deployment through /model/new and return (model_name, model_id). The name is marker-unique so it never collides with a @@ -393,6 +396,7 @@ class RealtimeClient: ) return model_name, model_id + @step("Open a /v1/realtime websocket session to {model}") @contextmanager def connect( self, *, key: str, model: str, timeout: float = 15.0 diff --git a/tests/e2e/llm_translation/realtime/test_realtime_bedrock_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_bedrock_e2e.py index 656882a4d92..a86f1daff63 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_bedrock_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_bedrock_e2e.py @@ -19,6 +19,7 @@ from __future__ import annotations import pytest from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from realtime_client import ( @@ -42,6 +43,15 @@ class TestNovaSonicRealtime: "llm.realtime.bedrock_converse.basic.stream.works", exercised_on=["realtime"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider.BEDROCK,), + models=(NOVA_SONIC,), + mode=Mode.WEBSOCKET, + ) + ) def test_nova_sonic_response_create_completes( self, client: RealtimeClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py index d7870b26497..622ed9d507f 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py @@ -12,7 +12,10 @@ hard failure, not a skip; once configured, a protocol failure is likewise a hard failure. See REALTIME_COVERAGE_MATRIX.md. """ +from typing import Final + import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from pydantic import BaseModel @@ -42,7 +45,42 @@ from websockets.exceptions import ConnectionClosedError pytestmark = pytest.mark.e2e -PROVIDER_PARAMS = [pytest.param(p, id=p.id) for p in PROVIDERS] +AZURE_REALTIME_MODEL: Final = "azure/gpt-realtime" + +TEXT_PARAMS: Final = tuple( + pytest.param( + p, + id=p.id, + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider(p.id),), + models=(p.litellm_params.model,), + mode=Mode.WEBSOCKET, + ) + ), + ) + for p in PROVIDERS +) + +TOOL_PARAMS: Final = tuple( + pytest.param( + p, + id=p.id, + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider(p.id),), + models=(p.litellm_params.model,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.WEBSOCKET, + ) + ), + ) + for p in PROVIDERS +) WEATHER_TOOL = FunctionTool( name="get_weather", @@ -62,7 +100,7 @@ class WeatherResult(BaseModel): temperature_f: int -@pytest.mark.parametrize("provider", PROVIDER_PARAMS) +@pytest.mark.parametrize("provider", TEXT_PARAMS) def test_text_conversation( client: RealtimeClient, scoped_key: str, @@ -99,7 +137,7 @@ def test_text_conversation( assert done.response.usage is not None, "response.done missing normalized usage" -@pytest.mark.parametrize("provider", PROVIDER_PARAMS) +@pytest.mark.parametrize("provider", TOOL_PARAMS) def test_tool_call_round_trip( client: RealtimeClient, scoped_key: str, @@ -158,7 +196,7 @@ _REFUSED_UPSTREAMS = ( "azure-bad-key", "azure-realtime-refused", LiteLLMParamsBody( - model="azure/gpt-realtime", + model=AZURE_REALTIME_MODEL, api_key="invalid-e2e-key", api_version="2025-08-28", realtime_protocol="GA", @@ -168,6 +206,15 @@ _REFUSED_UPSTREAMS = ( @pytest.mark.parametrize("provider", _REFUSED_UPSTREAMS, ids=[p.id for p in _REFUSED_UPSTREAMS]) +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider.AZURE,), + models=(AZURE_REALTIME_MODEL,), + mode=Mode.WEBSOCKET, + ) +) def test_upstream_handshake_refusal_is_an_error_event_and_policy_close( client: RealtimeClient, resources: ResourceManager, diff --git a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py index 2e9cfcfe648..27d5abd1828 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py @@ -24,10 +24,12 @@ Three test scenarios per provider: import asyncio import wave from pathlib import Path +from typing import Final import pytest from e2e_config import ws_base_url +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from realtime_client import ( PROVIDERS, RealtimeProvider, @@ -73,7 +75,59 @@ from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService # noq from pipecat_service import LiteLLMRealtimeLLMService # noqa: E402 -PROVIDER_PARAMS = [pytest.param(p, id=p.id) for p in PROVIDERS] +TOOL_PARAMS: Final = tuple( + pytest.param( + p, + id=p.id, + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider(p.id),), + models=(p.litellm_params.model,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.WEBSOCKET, + ) + ), + ) + for p in PROVIDERS +) + +AUDIO_OUTPUT_PARAMS: Final = tuple( + pytest.param( + p, + id=p.id, + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider(p.id),), + models=(p.litellm_params.model,), + capabilities=(Capability.AUDIO_OUTPUT,), + mode=Mode.WEBSOCKET, + ) + ), + ) + for p in PROVIDERS +) + +AUDIO_INPUT_PARAMS: Final = tuple( + pytest.param( + p, + id=p.id, + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider(p.id),), + models=(p.litellm_params.model,), + capabilities=(Capability.AUDIO_INPUT, Capability.AUDIO_OUTPUT), + mode=Mode.WEBSOCKET, + ) + ), + ) + for p in PROVIDERS +) # PCM16 24 kHz mono WAV of "What is the weather in Paris?" (generated via macOS # `say` and resampled with audioop). Used by the server-VAD audio-input test. @@ -193,7 +247,7 @@ async def _run_pipeline( # --------------------------------------------------------------------------- -@pytest.mark.parametrize("provider", PROVIDER_PARAMS) +@pytest.mark.parametrize("provider", TOOL_PARAMS) def test_pipecat_server_vad( scoped_key: str, realtime_models: dict[str, str], @@ -208,7 +262,7 @@ def test_pipecat_server_vad( assert got_text, "no assistant text frames produced" -@pytest.mark.parametrize("provider", PROVIDER_PARAMS) +@pytest.mark.parametrize("provider", AUDIO_OUTPUT_PARAMS) def test_pipecat_audio_output( scoped_key: str, realtime_models: dict[str, str], @@ -332,7 +386,7 @@ async def _run_audio_input_pipeline( return bool(capture.texts), capture.audio_bytes -@pytest.mark.parametrize("provider", PROVIDER_PARAMS) +@pytest.mark.parametrize("provider", AUDIO_INPUT_PARAMS) def test_pipecat_server_vad_audio_input( scoped_key: str, realtime_models: dict[str, str], diff --git a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py index f84ce197f88..3c22ae1a928 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py @@ -26,6 +26,7 @@ import asyncio import pytest from e2e_config import ws_base_url +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from realtime_client import ( PROVIDERS, RealtimeProvider, @@ -64,7 +65,21 @@ from pipecat_service import LiteLLMRealtimeLLMService # noqa: E402 # pipecat-ai/pipecat#2544); raw-ws tool_call_round_trip[vertex_ai] is the # source of truth for that provider. Keep openai/azure/gemini here. PROVIDER_PARAMS = [ - pytest.param(p, id=p.id) for p in PROVIDERS if p.id != "vertex_ai" + pytest.param( + p, + id=p.id, + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider(p.id),), + models=(p.litellm_params.model,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.WEBSOCKET, + ) + ), + ) + for p in PROVIDERS if p.id != "vertex_ai" ] WEATHER_TOOL = ToolsSchema( diff --git a/tests/e2e/llm_translation/test_audio_speech_e2e.py b/tests/e2e/llm_translation/test_audio_speech_e2e.py index 75a3de86d13..c4c3f751828 100644 --- a/tests/e2e/llm_translation/test_audio_speech_e2e.py +++ b/tests/e2e/llm_translation/test_audio_speech_e2e.py @@ -10,8 +10,11 @@ the SDK refuses to send a request missing its required fields. from __future__ import annotations +from typing import Final + import pytest from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from e2e_http import assert_client_error from lifecycle import ResourceManager from models import LiteLLMParamsBody @@ -21,6 +24,9 @@ from sdk_clients import SdkClients, response_header pytestmark = pytest.mark.e2e +OPENAI_TTS_MODEL: Final = "openai/gpt-4o-mini-tts" +AWS_POLLY_MODEL: Final = "aws_polly/generative" + class _OptionalSpeechBody(BaseModel): model: str | None = None @@ -32,7 +38,7 @@ def _register_tts(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, model = f"e2e-speech-{unique_marker()}" model_id = proxy.create_model( model, - LiteLLMParamsBody(model="openai/gpt-4o-mini-tts", api_key="os.environ/OPENAI_API_KEY"), + LiteLLMParamsBody(model=OPENAI_TTS_MODEL, api_key="os.environ/OPENAI_API_KEY"), ) resources.defer(lambda: proxy.delete_model(model_id)) return model, resources.key() @@ -40,6 +46,15 @@ def _register_tts(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, class TestAudioSpeech: @pytest.mark.covers("llm.audio_speech.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(Provider.OPENAI,), + models=(OPENAI_TTS_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_audio_speech_returns_audio( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -56,6 +71,15 @@ class TestAudioSpeech: assert response.content, "/audio/speech returned an empty body" @pytest.mark.covers("llm.audio_speech.openai.basic.stream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(Provider.OPENAI,), + models=(OPENAI_TTS_MODEL,), + mode=Mode.STREAM, + ) + ) def test_audio_speech_streams_audio_chunks( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -90,6 +114,15 @@ class TestAudioSpeech: @pytest.mark.skip(reason="stage red: product gap, /v1/audio/speech 500s on missing input instead of 400") @pytest.mark.covers("llm.audio_speech.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(Provider.OPENAI,), + models=(OPENAI_TTS_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_missing_input_returns_error( self, proxy: ProxyClient, resources: ResourceManager ) -> None: @@ -103,6 +136,12 @@ class TestAudioSpeech: @pytest.mark.skip(reason="stage red: product gap, /v1/audio/speech 500s on missing model instead of 400") @pytest.mark.covers("llm.audio_speech.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + ) + ) def test_missing_model_returns_error( self, proxy: ProxyClient, resources: ResourceManager ) -> None: @@ -116,6 +155,15 @@ class TestAudioSpeech: @pytest.mark.skip(reason="stage red: product gap, /v1/audio/speech 500s on invalid voice instead of surfacing the provider 4xx") @pytest.mark.covers("llm.audio_speech.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(Provider.OPENAI,), + models=(OPENAI_TTS_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_invalid_voice_returns_error( self, proxy: ProxyClient, resources: ResourceManager ) -> None: @@ -129,6 +177,15 @@ class TestAudioSpeech: @pytest.mark.skip(reason="stage red: product gap, /v1/audio/speech 500s on empty input instead of surfacing the provider 4xx") @pytest.mark.covers("llm.audio_speech.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(Provider.OPENAI,), + models=(OPENAI_TTS_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_empty_input_returns_error( self, proxy: ProxyClient, resources: ResourceManager ) -> None: @@ -145,6 +202,15 @@ MP3_PREFIXES = (b"ID3", b"\xff\xfb", b"\xff\xf3", b"\xff\xf2") class TestAwsPollySpeech: + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(Provider.AWS_POLLY,), + models=(AWS_POLLY_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_polly_generative_voice_returns_mp3( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -152,7 +218,7 @@ class TestAwsPollySpeech: model_id = proxy.create_model( model, LiteLLMParamsBody( - model="aws_polly/generative", + model=AWS_POLLY_MODEL, aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", aws_region_name="os.environ/AWS_REGION", diff --git a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py index 725e15a0209..cd1ee8fb03c 100644 --- a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py +++ b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py @@ -17,6 +17,7 @@ from typing import Final import pytest from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from e2e_http import UnknownApiError, unwrap from lifecycle import ResourceManager from models import LiteLLMParamsBody @@ -30,6 +31,8 @@ WEATHER_WAV = ( Path(__file__).resolve().parent / "realtime" / "fixtures" / "weather_question_24k.wav" ) +OPENAI_TRANSCRIBE_MODEL: Final = "openai/gpt-4o-mini-transcribe" +OPENAI_WHISPER_MODEL: Final = "openai/whisper-1" MISSING_MODEL_PHRASES: Final = ("model=none", "invalid model", "model is required") @@ -47,7 +50,7 @@ def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str] model_id = proxy.create_model( model, LiteLLMParamsBody( - model="openai/gpt-4o-mini-transcribe", api_key="os.environ/OPENAI_API_KEY" + model=OPENAI_TRANSCRIBE_MODEL, api_key="os.environ/OPENAI_API_KEY" ), ) resources.defer(lambda: proxy.delete_model(model_id)) @@ -56,6 +59,15 @@ def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str] class TestAudioTranscriptions: @pytest.mark.covers("llm.audio_transcriptions.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(Provider.OPENAI,), + models=(OPENAI_TRANSCRIBE_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_audio_transcriptions_returns_text( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -72,6 +84,15 @@ class TestAudioTranscriptions: ) @pytest.mark.covers("llm.audio_transcriptions.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(Provider.OPENAI,), + models=(OPENAI_TRANSCRIBE_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_missing_file_returns_error( self, proxy: ProxyClient, resources: ResourceManager ) -> None: @@ -98,6 +119,12 @@ class TestAudioTranscriptions: pytest.fail(f"empty audio expected a file-specific 400, got {other!r}") @pytest.mark.covers("llm.audio_transcriptions.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + ) + ) def test_missing_model_returns_error( self, proxy: ProxyClient, resources: ResourceManager ) -> None: @@ -143,7 +170,7 @@ class TestWhisperTranscriptionFormats: self, proxy: ProxyClient, resources: ResourceManager, form: _WhisperForm, response_type: type[R] ) -> R: model_id = proxy.create_model( - form.model, LiteLLMParamsBody(model="openai/whisper-1", api_key="os.environ/OPENAI_API_KEY") + form.model, LiteLLMParamsBody(model=OPENAI_WHISPER_MODEL, api_key="os.environ/OPENAI_API_KEY") ) resources.defer(lambda: proxy.delete_model(model_id)) return unwrap( @@ -158,12 +185,30 @@ class TestWhisperTranscriptionFormats: ) ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(Provider.OPENAI,), + models=(OPENAI_WHISPER_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_vtt_format_returns_webvtt_transcript(self, proxy: ProxyClient, resources: ResourceManager) -> None: form = _WhisperForm(model=f"e2e-whisper-vtt-{unique_marker()}", response_format="vtt") transcript = self._upload(proxy, resources, form, _TranscriptionResult) assert transcript.text.lstrip().startswith("WEBVTT"), f"vtt transcript is not WebVTT: {transcript.text[:200]!r}" assert "weather" in transcript.text.lower(), f"vtt transcript lost the spoken words: {transcript.text!r}" + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(Provider.OPENAI,), + models=(OPENAI_WHISPER_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_verbose_json_returns_word_timestamps(self, proxy: ProxyClient, resources: ResourceManager) -> None: form = _WhisperForm( model=f"e2e-whisper-verbose-{unique_marker()}", diff --git a/tests/e2e/llm_translation/test_bedrock_native_e2e.py b/tests/e2e/llm_translation/test_bedrock_native_e2e.py index 19c1be7b6db..a9b9e45858d 100644 --- a/tests/e2e/llm_translation/test_bedrock_native_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_native_e2e.py @@ -8,6 +8,7 @@ from __future__ import annotations import pytest from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from e2e_http import ( assert_client_error, require_successful_call, @@ -100,6 +101,15 @@ def _default_invoke() -> InvokeBody: class TestBedrockNative: @pytest.mark.covers("llm.bedrock_native.bedrock_converse.basic.nonstream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + models=(BEDROCK_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_converse_returns_assistant(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) result = proxy.transport.send( @@ -113,6 +123,15 @@ class TestBedrockNative: assert any(part.text.strip() for part in response.output.message.content) @pytest.mark.covers("llm.bedrock_native.bedrock_converse.basic.stream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + models=(BEDROCK_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_converse_stream_returns_chunks(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) result = proxy.transport.send( @@ -126,6 +145,15 @@ class TestBedrockNative: assert result.chunks > 0, "converse-stream returned no events" @pytest.mark.covers("llm.bedrock_native.bedrock_invoke.basic.nonstream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + models=(BEDROCK_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_invoke_returns_message(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) result = proxy.transport.send( @@ -138,6 +166,15 @@ class TestBedrockNative: assert any(part.text.strip() for part in response.content) @pytest.mark.covers("llm.bedrock_native.bedrock_invoke.basic.stream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + models=(BEDROCK_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_invoke_stream_returns_chunks(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) result = proxy.transport.send( @@ -151,6 +188,15 @@ class TestBedrockNative: assert result.chunks > 0, "invoke stream returned no events" @pytest.mark.covers("llm.bedrock_native.bedrock_converse.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + models=(BEDROCK_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_converse_missing_messages_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) result = proxy.transport.send( @@ -161,6 +207,15 @@ class TestBedrockNative: assert_client_error(result, "converse missing messages") @pytest.mark.covers("llm.bedrock_native.bedrock_converse.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + models=(BEDROCK_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_converse_empty_messages_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) result = proxy.transport.send( @@ -171,6 +226,14 @@ class TestBedrockNative: assert_client_error(result, "converse empty messages") @pytest.mark.covers("llm.bedrock_native.bedrock_converse.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + mode=Mode.NONSTREAM, + ) + ) def test_converse_invalid_model_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: _, key = _register(proxy, resources) result = proxy.transport.send( @@ -183,6 +246,15 @@ class TestBedrockNative: ) @pytest.mark.covers("llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + models=(BEDROCK_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_invoke_missing_messages_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) result = proxy.transport.send( @@ -193,6 +265,15 @@ class TestBedrockNative: assert_client_error(result, "invoke missing messages") @pytest.mark.covers("llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + models=(BEDROCK_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_invoke_missing_max_tokens_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) result = proxy.transport.send( @@ -206,6 +287,15 @@ class TestBedrockNative: assert_client_error(result, "invoke missing max_tokens") @pytest.mark.covers("llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + models=(BEDROCK_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_invoke_invalid_temperature_returns_client_error( self, proxy: ProxyClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_bedrock_provider_matrix_e2e.py b/tests/e2e/llm_translation/test_bedrock_provider_matrix_e2e.py index 21333d39849..c80609edcdd 100644 --- a/tests/e2e/llm_translation/test_bedrock_provider_matrix_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_provider_matrix_e2e.py @@ -18,6 +18,7 @@ import pytest from pydantic import BaseModel from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from e2e_http import StreamingResponse, unwrap from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody @@ -98,6 +99,15 @@ class TestBedrockResponseHeaders: "llm.chat_completions.bedrock_converse.response_headers.nonstream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(CONVERSE_REGIONAL_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_request_id_header_surfaces( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -117,6 +127,15 @@ class TestBedrockResponseHeaders: "llm.chat_completions.bedrock_converse.response_headers.stream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(CONVERSE_REGIONAL_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_bedrock_request_id_header_surfaces_on_stream( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -158,6 +177,15 @@ class TestBedrockBatchDeploymentServesChat: "llm.chat_completions.bedrock_converse.batch_deployment.nonstream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(CONVERSE_REGIONAL_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_batch_s3_keys_do_not_break_chat( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -179,6 +207,15 @@ class TestBedrockBatchDeploymentServesChat: class TestBedrockInvokeRegionalModelIds: @pytest.mark.covers("llm.chat_completions.bedrock_invoke.basic.nonstream.works", exercised_on=[]) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(INVOKE_REGIONAL_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_invoke_regional_id_completes( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -190,6 +227,15 @@ class TestBedrockInvokeRegionalModelIds: _assert_completion(response) @pytest.mark.covers("llm.chat_completions.bedrock_invoke.basic.stream.works", exercised_on=[]) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(INVOKE_REGIONAL_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_invoke_regional_id_streams( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -205,6 +251,15 @@ class TestBedrockInvokeRegionalModelIds: class TestBedrockOpenAIFamilyDefaultRoute: @pytest.mark.covers("llm.chat_completions.bedrock_converse.basic.nonstream.works", exercised_on=[]) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(OPENAI_FAMILY_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_openai_family_model_id_completes_with_max_tokens( self, client: PassthroughClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py b/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py index b4253a82dd8..6133fa960ff 100644 --- a/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py @@ -36,6 +36,7 @@ from __future__ import annotations import pytest from anthropic.types import WebSearchTool20250305Param from e2e_config import unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient @@ -62,6 +63,16 @@ class TestBedrockWebSearchServerTool: "ephemeral stack ships the config in this module's docstring." ) @pytest.mark.covers("llm.messages.bedrock_invoke.web_search_server_tool.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=(BEDROCK_INVOKE_BACKEND,), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) + ) def test_web_search_server_tool_is_served( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index bec65144c9f..33c52efe37e 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -43,6 +43,7 @@ import pytest from pydantic import BaseModel from e2e_config import unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from e2e_http import Result, UnknownApiError, unwrap from lifecycle import ResourceManager from models import CacheControl, ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody, RichMessage, TextBlock, Usage @@ -226,6 +227,16 @@ class TestCacheControl: "llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_MODEL,), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_prompt_caching_reads_cache( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -242,6 +253,16 @@ class TestCacheControl: "llm.chat_completions.vertex.prompt_cache_5m.nonstream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_MODEL,), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_vertex_prompt_caching_reads_cache( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -266,6 +287,16 @@ class TestCacheControl: "llm.chat_completions.anthropic.prompt_cache_5m.nonstream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_MODEL,), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_anthropic_prompt_caching_reads_cache( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -282,6 +313,16 @@ class TestCacheControl: "llm.chat_completions.openai.prompt_cache_5m.nonstream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_MODEL,), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_openai_prompt_caching_reads_cache( self, client: PassthroughClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_cache_control_injection_tool_calls_e2e.py b/tests/e2e/llm_translation/test_cache_control_injection_tool_calls_e2e.py index 1481185a601..0d796a32ef3 100644 --- a/tests/e2e/llm_translation/test_cache_control_injection_tool_calls_e2e.py +++ b/tests/e2e/llm_translation/test_cache_control_injection_tool_calls_e2e.py @@ -5,6 +5,7 @@ from typing import Final, Literal, TypeAlias import pytest from e2e_config import unique_marker from e2e_http import Result, unwrap +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ( CacheControl, @@ -162,7 +163,36 @@ def _assert_normal_completion(response: ChatResponse, model_name: str) -> None: @pytest.mark.parametrize( "backend", - (pytest.param("azure_foundry", id="azure-foundry"), pytest.param("vertex", id="vertex")), + ( + pytest.param( + "azure_foundry", + id="azure-foundry", + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.AZURE_AI,), + models=(AZURE_MODEL,), + capabilities=(Capability.FUNCTION_CALLING, Capability.PROMPT_CACHING), + mode=Mode.NONSTREAM, + ) + ), + ), + pytest.param( + "vertex", + id="vertex", + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_MODEL,), + capabilities=(Capability.FUNCTION_CALLING, Capability.PROMPT_CACHING), + mode=Mode.NONSTREAM, + ) + ), + ), + ), ) @pytest.mark.provider_live @pytest.mark.covers("llm.chat_completions.azure_foundry.basic.nonstream.works") diff --git a/tests/e2e/llm_translation/test_chat_completions_contract_e2e.py b/tests/e2e/llm_translation/test_chat_completions_contract_e2e.py index 09b484eb120..45b9fa44e52 100644 --- a/tests/e2e/llm_translation/test_chat_completions_contract_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_contract_e2e.py @@ -8,6 +8,7 @@ from __future__ import annotations import pytest from e2e_config import provider_edge_base, unique_marker from e2e_http import StreamingResponse, assert_client_error, require_successful_call, unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody from proxy_client import ProxyClient @@ -62,6 +63,15 @@ def _chat_status(proxy: ProxyClient, key: str, body: BaseModel) -> StreamingResp class TestChatCompletionsContract: @pytest.mark.covers("llm.chat_completions.openai.multi_turn.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_multi_turn_history_is_honored(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register_chat_model(proxy, resources) turn1 = unwrap( @@ -106,6 +116,15 @@ class TestChatCompletionsContract: assert "84" in second, f"turn2 must answer 84 from history, got: {second!r}" @pytest.mark.covers("llm.chat_completions.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_success_response_matches_chat_completion_contract( self, proxy: ProxyClient, resources: ResourceManager ) -> None: @@ -131,6 +150,12 @@ class TestChatCompletionsContract: assert (message.content or "").strip(), f"content must be non-empty: {result.body[:300]}" @pytest.mark.covers("llm.chat_completions.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + ) + ) def test_missing_model_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: _, key = _register_chat_model(proxy, resources) result = _chat_status( @@ -145,12 +170,30 @@ class TestChatCompletionsContract: ) @pytest.mark.covers("llm.chat_completions.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_missing_messages_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register_chat_model(proxy, resources) result = _chat_status(proxy, key, ChatMissingMessagesBody(model=model)) assert_client_error(result, "missing messages") @pytest.mark.covers("llm.chat_completions.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_empty_messages_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register_chat_model(proxy, resources) result = _chat_status( @@ -161,6 +204,15 @@ class TestChatCompletionsContract: assert_client_error(result, "empty messages") @pytest.mark.covers("llm.chat_completions.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_invalid_role_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register_chat_model(proxy, resources) result = _chat_status( @@ -175,6 +227,15 @@ class TestChatCompletionsContract: assert_client_error(result, "invalid role") @pytest.mark.covers("llm.chat_completions.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_invalid_temperatures_return_client_errors(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register_chat_model(proxy, resources) for temperature in (-0.1, 2.1, 3.0, 100.0): @@ -191,6 +252,15 @@ class TestChatCompletionsContract: assert_client_error(result, f"temperature={temperature}") @pytest.mark.covers("llm.chat_completions.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_invalid_max_completion_tokens_return_client_errors( self, proxy: ProxyClient, resources: ResourceManager ) -> None: @@ -208,6 +278,15 @@ class TestChatCompletionsContract: assert_client_error(result, f"max_completion_tokens={max_completion_tokens}") @pytest.mark.covers("llm.chat_completions.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_temperature_boundaries_succeed(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register_chat_model(proxy, resources) for temperature in (0.0, 2.0): diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 90ac18c16ac..1e0a8cbb412 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -26,6 +26,7 @@ from pydantic import BaseModel from e2e_config import unique_marker from e2e_http import StreamingResponse, unwrap +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ( ChatBody, @@ -53,16 +54,40 @@ OPENAI_BACKEND = "openai/gpt-5.6" ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5-20251001" BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" BEDROCK_NOVA_BACKEND: Final = "bedrock/us.amazon.nova-2-lite-v1:0" +VERTEX_MISTRAL_BACKEND: Final = "vertex_ai/mistral-small-2503" +VERTEX_GPT_OSS_BACKEND: Final = "vertex_ai/openai/gpt-oss-120b-maas" VERTEX_PARTNER_BACKENDS: Final = ( pytest.param( - "vertex_ai/mistral-small-2503", - marks=pytest.mark.skip( - reason="the e2e Vertex project has no access to mistral-small-2503 (404 publisher model not found)" + VERTEX_MISTRAL_BACKEND, + marks=( + pytest.mark.skip( + reason="the e2e Vertex project has no access to mistral-small-2503 (404 publisher model not found)" + ), + meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_MISTRAL_BACKEND,), + mode=Mode.STREAM, + ) + ), ), ), pytest.param( - "vertex_ai/openai/gpt-oss-120b-maas", - marks=pytest.mark.skip(reason="never served by the e2e Vertex project (60s read timeout, no headers)"), + VERTEX_GPT_OSS_BACKEND, + marks=( + pytest.mark.skip(reason="never served by the e2e Vertex project (60s read timeout, no headers)"), + meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_GPT_OSS_BACKEND,), + mode=Mode.STREAM, + ) + ), + ), ), ) PDF_DOCUMENT_URL: Final = ( @@ -216,19 +241,58 @@ _PERSON_SCHEMA: dict[str, object] = { }, } -CHAT_MODELS: tuple[tuple[str, str], ...] = ( - ("gpt-5.5", "openai"), - ("claude-haiku-4-5", "anthropic"), - ("gemini-2.5-flash", "gemini"), +OPENAI_CHAT_MODEL: Final = "gpt-5.5" +ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5" +GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash" + +CHAT_MODELS: Final = ( + pytest.param( + OPENAI_CHAT_MODEL, + "openai", + id=f"{OPENAI_CHAT_MODEL}-openai", + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_CHAT_MODEL,), + mode=Mode.NONSTREAM, + ) + ), + ), + pytest.param( + ANTHROPIC_CHAT_MODEL, + "anthropic", + id=f"{ANTHROPIC_CHAT_MODEL}-anthropic", + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_CHAT_MODEL,), + mode=Mode.NONSTREAM, + ) + ), + ), + pytest.param( + GEMINI_FLASH_MODEL, + "gemini", + id=f"{GEMINI_FLASH_MODEL}-gemini", + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(GEMINI_FLASH_MODEL,), + mode=Mode.NONSTREAM, + ) + ), + ), ) class TestChatCompletionsRegression: - @pytest.mark.parametrize( - ("model", "route"), - CHAT_MODELS, - ids=[f"{model}-{route}" for model, route in CHAT_MODELS], - ) + @pytest.mark.parametrize(("model", "route"), CHAT_MODELS) @pytest.mark.covers( "llm.chat_completions.openai.basic.nonstream.works", "llm.chat_completions.anthropic.basic.nonstream.works", @@ -272,6 +336,15 @@ class TestCohereChat: "llm.chat_completions.cohere.basic.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.COHERE,), + models=(COHERE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_cohere_chat_returns_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -316,6 +389,15 @@ class TestGeminiChatCompletions: "llm.chat_completions.gemini.basic.nonstream.cost_logged", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(GEMINI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_gemini_chat_returns_content_and_logs_cost( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -377,6 +459,15 @@ class TestVertexChatCompletions: "llm.chat_completions.vertex.basic.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_vertex_chat_returns_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -406,6 +497,16 @@ class TestVertexChatCompletions: "llm.chat_completions.vertex.tool_use.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_vertex_chat_returns_tool_call( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -435,6 +536,16 @@ class TestVertexChatCompletions: "llm.chat_completions.vertex.vision.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_BACKEND,), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) + ) def test_vertex_chat_vision_describes_image( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -452,6 +563,15 @@ class TestVertexChatCompletions: "llm.chat_completions.vertex.basic.stream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_vertex_chat_streams_real_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -494,6 +614,15 @@ class TestAzureOpenAIChatCompletions: "llm.chat_completions.azure_openai.basic.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.AZURE,), + models=(AZURE_OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_azure_openai_chat_returns_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -523,6 +652,16 @@ class TestAzureOpenAIChatCompletions: "llm.chat_completions.azure_openai.tool_use.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.AZURE,), + models=(AZURE_OPENAI_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_azure_openai_chat_returns_tool_call( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -554,6 +693,15 @@ class TestAzureFoundryChatCompletions: "llm.chat_completions.azure_foundry.basic.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.AZURE_AI,), + models=(AZURE_FOUNDRY_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_azure_foundry_chat_returns_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -596,6 +744,14 @@ class TestHostedVllmChat: "llm.chat_completions.hosted_vllm.basic.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.HOSTED_VLLM,), + mode=Mode.NONSTREAM, + ) + ) def test_hosted_vllm_chat_returns_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -651,6 +807,15 @@ class TestOpenAIChatCompletions: "llm.chat_completions.openai.basic.stream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_openai_chat_streams_real_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -678,6 +843,15 @@ class TestOpenAIChatCompletions: "llm.chat_completions.openai.basic.nonstream.cost_logged", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_openai_chat_logs_cost( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -711,6 +885,16 @@ class TestOpenAIChatCompletions: "llm.chat_completions.openai.tool_use.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_openai_chat_returns_tool_call( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -742,6 +926,16 @@ class TestOpenAIChatCompletions: "llm.chat_completions.openai.structured_output.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) + ) def test_openai_chat_structured_output_conforms_to_schema( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -775,6 +969,16 @@ class TestOpenAIChatCompletions: "llm.chat_completions.openai.thinking.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_openai_chat_reasoning_reports_reasoning_tokens( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -825,6 +1029,16 @@ class TestOpenAIChatCompletions: "llm.chat_completions.openai.vision.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_VISION_BACKEND,), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) + ) def test_openai_chat_vision_describes_image( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -842,6 +1056,16 @@ class TestOpenAIChatCompletions: "llm.chat_completions.openai.tool_use.stream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) + ) def test_openai_chat_streams_tool_call( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -889,6 +1113,15 @@ class TestBedrockConverseChatCompletions: "llm.chat_completions.bedrock_converse.basic.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_CONVERSE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_converse_chat_returns_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -913,6 +1146,15 @@ class TestBedrockConverseChatCompletions: "llm.chat_completions.bedrock_converse.basic.stream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_CONVERSE_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_bedrock_converse_chat_streams_real_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -936,6 +1178,16 @@ class TestBedrockConverseChatCompletions: "llm.chat_completions.bedrock_converse.tool_use.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_CONVERSE_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_converse_chat_returns_tool_call( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -962,6 +1214,16 @@ class TestBedrockConverseChatCompletions: "llm.chat_completions.bedrock_converse.thinking.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_CONVERSE_BACKEND,), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_converse_chat_returns_reasoning( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -992,6 +1254,16 @@ class TestBedrockConverseChatCompletions: "llm.chat_completions.bedrock_converse.vision.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_CONVERSE_BACKEND,), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_converse_chat_vision_describes_image( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -1001,6 +1273,16 @@ class TestBedrockConverseChatCompletions: response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32))) _assert_describes_cat(response) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_NOVA_BACKEND,), + capabilities=(Capability.PDF_INPUT,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_converse_reads_a_pdf_sent_by_url( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -1107,6 +1389,16 @@ class TestAnthropicChatCompletions: "llm.chat_completions.anthropic.structured_output.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) + ) def test_anthropic_chat_structured_output_conforms_to_schema( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -1140,6 +1432,16 @@ class TestAnthropicChatCompletions: "llm.chat_completions.anthropic.thinking.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_anthropic_chat_returns_thinking_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -1178,6 +1480,16 @@ class TestAnthropicChatCompletions: "llm.chat_completions.anthropic.vision.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) + ) def test_anthropic_chat_vision_describes_image( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -1191,6 +1503,15 @@ class TestAnthropicChatCompletions: "llm.chat_completions.anthropic.basic.stream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_anthropic_chat_streams_real_content( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -1214,6 +1535,16 @@ class TestAnthropicChatCompletions: "llm.chat_completions.anthropic.tool_use.nonstream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_anthropic_chat_returns_tool_call( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -1240,6 +1571,16 @@ class TestAnthropicChatCompletions: "llm.chat_completions.anthropic.tool_use.stream.works", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) + ) def test_anthropic_chat_streams_tool_call( self, client: PassthroughClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_chat_mid_conversation_system_e2e.py b/tests/e2e/llm_translation/test_chat_mid_conversation_system_e2e.py index 480225b502e..71e716b2255 100644 --- a/tests/e2e/llm_translation/test_chat_mid_conversation_system_e2e.py +++ b/tests/e2e/llm_translation/test_chat_mid_conversation_system_e2e.py @@ -37,6 +37,7 @@ from pydantic import BaseModel from e2e_config import unique_marker from e2e_http import Result, unwrap +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import CacheControl, ChatResponse, LiteLLMParamsBody, RichMessage, TextBlock, Usage from passthrough_client import PassthroughClient @@ -281,6 +282,16 @@ class TestAnthropicChatMidConversationSystem: "llm.chat_completions.anthropic.mid_conversation_system.nonstream.cache_hit", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(FLAGGED_MODEL,), + capabilities=(Capability.MID_CONVERSATION_SYSTEM, Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_flagged_model_keeps_prompt_cache_across_system_reminder( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -290,6 +301,16 @@ class TestAnthropicChatMidConversationSystem: "llm.chat_completions.anthropic.mid_conversation_system.nonstream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(UNFLAGGED_MODEL,), + capabilities=(Capability.MID_CONVERSATION_SYSTEM, Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_unflagged_model_converts_system_reminder_and_succeeds( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -305,6 +326,16 @@ class TestBedrockInvokeChatMidConversationSystem: "llm.chat_completions.bedrock_invoke.mid_conversation_system.nonstream.cache_hit", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(FLAGGED_MODEL,), + capabilities=(Capability.MID_CONVERSATION_SYSTEM, Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_flagged_model_keeps_prompt_cache_across_system_reminder( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -314,6 +345,16 @@ class TestBedrockInvokeChatMidConversationSystem: "llm.chat_completions.bedrock_invoke.mid_conversation_system.nonstream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(UNFLAGGED_MODEL,), + capabilities=(Capability.MID_CONVERSATION_SYSTEM, Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_unflagged_model_converts_system_reminder_and_succeeds( self, client: PassthroughClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_chat_stream_contract_e2e.py b/tests/e2e/llm_translation/test_chat_stream_contract_e2e.py index fdb76df703d..1e353d11283 100644 --- a/tests/e2e/llm_translation/test_chat_stream_contract_e2e.py +++ b/tests/e2e/llm_translation/test_chat_stream_contract_e2e.py @@ -5,6 +5,7 @@ from typing import Final import pytest from e2e_config import provider_edge_base, unique_marker from e2e_http import require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatStreamOptions, LiteLLMParamsBody, Usage from proxy_client import ProxyClient @@ -12,6 +13,8 @@ from pydantic import BaseModel pytestmark = [pytest.mark.e2e, pytest.mark.replayable] +OPENAI_BACKEND: Final = "openai/gpt-5.6" + class _Delta(BaseModel): content: str | None = None @@ -30,13 +33,22 @@ class _Chunk(BaseModel): class TestChatStreamContract: @pytest.mark.covers("llm.chat_completions.openai.basic.stream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_chat_stream_is_sse_and_ends_with_done(self, proxy: ProxyClient, resources: ResourceManager) -> None: model: Final = f"e2e-chat-stream-{unique_marker()}" base: Final = provider_edge_base("openai") model_id: Final = proxy.create_model( model, LiteLLMParamsBody( - model="openai/gpt-5.6", + model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY", api_base=f"{base}/v1" if base else None, ), diff --git a/tests/e2e/llm_translation/test_chat_tool_round_trip_e2e.py b/tests/e2e/llm_translation/test_chat_tool_round_trip_e2e.py index 75ba5c23ff8..01d15aed443 100644 --- a/tests/e2e/llm_translation/test_chat_tool_round_trip_e2e.py +++ b/tests/e2e/llm_translation/test_chat_tool_round_trip_e2e.py @@ -6,6 +6,7 @@ from typing import Final import pytest from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ( ChatAssistantTurn, @@ -138,22 +139,72 @@ def _assert_tool_results_reach_the_model( class TestChatToolResultRoundTrip: + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(GEMINI_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_gemini(self, client: PassthroughClient, resources: ResourceManager) -> None: model, key = _register(client, resources, _api_key_params(GEMINI_BACKEND, "GEMINI_API_KEY")) _assert_tool_results_reach_the_model(client, key, model, thinking=None, tool_choice="required") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.MISTRAL,), + models=(MISTRAL_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_mistral(self, client: PassthroughClient, resources: ResourceManager) -> None: model, key = _register(client, resources, _api_key_params(MISTRAL_BACKEND, "MISTRAL_API_KEY")) _assert_tool_results_reach_the_model(client, key, model, thinking=None, tool_choice="required") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_CONVERSE_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_converse(self, client: PassthroughClient, resources: ResourceManager) -> None: model, key = _register(client, resources, _bedrock_params(BEDROCK_CONVERSE_BACKEND)) _assert_tool_results_reach_the_model(client, key, model, thinking=None, tool_choice="required") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING, Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_anthropic_with_extended_thinking(self, client: PassthroughClient, resources: ResourceManager) -> None: model, key = _register(client, resources, _api_key_params(ANTHROPIC_BACKEND, "ANTHROPIC_API_KEY")) _assert_tool_results_reach_the_model(client, key, model, thinking=THINKING, tool_choice=None) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_LEGACY_THINKING_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING, Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_converse_with_extended_thinking( self, client: PassthroughClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_completions_endpoint_e2e.py b/tests/e2e/llm_translation/test_completions_endpoint_e2e.py index 6202dada599..7f89d8a7ed8 100644 --- a/tests/e2e/llm_translation/test_completions_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_completions_endpoint_e2e.py @@ -10,8 +10,11 @@ the completion fails here. from __future__ import annotations +from typing import Final + import pytest from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient @@ -19,9 +22,20 @@ from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e +OPENAI_COMPLETIONS_BACKEND: Final = "openai/gpt-5.4-nano" + class TestCompletionsEndpoint: @pytest.mark.covers("llm.completions.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_COMPLETIONS_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_text_completion_returns_text( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -29,7 +43,7 @@ class TestCompletionsEndpoint: model_id = proxy.create_model( model, LiteLLMParamsBody( - model="openai/gpt-5.4-nano", + model=OPENAI_COMPLETIONS_BACKEND, api_key="os.environ/OPENAI_API_KEY", ), ) diff --git a/tests/e2e/llm_translation/test_containers_e2e.py b/tests/e2e/llm_translation/test_containers_e2e.py index 1c3e37ec8bb..0a494d11108 100644 --- a/tests/e2e/llm_translation/test_containers_e2e.py +++ b/tests/e2e/llm_translation/test_containers_e2e.py @@ -50,6 +50,7 @@ import openai import pytest from e2e_config import REQUEST_TIMEOUT, unique_marker from e2e_http import unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from management.management_client import ManagementClient, build_client from models import KeyGenerateBody, KeyGenerateResponse, LiteLLMParamsBody, TeamNewBody, UserNewBody @@ -172,6 +173,15 @@ def _assert_file_round_trip(client: OpenAI, native_id: str, marker: str) -> None class TestAzureContainerFiles: @pytest.mark.covers("llm.responses.azure_openai.code_interpreter.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CONTAINERS, + providers=(Provider.AZURE,), + models=(AZURE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_service_account_key_reads_container_file_by_native_id( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -187,6 +197,15 @@ class TestAzureContainerFiles: _assert_file_round_trip(client, native_id, marker) @pytest.mark.covers("llm.responses.azure_openai.code_interpreter.stream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CONTAINERS, + providers=(Provider.AZURE,), + models=(AZURE_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_service_account_key_reads_container_file_created_by_a_streamed_response( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -202,6 +221,13 @@ class TestAzureContainerFiles: class TestOpenAIContainerFiles: + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CONTAINERS, + providers=(Provider.OPENAI,), + ) + ) def test_container_file_lifecycle_through_the_gateway(self, resources: ResourceManager, sdk: SdkClients) -> None: client: Final = sdk.openai(resources.key()) marker: Final = unique_marker() diff --git a/tests/e2e/llm_translation/test_credential_messages_e2e.py b/tests/e2e/llm_translation/test_credential_messages_e2e.py index 58e17f20bb8..55fd1cf7f2f 100644 --- a/tests/e2e/llm_translation/test_credential_messages_e2e.py +++ b/tests/e2e/llm_translation/test_credential_messages_e2e.py @@ -3,10 +3,12 @@ from __future__ import annotations import os +from typing import Final import pytest from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import CredentialCreateBody, LiteLLMParamsBody from proxy_client import ProxyClient @@ -14,9 +16,20 @@ from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e +CLAUDE_BACKEND: Final = "anthropic/claude-haiku-4-5" + class TestCredentialBackedMessages: @pytest.mark.covers("mgmt.credential.new.serves_request") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(CLAUDE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_credential_backed_messages(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: marker = unique_marker() credential_name = f"e2e-cred-{marker}" @@ -35,7 +48,7 @@ class TestCredentialBackedMessages: model_id = proxy.create_model( model, LiteLLMParamsBody( - model="anthropic/claude-haiku-4-5", + model=CLAUDE_BACKEND, litellm_credential_name=credential_name, ), ) diff --git a/tests/e2e/llm_translation/test_custom_pricing_e2e.py b/tests/e2e/llm_translation/test_custom_pricing_e2e.py index 1cebf90fa21..6d8cd24648e 100644 --- a/tests/e2e/llm_translation/test_custom_pricing_e2e.py +++ b/tests/e2e/llm_translation/test_custom_pricing_e2e.py @@ -23,6 +23,7 @@ import pytest from pydantic import BaseModel, RootModel from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from proxy_client import ProxyClient from e2e_http import Success, unwrap from lifecycle import ResourceManager @@ -148,6 +149,15 @@ def _poll_breakdown_row(proxy: ProxyClient, key: str, response_id: str | None) - class TestCustomPricing: + @meta( + Subject( + domain=Domain.COST_MAP, + route=Route.SPEND_REPORTING, + providers=(Provider.GEMINI,), + models=(BACKEND_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_custom_pricing_is_billed_at_configured_rate( self, proxy: ProxyClient, @@ -193,6 +203,12 @@ class TestCustomPricing: f"= {completion * CUSTOM_OUTPUT_RATE}" ) + @meta( + Subject( + domain=Domain.COST_MAP, + route=Route.MODEL_MANAGEMENT, + ) + ) def test_model_info_reports_custom_pricing( self, proxy: ProxyClient, resources: ResourceManager ) -> None: @@ -208,6 +224,12 @@ class TestCustomPricing: f"{entry.litellm_params.output_cost_per_token} != configured {CUSTOM_OUTPUT_RATE}" ) + @meta( + Subject( + domain=Domain.COST_MAP, + route=Route.MODEL_MANAGEMENT, + ) + ) def test_custom_pricing_is_isolated_from_sibling_deployment( self, proxy: ProxyClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py b/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py index 8dfccf0d74b..8008345a3b6 100644 --- a/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py +++ b/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py @@ -20,6 +20,7 @@ from __future__ import annotations import pytest from e2e_config import unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from e2e_http import unwrap from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody, ThinkingParam @@ -49,6 +50,16 @@ def _reasoning_content(response: ChatResponse) -> str | None: class TestDeepSeekReasoningDisable: + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.DEEPSEEK,), + models=(REASONER,), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_reasoner_returns_reasoning_by_default( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -71,6 +82,16 @@ class TestDeepSeekReasoningDisable: f"disable param, so the disable assertions below can't be trusted: {response}" ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.DEEPSEEK,), + models=(REASONER,), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_reasoning_effort_none_disables_reasoning( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -93,6 +114,16 @@ class TestDeepSeekReasoningDisable: f"is still present: {response}" ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.DEEPSEEK,), + models=(REASONER,), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_thinking_disabled_disables_reasoning( self, client: PassthroughClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index 0e5bac556cc..7b59ffbf15f 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -16,6 +16,7 @@ from typing import Final import pytest from e2e_config import provider_edge_base, unique_marker from e2e_http import assert_client_error +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient @@ -24,6 +25,10 @@ from sdk_clients import NO_PROXY_CACHE, SdkClients, response_header pytestmark = pytest.mark.e2e +OPENAI_EMBEDDING: Final = "openai/text-embedding-3-small" +BEDROCK_TITAN_EMBEDDING: Final = "bedrock/amazon.titan-embed-text-v2:0" +COHERE_EMBEDDING: Final = "cohere/embed-v4.0" +MISTRAL_EMBEDDING: Final = "mistral/mistral-embed" VERTEX_TEXT_EMBEDDING: Final = "vertex_ai/text-embedding-005" VERTEX_MULTIMODAL_EMBEDDING: Final = "vertex_ai/multimodalembedding@001" TOKENS_TEXT: Final = "The quick brown fox jumps over the lazy dog" @@ -45,7 +50,7 @@ def _cosine(left: list[float], right: list[float]) -> float: def _titan_params() -> LiteLLMParamsBody: return LiteLLMParamsBody( - model="bedrock/amazon.titan-embed-text-v2:0", + model=BEDROCK_TITAN_EMBEDDING, aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", aws_region_name="os.environ/AWS_REGION", @@ -62,7 +67,7 @@ def _openai_embeddings_params() -> LiteLLMParamsBody: Vertex stay live: SigV4 signs the Host header, and neither has an edge mount.""" base = provider_edge_base("openai") return LiteLLMParamsBody( - model="openai/text-embedding-3-small", + model=OPENAI_EMBEDDING, api_key="os.environ/OPENAI_API_KEY", api_base=None if base is None else f"{base}/v1", ) @@ -97,10 +102,28 @@ def _assert_embedding_vector( class TestEmbeddingsEndpoint: @pytest.mark.replayable @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + providers=(Provider.OPENAI,), + models=(OPENAI_EMBEDDING,), + mode=Mode.NONSTREAM, + ) + ) def test_embeddings_returns_vector(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: _assert_embedding_vector(proxy, resources, sdk, "e2e-embeddings", _openai_embeddings_params()) @pytest.mark.covers("llm.embeddings.bedrock.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_TITAN_EMBEDDING,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_embeddings_returns_vector( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -113,6 +136,15 @@ class TestEmbeddingsEndpoint: ) @pytest.mark.covers("llm.embeddings.cohere.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + providers=(Provider.COHERE,), + models=(COHERE_EMBEDDING,), + mode=Mode.NONSTREAM, + ) + ) def test_cohere_embeddings_returns_vector( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -121,10 +153,19 @@ class TestEmbeddingsEndpoint: resources, sdk, "e2e-embeddings-cohere", - LiteLLMParamsBody(model="cohere/embed-v4.0", api_key="os.environ/COHERE_API_KEY"), + LiteLLMParamsBody(model=COHERE_EMBEDDING, api_key="os.environ/COHERE_API_KEY"), ) @pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_TEXT_EMBEDDING,), + mode=Mode.NONSTREAM, + ) + ) def test_vertex_embeddings_returns_vector( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -134,12 +175,21 @@ class TestEmbeddingsEndpoint: sdk, "e2e-embeddings-vertex", LiteLLMParamsBody( - model="vertex_ai/text-embedding-005", + model=VERTEX_TEXT_EMBEDDING, vertex_project="os.environ/VERTEXAI_PROJECT", vertex_location="us-central1", ), ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + providers=(Provider.MISTRAL,), + models=(MISTRAL_EMBEDDING,), + mode=Mode.NONSTREAM, + ) + ) def test_mistral_embeddings_returns_vector( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -148,10 +198,19 @@ class TestEmbeddingsEndpoint: resources, sdk, "e2e-embeddings-mistral", - LiteLLMParamsBody(model="mistral/mistral-embed", api_key="os.environ/MISTRAL_API_KEY"), + LiteLLMParamsBody(model=MISTRAL_EMBEDDING, api_key="os.environ/MISTRAL_API_KEY"), ) @pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_TEXT_EMBEDDING,), + mode=Mode.NONSTREAM, + ) + ) def test_vertex_embeddings_honor_requested_dimensions( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -166,6 +225,15 @@ class TestEmbeddingsEndpoint: assert len(embeddings.data[0].embedding) == 8, f"dimensions=8 was not honored: {embeddings!r}" assert embeddings.usage.prompt_tokens > 0, f"vertex embeddings reported no prompt usage: {embeddings.usage!r}" + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_MULTIMODAL_EMBEDDING,), + mode=Mode.NONSTREAM, + ) + ) def test_vertex_multimodal_embeddings_honor_dimensions_and_are_costed( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -181,6 +249,15 @@ class TestEmbeddingsEndpoint: cost = response_header(raw.headers, "x-litellm-response-cost") assert cost is not None and float(cost) > 0, f"multimodal embedding was not costed: {cost!r}" + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_TITAN_EMBEDDING,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_titan_embeds_token_array_input_as_its_decoded_text( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -199,6 +276,15 @@ class TestEmbeddingsEndpoint: @pytest.mark.replayable @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + providers=(Provider.OPENAI,), + models=(OPENAI_EMBEDDING,), + mode=Mode.NONSTREAM, + ) + ) def test_array_input_returns_vectors(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model, key = _register(proxy, resources, "e2e-embeddings-array", _openai_embeddings_params()) embeddings = sdk.openai(key).embeddings.create( @@ -208,6 +294,12 @@ class TestEmbeddingsEndpoint: @pytest.mark.replayable @pytest.mark.covers("llm.embeddings.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + ) + ) def test_missing_model_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: key = resources.key() result = proxy.transport.send( @@ -219,6 +311,12 @@ class TestEmbeddingsEndpoint: @pytest.mark.replayable @pytest.mark.covers("llm.embeddings.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + ) + ) def test_missing_input_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources, "e2e-embeddings-missin", _openai_embeddings_params()) result = proxy.transport.send( diff --git a/tests/e2e/llm_translation/test_files_batches_contract_e2e.py b/tests/e2e/llm_translation/test_files_batches_contract_e2e.py index 5627fa1c0bf..5e925f222b7 100644 --- a/tests/e2e/llm_translation/test_files_batches_contract_e2e.py +++ b/tests/e2e/llm_translation/test_files_batches_contract_e2e.py @@ -8,6 +8,7 @@ from __future__ import annotations import pytest from e2e_http import NoBody, Success, UnknownApiError, assert_client_error +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from proxy_client import ProxyClient from pydantic import BaseModel @@ -28,6 +29,13 @@ class BatchObject(BaseModel): class TestFilesBatchesContract: @pytest.mark.covers("llm.files.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.FILES, + mode=Mode.BATCH, + ) + ) def test_upload_without_purpose_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: key = resources.key() result = proxy.transport.upload( @@ -47,6 +55,13 @@ class TestFilesBatchesContract: pytest.fail(f"upload without purpose expected 4xx, got {other!r}") @pytest.mark.covers("llm.batches.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + mode=Mode.BATCH, + ) + ) def test_create_batch_missing_input_file_id_returns_error( self, proxy: ProxyClient, resources: ResourceManager ) -> None: @@ -59,6 +74,14 @@ class TestFilesBatchesContract: assert_client_error(result, "batch missing input_file_id") @pytest.mark.covers("llm.batches.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + providers=(Provider.OPENAI,), + mode=Mode.BATCH, + ) + ) def test_retrieve_invalid_batch_id_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: key = resources.key() result = proxy.transport.get( diff --git a/tests/e2e/llm_translation/test_google_native_e2e.py b/tests/e2e/llm_translation/test_google_native_e2e.py index 6910519c6df..b18683bd4ad 100644 --- a/tests/e2e/llm_translation/test_google_native_e2e.py +++ b/tests/e2e/llm_translation/test_google_native_e2e.py @@ -13,6 +13,7 @@ from typing import Literal import pytest from e2e_config import unique_marker from e2e_http import StreamingResponse, require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient @@ -85,6 +86,15 @@ def _streamed_text(result: StreamingResponse) -> str: class TestGoogleNativeGenerateContent: @pytest.mark.covers("llm.google_native.gemini.basic.nonstream.cost_logged") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.GOOGLE_GENAI, + providers=(Provider.GEMINI,), + models=(UPSTREAM_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_generate_content_returns_response_cost_header( self, proxy: ProxyClient, @@ -104,6 +114,15 @@ class TestGoogleNativeGenerateContent: assert result.response_cost > 0, f"x-litellm-response-cost must be a real cost, got {result.response_cost}" @pytest.mark.covers("llm.google_native.gemini.basic.stream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.GOOGLE_GENAI, + providers=(Provider.GEMINI,), + models=(UPSTREAM_MODEL,), + mode=Mode.STREAM, + ) + ) def test_stream_generate_content_frames_sse_the_way_google_sdks_expect( self, proxy: ProxyClient, diff --git a/tests/e2e/llm_translation/test_image_edits_e2e.py b/tests/e2e/llm_translation/test_image_edits_e2e.py index e95b054862e..66b0ebfc538 100644 --- a/tests/e2e/llm_translation/test_image_edits_e2e.py +++ b/tests/e2e/llm_translation/test_image_edits_e2e.py @@ -11,10 +11,12 @@ as the `image` part, not a JSON body. The fixture image is a small generated from __future__ import annotations import base64 +from typing import Final import openai import pytest from e2e_config import SLOW_PROVIDER_TIMEOUT_SECONDS, unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient @@ -22,6 +24,8 @@ from sdk_clients import SdkClients pytestmark = pytest.mark.e2e +IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-1" + _TEST_PNG = base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3" "3UrYvQQckD4XAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB" @@ -33,7 +37,7 @@ def _register_image_model(proxy: ProxyClient, resources: ResourceManager) -> tup model = f"e2e-image-edit-{unique_marker()}" model_id = proxy.create_model( model, - LiteLLMParamsBody(model="openai/gpt-image-1", api_key="os.environ/OPENAI_API_KEY"), + LiteLLMParamsBody(model=IMAGE_EDIT_BACKEND, api_key="os.environ/OPENAI_API_KEY"), ) resources.defer(lambda: proxy.delete_model(model_id)) return model, resources.key() @@ -49,6 +53,15 @@ def _assert_client_error(error: openai.APIStatusError, context: str) -> None: class TestImageEdit: @pytest.mark.covers("llm.images_edits.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.IMAGES, + providers=(Provider.OPENAI,), + models=(IMAGE_EDIT_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_image_edit_returns_image(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model, key = _register_image_model(proxy, resources) client = sdk.openai(key) @@ -64,6 +77,15 @@ class TestImageEdit: assert first.b64_json or first.url, f"edited image has neither b64_json nor url: {first!r}" @pytest.mark.covers("llm.images_edits.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.IMAGES, + providers=(Provider.OPENAI,), + models=(IMAGE_EDIT_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_empty_prompt_returns_error(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model, key = _register_image_model(proxy, resources) client = sdk.openai(key) @@ -73,6 +95,15 @@ class TestImageEdit: _assert_client_error(raised.value, "empty image-edit prompt") @pytest.mark.covers("llm.images_edits.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.IMAGES, + providers=(Provider.OPENAI,), + models=(IMAGE_EDIT_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_empty_image_returns_error(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model, key = _register_image_model(proxy, resources) client = sdk.openai(key) diff --git a/tests/e2e/llm_translation/test_image_generation_e2e.py b/tests/e2e/llm_translation/test_image_generation_e2e.py index 1db40e7e15a..15cf20aa65a 100644 --- a/tests/e2e/llm_translation/test_image_generation_e2e.py +++ b/tests/e2e/llm_translation/test_image_generation_e2e.py @@ -8,9 +8,12 @@ from litellm-regression-tests/tests/test_inference_endpoints.py. from __future__ import annotations +from typing import Final + import pytest from e2e_config import unique_marker from e2e_http import assert_client_error +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from openai.types import ImagesResponse @@ -20,6 +23,9 @@ from sdk_clients import SdkClients pytestmark = pytest.mark.e2e +OPENAI_IMAGE_BACKEND: Final = "openai/gpt-image-1-mini" +BEDROCK_IMAGE_BACKEND: Final = "bedrock/amazon.nova-canvas-v1:0" + class _OptionalImageBody(BaseModel): model: str | None = None @@ -47,12 +53,21 @@ def _register_openai_image(proxy: ProxyClient, resources: ResourceManager) -> tu proxy, resources, "e2e-image", - LiteLLMParamsBody(model="openai/gpt-image-1-mini", api_key="os.environ/OPENAI_API_KEY"), + LiteLLMParamsBody(model=OPENAI_IMAGE_BACKEND, api_key="os.environ/OPENAI_API_KEY"), ) class TestImageGeneration: @pytest.mark.covers("llm.images_generations.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.IMAGES, + providers=(Provider.OPENAI,), + models=(OPENAI_IMAGE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_image_generation_returns_image( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -61,6 +76,15 @@ class TestImageGeneration: _assert_image_returned(images) @pytest.mark.covers("llm.images_generations.bedrock.basic.nonstream.works", exercised_on=["images_generations"]) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.IMAGES, + providers=(Provider.BEDROCK,), + models=(BEDROCK_IMAGE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_image_generation_returns_image( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -69,7 +93,7 @@ class TestImageGeneration: resources, "e2e-bedrock-image", LiteLLMParamsBody( - model="bedrock/amazon.nova-canvas-v1:0", + model=BEDROCK_IMAGE_BACKEND, aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", aws_region_name="os.environ/AWS_REGION", @@ -80,6 +104,12 @@ class TestImageGeneration: @pytest.mark.skip(reason="stage red: product gap, /v1/images/generations 500s (aimage_generation TypeError) on missing prompt instead of 400") @pytest.mark.covers("llm.images_generations.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.IMAGES, + ) + ) def test_missing_prompt_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register_openai_image(proxy, resources) result = proxy.transport.send( @@ -90,6 +120,15 @@ class TestImageGeneration: assert_client_error(result, "images missing prompt") @pytest.mark.covers("llm.images_generations.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.IMAGES, + providers=(Provider.OPENAI,), + models=(OPENAI_IMAGE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_empty_prompt_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register_openai_image(proxy, resources) result = proxy.transport.send( @@ -100,6 +139,15 @@ class TestImageGeneration: assert_client_error(result, "images empty prompt") @pytest.mark.covers("llm.images_generations.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.IMAGES, + providers=(Provider.OPENAI,), + models=(OPENAI_IMAGE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_invalid_size_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register_openai_image(proxy, resources) result = proxy.transport.send( @@ -110,6 +158,15 @@ class TestImageGeneration: assert_client_error(result, "images invalid size") @pytest.mark.covers("llm.images_generations.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.IMAGES, + providers=(Provider.OPENAI,), + models=(OPENAI_IMAGE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_invalid_n_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register_openai_image(proxy, resources) result = proxy.transport.send( diff --git a/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py b/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py index 7a99f9c45e1..b10b8fe8bea 100644 --- a/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py +++ b/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py @@ -14,6 +14,7 @@ import pytest from anthropic.types import RawMessageStreamEvent, ToolParam from e2e_config import unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient @@ -56,6 +57,15 @@ class TestAzureFoundryMessages: return model @pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=(AZURE_FOUNDRY_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_basic_nonstream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -71,6 +81,15 @@ class TestAzureFoundryMessages: assert text.strip(), f"/v1/messages returned no text: {message.content!r}" @pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=(AZURE_FOUNDRY_MODEL,), + mode=Mode.STREAM, + ) + ) def test_basic_stream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -85,6 +104,16 @@ class TestAzureFoundryMessages: _assert_streamed_ok([event.type for event in stream]) @pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=(AZURE_FOUNDRY_MODEL,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_tool_use_nonstream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -102,6 +131,16 @@ class TestAzureFoundryMessages: ) @pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=(AZURE_FOUNDRY_MODEL,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) + ) def test_tool_use_stream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -122,6 +161,16 @@ class TestAzureFoundryMessages: ), "stream carried no tool_use block" assert "message_stop" in event_types, "stream never reached message_stop" + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=(AZURE_FOUNDRY_MODEL,), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) + ) def test_output_format_returns_schema_json( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: diff --git a/tests/e2e/llm_translation/test_messages_bedrock_e2e.py b/tests/e2e/llm_translation/test_messages_bedrock_e2e.py index 61a37af7fd0..4e37f01886a 100644 --- a/tests/e2e/llm_translation/test_messages_bedrock_e2e.py +++ b/tests/e2e/llm_translation/test_messages_bedrock_e2e.py @@ -5,6 +5,7 @@ from typing import Final import pytest from anthropic.types import RawContentBlockDeltaEvent, RawMessageDeltaEvent, TextBlock, TextDelta from e2e_config import unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient @@ -33,6 +34,16 @@ def _register(proxy: ProxyClient, resources: ResourceManager, backend: str) -> s class TestBedrockMessages: + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=(CONVERSE_CLAUDE_BACKEND,), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) + ) def test_converse_output_format_returns_schema_json_text( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -50,6 +61,15 @@ class TestBedrockMessages: assert_sentiment_json("".join(texts)) @pytest.mark.covers("llm.messages.bedrock_converse.basic.stream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=(NOVA_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_nova_stream_relays_text_usage_and_stop( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index 90e74474d3b..6d2f690b947 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -17,6 +17,7 @@ from typing import Final import anthropic import pytest +from _pytest.mark.structures import ParameterSet from anthropic import Anthropic from anthropic.types import ( InputJSONDelta, @@ -42,6 +43,7 @@ from e2e_config import ( unique_marker, ) from e2e_http import assert_client_error +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import AnthropicErrorEvent, AnthropicMessagesBody, ChatMessage, LiteLLMParamsBody, SpendLogRow from provider_edge import EDGE_MOUNTS, LiveEdge, RunningEdge, StreamCut, start_provider_edge @@ -61,6 +63,7 @@ class _OptionalMessagesBody(BaseModel): ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5" +OPENAI_BRIDGE_BACKEND: Final = "openai/gpt-5.6" WEATHER_TOOL: ToolParam = { "name": "get_weather", @@ -110,6 +113,15 @@ def _user_turn(text: str) -> MessageParam: class TestAnthropicMessages: @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_messages_returns_completion(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model, key = _register(proxy, resources) client = sdk.anthropic(key) @@ -121,6 +133,15 @@ class TestAnthropicMessages: assert _text(message).strip(), f"/v1/messages returned no text: {message.content!r}" @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.cost_logged") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_messages_logs_cost_matching_the_response_header( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -169,6 +190,15 @@ class TestAnthropicMessages: @pytest.mark.covers("llm.messages.anthropic.basic.stream.works") @pytest.mark.provider_live + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_messages_streams_completion(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: """Edge-wired like its non-streaming siblings, so record and replay both carry the streamed response. @@ -226,6 +256,16 @@ class TestAnthropicMessages: ) @pytest.mark.covers("llm.messages.anthropic.tool_use.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_messages_tool_use(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model, key = _register(proxy, resources) client = sdk.anthropic(key) @@ -243,6 +283,16 @@ class TestAnthropicMessages: ) @pytest.mark.covers("llm.messages.anthropic.structured_output.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) + ) def test_messages_output_format_returns_schema_json( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -261,6 +311,14 @@ class TestAnthropicMessages: reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing messages instead of 400" ) @pytest.mark.covers("llm.messages.anthropic.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(), + models=(), + ) + ) def test_missing_messages_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) result = proxy.transport.send( @@ -274,6 +332,14 @@ class TestAnthropicMessages: reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing max_tokens instead of 400" ) @pytest.mark.covers("llm.messages.anthropic.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(), + models=(), + ) + ) def test_missing_max_tokens_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) result = proxy.transport.send( @@ -284,6 +350,14 @@ class TestAnthropicMessages: assert_client_error(result, "messages missing max_tokens") @pytest.mark.covers("llm.messages.anthropic.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(), + models=(), + ) + ) def test_missing_model_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: _, key = _register(proxy, resources) result = proxy.transport.send( @@ -364,9 +438,26 @@ def _request_tool(client: Anthropic, model: str, question: MessageParam, tool: T return blocks[0] +def _openai_bridge_subject(mode: Mode) -> Subject: + return Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.OPENAI,), + models=(OPENAI_BRIDGE_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=mode, + ) + + class TestOpenAIMessagesToolContinuation: @pytest.mark.provider_live - @pytest.mark.parametrize("stream", [True, False], ids=["stream", "nonstream"]) + @pytest.mark.parametrize( + "stream", + [ + pytest.param(stream, marks=meta(_openai_bridge_subject(mode)), id=name) + for stream, name, mode in ((True, "stream", Mode.STREAM), (False, "nonstream", Mode.NONSTREAM)) + ], + ) def test_required_tool_arguments_and_correlated_result( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, stream: bool ) -> None: @@ -375,7 +466,7 @@ class TestOpenAIMessagesToolContinuation: model_id: Final = proxy.create_model( model, LiteLLMParamsBody( - model="openai/gpt-5.6", api_key="os.environ/OPENAI_API_KEY", api_base=f"{base}/v1" if base else None + model=OPENAI_BRIDGE_BACKEND, api_key="os.environ/OPENAI_API_KEY", api_base=f"{base}/v1" if base else None ), ) resources.defer(lambda: proxy.delete_model(model_id)) @@ -478,6 +569,30 @@ _DROPPED_BEFORE_FIRST_BYTE: Final[tuple[tuple[str, _CutRegistration, StreamCut], ) +_CUT_SUBJECTS: Final[MappingProxyType[_CutRegistration, Subject]] = MappingProxyType( + { + _register_cut_bedrock: Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=(BEDROCK_BACKEND,), + mode=Mode.STREAM, + ), + _register_cut_anthropic: Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + mode=Mode.STREAM, + ), + } +) + + +def _cut_params(cases: tuple[tuple[str, _CutRegistration, StreamCut], ...]) -> list[ParameterSet]: + return [pytest.param(register, cut, id=name, marks=meta(_CUT_SUBJECTS[register])) for name, register, cut in cases] + + def _payload(frame: str) -> JsonValue | None: try: return _FRAME_PAYLOAD.validate_json(frame) @@ -495,7 +610,7 @@ def _bare_error_frame(frame: str) -> bool: class TestMessagesUpstreamStreamFailure: @pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_event") @pytest.mark.parametrize( - ("register", "cut"), [case[1:] for case in _DROPPED_UPSTREAMS], ids=[case[0] for case in _DROPPED_UPSTREAMS] + ("register", "cut"), _cut_params(_DROPPED_UPSTREAMS) ) def test_interrupted_upstream_stream_raises_in_the_anthropic_sdk( self, @@ -533,7 +648,7 @@ class TestMessagesUpstreamStreamFailure: @pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_event") @pytest.mark.parametrize( - ("register", "cut"), [case[1:] for case in _DROPPED_UPSTREAMS], ids=[case[0] for case in _DROPPED_UPSTREAMS] + ("register", "cut"), _cut_params(_DROPPED_UPSTREAMS) ) def test_interrupted_upstream_stream_is_an_anthropic_error_event( self, proxy: ProxyClient, resources: ResourceManager, register: _CutRegistration, cut: StreamCut @@ -585,11 +700,7 @@ class TestMessagesUpstreamStreamFailure: ) @pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_status") - @pytest.mark.parametrize( - ("register", "cut"), - [case[1:] for case in _DROPPED_BEFORE_FIRST_BYTE], - ids=[case[0] for case in _DROPPED_BEFORE_FIRST_BYTE], - ) + @pytest.mark.parametrize(("register", "cut"), _cut_params(_DROPPED_BEFORE_FIRST_BYTE)) def test_upstream_that_hangs_up_before_the_first_byte_raises_with_its_status_in_the_anthropic_sdk( self, proxy: ProxyClient, @@ -622,11 +733,7 @@ class TestMessagesUpstreamStreamFailure: ) @pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_status") - @pytest.mark.parametrize( - ("register", "cut"), - [case[1:] for case in _DROPPED_BEFORE_FIRST_BYTE], - ids=[case[0] for case in _DROPPED_BEFORE_FIRST_BYTE], - ) + @pytest.mark.parametrize(("register", "cut"), _cut_params(_DROPPED_BEFORE_FIRST_BYTE)) def test_upstream_that_hangs_up_before_the_first_byte_is_a_json_error_with_its_status( self, proxy: ProxyClient, resources: ResourceManager, register: _CutRegistration, cut: StreamCut ) -> None: diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py index e9b4b394996..84e0674d959 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py @@ -34,6 +34,7 @@ import pytest from anthropic import Anthropic from anthropic.types import Message, MessageParam, TextBlockParam from e2e_config import unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient @@ -208,6 +209,16 @@ class TestBedrockInvokeMidConversationSystem: "llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=(FLAGGED_INVOKE_MODEL,), + capabilities=(Capability.MID_CONVERSATION_SYSTEM, Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_flagged_model_keeps_prompt_cache_across_system_reminder( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -234,6 +245,16 @@ class TestBedrockInvokeMidConversationSystem: "llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=(UNFLAGGED_INVOKE_MODEL,), + capabilities=(Capability.MID_CONVERSATION_SYSTEM, Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_unflagged_model_converts_system_reminder_and_succeeds( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py index 9f5ed8b05da..44701b827f0 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py @@ -41,6 +41,7 @@ import pytest from anthropic import Anthropic from anthropic.types import Message, MessageParam, TextBlockParam from e2e_config import unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient @@ -290,6 +291,16 @@ class TestAzureFoundryMidConversationSystem: "llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=(FLAGGED_MODEL,), + capabilities=(Capability.MID_CONVERSATION_SYSTEM, Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_flagged_model_keeps_prompt_cache_across_system_reminder( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -299,6 +310,16 @@ class TestAzureFoundryMidConversationSystem: "llm.messages.azure_foundry.mid_conversation_system.nonstream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=(UNFLAGGED_MODEL,), + capabilities=(Capability.MID_CONVERSATION_SYSTEM, Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_unflagged_model_converts_system_reminder_and_succeeds( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -322,6 +343,16 @@ class TestVertexMidConversationSystem: "llm.messages.vertex.mid_conversation_system.nonstream.cache_hit", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=(FLAGGED_MODEL,), + capabilities=(Capability.MID_CONVERSATION_SYSTEM, Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_flagged_model_keeps_prompt_cache_across_system_reminder( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -333,6 +364,16 @@ class TestVertexMidConversationSystem: "llm.messages.vertex.mid_conversation_system.nonstream.works", exercised_on=[], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=(UNFLAGGED_MODEL,), + capabilities=(Capability.MID_CONVERSATION_SYSTEM, Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_unflagged_model_converts_system_reminder_and_succeeds( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: diff --git a/tests/e2e/llm_translation/test_moderations_e2e.py b/tests/e2e/llm_translation/test_moderations_e2e.py index e936f7b335a..4d2d5ea8b79 100644 --- a/tests/e2e/llm_translation/test_moderations_e2e.py +++ b/tests/e2e/llm_translation/test_moderations_e2e.py @@ -9,9 +9,12 @@ negative stays on the shared transport, since the SDK refuses to send it. from __future__ import annotations +from typing import Final + import pytest from e2e_config import unique_marker from e2e_http import assert_client_error +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from openai.types import Moderation @@ -21,6 +24,7 @@ from sdk_clients import SdkClients pytestmark = pytest.mark.e2e +OPENAI_MODERATION_BACKEND: Final = "openai/omni-moderation-latest" VIOLENT_TEXT = "I am going to find you and kill you, and I will hurt everyone you love." BENIGN_TEXT = "I enjoyed the sunny afternoon and a relaxing walk in the park today." @@ -35,7 +39,7 @@ def _register_moderation_model(proxy: ProxyClient, resources: ResourceManager) - model_id = proxy.create_model( model, LiteLLMParamsBody( - model="openai/omni-moderation-latest", api_key="os.environ/OPENAI_API_KEY" + model=OPENAI_MODERATION_BACKEND, api_key="os.environ/OPENAI_API_KEY" ), ) resources.defer(lambda: proxy.delete_model(model_id)) @@ -52,6 +56,15 @@ def _flagged_categories(item: Moderation) -> tuple[str, ...]: class TestModerations: @pytest.mark.covers("llm.moderations.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MODERATIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_MODERATION_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_moderations_flags_violent_content( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -64,6 +77,15 @@ class TestModerations: assert item.flagged, f"violent text was not flagged: {item!r}" assert _flagged_categories(item), f"flagged result reported no true category: {item!r}" + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MODERATIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_MODERATION_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_moderations_passes_benign_content( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -79,6 +101,14 @@ class TestModerations: @pytest.mark.skip(reason="stage red: product gap, /v1/moderations 500s (KeyError 'input') on missing input instead of 400") @pytest.mark.covers("llm.moderations.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MODERATIONS, + providers=(), + models=(), + ) + ) def test_missing_input_returns_error( self, proxy: ProxyClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_ocr_rust_e2e.py b/tests/e2e/llm_translation/test_ocr_rust_e2e.py index b54a6ea010b..cdc3fe87431 100644 --- a/tests/e2e/llm_translation/test_ocr_rust_e2e.py +++ b/tests/e2e/llm_translation/test_ocr_rust_e2e.py @@ -25,6 +25,7 @@ from typing import Final, Protocol import pytest from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from e2e_http import ( PROVIDER_RATE_LIMIT_ATTEMPTS, RateLimitedError, @@ -60,6 +61,13 @@ TEST_IMAGE_URL = ( ) +MISTRAL_OCR_MODEL: Final = "mistral/mistral-ocr-latest" +AZURE_AI_OCR_MODEL: Final = "azure_ai/mistral-document-ai-2512" +AZURE_DOC_INTELLIGENCE_MODEL: Final = "azure_ai/doc-intelligence/prebuilt-layout" +VERTEX_OCR_MODEL: Final = "vertex_ai/mistral-ocr-2505" +COHERE_OCR_MODEL: Final = "cohere/parse-v5.0" + + class OcrProvider(Protocol): """One OCR provider's deployment config: its model id plus the os.environ/* credential references the proxy resolves at call time. Each provider owns which @@ -70,7 +78,7 @@ class OcrProvider(Protocol): @dataclass(frozen=True, slots=True) class MistralOcr: - model: str = "mistral/mistral-ocr-latest" + model: str = MISTRAL_OCR_MODEL def litellm_params(self) -> LiteLLMParamsBody: return LiteLLMParamsBody(model=self.model, api_key="os.environ/MISTRAL_API_KEY") @@ -96,7 +104,7 @@ class AzureDocIntelligenceOcr: AZURE_DOCUMENT_INTELLIGENCE_API_KEY, which the OCR config resolves from the doc-intelligence model name when api_base/api_key are left unset.""" - model: str = "azure_ai/doc-intelligence/prebuilt-layout" + model: str = AZURE_DOC_INTELLIGENCE_MODEL def litellm_params(self) -> LiteLLMParamsBody: return LiteLLMParamsBody(model=self.model) @@ -120,7 +128,7 @@ class VertexOcr: @dataclass(frozen=True, slots=True) class CohereOcr: - model: str = "cohere/parse-v5.0" + model: str = COHERE_OCR_MODEL def litellm_params(self) -> LiteLLMParamsBody: return LiteLLMParamsBody(model=self.model, api_key="os.environ/COHERE_API_KEY") @@ -141,7 +149,7 @@ RUST_OCR_CASES: tuple[_OcrCase, ...] = ( ), _OcrCase( "azure-ai", - AzureAiOcr("azure_ai/mistral-document-ai-2512"), + AzureAiOcr(AZURE_AI_OCR_MODEL), OcrDocument(type="document_url", document_url=TEST_PDF_URL), ), _OcrCase( @@ -151,13 +159,11 @@ RUST_OCR_CASES: tuple[_OcrCase, ...] = ( ), _OcrCase( "vertex-mistral", - VertexOcr("vertex_ai/mistral-ocr-2505", "us-central1"), + VertexOcr(VERTEX_OCR_MODEL, "us-central1"), OcrDocument(type="document_url", document_url=TEST_PDF_URL), ), ) -_CASE_IDS = tuple(case.suffix for case in RUST_OCR_CASES) - PDF_TEXT: Final = "test pdf file" IMAGE_TEXT: Final = "litellm" PDF_DOCUMENT: Final = OcrDocument(type="document_url", document_url=TEST_PDF_URL) @@ -175,14 +181,37 @@ class _OcrContentCase: OCR_CONTENT_CASES: Final = ( _OcrContentCase("mistral-pdf", MistralOcr(), PDF_DOCUMENT, PDF_TEXT), _OcrContentCase("mistral-image", MistralOcr(), IMAGE_DOCUMENT, IMAGE_TEXT), - _OcrContentCase("azure-ai-image", AzureAiOcr("azure_ai/mistral-document-ai-2512"), IMAGE_DOCUMENT, IMAGE_TEXT), + _OcrContentCase("azure-ai-image", AzureAiOcr(AZURE_AI_OCR_MODEL), IMAGE_DOCUMENT, IMAGE_TEXT), _OcrContentCase( - "vertex-mistral-image", VertexOcr("vertex_ai/mistral-ocr-2505", "us-central1"), IMAGE_DOCUMENT, IMAGE_TEXT + "vertex-mistral-image", VertexOcr(VERTEX_OCR_MODEL, "us-central1"), IMAGE_DOCUMENT, IMAGE_TEXT ), _OcrContentCase("cohere-image", CohereOcr(), IMAGE_DOCUMENT, IMAGE_TEXT), ) +def _ocr_subject(provider: OcrProvider) -> Subject: + match provider: + case MistralOcr(): + vendor, model = Provider.MISTRAL, MISTRAL_OCR_MODEL + case AzureAiOcr(): + vendor, model = Provider.AZURE_AI, AZURE_AI_OCR_MODEL + case AzureDocIntelligenceOcr(): + vendor, model = Provider.AZURE_AI, AZURE_DOC_INTELLIGENCE_MODEL + case VertexOcr(): + vendor, model = Provider.VERTEX_AI, VERTEX_OCR_MODEL + case CohereOcr(): + vendor, model = Provider.COHERE, COHERE_OCR_MODEL + case _: + raise TypeError(f"no OCR subject for {provider!r}") + return Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.OCR, + providers=(vendor,), + models=(model,), + mode=Mode.NONSTREAM, + ) + + def _assert_ocr_document(response: OcrResponse) -> None: assert response.object == "ocr", f"expected object='ocr', got {response.object!r}" assert response.model, "response missing the resolved model name" @@ -202,7 +231,10 @@ def _assert_provider_rate_limit_relayed(model: str, outcome: RateLimitedError) - class TestRustOcrGateway: - @pytest.mark.parametrize("case", RUST_OCR_CASES, ids=_CASE_IDS) + @pytest.mark.parametrize( + "case", + [pytest.param(case, marks=meta(_ocr_subject(case.provider)), id=case.suffix) for case in RUST_OCR_CASES], + ) def test_rust_ocr_response(self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase) -> None: model = f"rust-ocr-{case.suffix}-{unique_marker()}" model_id = proxy.create_model(model, case.provider.litellm_params()) @@ -219,6 +251,14 @@ class TestRustOcrGateway: @pytest.mark.skip(reason="stage red: product gap, /v1/ocr 500s (aocr TypeError) on missing document instead of 400") @pytest.mark.covers("llm.ocr.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.OCR, + providers=(), + models=(), + ) + ) def test_missing_document_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model = f"rust-ocr-val-{unique_marker()}" model_id = proxy.create_model(model, MistralOcr().litellm_params()) @@ -233,7 +273,13 @@ class TestRustOcrGateway: class TestOcrDocumentContent: - @pytest.mark.parametrize("case", OCR_CONTENT_CASES, ids=tuple(case.suffix for case in OCR_CONTENT_CASES)) + @pytest.mark.parametrize( + "case", + [ + pytest.param(case, marks=meta(_ocr_subject(case.provider)), id=case.suffix) + for case in OCR_CONTENT_CASES + ], + ) def test_ocr_reads_the_document_and_bills_its_pages( self, proxy: ProxyClient, resources: ResourceManager, case: _OcrContentCase ) -> None: diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index 447fe7d30d9..52d1c10d829 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -12,10 +12,13 @@ A passthrough call returning non-2xx fails hard (never a skip); once it returns 2xx, a missing or zero-cost SpendLogs row fails too. """ +from typing import Final + import pytest from e2e_config import CHEAP_OPENAI_MODEL, unique_marker from e2e_http import require_successful_call, unwrap +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatResponse, KeyGenerateBody, SpendLogRow from passthrough_client import ( @@ -29,6 +32,8 @@ from passthrough_client import ( completed_responses_object, ) +GEMINI_MODEL: Final = "gemini-2.5-flash" +ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5" EMBEDDING_MODEL = "text-embedding-3-small" REALTIME_MODEL = "gpt-realtime-2" @@ -57,12 +62,21 @@ def _fetch_cost_breakdown(client: PassthroughClient, request_id: str | None) -> # ---- Gemini passthrough ------------------------------------------------ +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.GEMINI,), + models=(GEMINI_MODEL,), + mode=Mode.NONSTREAM, + ) +) def test_gemini_passthrough_nonstreaming_logs_cost( client: PassthroughClient, scoped_key: str ) -> None: tag = f"e2e-passthrough-{unique_marker()}" result = client.gemini_generate( - scoped_key, "gemini-2.5-flash", "Say hello in one word", tags=[tag, "gemini"] + scoped_key, GEMINI_MODEL, "Say hello in one word", tags=[tag, "gemini"] ) require_successful_call(result) @@ -73,6 +87,15 @@ def test_gemini_passthrough_nonstreaming_logs_cost( @pytest.mark.skip(reason="stage red: product gap, native passthrough returns no x-litellm-response-cost or x-ratelimit-* headers") +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.GEMINI,), + models=(GEMINI_MODEL,), + mode=Mode.NONSTREAM, + ) +) def test_gemini_passthrough_returns_the_same_header_contract_as_the_managed_route( client: PassthroughClient, scoped_key: str ) -> None: @@ -82,7 +105,7 @@ def test_gemini_passthrough_returns_the_same_header_contract_as_the_managed_rout today, which makes native traffic invisible to the same tooling. """ result = client.gemini_generate( - scoped_key, "gemini-2.5-flash", f"Say hello in one word. {unique_marker()}" + scoped_key, GEMINI_MODEL, f"Say hello in one word. {unique_marker()}" ) require_successful_call(result) @@ -102,10 +125,19 @@ def test_gemini_passthrough_returns_the_same_header_contract_as_the_managed_rout ) +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.GEMINI,), + models=(GEMINI_MODEL,), + mode=Mode.STREAM, + ) +) def test_gemini_passthrough_streaming_logs_cost( client: PassthroughClient, scoped_key: str ) -> None: - result = client.gemini_stream(scoped_key, "gemini-2.5-flash", "Count to five") + result = client.gemini_stream(scoped_key, GEMINI_MODEL, "Count to five") require_successful_call(result) assert result.chunks > 0, "streaming passthrough produced no events" @@ -113,12 +145,22 @@ def test_gemini_passthrough_streaming_logs_cost( assert row.custom_llm_provider == "gemini" +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.GEMINI,), + models=(GEMINI_MODEL,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_gemini_passthrough_tool_call_logs_cost( client: PassthroughClient, scoped_key: str ) -> None: result = client.gemini_generate( scoped_key, - "gemini-2.5-flash", + GEMINI_MODEL, "What is the weather in Paris? Use the get_weather tool.", tools=[ GeminiTool( @@ -146,10 +188,19 @@ def test_gemini_passthrough_tool_call_logs_cost( # ---- Anthropic passthrough --------------------------------------------- +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_PASSTHROUGH_MODEL,), + mode=Mode.NONSTREAM, + ) +) def test_anthropic_passthrough_nonstreaming_logs_cost( client: PassthroughClient, scoped_key: str ) -> None: - result = client.anthropic_message(scoped_key, "claude-haiku-4-5", "Say hello") + result = client.anthropic_message(scoped_key, ANTHROPIC_PASSTHROUGH_MODEL, "Say hello") require_successful_call(result) row = _fetch_cost_breakdown(client, anthropic_message_id(result)) @@ -157,11 +208,20 @@ def test_anthropic_passthrough_nonstreaming_logs_cost( assert "claude" in (row.model or "") +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_PASSTHROUGH_MODEL,), + mode=Mode.STREAM, + ) +) def test_anthropic_passthrough_streaming_logs_cost( client: PassthroughClient, scoped_key: str ) -> None: result = client.anthropic_message( - scoped_key, "claude-haiku-4-5", "Count to five", stream=True + scoped_key, ANTHROPIC_PASSTHROUGH_MODEL, "Count to five", stream=True ) require_successful_call(result) assert result.chunks > 0, "streaming passthrough produced no events" @@ -170,12 +230,22 @@ def test_anthropic_passthrough_streaming_logs_cost( assert row.custom_llm_provider == "anthropic" +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_PASSTHROUGH_MODEL,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_anthropic_passthrough_tool_call_logs_cost( client: PassthroughClient, scoped_key: str ) -> None: result = client.anthropic_message( scoped_key, - "claude-haiku-4-5", + ANTHROPIC_PASSTHROUGH_MODEL, "What is the weather in Paris? Use the get_weather tool.", tools=[ AnthropicTool( @@ -205,13 +275,21 @@ class TestPassthroughModelAllowlist: """ @pytest.mark.covers("other.auth.passthrough.model_allowlist_enforced") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.PASSTHROUGH, + providers=(), + models=(), + ) + ) def test_passthrough_denies_model_outside_key_allowlist( self, client: PassthroughClient, resources: ResourceManager ) -> None: - key = client.proxy.generate_key(KeyGenerateBody(models=["gemini-2.5-flash"])) + key = client.proxy.generate_key(KeyGenerateBody(models=[GEMINI_MODEL])) resources.defer(lambda: client.proxy.delete_key(key)) - result = client.anthropic_message(key, "claude-haiku-4-5", f"say hi {unique_marker()}") + result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}") assert result.status_code == 403, ( "a key restricted to gemini-2.5-flash must be denied a claude passthrough call, " f"got {result.status_code}: {result.body[:300]}" @@ -230,6 +308,14 @@ class TestOpenAIPassthroughPrefix: """ @pytest.mark.covers("llm.files.openai.passthrough.nonstream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.OPENAI,), + models=(), + ) + ) def test_passthrough_prefix_uploads_a_file_to_openai( self, client: PassthroughClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -252,6 +338,14 @@ class TestOpenAIPassthroughPrefix: assert uploaded.bytes == len(content) @pytest.mark.covers("llm.batches.openai.passthrough.nonstream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.OPENAI,), + models=(), + ) + ) def test_passthrough_prefix_lists_batches_from_openai( self, client: PassthroughClient, scoped_key: str ) -> None: @@ -274,6 +368,15 @@ class TestOpenAIPassthroughSpend: """ @pytest.mark.covers("llm.responses.openai.passthrough.stream.cost_logged") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.STREAM, + ) + ) def test_streamed_responses_call_logs_its_cost( self, client: PassthroughClient, scoped_key: str ) -> None: @@ -318,6 +421,15 @@ class TestOpenAIPassthroughSpend: ) @pytest.mark.covers("llm.embeddings.openai.passthrough.nonstream.cost_logged") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.OPENAI,), + models=(EMBEDDING_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_embeddings_call_logs_its_cost( self, client: PassthroughClient, scoped_key: str ) -> None: @@ -353,6 +465,15 @@ class TestOpenAIProviderPrefixChat: """ @pytest.mark.covers("llm.chat_completions.openai.passthrough.nonstream.cost_logged") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_openai_prefix_chat_returns_completion_and_logs_its_cost( self, client: PassthroughClient, scoped_key: str ) -> None: @@ -393,6 +514,15 @@ class TestOpenAIPassthroughWebsocket: """ @pytest.mark.covers("llm.realtime.openai.passthrough.stream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.OPENAI,), + models=(REALTIME_MODEL,), + mode=Mode.WEBSOCKET, + ) + ) def test_realtime_upgrade_reaches_openai_through_the_passthrough_prefix( self, client: PassthroughClient, scoped_key: str ) -> None: @@ -414,6 +544,15 @@ class TestOpenAIPassthroughWebsocket: ) @pytest.mark.covers("llm.responses.openai.passthrough_websocket.stream.works") + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.OPENAI,), + models=(), + mode=Mode.WEBSOCKET, + ) + ) def test_responses_upgrade_is_accepted_on_the_openai_prefix( self, client: PassthroughClient, scoped_key: str ) -> None: diff --git a/tests/e2e/llm_translation/test_passthrough_headers_e2e.py b/tests/e2e/llm_translation/test_passthrough_headers_e2e.py index 26f98774c63..66d51d122da 100644 --- a/tests/e2e/llm_translation/test_passthrough_headers_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_headers_e2e.py @@ -20,6 +20,7 @@ from pydantic import BaseModel, Field from e2e_config import unique_marker from e2e_http import AuthHeaders, NoBody, require_successful_call, unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import AnthropicMessagesResponse, ChatMessage, KeyGenerateBody from passthrough_client import PassthroughClient @@ -136,6 +137,15 @@ class TestPassthroughHeaders: "other.config.passthrough.headers_forwarded", exercised_on=[], ) + @meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.ANTHROPIC,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_static_and_x_pass_headers_reach_upstream( self, client: PassthroughClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_provider_features_e2e.py b/tests/e2e/llm_translation/test_provider_features_e2e.py index 2ea1d28748d..9a3de0aba7d 100644 --- a/tests/e2e/llm_translation/test_provider_features_e2e.py +++ b/tests/e2e/llm_translation/test_provider_features_e2e.py @@ -16,10 +16,13 @@ Prompt caching lives in test_cache_control.py. from __future__ import annotations +from typing import Final + import pytest from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, LiteLLMParamsBody from passthrough_client import PassthroughClient @@ -27,12 +30,22 @@ from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e SERVICE_TIER = "priority" +OPENAI_BACKEND: Final = "openai/gpt-5.5" class TestServiceTier: @pytest.mark.covers( "llm.chat_completions.openai.service_tier.nonstream.works", exercised_on=[] ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_openai_service_tier_is_echoed( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -40,7 +53,7 @@ class TestServiceTier: model_id = client.proxy.create_model( model, LiteLLMParamsBody( - model="openai/gpt-5.5", api_key="os.environ/OPENAI_API_KEY" + model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY" ), ) resources.defer(lambda: client.proxy.delete_model(model_id)) diff --git a/tests/e2e/llm_translation/test_realtime_http_e2e.py b/tests/e2e/llm_translation/test_realtime_http_e2e.py index 9579ae13bbc..ee3ce3893c2 100644 --- a/tests/e2e/llm_translation/test_realtime_http_e2e.py +++ b/tests/e2e/llm_translation/test_realtime_http_e2e.py @@ -9,6 +9,7 @@ from __future__ import annotations import pytest from e2e_config import unique_marker from e2e_http import NoBody, assert_auth_denied, unwrap +from e2e_metadata import Domain, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient @@ -59,6 +60,14 @@ def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str] class TestRealtimeHttp: @pytest.mark.covers("llm.realtime.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider.OPENAI,), + models=(REALTIME_BACKEND,), + ) + ) def test_create_client_secret(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) secret = unwrap( @@ -82,6 +91,7 @@ class TestRealtimeHttp: assert secret.session.type in (None, "realtime"), f"unexpected session type: {secret.session.type}" @pytest.mark.covers("other.auth.realtime.missing_header_denied") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.REALTIME)) def test_client_secret_missing_auth_is_denied(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, _ = _register(proxy, resources) result = proxy.transport.send( @@ -92,6 +102,7 @@ class TestRealtimeHttp: assert_auth_denied(result, "realtime client_secrets missing auth") @pytest.mark.covers("other.auth.realtime.missing_header_denied") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.REALTIME)) def test_calls_without_auth_is_denied(self, proxy: ProxyClient) -> None: result = proxy.transport.send( "/v1/realtime/calls", diff --git a/tests/e2e/llm_translation/test_rerank_e2e.py b/tests/e2e/llm_translation/test_rerank_e2e.py index 87b8618e6fb..9fafff03004 100644 --- a/tests/e2e/llm_translation/test_rerank_e2e.py +++ b/tests/e2e/llm_translation/test_rerank_e2e.py @@ -9,9 +9,12 @@ litellm-regression-tests/tests/test_inference_endpoints.py. from __future__ import annotations +from typing import Final + import pytest from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody, RerankBody, RerankResponse from proxy_client import ProxyClient @@ -24,6 +27,8 @@ DOCUMENTS = [ "Washington, D.C. is the capital of the United States.", "Capital punishment has existed in the United States since before it was a country.", ] +COHERE_RERANK_BACKEND: Final = "cohere/rerank-v3.5" +BEDROCK_RERANK_BACKEND: Final = "bedrock/arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0" QUERY = "What is the capital of the United States?" @@ -43,11 +48,20 @@ def _rerank_top_3(proxy: ProxyClient, key: str, model: str) -> RerankResponse: class TestRerank: @pytest.mark.covers("llm.rerank.cohere.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RERANK, + providers=(Provider.COHERE,), + models=(COHERE_RERANK_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_rerank_scores_top_n(self, proxy: ProxyClient, resources: ResourceManager) -> None: model = f"e2e-rerank-{unique_marker()}" model_id = proxy.create_model( model, - LiteLLMParamsBody(model="cohere/rerank-v3.5", api_key="os.environ/COHERE_API_KEY"), + LiteLLMParamsBody(model=COHERE_RERANK_BACKEND, api_key="os.environ/COHERE_API_KEY"), ) resources.defer(lambda: proxy.delete_model(model_id)) key = resources.key() @@ -55,6 +69,15 @@ class TestRerank: _assert_top_n_scored(_rerank_top_3(proxy, key, model)) @pytest.mark.covers("llm.rerank.bedrock.basic.nonstream.works", exercised_on=["rerank"]) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RERANK, + providers=(Provider.BEDROCK,), + models=(BEDROCK_RERANK_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_rerank_scores_top_n( self, proxy: ProxyClient, resources: ResourceManager ) -> None: @@ -62,7 +85,7 @@ class TestRerank: model_id = proxy.create_model( model, LiteLLMParamsBody( - model="bedrock/arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0", + model=BEDROCK_RERANK_BACKEND, aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", aws_region_name="os.environ/AWS_REGION", diff --git a/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py b/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py index 75817340876..7826dc388ea 100644 --- a/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py +++ b/tests/e2e/llm_translation/test_responses_bridge_streaming_e2e.py @@ -23,13 +23,14 @@ from pydantic import BaseModel, Field from e2e_config import unique_marker from e2e_http import StreamingResponse +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatTool, ChatToolFunction, LiteLLMParamsBody from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e -RESPONSES_ONLY_BACKEND = "openai/gpt-5.3-codex" +RESPONSES_ONLY_BACKEND: Final = "openai/gpt-5.3-codex" class _BridgeToolCallFunction(BaseModel): @@ -99,6 +100,15 @@ class TestResponsesBridgeChatCompletionsStreaming: "llm.chat_completions.openai.basic.stream.bridge_shares_chunk_id", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(RESPONSES_ONLY_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_bridged_stream_shares_one_chunk_id( self, client: PassthroughClient, resources: ResourceManager, bridged_model: str ) -> None: @@ -124,6 +134,15 @@ class TestResponsesBridgeChatCompletionsStreaming: "llm.chat_completions.openai.basic.stream.bridge_streams_sse", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(RESPONSES_ONLY_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_bridged_stream_delivers_content_finish_reason_and_done( self, client: PassthroughClient, resources: ResourceManager, bridged_model: str ) -> None: @@ -149,6 +168,16 @@ class TestResponsesBridgeChatCompletionsStreaming: "llm.chat_completions.openai.tool_use.stream.bridge_streams_tool_call", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(RESPONSES_ONLY_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) + ) def test_bridged_stream_reassembles_tool_call( self, client: PassthroughClient, resources: ResourceManager, bridged_model: str ) -> None: diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index cc6f98dad50..00a1d3611f9 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -21,6 +21,7 @@ import openai import pytest from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker from e2e_http import assert_client_error +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, LiteLLMParamsBody from openai.types.responses import ( @@ -52,7 +53,10 @@ class _OptionalResponsesBody(BaseModel): max_output_tokens: int | None = None -BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" +OPENAI_MINI_BACKEND: Final = "openai/gpt-4o-mini" +OPENAI_VISION_BACKEND: Final = "openai/gpt-4o" +ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5" +BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" AZURE_OPENAI_API_VERSION: Final = "v1" @@ -100,11 +104,11 @@ WEATHER_TOOL: FunctionToolParam = { def _openai_params() -> LiteLLMParamsBody: - return LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY") + return LiteLLMParamsBody(model=OPENAI_MINI_BACKEND, api_key="os.environ/OPENAI_API_KEY") def _anthropic_params() -> LiteLLMParamsBody: - return LiteLLMParamsBody(model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY") + return LiteLLMParamsBody(model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY") def _bedrock_params() -> LiteLLMParamsBody: @@ -160,6 +164,15 @@ class WeatherArguments(BaseModel): class TestResponses: @pytest.mark.covers("llm.responses.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_MINI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_returns_completion( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -172,6 +185,15 @@ class TestResponses: assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.openai.basic.stream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_MINI_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_responses_streaming_returns_completion( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -194,6 +216,15 @@ class TestResponses: ) @pytest.mark.covers("llm.responses.openai.basic.nonstream.cost_logged") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_MINI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_logs_cost(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = _register(proxy, resources, _openai_params()) client = sdk.openai(resources.key()) @@ -219,6 +250,16 @@ class TestResponses: assert "gpt-4o-mini" in (row.model or ""), f"unexpected spend row model: {row.model}" @pytest.mark.covers("llm.responses.openai.tool_use.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_MINI_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_returns_function_call( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -235,13 +276,23 @@ class TestResponses: _assert_weather_call(response) @pytest.mark.covers("llm.responses.openai.vision.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_VISION_BACKEND,), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_vision_describes_image( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: model = _register( proxy, resources, - LiteLLMParamsBody(model="openai/gpt-4o", api_key="os.environ/OPENAI_API_KEY"), + LiteLLMParamsBody(model=OPENAI_VISION_BACKEND, api_key="os.environ/OPENAI_API_KEY"), ) client = sdk.openai(resources.key()) @@ -264,6 +315,15 @@ class TestResponses: ) @pytest.mark.covers("llm.responses.anthropic.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_anthropic_returns_completion( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -276,6 +336,16 @@ class TestResponses: assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.anthropic.tool_use.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_anthropic_returns_function_call( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -292,6 +362,15 @@ class TestResponses: _assert_weather_call(response) @pytest.mark.covers("llm.responses.bedrock_converse.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK,), + models=(BEDROCK_CONVERSE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_bedrock_returns_completion( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -304,6 +383,16 @@ class TestResponses: assert response.output_text.strip(), f"/responses over bedrock returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK,), + models=(BEDROCK_CONVERSE_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_bedrock_returns_function_call( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -320,6 +409,15 @@ class TestResponses: _assert_weather_call(response) @pytest.mark.covers("llm.responses.vertex.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_vertex_returns_completion( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -332,6 +430,16 @@ class TestResponses: assert response.output_text.strip(), f"/responses over vertex returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.vertex.tool_use.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_vertex_returns_function_call( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -349,6 +457,15 @@ class TestResponses: _assert_weather_call(response) @pytest.mark.covers("llm.responses.azure_openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=(AZURE_OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_azure_openai_returns_completion( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -363,6 +480,16 @@ class TestResponses: ) @pytest.mark.covers("llm.responses.azure_openai.tool_use.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=(AZURE_OPENAI_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_azure_openai_returns_function_call( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -380,7 +507,37 @@ class TestResponses: _assert_weather_call(response) @pytest.mark.provider_edge_host - @pytest.mark.parametrize("endpoint", ["/v1/responses", "/v1/chat/completions"]) + @pytest.mark.parametrize( + "endpoint", + [ + pytest.param( + "/v1/responses", + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK,), + models=(BEDROCK_CONVERSE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ), + id="/v1/responses", + ), + pytest.param( + "/v1/chat/completions", + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_CONVERSE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ), + id="/v1/chat/completions", + ), + ], + ) def test_bedrock_forwards_allowed_safety_identifier_as_additional_model_request_field( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, endpoint: str ) -> None: @@ -442,6 +599,14 @@ class TestResponses: reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400" ) @pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_MINI_BACKEND,), + ) + ) def test_missing_input_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model = _register(proxy, resources, _openai_params(), prefix="e2e-responses-val") key = resources.key() @@ -453,6 +618,12 @@ class TestResponses: assert_client_error(result, "responses missing input") @pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + ) + ) def test_missing_model_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: key = resources.key() result = proxy.transport.send( @@ -463,6 +634,14 @@ class TestResponses: assert_client_error(result, "responses missing model") @pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_MINI_BACKEND,), + ) + ) def test_empty_input_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model = _register(proxy, resources, _openai_params(), prefix="e2e-responses-val") key = resources.key() @@ -509,6 +688,16 @@ class TodayReport(BaseModel): class TestResponsesOpenAIHostedFeatures: + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(REASONING_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING, Capability.REASONING, Capability.RESPONSE_SCHEMA), + mode=Mode.NONSTREAM, + ) + ) def test_reasoning_items_replay_into_structured_output_after_tool_call( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -561,6 +750,15 @@ class TestResponsesOpenAIHostedFeatures: assert TOOL_DATE in report.today, f"structured output ignored the tool result: {report!r}" @pytest.mark.provider_live + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(SHELL_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_shell_tool_stream_surfaces_shell_call_and_its_output( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: diff --git a/tests/e2e/llm_translation/test_responses_retrieve_e2e.py b/tests/e2e/llm_translation/test_responses_retrieve_e2e.py index 063d014d1f5..b4a9634fbef 100644 --- a/tests/e2e/llm_translation/test_responses_retrieve_e2e.py +++ b/tests/e2e/llm_translation/test_responses_retrieve_e2e.py @@ -12,6 +12,7 @@ import openai import pytest from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker from e2e_http import NoBody, Success, UnknownApiError, unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from openai.types.responses import ( @@ -27,6 +28,7 @@ from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e OPENAI_BACKEND: Final = "openai/gpt-5.5" +OPENAI_MINI_BACKEND: Final = "openai/gpt-4o-mini" LONG_TASK: Final = "Write a numbered list counting from 1 to 400, one number per line, with a short word after each." CANCELLABLE_STATUSES: Final = frozenset({"queued", "in_progress"}) @@ -69,11 +71,20 @@ class TestResponsesRetrieve: reason="stage red: product gap (LIT-5446), retrieve returns a different id than the stored response (non-idempotent response-id re-encryption)" ) @pytest.mark.covers("llm.responses.openai.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_MINI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_store_and_retrieve_by_id(self, proxy: ProxyClient, resources: ResourceManager) -> None: model = f"e2e-resp-store-{unique_marker()}" model_id = proxy.create_model( model, - LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"), + LiteLLMParamsBody(model=OPENAI_MINI_BACKEND, api_key="os.environ/OPENAI_API_KEY"), ) resources.defer(lambda: proxy.delete_model(model_id)) key = resources.key() @@ -103,6 +114,12 @@ class TestResponsesRetrieve: reason="stage red: product gap (LIT-5447), retrieving an unknown response id returns 400 (model=None) instead of 404" ) @pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + ) + ) def test_invalid_response_id_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: key = resources.key() get_result = proxy.transport.get( @@ -135,6 +152,15 @@ def _input_texts(item: object) -> tuple[str, ...]: @pytest.mark.provider_live class TestStoredResponseLifecycle: + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_input_items_list_the_stored_prompt( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -150,6 +176,15 @@ class TestStoredResponseLifecycle: texts = tuple(text for item in items for text in _input_texts(item)) assert any(marker in text for text in texts), f"input_items did not list the stored prompt: {items!r}" + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_deleted_response_is_no_longer_retrievable( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -171,6 +206,15 @@ class TestStoredResponseLifecycle: @pytest.mark.provider_live class TestBackgroundResponseCancel: + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_cancel_background_response( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -185,6 +229,15 @@ class TestBackgroundResponseCancel: cancelled = client.responses.cancel(created.id) assert cancelled.status == "cancelled", f"cancel did not stop the response: {cancelled.status}" + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_cancel_background_streaming_response_by_streamed_id( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: diff --git a/tests/e2e/llm_translation/test_sail_e2e.py b/tests/e2e/llm_translation/test_sail_e2e.py index 7267052e12c..77003ec4198 100644 --- a/tests/e2e/llm_translation/test_sail_e2e.py +++ b/tests/e2e/llm_translation/test_sail_e2e.py @@ -15,6 +15,7 @@ from typing import Final, Literal import pytest from e2e_config import SLOW_PROVIDER_TIMEOUT_SECONDS, unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody, SpendLogRow from openai import OpenAI @@ -116,6 +117,15 @@ def _assert_spend_row_matches(proxy: ProxyClient, key: str, header_cost: float) class TestSailChatCompletions: @pytest.mark.covers("llm.chat_completions.sail.service_tier.nonstream.cost_logged") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.SAIL,), + models=(BACKEND,), + mode=Mode.NONSTREAM, + ) + ) @pytest.mark.parametrize( ("service_tier", "billed_tier"), [("balanced", "balanced"), ("auto", "base")] ) @@ -150,6 +160,15 @@ class TestSailChatCompletions: _assert_spend_row_matches(proxy, key, header_cost) @pytest.mark.covers("llm.chat_completions.sail.service_tier.nonstream.drops_unknown_tier_and_bills_asap") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.SAIL,), + models=(BACKEND,), + mode=Mode.NONSTREAM, + ) + ) @pytest.mark.parametrize("service_tier", ["bogus", 5]) def test_unknown_service_tier_is_dropped_and_billed_asap( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, service_tier: str | int @@ -176,6 +195,15 @@ class TestSailChatCompletions: class TestSailResponses: @pytest.mark.covers("llm.responses.sail.service_tier.nonstream.cost_logged") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.RESPONSES, + providers=(Provider.SAIL,), + models=(BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_caller_completion_window_bills_its_rates( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: @@ -204,6 +232,15 @@ class TestSailResponses: class TestSailMessages: @pytest.mark.covers("llm.messages.sail.basic.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.SAIL,), + models=(BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_plain_call_returns_a_message( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: diff --git a/tests/e2e/llm_translation/test_together_ai_e2e.py b/tests/e2e/llm_translation/test_together_ai_e2e.py index 874c6d77d19..d1ec2764a70 100644 --- a/tests/e2e/llm_translation/test_together_ai_e2e.py +++ b/tests/e2e/llm_translation/test_together_ai_e2e.py @@ -26,6 +26,7 @@ from typing import Final import pytest from e2e_config import STREAM_MIN_LEAD_SECONDS, provider_paces_stream, unique_marker from e2e_http import StreamingResponse, require_successful_call, unwrap +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ( AnthropicAssistantTurn, @@ -324,6 +325,15 @@ def _weather_call(client: PassthroughClient, key: str, model: str) -> OutMessage class TestTogetherChatCompletions: @pytest.mark.covers("llm.chat_completions.together_ai.thinking.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.TOGETHER_AI,), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_reasoning_surfaces_as_reasoning_content( self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str ) -> None: @@ -347,6 +357,15 @@ class TestTogetherChatCompletions: assert message.content and "43" in message.content, f"answer lost: {message}" @pytest.mark.covers("llm.chat_completions.together_ai.thinking.stream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.TOGETHER_AI,), + capabilities=(Capability.REASONING,), + mode=Mode.STREAM, + ) + ) def test_reasoning_streams_as_reasoning_content_deltas( self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str ) -> None: @@ -369,6 +388,15 @@ class TestTogetherChatCompletions: assert "43" in content, f"streamed answer lost: {content!r}" @pytest.mark.covers("llm.chat_completions.together_ai.tool_use.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.TOGETHER_AI,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_tool_call_is_returned( self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str ) -> None: @@ -376,6 +404,15 @@ class TestTogetherChatCompletions: _ = _weather_call_ids(_weather_call(client, key, model)) @pytest.mark.covers("llm.chat_completions.together_ai.tool_use.stream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.TOGETHER_AI,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) + ) def test_tool_call_is_streamed( self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str ) -> None: @@ -407,6 +444,15 @@ class TestTogetherChatCompletions: assert "paris" in args.location.lower(), f"streamed tool arguments lost the location: {args}" @pytest.mark.covers("llm.chat_completions.together_ai.multi_turn.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.TOGETHER_AI,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_tool_result_round_trip( self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str ) -> None: @@ -443,6 +489,16 @@ class TestTogetherChatCompletions: ) @pytest.mark.covers("llm.chat_completions.together_ai.thinking.nonstream.template_kwargs_forwarded") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.TOGETHER_AI,), + models=(HYBRID_REASONING_BACKEND,), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_chat_template_kwargs_reach_together( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -476,6 +532,16 @@ class TestTogetherChatCompletions: assert treatment.content and "43" in treatment.content, f"answer lost: {treatment}" @pytest.mark.covers("llm.chat_completions.together_ai.thinking.nonstream.replayed_reasoning_forwarded") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.TOGETHER_AI,), + models=(REASONING_REPLAY_BACKEND,), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_replayed_reasoning_content_reaches_together( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -496,6 +562,14 @@ class TestTogetherChatCompletions: ) @pytest.mark.covers("llm.chat_completions.together_ai.basic.nonstream.cost_logged") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.TOGETHER_AI,), + mode=Mode.NONSTREAM, + ) + ) def test_cost_header_and_spend_row_match_the_registry_price( self, client: PassthroughClient, @@ -551,6 +625,16 @@ class TestTogetherChatCompletions: ) @pytest.mark.covers("llm.chat_completions.together_ai.thinking.nonstream.effort_none_disables") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.TOGETHER_AI,), + models=(HYBRID_REASONING_BACKEND,), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_reasoning_effort_none_reaches_together( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -584,6 +668,16 @@ class TestTogetherChatCompletions: assert treatment.content and "43" in treatment.content, f"answer lost: {treatment}" @pytest.mark.covers("llm.chat_completions.together_ai.structured_output.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.TOGETHER_AI,), + models=(HYBRID_REASONING_BACKEND,), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) + ) def test_response_format_json_schema_shapes_the_reply( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -608,6 +702,15 @@ class TestTogetherChatCompletions: assert person.name, f"schema-shaped reply carries an empty name: {message.content!r}" @pytest.mark.covers("llm.chat_completions.together_ai.prompt_cache_5m.nonstream.cost_logged") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.TOGETHER_AI,), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_cache_read_tokens_bill_at_the_cache_read_rate( self, client: PassthroughClient, @@ -705,6 +808,15 @@ def _messages_weather_call( class TestTogetherMessages: @pytest.mark.covers("llm.messages.together_ai.tool_use.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.TOGETHER_AI,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_tool_use_block_is_returned( self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str ) -> None: @@ -712,6 +824,15 @@ class TestTogetherMessages: _messages_weather_call(client, key, model) @pytest.mark.covers("llm.messages.together_ai.multi_turn.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.TOGETHER_AI,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_tool_result_round_trip( self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str ) -> None: @@ -744,6 +865,14 @@ class TestTogetherMessages: @pytest.mark.covers("llm.messages.together_ai.basic.stream.works") @pytest.mark.provider_live + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.TOGETHER_AI,), + mode=Mode.STREAM, + ) + ) def test_streams_text_deltas( self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str ) -> None: diff --git a/tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py b/tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py index b2b8f46ab0a..d50a7bae9d1 100644 --- a/tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py +++ b/tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py @@ -9,15 +9,43 @@ route the claude_code rows never reach from __future__ import annotations +from typing import Final + import pytest from e2e_config import unique_marker from e2e_http import require_successful_call +from e2e_metadata import Domain, Provider, Route, Subject, meta from proxy_client import ProxyClient from pydantic import BaseModel pytestmark = pytest.mark.e2e -GEMINI_DEPLOYMENTS = ("gemini-2.5-flash", "gemini-2.5-flash-vertex") +GEMINI_STUDIO_DEPLOYMENT: Final = "gemini-2.5-flash" +GEMINI_VERTEX_DEPLOYMENT: Final = "gemini-2.5-flash-vertex" +GEMINI_DEPLOYMENTS = ( + pytest.param( + GEMINI_STUDIO_DEPLOYMENT, + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.GEMINI,), + models=(GEMINI_STUDIO_DEPLOYMENT,), + ) + ), + ), + pytest.param( + GEMINI_VERTEX_DEPLOYMENT, + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.VERTEX_AI,), + models=(GEMINI_VERTEX_DEPLOYMENT,), + ) + ), + ), +) class _Part(BaseModel): diff --git a/tests/e2e/llm_translation/test_vector_stores_e2e.py b/tests/e2e/llm_translation/test_vector_stores_e2e.py index 71015d28d9f..2ae56594335 100644 --- a/tests/e2e/llm_translation/test_vector_stores_e2e.py +++ b/tests/e2e/llm_translation/test_vector_stores_e2e.py @@ -12,6 +12,7 @@ from typing import Literal import pytest from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker +from e2e_metadata import Domain, Provider, Route, Subject, meta from e2e_http import ( FileUploadForm, NoBody, @@ -162,6 +163,7 @@ def _await_store_in_list(proxy: ProxyClient, key: str, store_id: str) -> None: class TestVectorStores: @pytest.mark.covers("llm.vector_stores.openai.basic.nonstream.works") + @meta(Subject(domain=Domain.LLM_TRANSLATION, route=Route.VECTOR_STORES, providers=(Provider.OPENAI,))) def test_create_list_retrieve_delete_lifecycle(self, proxy: ProxyClient, resources: ResourceManager) -> None: key = resources.key() name = f"e2e-vector-store-{unique_marker()}" @@ -204,6 +206,7 @@ class TestVectorStores: reason="stage red: product gap, vector store search 500s (asearch TypeError) on missing query instead of 400" ) @pytest.mark.covers("llm.vector_stores.openai.input_validation.nonstream.works") + @meta(Subject(domain=Domain.LLM_TRANSLATION, route=Route.VECTOR_STORES, providers=(Provider.OPENAI,))) def test_search_missing_query_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: key = resources.key() created = unwrap( @@ -223,6 +226,7 @@ class TestVectorStores: assert_client_error(result, "vector store search missing query") @pytest.mark.covers("llm.vector_stores.openai.basic.nonstream.works") + @meta(Subject(domain=Domain.LLM_TRANSLATION, route=Route.VECTOR_STORES, providers=(Provider.OPENAI,))) def test_file_attach_poll_and_search(self, proxy: ProxyClient, resources: ResourceManager) -> None: key = resources.key() marker = f"azure-falcon-{unique_marker()}" @@ -309,6 +313,7 @@ class TestVectorStores: reason="stage red: product gap, retrieving a nonexistent vector store returns 2xx with an error envelope in the body instead of 404" ) @pytest.mark.covers("llm.vector_stores.openai.input_validation.nonstream.works") + @meta(Subject(domain=Domain.LLM_TRANSLATION, route=Route.VECTOR_STORES, providers=(Provider.OPENAI,))) def test_retrieve_invalid_id_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: key = resources.key() result = proxy.transport.get( @@ -328,6 +333,7 @@ class TestVectorStores: pytest.fail(f"invalid vector store id must be a client error, got {other!r}") @pytest.mark.covers("llm.vector_stores.openai.input_validation.nonstream.works") + @meta(Subject(domain=Domain.LLM_TRANSLATION, route=Route.VECTOR_STORES, providers=(Provider.OPENAI,))) def test_invalid_chunking_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: key = resources.key() result = proxy.transport.send( diff --git a/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py b/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py index 5e9c9f614e5..3daf78a31b2 100644 --- a/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py @@ -30,6 +30,7 @@ from pydantic import BaseModel from e2e_config import settle_propagation, unique_marker from e2e_http import NoBody, require_successful_call, unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import SpendLogRow from passthrough_client import PassthroughClient @@ -149,6 +150,15 @@ def _costed_row(client: PassthroughClient, call_id: str | None) -> SpendLogRow: class TestVertexPassthroughSpendTracking: + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.PASSTHROUGH, + providers=(Provider.VERTEX_AI,), + models=(VERTEX_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_vertex_passthrough_via_managed_model_logs_cost( self, client: PassthroughClient, From 0ae55bdf7ad0bc81a73399f5e8e820ad2a109917 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 7 Oct 2026 10:31:54 -0700 Subject: [PATCH 03/35] test(e2e): tag claude_code cells with Subject metadata and record CLI driver steps (#44961) * test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata * test(e2e): tag claude_code tests with Subject metadata and record CLI driver steps * docs(e2e): name every markerless harness test file that carries no Subject * test(e2e): keep the step discovery comprehensions to one for clause * test(e2e): decorate run_claude directly so the label gate discovers its step --- tests/e2e/claude_code/_basic_messaging.py | 3 +++ tests/e2e/claude_code/_passthrough.py | 6 +++++ .../test_anthropic.py | 10 +++++++++ .../test_azure.py | 10 +++++++++ .../test_azure_openai.py | 10 +++++++++ .../test_bedrock_converse.py | 10 +++++++++ .../test_bedrock_invoke.py | 10 +++++++++ .../test_bedrock_mantle.py | 10 +++++++++ .../test_openai.py | 10 +++++++++ .../test_vertex_ai.py | 10 +++++++++ .../test_vertex_ai_gpt.py | 2 ++ .../test_anthropic.py | 10 +++++++++ .../basic_messaging_streaming/test_azure.py | 10 +++++++++ .../test_azure_openai.py | 10 +++++++++ .../test_bedrock_converse.py | 10 +++++++++ .../test_bedrock_invoke.py | 10 +++++++++ .../test_bedrock_mantle.py | 10 +++++++++ .../basic_messaging_streaming/test_openai.py | 10 +++++++++ .../test_vertex_ai.py | 10 +++++++++ .../test_vertex_ai_gpt.py | 2 ++ tests/e2e/claude_code/cli_driver.py | 4 ++++ .../count_tokens/test_anthropic.py | 10 +++++++++ .../claude_code/count_tokens/test_azure.py | 10 +++++++++ .../count_tokens/test_bedrock_converse.py | 10 +++++++++ .../count_tokens/test_bedrock_invoke.py | 10 +++++++++ .../count_tokens/test_vertex_ai.py | 10 +++++++++ tests/e2e/claude_code/http_probe.py | 7 ++++++ .../long_context_1m/test_anthropic.py | 10 +++++++++ .../claude_code/long_context_1m/test_azure.py | 10 +++++++++ .../long_context_1m/test_bedrock_converse.py | 10 +++++++++ .../long_context_1m/test_bedrock_invoke.py | 10 +++++++++ .../long_context_1m/test_vertex_ai.py | 10 +++++++++ .../claude_code/passthrough/test_anthropic.py | 10 +++++++++ .../e2e/claude_code/passthrough/test_azure.py | 10 +++++++++ .../passthrough/test_bedrock_converse.py | 3 +++ .../passthrough/test_bedrock_invoke.py | 10 +++++++++ .../claude_code/passthrough/test_vertex_ai.py | 10 +++++++++ .../claude_code/pdf_input/test_anthropic.py | 11 ++++++++++ tests/e2e/claude_code/pdf_input/test_azure.py | 11 ++++++++++ .../pdf_input/test_bedrock_converse.py | 11 ++++++++++ .../pdf_input/test_bedrock_invoke.py | 11 ++++++++++ .../claude_code/pdf_input/test_vertex_ai.py | 11 ++++++++++ .../prompt_caching_1h/test_anthropic.py | 11 ++++++++++ .../prompt_caching_1h/test_azure.py | 11 ++++++++++ .../test_bedrock_converse.py | 11 ++++++++++ .../prompt_caching_1h/test_bedrock_invoke.py | 11 ++++++++++ .../prompt_caching_1h/test_vertex_ai.py | 11 ++++++++++ .../prompt_caching_5m/test_anthropic.py | 11 ++++++++++ .../prompt_caching_5m/test_azure.py | 11 ++++++++++ .../test_bedrock_converse.py | 11 ++++++++++ .../prompt_caching_5m/test_bedrock_invoke.py | 11 ++++++++++ .../prompt_caching_5m/test_vertex_ai.py | 11 ++++++++++ .../structured_outputs/test_anthropic.py | 11 ++++++++++ .../structured_outputs/test_azure.py | 11 ++++++++++ .../test_bedrock_converse.py | 12 ++++++++++ .../structured_outputs/test_bedrock_invoke.py | 12 ++++++++++ .../structured_outputs/test_vertex_ai.py | 12 ++++++++++ .../claude_code/thinking/test_anthropic.py | 12 ++++++++++ tests/e2e/claude_code/thinking/test_azure.py | 12 ++++++++++ .../thinking/test_bedrock_converse.py | 12 ++++++++++ .../thinking/test_bedrock_invoke.py | 12 ++++++++++ .../claude_code/thinking/test_vertex_ai.py | 12 ++++++++++ .../thinking_with_tool_use/test_anthropic.py | 12 ++++++++++ .../thinking_with_tool_use/test_azure.py | 12 ++++++++++ .../test_bedrock_converse.py | 12 ++++++++++ .../test_bedrock_invoke.py | 12 ++++++++++ .../thinking_with_tool_use/test_vertex_ai.py | 12 ++++++++++ .../claude_code/tool_search/test_anthropic.py | 12 ++++++++++ .../e2e/claude_code/tool_search/test_azure.py | 12 ++++++++++ .../tool_search/test_bedrock_converse.py | 12 ++++++++++ .../tool_search/test_bedrock_invoke.py | 22 +++++++++++++++++++ .../claude_code/tool_search/test_vertex_ai.py | 12 ++++++++++ .../claude_code/tool_use/test_anthropic.py | 11 ++++++++++ tests/e2e/claude_code/tool_use/test_azure.py | 11 ++++++++++ .../claude_code/tool_use/test_azure_openai.py | 11 ++++++++++ .../tool_use/test_bedrock_converse.py | 11 ++++++++++ .../tool_use/test_bedrock_invoke.py | 11 ++++++++++ .../tool_use/test_bedrock_mantle.py | 11 ++++++++++ tests/e2e/claude_code/tool_use/test_openai.py | 11 ++++++++++ .../claude_code/tool_use/test_vertex_ai.py | 11 ++++++++++ .../tool_use/test_vertex_ai_gpt.py | 2 ++ .../tool_use_streaming/test_anthropic.py | 11 ++++++++++ .../tool_use_streaming/test_azure.py | 11 ++++++++++ .../tool_use_streaming/test_azure_openai.py | 11 ++++++++++ .../test_bedrock_converse.py | 11 ++++++++++ .../tool_use_streaming/test_bedrock_invoke.py | 11 ++++++++++ .../tool_use_streaming/test_bedrock_mantle.py | 11 ++++++++++ .../tool_use_streaming/test_openai.py | 11 ++++++++++ .../tool_use_streaming/test_vertex_ai.py | 11 ++++++++++ .../tool_use_streaming/test_vertex_ai_gpt.py | 2 ++ .../e2e/claude_code/vision/test_anthropic.py | 11 ++++++++++ tests/e2e/claude_code/vision/test_azure.py | 11 ++++++++++ .../vision/test_bedrock_converse.py | 11 ++++++++++ .../claude_code/vision/test_bedrock_invoke.py | 11 ++++++++++ .../e2e/claude_code/vision/test_vertex_ai.py | 11 ++++++++++ .../claude_code/web_search/test_anthropic.py | 11 ++++++++++ .../e2e/claude_code/web_search/test_azure.py | 11 ++++++++++ .../web_search/test_bedrock_converse.py | 11 ++++++++++ .../web_search/test_bedrock_invoke.py | 11 ++++++++++ .../claude_code/web_search/test_vertex_ai.py | 11 ++++++++++ 100 files changed, 1030 insertions(+) diff --git a/tests/e2e/claude_code/_basic_messaging.py b/tests/e2e/claude_code/_basic_messaging.py index 7c581cc5e38..b207bb6808f 100644 --- a/tests/e2e/claude_code/_basic_messaging.py +++ b/tests/e2e/claude_code/_basic_messaging.py @@ -31,6 +31,8 @@ from typing import Any, Callable, Mapping, Sequence import pytest +from e2e_metadata import step + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -74,6 +76,7 @@ def _count_stream_event_deltas(events: Sequence[Mapping[str, Any]]) -> int: return count +@step("Run Claude Code headless against {models} through the proxy and check every model replies") def run_basic_messaging_cell( *, compat_result, diff --git a/tests/e2e/claude_code/_passthrough.py b/tests/e2e/claude_code/_passthrough.py index be7a475dff7..3693ce25a9c 100644 --- a/tests/e2e/claude_code/_passthrough.py +++ b/tests/e2e/claude_code/_passthrough.py @@ -58,6 +58,8 @@ from typing import Any, Callable, Dict, Mapping, Optional, Sequence import pytest +from e2e_metadata import step + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -118,6 +120,10 @@ def foundry_extra_env(proxy_base_url: str) -> Dict[str, str]: } +@step( + "Run Claude Code headless against {models} through the proxy's native provider passthrough route" + " and check every model replies" +) def run_passthrough_cell( *, compat_result, diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py index 21383b85da5..1a7e26d37c7 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell # Per the PRD: each cell is exercised against three Claude tiers via the @@ -34,6 +35,15 @@ ANTHROPIC_MODELS = [ @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py index 19e88dbe3cb..1fb1cc6fac7 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py @@ -26,6 +26,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -40,6 +41,15 @@ AZURE_MODELS = [ @pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py index 77876c8f7ee..2b90b4e9f02 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure_openai.py @@ -25,6 +25,7 @@ green if all three pass. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell AZURE_OPENAI_MODELS = [ @@ -34,6 +35,15 @@ AZURE_OPENAI_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE,), + models=tuple(AZURE_OPENAI_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_azure_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py index 2b0f49bc205..4b02f10e6a9 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -35,6 +36,15 @@ BEDROCK_CONVERSE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_converse.basic.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py index 937ea5ee27e..874dcebdc35 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -35,6 +36,15 @@ BEDROCK_INVOKE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_invoke.basic.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py index 8a64547a732..28e723bb450 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_mantle.py @@ -25,6 +25,7 @@ COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`). from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell from claude_code._gpt_cells import skip_unless_mantle_cells_enabled @@ -35,6 +36,15 @@ BEDROCK_MANTLE_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK_MANTLE,), + models=tuple(BEDROCK_MANTLE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_bedrock_mantle(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py index 323c2f11173..27ab244c52a 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_openai.py @@ -22,6 +22,7 @@ green if all three pass. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled @@ -32,6 +33,15 @@ OPENAI_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.OPENAI,), + models=tuple(OPENAI_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py index c46e5a8f762..b0b41dc3f87 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell # Per-model aliases registered in the LiteLLM proxy's routing config to @@ -35,6 +36,15 @@ VERTEX_AI_MODELS = [ @pytest.mark.covers("llm.messages.vertex.basic.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_basic_messaging_non_streaming_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py index 3b155b6ac9d..7e8358a7975 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai_gpt.py @@ -16,9 +16,11 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +from e2e_metadata import Domain, Subject, meta from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON +@meta(Subject(domain=Domain.LLM_TRANSLATION)) def test_basic_messaging_non_streaming_vertex_ai_gpt(compat_result): """Record the static not_applicable outcome for this cell.""" compat_result.set( diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py index ce453f3e523..69ca0ee475a 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py @@ -26,6 +26,7 @@ sees three rows for this (feature, provider). from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell ANTHROPIC_MODELS = [ @@ -36,6 +37,15 @@ ANTHROPIC_MODELS = [ @pytest.mark.covers("llm.messages.anthropic.basic.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py index 3307194e862..f9fc39c8f65 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py @@ -20,6 +20,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell AZURE_MODELS = [ @@ -30,6 +31,15 @@ AZURE_MODELS = [ @pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py index 357596590c7..d1ff8578f09 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_azure_openai.py @@ -24,6 +24,7 @@ green if all three pass. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell AZURE_OPENAI_MODELS = [ @@ -33,6 +34,15 @@ AZURE_OPENAI_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE,), + models=tuple(AZURE_OPENAI_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_azure_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py index a8bc0b77a5d..3ad0b930df4 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py @@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell BEDROCK_CONVERSE_MODELS = [ @@ -26,6 +27,15 @@ BEDROCK_CONVERSE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_converse.basic.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py index c0ece0e0721..1de3236d2aa 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py @@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell BEDROCK_INVOKE_MODELS = [ @@ -26,6 +27,15 @@ BEDROCK_INVOKE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_invoke.basic.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py index 38297e6a3e5..679731003bb 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_mantle.py @@ -25,6 +25,7 @@ COMPAT_MANTLE_CELLS=1 (see `claude_code._gpt_cells`). from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell from claude_code._gpt_cells import skip_unless_mantle_cells_enabled @@ -35,6 +36,15 @@ BEDROCK_MANTLE_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK_MANTLE,), + models=tuple(BEDROCK_MANTLE_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_bedrock_mantle(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py index a7945fb92c0..c1617253f39 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_openai.py @@ -24,6 +24,7 @@ green if all three pass. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled @@ -34,6 +35,15 @@ OPENAI_MODELS = [ ] +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.OPENAI,), + models=tuple(OPENAI_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply from each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py index 13f1a0abf40..1201c71dce6 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py @@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._basic_messaging import run_basic_messaging_cell VERTEX_AI_MODELS = [ @@ -26,6 +27,15 @@ VERTEX_AI_MODELS = [ @pytest.mark.covers("llm.messages.vertex.basic.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + mode=Mode.STREAM, + ) +) def test_basic_messaging_streaming_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py index f6aa01de521..6a6cec8411a 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai_gpt.py @@ -16,9 +16,11 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +from e2e_metadata import Domain, Subject, meta from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON +@meta(Subject(domain=Domain.LLM_TRANSLATION)) def test_basic_messaging_streaming_vertex_ai_gpt(compat_result): """Record the static not_applicable outcome for this cell.""" compat_result.set( diff --git a/tests/e2e/claude_code/cli_driver.py b/tests/e2e/claude_code/cli_driver.py index 6996849de80..6f7a7b39a39 100644 --- a/tests/e2e/claude_code/cli_driver.py +++ b/tests/e2e/claude_code/cli_driver.py @@ -25,6 +25,8 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union +from e2e_metadata import step + from claude_code.rate_limiter import ( RateLimiter, get_default_limiter, @@ -211,6 +213,7 @@ class DriverResult: duration_ms: Optional[int] = None +@step("Run Claude Code headless against {model} through the proxy") def run_claude( *, prompt: Optional[str], @@ -395,6 +398,7 @@ def _matches_failure_shape(outcome: ModelResult, pattern: "re.Pattern[str]") -> return bool(pattern.search(failure_diagnostic(outcome))) +@step("Run Claude Code headless against {models} in parallel through the proxy") def run_claude_models_parallel( *, models: Sequence[str], diff --git a/tests/e2e/claude_code/count_tokens/test_anthropic.py b/tests/e2e/claude_code/count_tokens/test_anthropic.py index 05110d24e86..19e7bcfbc81 100644 --- a/tests/e2e/claude_code/count_tokens/test_anthropic.py +++ b/tests/e2e/claude_code/count_tokens/test_anthropic.py @@ -39,6 +39,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_count_tokens_shape, @@ -54,6 +55,15 @@ ANTHROPIC_MODELS = [ @pytest.mark.covers("llm.messages.anthropic.count_tokens.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_count_tokens_anthropic(compat_result): """Probe `/v1/messages/count_tokens` for each Anthropic tier and assert the response shape.""" diff --git a/tests/e2e/claude_code/count_tokens/test_azure.py b/tests/e2e/claude_code/count_tokens/test_azure.py index c60c623ae89..256babf3cd7 100644 --- a/tests/e2e/claude_code/count_tokens/test_azure.py +++ b/tests/e2e/claude_code/count_tokens/test_azure.py @@ -39,6 +39,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_count_tokens_shape, @@ -54,6 +55,15 @@ AZURE_MODELS = [ @pytest.mark.covers("llm.messages.azure_foundry.count_tokens.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_count_tokens_azure(compat_result): """Probe `/v1/messages/count_tokens` for each Azure (Microsoft Foundry) tier and assert the response shape.""" diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py index 0cb4766bb31..de5f4eae3ee 100644 --- a/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py @@ -39,6 +39,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_count_tokens_shape, @@ -54,6 +55,15 @@ BEDROCK_CONVERSE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_converse.count_tokens.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_count_tokens_bedrock_converse(compat_result): """Probe `/v1/messages/count_tokens` for each Bedrock (Converse) tier and assert the response shape.""" diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py index f1389574527..56dea8a5ad6 100644 --- a/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py @@ -39,6 +39,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_count_tokens_shape, @@ -54,6 +55,15 @@ BEDROCK_INVOKE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_invoke.count_tokens.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_count_tokens_bedrock_invoke(compat_result): """Probe `/v1/messages/count_tokens` for each Bedrock (Invoke) tier and assert the response shape.""" diff --git a/tests/e2e/claude_code/count_tokens/test_vertex_ai.py b/tests/e2e/claude_code/count_tokens/test_vertex_ai.py index 0894214d4f0..63d12fc5dae 100644 --- a/tests/e2e/claude_code/count_tokens/test_vertex_ai.py +++ b/tests/e2e/claude_code/count_tokens/test_vertex_ai.py @@ -39,6 +39,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_count_tokens_shape, @@ -55,6 +56,15 @@ VERTEX_AI_MODELS = [ @pytest.mark.skip(reason="stage red: Vertex returns not supported for token counting for Claude aliases") @pytest.mark.covers("llm.messages.vertex.count_tokens.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.COUNT_TOKENS, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_count_tokens_vertex_ai(compat_result): """Probe `/v1/messages/count_tokens` for each Vertex AI tier and assert the response shape.""" diff --git a/tests/e2e/claude_code/http_probe.py b/tests/e2e/claude_code/http_probe.py index 8aba54576c4..a6c587f98cf 100644 --- a/tests/e2e/claude_code/http_probe.py +++ b/tests/e2e/claude_code/http_probe.py @@ -42,6 +42,7 @@ from e2e_http import ( UnknownApiError, ValidationError, ) +from e2e_metadata import step from models import ( AnthropicAssistantTurn, AnthropicCustomTool, @@ -109,6 +110,7 @@ def _acquire(model: str, rate_limiter: RateLimiter | None) -> None: limiter.acquire(infer_provider(model)) +@step('Count tokens with /v1/messages/count_tokens for {model} on the message "{message}"') def probe_count_tokens( *, client: ProxyClient, @@ -132,6 +134,7 @@ def probe_count_tokens( ) +@step("Send a /v1/messages request to {model} with the tool_search tool declared") def probe_tool_search( *, client: ProxyClient, @@ -228,6 +231,10 @@ def _replay_history(answer: AnthropicMessagesResponse) -> tuple[AnthropicMessage ) +@step( + "Send a /v1/messages request to {model} with the tool_search tool declared," + " then send its answer back as history in a second request" +) def probe_tool_search_multiturn( *, client: ProxyClient, diff --git a/tests/e2e/claude_code/long_context_1m/test_anthropic.py b/tests/e2e/claude_code/long_context_1m/test_anthropic.py index 0f53e512ace..1de515c11f0 100644 --- a/tests/e2e/claude_code/long_context_1m/test_anthropic.py +++ b/tests/e2e/claude_code/long_context_1m/test_anthropic.py @@ -58,6 +58,7 @@ from typing import Sequence import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: @pytest.mark.skip(reason="stage red: 1M long_context not green on stage Anthropic path yet (200k sonnet / model alias)") @pytest.mark.covers("llm.messages.anthropic.long_context_1m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_long_context_1m_anthropic(compat_result): """Drive the `claude` CLI with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a diff --git a/tests/e2e/claude_code/long_context_1m/test_azure.py b/tests/e2e/claude_code/long_context_1m/test_azure.py index cdaa7f08178..eaad5e3de9f 100644 --- a/tests/e2e/claude_code/long_context_1m/test_azure.py +++ b/tests/e2e/claude_code/long_context_1m/test_azure.py @@ -58,6 +58,7 @@ from typing import Sequence import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: @pytest.mark.skip(reason="stage red: 1M long_context not green on stage Azure Foundry deployments yet") @pytest.mark.covers("llm.messages.azure_foundry.long_context_1m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_long_context_1m_azure(compat_result): """Drive the `claude` CLI (Azure (Microsoft Foundry)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py index 38aeef2ae63..eb72a36bf14 100644 --- a/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py @@ -58,6 +58,7 @@ from typing import Sequence import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: @pytest.mark.skip(reason="stage red: 1M long_context not green on stage Bedrock Converse deployments yet") @pytest.mark.covers("llm.messages.bedrock_converse.long_context_1m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_long_context_1m_bedrock_converse(compat_result): """Drive the `claude` CLI (Bedrock (Converse)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py index f652af4aa22..dd960c15428 100644 --- a/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py @@ -58,6 +58,7 @@ from typing import Sequence import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: @pytest.mark.skip(reason="stage red: 1M long_context not green on stage Bedrock Invoke deployments yet") @pytest.mark.covers("llm.messages.bedrock_invoke.long_context_1m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_long_context_1m_bedrock_invoke(compat_result): """Drive the `claude` CLI (Bedrock (Invoke)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a diff --git a/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py b/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py index 0ad68aac138..1f344d80ba8 100644 --- a/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py +++ b/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py @@ -58,6 +58,7 @@ from typing import Sequence import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -155,6 +156,15 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: @pytest.mark.skip(reason="stage red: 1M long_context not green on stage Vertex deployments yet") @pytest.mark.covers("llm.messages.vertex.long_context_1m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + mode=Mode.NONSTREAM, + ) +) def test_long_context_1m_vertex_ai(compat_result): """Drive the `claude` CLI (Vertex AI) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a diff --git a/tests/e2e/claude_code/passthrough/test_anthropic.py b/tests/e2e/claude_code/passthrough/test_anthropic.py index 8382342ae12..bf6ab0524d0 100644 --- a/tests/e2e/claude_code/passthrough/test_anthropic.py +++ b/tests/e2e/claude_code/passthrough/test_anthropic.py @@ -22,6 +22,7 @@ no per-provider transformation is involved. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._passthrough import ( ANTHROPIC_PASSTHROUGH_BASE_PATH, run_passthrough_cell, @@ -34,6 +35,15 @@ ANTHROPIC_MODELS = [ ] +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + mode=Mode.STREAM, + ) +) def test_passthrough_anthropic(compat_result): """Drive the `claude` CLI through `{proxy}/anthropic` and assert a reply.""" run_passthrough_cell( diff --git a/tests/e2e/claude_code/passthrough/test_azure.py b/tests/e2e/claude_code/passthrough/test_azure.py index 7365b4f50da..d2690025f08 100644 --- a/tests/e2e/claude_code/passthrough/test_azure.py +++ b/tests/e2e/claude_code/passthrough/test_azure.py @@ -42,6 +42,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._passthrough import foundry_extra_env, run_passthrough_cell AZURE_MODELS = [ @@ -52,6 +53,15 @@ AZURE_MODELS = [ @pytest.mark.skip(reason="stage red: /azure passthrough drops client headers (e.g. anthropic-version); product gap") +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.AZURE,), + models=tuple(AZURE_MODELS), + mode=Mode.STREAM, + ) +) def test_passthrough_azure(compat_result): """Drive the `claude` CLI through `{proxy}/azure` and assert a reply.""" run_passthrough_cell( diff --git a/tests/e2e/claude_code/passthrough/test_bedrock_converse.py b/tests/e2e/claude_code/passthrough/test_bedrock_converse.py index d1093a7a958..e1605be9e49 100644 --- a/tests/e2e/claude_code/passthrough/test_bedrock_converse.py +++ b/tests/e2e/claude_code/passthrough/test_bedrock_converse.py @@ -18,7 +18,10 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +from e2e_metadata import Domain, Subject, meta + +@meta(Subject(domain=Domain.PASSTHROUGH)) def test_passthrough_bedrock_converse(compat_result): """Report not_applicable: Claude Code has no Converse-wire mode.""" compat_result.set( diff --git a/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py b/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py index f1f28ab5b4c..d95118991fa 100644 --- a/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/passthrough/test_bedrock_invoke.py @@ -23,6 +23,7 @@ cell. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._passthrough import bedrock_extra_env, run_passthrough_cell BEDROCK_INVOKE_MODELS = [ @@ -32,6 +33,15 @@ BEDROCK_INVOKE_MODELS = [ ] +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + mode=Mode.STREAM, + ) +) def test_passthrough_bedrock_invoke(compat_result): """Drive the `claude` CLI through `{proxy}/bedrock` and assert a reply.""" run_passthrough_cell( diff --git a/tests/e2e/claude_code/passthrough/test_vertex_ai.py b/tests/e2e/claude_code/passthrough/test_vertex_ai.py index 790f8b60c8f..3f84cdf02fa 100644 --- a/tests/e2e/claude_code/passthrough/test_vertex_ai.py +++ b/tests/e2e/claude_code/passthrough/test_vertex_ai.py @@ -26,6 +26,7 @@ Google and every tier fails with a 401. from __future__ import annotations +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from claude_code._passthrough import run_passthrough_cell, vertex_extra_env VERTEX_MODELS = [ @@ -35,6 +36,15 @@ VERTEX_MODELS = [ ] +@meta( + Subject( + domain=Domain.PASSTHROUGH, + route=Route.PASSTHROUGH, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_MODELS), + mode=Mode.STREAM, + ) +) def test_passthrough_vertex_ai(compat_result): """Drive the `claude` CLI through `{proxy}/vertex_ai` and assert a reply.""" run_passthrough_cell( diff --git a/tests/e2e/claude_code/pdf_input/test_anthropic.py b/tests/e2e/claude_code/pdf_input/test_anthropic.py index 21c8028ef1c..59a91036f26 100644 --- a/tests/e2e/claude_code/pdf_input/test_anthropic.py +++ b/tests/e2e/claude_code/pdf_input/test_anthropic.py @@ -24,6 +24,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -106,6 +107,16 @@ def _build_minimal_pdf(marker: str) -> bytes: @pytest.mark.covers("llm.messages.anthropic.pdf_input.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.PDF_INPUT,), + mode=Mode.NONSTREAM, + ) +) def test_pdf_input_anthropic(compat_result, tmp_path): """Drive the `claude` CLI against the LiteLLM proxy with a PDF attached via the Read tool and assert the reply references it.""" diff --git a/tests/e2e/claude_code/pdf_input/test_azure.py b/tests/e2e/claude_code/pdf_input/test_azure.py index 34ae3732b99..3f7642321cf 100644 --- a/tests/e2e/claude_code/pdf_input/test_azure.py +++ b/tests/e2e/claude_code/pdf_input/test_azure.py @@ -17,6 +17,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_minimal_pdf(marker: str) -> bytes: @pytest.mark.covers("llm.messages.azure_foundry.pdf_input.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.PDF_INPUT,), + mode=Mode.NONSTREAM, + ) +) def test_pdf_input_azure(compat_result, tmp_path): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py index 76aa84f0f47..14020799c21 100644 --- a/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py @@ -23,6 +23,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -89,6 +90,16 @@ def _build_minimal_pdf(marker: str) -> bytes: @pytest.mark.covers("llm.messages.bedrock_converse.pdf_input.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.PDF_INPUT,), + mode=Mode.NONSTREAM, + ) +) def test_pdf_input_bedrock_converse(compat_result, tmp_path): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py index 4450266bb6b..5b3ba208421 100644 --- a/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py @@ -22,6 +22,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -88,6 +89,16 @@ def _build_minimal_pdf(marker: str) -> bytes: @pytest.mark.covers("llm.messages.bedrock_invoke.pdf_input.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.PDF_INPUT,), + mode=Mode.NONSTREAM, + ) +) def test_pdf_input_bedrock_invoke(compat_result, tmp_path): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/pdf_input/test_vertex_ai.py b/tests/e2e/claude_code/pdf_input/test_vertex_ai.py index b78f58cfda1..5f42c00efcb 100644 --- a/tests/e2e/claude_code/pdf_input/test_vertex_ai.py +++ b/tests/e2e/claude_code/pdf_input/test_vertex_ai.py @@ -17,6 +17,7 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_minimal_pdf(marker: str) -> bytes: @pytest.mark.covers("llm.messages.vertex.pdf_input.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.PDF_INPUT,), + mode=Mode.NONSTREAM, + ) +) def test_pdf_input_vertex_ai(compat_result, tmp_path): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py index 637be1c551d..60045eff8b9 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py @@ -28,6 +28,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -64,6 +65,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.anthropic.prompt_cache_1h.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_1h_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with the 1h TTL opt-in env var set, and assert the upstream usage block diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_azure.py b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py index f34557b3c5f..86dc4c8e50a 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_azure.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -48,6 +49,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.azure_foundry.prompt_cache_1h.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_1h_azure(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py index bf62a49444c..d1106ca4350 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py @@ -24,6 +24,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -56,6 +57,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_converse.prompt_cache_1h.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_1h_bedrock_converse(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py index dc3468702d4..9c1166eb3b6 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py @@ -26,6 +26,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -60,6 +61,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_invoke.prompt_cache_1h.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_1h_bedrock_invoke(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py b/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py index 66cf961fcfc..89b6d7e1d4e 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -48,6 +49,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.vertex.prompt_cache_1h.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_1h_vertex_ai(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py index ef551beb45c..c97df9afe2c 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py @@ -26,6 +26,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -55,6 +56,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.anthropic.prompt_cache_5m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_5m_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_azure.py b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py index 9d4137e0726..7705af62ab4 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_azure.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py @@ -26,6 +26,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -53,6 +54,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.azure_foundry.prompt_cache_5m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_5m_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py index c9b34c010b0..4a5a7dfd660 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -46,6 +47,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_converse.prompt_cache_5m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_5m_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py index b95c509ba3c..bbe483242bd 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -46,6 +47,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_invoke.prompt_cache_5m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_5m_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py b/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py index f79377b7372..3f325efcc93 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Optional import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -46,6 +47,16 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.vertex.prompt_cache_5m.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) +) def test_prompt_caching_5m_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" diff --git a/tests/e2e/claude_code/structured_outputs/test_anthropic.py b/tests/e2e/claude_code/structured_outputs/test_anthropic.py index 3dc4c7ab8f2..c8cb7e7386c 100644 --- a/tests/e2e/claude_code/structured_outputs/test_anthropic.py +++ b/tests/e2e/claude_code/structured_outputs/test_anthropic.py @@ -53,6 +53,7 @@ from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -151,6 +152,16 @@ def _validate_against_schema( @pytest.mark.covers("llm.messages.anthropic.structured_output.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) +) def test_structured_outputs_anthropic(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming diff --git a/tests/e2e/claude_code/structured_outputs/test_azure.py b/tests/e2e/claude_code/structured_outputs/test_azure.py index 7a776ed55ad..0708920f5e7 100644 --- a/tests/e2e/claude_code/structured_outputs/test_azure.py +++ b/tests/e2e/claude_code/structured_outputs/test_azure.py @@ -53,6 +53,7 @@ from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -151,6 +152,16 @@ def _validate_against_schema( @pytest.mark.covers("llm.messages.azure_foundry.structured_output.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) +) def test_structured_outputs_azure(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py index 345d7c327cf..7228c194f85 100644 --- a/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py @@ -53,6 +53,8 @@ from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -151,6 +153,16 @@ def _validate_against_schema( @pytest.mark.covers("llm.messages.bedrock_converse.structured_output.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) +) def test_structured_outputs_bedrock_converse(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py index 0cf48c72d4f..c415de9e396 100644 --- a/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py @@ -53,6 +53,8 @@ from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -151,6 +153,16 @@ def _validate_against_schema( @pytest.mark.covers("llm.messages.bedrock_invoke.structured_output.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) +) def test_structured_outputs_bedrock_invoke(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming diff --git a/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py b/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py index 24f5a0c35d4..a4cf669c19f 100644 --- a/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py +++ b/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py @@ -53,6 +53,8 @@ from typing import Any, Mapping, Optional, Sequence, Tuple import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -151,6 +153,16 @@ def _validate_against_schema( @pytest.mark.covers("llm.messages.vertex.structured_output.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) +) def test_structured_outputs_vertex_ai(compat_result): """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming diff --git a/tests/e2e/claude_code/thinking/test_anthropic.py b/tests/e2e/claude_code/thinking/test_anthropic.py index ebb2445fb6d..2c3a8f0a8a9 100644 --- a/tests/e2e/claude_code/thinking/test_anthropic.py +++ b/tests/e2e/claude_code/thinking/test_anthropic.py @@ -24,6 +24,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -75,6 +77,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.anthropic.thinking.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" diff --git a/tests/e2e/claude_code/thinking/test_azure.py b/tests/e2e/claude_code/thinking/test_azure.py index ffd5ca92df0..0828dd033f4 100644 --- a/tests/e2e/claude_code/thinking/test_azure.py +++ b/tests/e2e/claude_code/thinking/test_azure.py @@ -27,6 +27,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -63,6 +65,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.azure_foundry.thinking.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" diff --git a/tests/e2e/claude_code/thinking/test_bedrock_converse.py b/tests/e2e/claude_code/thinking/test_bedrock_converse.py index 0b409f18ea7..fd075298234 100644 --- a/tests/e2e/claude_code/thinking/test_bedrock_converse.py +++ b/tests/e2e/claude_code/thinking/test_bedrock_converse.py @@ -19,6 +19,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -55,6 +57,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_converse.thinking.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" diff --git a/tests/e2e/claude_code/thinking/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py index a2c97eae321..c115ba9e408 100644 --- a/tests/e2e/claude_code/thinking/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py @@ -19,6 +19,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -55,6 +57,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_invoke.thinking.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" diff --git a/tests/e2e/claude_code/thinking/test_vertex_ai.py b/tests/e2e/claude_code/thinking/test_vertex_ai.py index f1a1c5b6cee..b947f97e7ac 100644 --- a/tests/e2e/claude_code/thinking/test_vertex_ai.py +++ b/tests/e2e/claude_code/thinking/test_vertex_ai.py @@ -19,6 +19,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -55,6 +57,16 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.vertex.thinking.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py index 7e39ea26d42..8ddbdba04f9 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py @@ -28,6 +28,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -89,6 +91,16 @@ def _has_block_type( @pytest.mark.covers("llm.messages.anthropic.thinking_with_tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_with_tool_use_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and tool use, and assert both `thinking` and `tool_use` diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py index 0371a10f8a6..6dbd6b27fde 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py @@ -22,6 +22,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -70,6 +72,16 @@ def _has_block_type( @pytest.mark.covers("llm.messages.azure_foundry.thinking_with_tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_with_tool_use_azure(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py index 026d2a3707f..b3869f157b5 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py @@ -27,6 +27,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -75,6 +77,16 @@ def _has_block_type( @pytest.mark.covers("llm.messages.bedrock_converse.thinking_with_tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_with_tool_use_bedrock_converse(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py index 1dd4cf0a73c..384ce0bb3c6 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py @@ -29,6 +29,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -77,6 +79,16 @@ def _has_block_type( @pytest.mark.covers("llm.messages.bedrock_invoke.thinking_with_tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_with_tool_use_bedrock_invoke(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py b/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py index b25228edb55..c3a3dd58533 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py @@ -27,6 +27,8 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -75,6 +77,16 @@ def _has_block_type( @pytest.mark.covers("llm.messages.vertex.thinking_with_tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.REASONING, Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_thinking_with_tool_use_vertex_ai(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_search/test_anthropic.py b/tests/e2e/claude_code/tool_search/test_anthropic.py index a23d1b3d2bb..f08c30b507e 100644 --- a/tests/e2e/claude_code/tool_search/test_anthropic.py +++ b/tests/e2e/claude_code/tool_search/test_anthropic.py @@ -45,6 +45,8 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_tool_search_shape, @@ -60,6 +62,16 @@ ANTHROPIC_MODELS = [ @pytest.mark.covers("llm.messages.anthropic.tool_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_anthropic(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Anthropic diff --git a/tests/e2e/claude_code/tool_search/test_azure.py b/tests/e2e/claude_code/tool_search/test_azure.py index b094a35ea63..87628bc02d5 100644 --- a/tests/e2e/claude_code/tool_search/test_azure.py +++ b/tests/e2e/claude_code/tool_search/test_azure.py @@ -45,6 +45,8 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_tool_search_shape, @@ -61,6 +63,16 @@ AZURE_MODELS = [ @pytest.mark.skip(reason="stage red: Azure Foundry tool_search_server not supported in workspace for probed models") @pytest.mark.covers("llm.messages.azure_foundry.tool_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_azure(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Azure (Microsoft Foundry) diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_converse.py b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py index f395122a5ab..84fd47bcb39 100644 --- a/tests/e2e/claude_code/tool_search/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py @@ -45,6 +45,8 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_tool_search_shape, @@ -60,6 +62,16 @@ BEDROCK_CONVERSE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_converse.tool_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_bedrock_converse(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Bedrock (Converse) diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py index 5b4c50e9dc5..b4a7f721a92 100644 --- a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py @@ -50,6 +50,8 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_tool_search_replay_shape, @@ -67,6 +69,16 @@ BEDROCK_INVOKE_MODELS = [ @pytest.mark.covers("llm.messages.bedrock_invoke.tool_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_bedrock_invoke(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Bedrock (Invoke) @@ -90,6 +102,16 @@ def test_tool_search_bedrock_invoke(compat_result): @pytest.mark.covers("llm.messages.bedrock_invoke.tool_search_history.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_history_bedrock_invoke(compat_result): """Send the tool-search request, take the real assistant turn back, and replay it as history with the tools still declared. diff --git a/tests/e2e/claude_code/tool_search/test_vertex_ai.py b/tests/e2e/claude_code/tool_search/test_vertex_ai.py index 7d0d35b1c1d..af629a7692e 100644 --- a/tests/e2e/claude_code/tool_search/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_search/test_vertex_ai.py @@ -45,6 +45,8 @@ from __future__ import annotations import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + from claude_code._env import require_proxy_client from claude_code.http_probe import ( assert_tool_search_shape, @@ -61,6 +63,16 @@ VERTEX_AI_MODELS = [ @pytest.mark.skip(reason="stage red: Vertex rejects tool_search when deployment extra_headers inject context-1m beta; product/config") @pytest.mark.covers("llm.messages.vertex.tool_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.TOOL_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_tool_search_vertex_ai(compat_result): """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Vertex AI diff --git a/tests/e2e/claude_code/tool_use/test_anthropic.py b/tests/e2e/claude_code/tool_use/test_anthropic.py index 9ff4c58907f..429557322f6 100644 --- a/tests/e2e/claude_code/tool_use/test_anthropic.py +++ b/tests/e2e/claude_code/tool_use/test_anthropic.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -72,6 +73,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.anthropic.tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" diff --git a/tests/e2e/claude_code/tool_use/test_azure.py b/tests/e2e/claude_code/tool_use/test_azure.py index 9e7398267c4..96946093d9d 100644 --- a/tests/e2e/claude_code/tool_use/test_azure.py +++ b/tests/e2e/claude_code/tool_use/test_azure.py @@ -23,6 +23,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -66,6 +67,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" diff --git a/tests/e2e/claude_code/tool_use/test_azure_openai.py b/tests/e2e/claude_code/tool_use/test_azure_openai.py index 7e1eecdbc03..3cb12b4023e 100644 --- a/tests/e2e/claude_code/tool_use/test_azure_openai.py +++ b/tests/e2e/claude_code/tool_use/test_azure_openai.py @@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -67,6 +68,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE,), + models=tuple(AZURE_OPENAI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_azure_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire by each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py index 33d4d3820d2..f361757042b 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -62,6 +63,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_converse.tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py index 47ae3aef1da..16150c5eb9d 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -62,6 +63,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_invoke.tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py b/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py index e9cb70e74e9..b04b19e5542 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_mantle.py @@ -33,6 +33,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code._gpt_cells import skip_unless_mantle_cells_enabled from claude_code.cli_driver import ( @@ -72,6 +73,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK_MANTLE,), + models=tuple(BEDROCK_MANTLE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_bedrock_mantle(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire by each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/tool_use/test_openai.py b/tests/e2e/claude_code/tool_use/test_openai.py index ffb7e795c2b..d8a9ba83480 100644 --- a/tests/e2e/claude_code/tool_use/test_openai.py +++ b/tests/e2e/claude_code/tool_use/test_openai.py @@ -28,6 +28,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled from claude_code.cli_driver import ( @@ -67,6 +68,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.OPENAI,), + models=tuple(OPENAI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_openai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire by each GPT-5.6 tier.""" diff --git a/tests/e2e/claude_code/tool_use/test_vertex_ai.py b/tests/e2e/claude_code/tool_use/test_vertex_ai.py index 79a3016345c..a082a61920e 100644 --- a/tests/e2e/claude_code/tool_use/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_use/test_vertex_ai.py @@ -19,6 +19,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -62,6 +63,16 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.vertex.tool_use.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) +) def test_tool_use_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" diff --git a/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py b/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py index d1ebbced9dc..c15ba00d325 100644 --- a/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py +++ b/tests/e2e/claude_code/tool_use/test_vertex_ai_gpt.py @@ -20,9 +20,11 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +from e2e_metadata import Domain, Subject, meta from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON +@meta(Subject(domain=Domain.LLM_TRANSLATION)) def test_tool_use_vertex_ai_gpt(compat_result): """Record the static not_applicable outcome for this cell.""" compat_result.set( diff --git a/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py index 152652dcf3c..91002e7ebf9 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py @@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -98,6 +99,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.anthropic.tool_use.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the proxy preserves fine-grained tool streaming end-to-end.""" diff --git a/tests/e2e/claude_code/tool_use_streaming/test_azure.py b/tests/e2e/claude_code/tool_use_streaming/test_azure.py index 8a1cc1852dd..4ec93f5a667 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_azure.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_azure.py @@ -21,6 +21,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_azure(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py b/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py index ad5d4e0f613..020f4e79eb4 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_azure_openai.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -88,6 +89,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: ) +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE,), + models=tuple(AZURE_OPENAI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_azure_openai(compat_result): proxy = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py index 3b04ed5962f..2d4b5764eb9 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py @@ -27,6 +27,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -89,6 +90,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_converse.tool_use.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_bedrock_converse(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py index c7b61129782..2cd7a68d57a 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py @@ -25,6 +25,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -87,6 +88,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.bedrock_invoke.tool_use.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_bedrock_invoke(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py index 20fae5d48db..190eb51adc0 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_mantle.py @@ -34,6 +34,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code._gpt_cells import skip_unless_mantle_cells_enabled from claude_code.cli_driver import ( @@ -92,6 +93,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: ) +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK_MANTLE,), + models=tuple(BEDROCK_MANTLE_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_bedrock_mantle(compat_result): skip_unless_mantle_cells_enabled() proxy = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_openai.py b/tests/e2e/claude_code/tool_use_streaming/test_openai.py index a5ce31b1fd6..3555741b197 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_openai.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_openai.py @@ -29,6 +29,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code._gpt_cells import skip_unless_openai_gpt_cells_enabled from claude_code.cli_driver import ( @@ -87,6 +88,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: ) +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.OPENAI,), + models=tuple(OPENAI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_openai(compat_result): skip_unless_openai_gpt_cells_enabled() proxy = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py index 2912e3aae3d..7bb29742cfd 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py @@ -24,6 +24,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -86,6 +87,16 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: @pytest.mark.covers("llm.messages.vertex.tool_use.stream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) def test_tool_use_streaming_vertex_ai(compat_result): base_url, api_key = require_proxy(compat_result) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py index 7037e91fee0..5e6c203cb57 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai_gpt.py @@ -20,9 +20,11 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +from e2e_metadata import Domain, Subject, meta from claude_code._gpt_cells import VERTEX_AI_GPT_NOT_APPLICABLE_REASON +@meta(Subject(domain=Domain.LLM_TRANSLATION)) def test_tool_use_streaming_vertex_ai_gpt(compat_result): """Record the static not_applicable outcome for this cell.""" compat_result.set( diff --git a/tests/e2e/claude_code/vision/test_anthropic.py b/tests/e2e/claude_code/vision/test_anthropic.py index f681b2be5ae..b37bbb4ebf5 100644 --- a/tests/e2e/claude_code/vision/test_anthropic.py +++ b/tests/e2e/claude_code/vision/test_anthropic.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_stdin_input() -> str: @pytest.mark.covers("llm.messages.anthropic.vision.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) +) def test_vision_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" diff --git a/tests/e2e/claude_code/vision/test_azure.py b/tests/e2e/claude_code/vision/test_azure.py index f0eaaad84a2..739c8aba74d 100644 --- a/tests/e2e/claude_code/vision/test_azure.py +++ b/tests/e2e/claude_code/vision/test_azure.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_stdin_input() -> str: @pytest.mark.covers("llm.messages.azure_foundry.vision.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) +) def test_vision_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" diff --git a/tests/e2e/claude_code/vision/test_bedrock_converse.py b/tests/e2e/claude_code/vision/test_bedrock_converse.py index 2a5aba5a393..07f21b7d657 100644 --- a/tests/e2e/claude_code/vision/test_bedrock_converse.py +++ b/tests/e2e/claude_code/vision/test_bedrock_converse.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_stdin_input() -> str: @pytest.mark.covers("llm.messages.bedrock_converse.vision.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) +) def test_vision_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" diff --git a/tests/e2e/claude_code/vision/test_bedrock_invoke.py b/tests/e2e/claude_code/vision/test_bedrock_invoke.py index 5c995cd479e..847f2c80484 100644 --- a/tests/e2e/claude_code/vision/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/vision/test_bedrock_invoke.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_stdin_input() -> str: @pytest.mark.covers("llm.messages.bedrock_invoke.vision.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) +) def test_vision_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" diff --git a/tests/e2e/claude_code/vision/test_vertex_ai.py b/tests/e2e/claude_code/vision/test_vertex_ai.py index 8d385e295d0..256b762f06a 100644 --- a/tests/e2e/claude_code/vision/test_vertex_ai.py +++ b/tests/e2e/claude_code/vision/test_vertex_ai.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -83,6 +84,16 @@ def _build_stdin_input() -> str: @pytest.mark.covers("llm.messages.vertex.vision.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) +) def test_vision_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" diff --git a/tests/e2e/claude_code/web_search/test_anthropic.py b/tests/e2e/claude_code/web_search/test_anthropic.py index a20a2133dc9..8b7f4259a68 100644 --- a/tests/e2e/claude_code/web_search/test_anthropic.py +++ b/tests/e2e/claude_code/web_search/test_anthropic.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.anthropic.web_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=tuple(ANTHROPIC_MODELS), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_web_search_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving diff --git a/tests/e2e/claude_code/web_search/test_azure.py b/tests/e2e/claude_code/web_search/test_azure.py index 8f9f638fbee..da3d1ebeb0b 100644 --- a/tests/e2e/claude_code/web_search/test_azure.py +++ b/tests/e2e/claude_code/web_search/test_azure.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.azure_foundry.web_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.AZURE_AI,), + models=tuple(AZURE_MODELS), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_web_search_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving diff --git a/tests/e2e/claude_code/web_search/test_bedrock_converse.py b/tests/e2e/claude_code/web_search/test_bedrock_converse.py index 32f37b2be79..36c163dd2d6 100644 --- a/tests/e2e/claude_code/web_search/test_bedrock_converse.py +++ b/tests/e2e/claude_code/web_search/test_bedrock_converse.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_converse.web_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_CONVERSE_MODELS), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_web_search_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving diff --git a/tests/e2e/claude_code/web_search/test_bedrock_invoke.py b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py index 68d1b30e83f..1328d361404 100644 --- a/tests/e2e/claude_code/web_search/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.bedrock_invoke.web_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=tuple(BEDROCK_INVOKE_MODELS), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_web_search_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving diff --git a/tests/e2e/claude_code/web_search/test_vertex_ai.py b/tests/e2e/claude_code/web_search/test_vertex_ai.py index 540a8396c98..7ab352b1953 100644 --- a/tests/e2e/claude_code/web_search/test_vertex_ai.py +++ b/tests/e2e/claude_code/web_search/test_vertex_ai.py @@ -31,6 +31,7 @@ from typing import Any, Mapping, Sequence import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from claude_code._env import require_proxy from claude_code.cli_driver import ( ClaudeCLIError, @@ -85,6 +86,16 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: @pytest.mark.covers("llm.messages.vertex.web_search.nonstream.works") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.VERTEX_AI,), + models=tuple(VERTEX_AI_MODELS), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) +) def test_web_search_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving From d7cdc88c66a89db4a837c26d5d0719c7b40823ba Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 7 Oct 2026 10:32:09 -0700 Subject: [PATCH 04/35] test(e2e): tag management tests with Subject metadata and record management client steps (#44962) * test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata * test(e2e): tag management tests with Subject metadata and record management client steps * docs(e2e): name every markerless harness test file that carries no Subject * test(e2e): keep the step discovery comprehensions to one for clause * test(e2e): keep the prompt out of the chat_status step so polled retries collapse --- tests/e2e/management/jwt_actors.py | 5 + tests/e2e/management/management_client.py | 68 +++- .../test_budget_customer_user_org_e2e.py | 14 + .../test_config_misc_endpoints_e2e.py | 12 + .../e2e/management/test_jwt_management_e2e.py | 10 + .../e2e/management/test_key_lifecycle_e2e.py | 22 ++ .../e2e/management/test_key_management_e2e.py | 29 +- tests/e2e/management/test_management_e2e.py | 308 +++++++++++++++--- .../e2e/management/test_mcp_lifecycle_e2e.py | 61 ++++ .../test_model_tag_accessgroup_e2e.py | 37 +++ .../test_model_test_connection_e2e.py | 10 + .../management/test_team_management_e2e.py | 15 + 12 files changed, 545 insertions(+), 46 deletions(-) diff --git a/tests/e2e/management/jwt_actors.py b/tests/e2e/management/jwt_actors.py index 2d23549fe71..ae33f583195 100644 --- a/tests/e2e/management/jwt_actors.py +++ b/tests/e2e/management/jwt_actors.py @@ -5,6 +5,7 @@ from typing import Final, Literal from e2e_config import unique_marker from e2e_http import NoBody, unwrap +from e2e_metadata import step from idp import ADMIN_CLIENT_ID, TESTS_CLIENT_ID, Identity, Keycloak from lifecycle import ResourceManager from management.management_client import ManagementClient @@ -53,6 +54,7 @@ class Actor: profile: ActorProfile tenants: tuple[Tenant, ...] + @step("Get a JWT from the identity provider for the actor in the {self.role} role") def mint_caller(self, idp: Keycloak) -> Caller: return Caller( credential=idp.access_token( @@ -74,6 +76,7 @@ class ActorFactory: if self.bootstrap.proxy.caller is not None: raise ValueError("Actor bootstrap requires a separately held master client") + @step("Generate a virtual key as the proxy admin") def key(self, tenant: Tenant | None = None, *, user_id: str | None = None) -> KeyGenerateResponse: created: Final = unwrap( self.bootstrap.generate_key( @@ -87,6 +90,7 @@ class ActorFactory: self.resources.defer(lambda: self.bootstrap.delete_key_strict(created.key, missing_ok=True)) return created + @step("Create an organization, a team in it and a matching identity provider group") def tenant(self) -> Tenant: marker: Final = unique_marker() organization_id: Final = self.bootstrap.create_org(OrgNewBody(organization_alias=f"e2e-organization-{marker}")) @@ -118,6 +122,7 @@ class ActorFactory: self.resources.defer(lambda: self.idp.with_strict_cleanup().delete_group(group_id)) return Tenant(organization_id=organization_id, team_id=team_id, group_id=group_id) + @step("Create an actor in the {role} role, with its identity provider user, internal user and any tenant memberships") def create( self, role: ActorRole, *, tenants: tuple[Tenant, ...] = (), profile: ActorProfile = "database_role" ) -> Actor: diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index 3da9bea12a3..ffa310a9f4a 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -24,6 +24,7 @@ from e2e_http import ( retry_attempts, unwrap, ) +from e2e_metadata import STEP_FRAMES, step from models import ( AuditLogPage, AuditLogParams, @@ -116,9 +117,11 @@ class ManagementClient: def with_caller(self, caller: Caller) -> ManagementClient: return replace(self, proxy=self.proxy.with_caller(caller)) + @step("Generate a virtual key limited to the LLM API routes") def llm_only_key(self) -> str: return self.proxy.generate_key(KeyGenerateBody(models=[], allowed_routes=["llm_api_routes"])) + @step("Generate a virtual key with {body}") def generate_key(self, body: KeyGenerateBody, *, caller_key: str | None = None) -> Result[KeyGenerateResponse]: """POST /key/generate. `caller_key` is who is creating the key: the master key by default, or a virtual key (an admin filling in Create New Key on the @@ -133,6 +136,7 @@ class ManagementClient: response_type=KeyGenerateResponse, ) + @step("Update the virtual key's settings with /key/update") def update_key(self, body: KeyUpdateBody, *, caller_key: str | None = None) -> Result[NoBody]: """POST /key/update. `caller_key` is who is editing: the master key by default, or a virtual key (the dashboard edits under the session key its @@ -152,16 +156,22 @@ class ManagementClient: case UnknownApiError(body=error_body) if any( marker in error_body.lower() for marker in _TRANSIENT_BACKEND_MARKERS ): - warnings.warn(f"Transient backend response on attempt {attempt + 1}", RuntimeWarning, stacklevel=2) + warnings.warn( + f"Transient backend response on attempt {attempt + 1}", + RuntimeWarning, + stacklevel=2 + STEP_FRAMES, + ) time.sleep(0.5 * (attempt + 1)) continue case _: break return last + @step("Set the virtual key's models to [{models}]") def update_key_models(self, key: str, models: list[str]) -> None: _ = unwrap(self.update_key(KeyUpdateBody(key=key, models=models))) + @step("Delete the virtual key with the alias {key_alias}") def delete_key_by_alias(self, key_alias: str) -> None: _ = unwrap( self.proxy.transport.post( @@ -172,6 +182,7 @@ class ManagementClient: ) ) + @step("Read the key's deletion entries from the /audit log") def key_deleted_audit_logs(self, token_hash: str) -> AuditLogPage: return unwrap( self.proxy.transport.get( @@ -187,6 +198,7 @@ class ManagementClient: ) ) + @step("Read the key's settings back from /key/info") def key_info_as(self, key: str, *, caller_key: str | None = None) -> Result[KeyInfoResponse]: return self.proxy.transport.get( "/key/info", @@ -195,6 +207,7 @@ class ManagementClient: response_type=KeyInfoResponse, ) + @step("Delete the virtual key") def delete_key_strict(self, key: str, *, caller_key: str | None = None, missing_ok: bool = False) -> None: """Strict delete for the act phase of a test: a failed delete is a hard failure, unlike the warn-only ProxyClient.delete_key used at teardown.""" @@ -208,6 +221,7 @@ class ManagementClient: return _ = unwrap(result) + @step("Delete the deployment") def delete_model_strict(self, model_id: str) -> None: """Strict delete for the act phase of a test: a failed delete is a hard failure, unlike the warn-only ProxyClient.delete_model used at teardown.""" @@ -220,6 +234,7 @@ class ManagementClient: ) ) + @step("Run Test Connection on {body.litellm_params.model} in {body.mode} mode with /health/test_connection") def connection_test(self, body: ConnectionTestBody) -> Result[ConnectionTestResponse]: """POST /health/test_connection, the call behind the Admin UI's Test Connection button, probing the live provider with the supplied params.""" @@ -231,6 +246,7 @@ class ManagementClient: timeout=120.0, ) + @step("Block the virtual key") def block_key(self, key: str) -> None: _ = unwrap( self.proxy.transport.post( @@ -240,6 +256,7 @@ class ManagementClient: response_type=NoBody, ) ) + @step("Regenerate the virtual key with /key/regenerate") def regenerate_key(self, key: str, *, grace_period: str | None = None) -> str: return unwrap( self.proxy.transport.post( @@ -250,6 +267,7 @@ class ManagementClient: ) ).key + @step("Reset the virtual key's spend to {reset_to}") def reset_key_spend(self, key: str, reset_to: float) -> KeyResetSpendResponse: return unwrap( self.proxy.transport.post( @@ -260,6 +278,7 @@ class ManagementClient: ) ) + @step("List the keys with the alias {key_alias} from /key/list") def key_list(self, key_alias: str, *, caller_key: str | None = None) -> Result[KeyListResponse]: """GET /key/list, the Virtual Keys page's own inventory call. `caller_key` is who is asking: the master key by default, or a virtual key.""" @@ -271,9 +290,11 @@ class ManagementClient: response_type=KeyListResponse, ) + @step("Count the keys with the alias {key_alias} in /key/list") def key_alias_count(self, key_alias: str) -> int: return unwrap(self.key_list(key_alias)).total_count + @step("Sign in to the Admin UI with /v2/login") def dashboard_login(self, username: str, password: str) -> DashboardSession: """POST /v2/login, the call the Admin UI's sign-in form makes. @@ -297,6 +318,7 @@ class ManagementClient: redirect_url=response.redirect_url, ) + @step("Create a team with {body}") def create_team(self, body: TeamNewBody) -> str: team_id = unwrap( self.proxy.transport.post( @@ -309,6 +331,7 @@ class ManagementClient: self._wait_for_team(team_id) return team_id + @step("Update a team with {body}") def update_team(self, body: TeamUpdateBody) -> None: last: Result[NoBody] | None = None for attempt in range(retry_attempts(5)): @@ -324,7 +347,11 @@ class ManagementClient: case UnknownApiError(body=body_text) if ( "connecting to redis" in body_text.lower() or "name resolution" in body_text.lower() ): - warnings.warn(f"Transient backend response on attempt {attempt + 1}", RuntimeWarning, stacklevel=2) + warnings.warn( + f"Transient backend response on attempt {attempt + 1}", + RuntimeWarning, + stacklevel=2 + STEP_FRAMES, + ) time.sleep(0.5 * (attempt + 1)) continue case _: @@ -332,6 +359,7 @@ class ManagementClient: assert last is not None raise AssertionError(last) + @step("Delete the team") def delete_team(self, team_id: str) -> None: _ = self.proxy.transport.post( "/team/delete", @@ -340,6 +368,7 @@ class ManagementClient: response_type=NoBody, ) + @step("Read the team back from /team/info") def team_info(self, team_id: str) -> TeamData: return unwrap( self.proxy.transport.get( @@ -350,6 +379,7 @@ class ManagementClient: ) ).team_info + @step("List the teams from /team/list") def team_list_ids(self) -> tuple[str, ...]: return tuple( entry.team_id @@ -363,6 +393,7 @@ class ManagementClient: ).root ) + @step("Check whether /team/info finds the team") def team_info_status(self, team_id: str) -> ProbeResult: return self.proxy.transport.probe( "/team/info", params=TeamInfoParams(team_id=team_id), headers=self.proxy.management_headers() @@ -386,6 +417,7 @@ class ManagementClient: assert last is not None raise AssertionError(last) + @step("Add a user to the team with /team/member_add") def add_team_member(self, team_id: str, user_id: str) -> None: last: Result[NoBody] | None = None for attempt in range(retry_attempts(_TEAM_READY_ATTEMPTS)): @@ -402,7 +434,9 @@ class ManagementClient: _TEAM_READY_ATTEMPTS ): warnings.warn( - "Retrying team membership while the team becomes available", RuntimeWarning, stacklevel=2 + "Retrying team membership while the team becomes available", + RuntimeWarning, + stacklevel=2 + STEP_FRAMES, ) time.sleep(_TEAM_READY_SLEEP_SECONDS) continue @@ -411,6 +445,7 @@ class ManagementClient: assert last is not None raise AssertionError(last) + @step("Add a roster of members to the team with /team/member_add") def add_team_members(self, team_id: str, members: list[TeamMemberEntry]) -> None: """Bulk form of /team/member_add: `member` accepts a list, so one call seeds a whole roster the way an admin import does.""" @@ -423,6 +458,7 @@ class ManagementClient: ) ) + @step("Try to delete the team with /team/delete") def delete_team_status(self, team_id: str) -> StreamingResponse: """POST /team/delete judged by HTTP outcome: the raw status and body, so a test can assert on what a caller actually sees when the delete fails.""" @@ -432,6 +468,7 @@ class ManagementClient: json=TeamDeleteBody(team_ids=[team_id]), ) + @step("Remove a user from the team with /team/member_delete") def delete_team_member(self, team_id: str, user_id: str) -> None: _ = unwrap( self.proxy.transport.post( @@ -442,6 +479,7 @@ class ManagementClient: ) ) + @step("Create an internal user with {body}") def create_user(self, body: UserNewBody) -> str: return unwrap( self.proxy.transport.post( @@ -452,6 +490,7 @@ class ManagementClient: ) ).user_id + @step("Create the end user {user_id}") def create_customer(self, user_id: str) -> str: _ = unwrap( self.proxy.transport.post( @@ -463,6 +502,7 @@ class ManagementClient: ) return user_id + @step("Read the end user {end_user_id} back from /customer/info") def customer_info(self, end_user_id: str) -> CustomerResponse: return unwrap( self.proxy.transport.get( @@ -473,6 +513,7 @@ class ManagementClient: ) ) + @step("Delete the end user {user_id}") def delete_customer(self, user_id: str) -> None: _ = self.proxy.transport.post( "/customer/delete", @@ -481,6 +522,7 @@ class ManagementClient: response_type=NoBody, ) + @step("Update an internal user with {body}") def update_user(self, body: UserUpdateBody) -> None: _ = unwrap( self.proxy.transport.post( @@ -491,6 +533,7 @@ class ManagementClient: ) ) + @step("Delete the internal user") def delete_user(self, user_id: str) -> None: _ = self.proxy.transport.post( "/user/delete", @@ -499,6 +542,7 @@ class ManagementClient: response_type=NoBody, ) + @step("Delete the internal user") def delete_user_strict(self, user_id: str) -> None: """Strict delete for the act phase of a test: a failed delete is a hard failure, unlike the warn-only delete_user used at teardown.""" @@ -511,6 +555,7 @@ class ManagementClient: ) ) + @step("Read the user back from /user/info") def user_info(self, user_id: str | None = None) -> UserInfoResponse: return unwrap( self.proxy.transport.get( @@ -521,6 +566,7 @@ class ManagementClient: ) ) + @step("Count the matching users in /user/list") def user_count(self, user_id: str) -> int: return unwrap( self.proxy.transport.get( @@ -531,6 +577,7 @@ class ManagementClient: ) ).total + @step("List the matching users from /user/list") def user_list_ids(self, user_id: str) -> tuple[str, ...]: listing = unwrap( self.proxy.transport.get( @@ -542,6 +589,7 @@ class ManagementClient: ) return tuple(row.user_id for row in listing.users) + @step("Create an organization with {body}") def create_org(self, body: OrgNewBody) -> str: return unwrap( self.proxy.transport.post( @@ -552,6 +600,7 @@ class ManagementClient: ) ).organization_id + @step("Update an organization with {body}") def update_org(self, body: OrgUpdateBody) -> None: _ = unwrap( self.proxy.transport.patch( @@ -562,6 +611,7 @@ class ManagementClient: ) ) + @step("Delete the organization") def delete_org(self, organization_id: str) -> None: _ = self.proxy.transport.delete( "/organization/delete", @@ -570,6 +620,7 @@ class ManagementClient: response_type=NoBody, ) + @step("Read the organization back from /organization/info") def org_info(self, organization_id: str) -> OrgInfoResponse: return unwrap( self.proxy.transport.get( @@ -580,6 +631,7 @@ class ManagementClient: ) ) + @step("Check whether /organization/info finds the organization") def org_info_status(self, organization_id: str) -> ProbeResult: return self.proxy.transport.probe( "/organization/info", @@ -587,6 +639,7 @@ class ManagementClient: headers=self.proxy.management_headers(), ) + @step("Create a tag with {body}") def create_tag(self, body: TagNewBody) -> None: _ = unwrap( self.proxy.transport.post( @@ -597,6 +650,7 @@ class ManagementClient: ) ) + @step("Delete the tag {name}") def delete_tag(self, name: str) -> None: _ = self.proxy.transport.post( "/tag/delete", @@ -605,6 +659,7 @@ class ManagementClient: response_type=NoBody, ) + @step("List the tags from /tag/list") def tag_list(self) -> tuple[TagListEntry, ...]: return tuple( unwrap( @@ -617,6 +672,7 @@ class ManagementClient: ).root ) + @step("Create an MCP server named {body.alias}") def create_mcp_server(self, body: McpServerCreateBody) -> McpServerRow: return unwrap( self.proxy.transport.post( @@ -627,6 +683,7 @@ class ManagementClient: ) ) + @step("Update the MCP server's settings with PUT /v1/mcp/server") def update_mcp_server(self, body: McpServerUpdateBody) -> McpServerRow: """PUT /v1/mcp/server, the call behind the dashboard's Save Changes: a partial update where a field left unset keeps its stored value and None clears it.""" @@ -639,6 +696,7 @@ class ManagementClient: ) ) + @step("Delete the MCP server") def delete_mcp_server(self, server_id: str) -> Result[NoBody]: """DELETE /v1/mcp/server/{server_id}. Returns the outcome so the act phase can unwrap it while a deferred teardown can ignore an already-deleted server.""" @@ -649,6 +707,7 @@ class ManagementClient: response_type=NoBody, ) + @step("Send a /chat/completions request to {model}") def chat_status(self, key: str, model: str, content: str) -> StreamingResponse: return self.proxy.transport.send( "/chat/completions", @@ -656,12 +715,15 @@ class ManagementClient: json=ChatBody(model=model, messages=[ChatMessage(role="user", content=content)], max_tokens=16), ) + @step("Try to generate a virtual key with {body}, calling with a virtual key") def key_generate_status(self, key: str, body: KeyGenerateBody) -> StreamingResponse: return self.proxy.transport.send("/key/generate", headers=self.proxy.transport.bearer(key), json=body) + @step("Try to create a team with {body}, calling with a virtual key") def team_new_status(self, key: str, body: TeamNewBody) -> StreamingResponse: return self.proxy.transport.send("/team/new", headers=self.proxy.transport.bearer(key), json=body) + @step("Try to create an internal user with {body}, calling with a virtual key") def user_new_status(self, key: str, body: UserNewBody) -> StreamingResponse: return self.proxy.transport.send("/user/new", headers=self.proxy.transport.bearer(key), json=body) diff --git a/tests/e2e/management/test_budget_customer_user_org_e2e.py b/tests/e2e/management/test_budget_customer_user_org_e2e.py index 6e14a2d5745..ae4e6db4e15 100644 --- a/tests/e2e/management/test_budget_customer_user_org_e2e.py +++ b/tests/e2e/management/test_budget_customer_user_org_e2e.py @@ -23,6 +23,7 @@ from pydantic import BaseModel, Field, RootModel from e2e_config import unique_marker from e2e_http import NoBody, Success, UnauthorizedError, UnknownApiError, is_ok, unwrap +from e2e_metadata import Domain, Route, Subject, meta from lifecycle import ResourceManager from management_client import ManagementClient from models import KeyGenerateBody, ModelBudgetEntry, OrgInfoParams, OrgNewBody, UserNewBody @@ -152,6 +153,7 @@ _UPDATED_MAX_BUDGET = 91.25 class TestBudgetManagement: @pytest.mark.covers("mgmt.budget.list.happy_path") + @meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT)) def test_created_budget_appears_in_budget_list( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -164,6 +166,7 @@ class TestBudgetManagement: ) @pytest.mark.covers("mgmt.budget.update.accepts_model_max_budget") + @meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT)) def test_update_accepts_per_model_budgets_including_punctuated_names( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -213,6 +216,7 @@ class TestBudgetManagement: ) @pytest.mark.covers("mgmt.budget.update.persists") + @meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT)) def test_update_max_budget_persists_to_budget_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -247,6 +251,7 @@ class TestBudgetManagement: ) @pytest.mark.covers("mgmt.budget.new.admin_only") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.BUDGET_MANAGEMENT)) def test_new_is_refused_for_a_non_admin_key( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -352,6 +357,7 @@ class TestBudgetListV1: """ @pytest.mark.covers("mgmt.budget.list_v1.happy_path") + @meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT)) def test_sorts_pages_and_filters_the_budgets_it_created( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -392,6 +398,7 @@ class TestBudgetListV1: assert [row.tpm_limit for row in limits] == [60000, 60000, 60000] @pytest.mark.covers("mgmt.budget.list_v1.happy_path") + @meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT)) def test_is_null_finds_the_budget_left_uncapped( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -409,11 +416,13 @@ class TestBudgetListV1: assert [row.max_budget for row in _list_budgets(client, found).data] == [None] @pytest.mark.covers("mgmt.budget.list_v1.happy_path") + @meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT)) def test_refuses_a_sort_field_and_a_parameter_it_does_not_support(self, client: ManagementClient) -> None: assert _list_status(client, BudgetPageParams(sort="budget_duration")) == 400 assert _list_status(client, BudgetPageParams(not_a_parameter="b-1")) == 400 @pytest.mark.covers("mgmt.budget.list_v1.admin_only") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.BUDGET_MANAGEMENT)) def test_is_refused_for_a_non_admin_key(self, client: ManagementClient, resources: ResourceManager) -> None: key = client.proxy.generate_key(KeyGenerateBody()) resources.defer(lambda: client.proxy.delete_key(key)) @@ -482,6 +491,7 @@ def _customer_info(client: ManagementClient, route: str, user_id: str) -> Custom class TestCustomerManagement: @pytest.mark.covers("mgmt.customer.new.happy_path") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.CUSTOMER_MANAGEMENT)) def test_new_persists_to_customer_info(self, client: ManagementClient, resources: ResourceManager) -> None: customer_id = f"e2e-mgmt-cust-{unique_marker()}" created = _create_customer( @@ -495,6 +505,7 @@ class TestCustomerManagement: ) @pytest.mark.covers("mgmt.customer.delete.persists") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.CUSTOMER_MANAGEMENT)) def test_delete_removes_the_customer(self, client: ManagementClient, resources: ResourceManager) -> None: """The teardown's deferred delete fires again on the already-deleted customer by design: it is the safety net if this test fails before the in-body delete, @@ -524,6 +535,7 @@ class TestCustomerManagement: _ = _poll(client, gone, f"customer {customer_id} still resolved on /customer/info after /customer/delete") @pytest.mark.covers("mgmt.end_user.new.happy_path") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.CUSTOMER_MANAGEMENT)) def test_end_user_new_persists_to_end_user_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -542,6 +554,7 @@ class TestCustomerManagement: class TestUserManagement: @pytest.mark.covers("mgmt.user.info.happy_path") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.USER_MANAGEMENT)) def test_new_user_is_readable_via_user_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -586,6 +599,7 @@ class OrgInfoMembersResponse(BaseModel): class TestOrganizationMembership: @pytest.mark.covers("mgmt.organization.member_add.happy_path") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.ORGANIZATION_MANAGEMENT)) def test_member_add_records_membership( self, client: ManagementClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/management/test_config_misc_endpoints_e2e.py b/tests/e2e/management/test_config_misc_endpoints_e2e.py index a3be0a64e7f..b7ab311cc18 100644 --- a/tests/e2e/management/test_config_misc_endpoints_e2e.py +++ b/tests/e2e/management/test_config_misc_endpoints_e2e.py @@ -32,6 +32,7 @@ from pydantic import BaseModel from e2e_config import unique_marker from e2e_http import NoBody, Success, unwrap, unwrap_status +from e2e_metadata import Domain, Route, Subject, meta from lifecycle import ResourceManager from management_client import ManagementClient from models import KeyGenerateBody, LiteLLMParamsBody, TeamNewBody @@ -231,6 +232,7 @@ class McpServerResponse(BaseModel): class TestInventoryRoutes: @pytest.mark.covers("mgmt.callback.list.happy_path") + @meta(Subject(domain=Domain.OBSERVABILITY)) def test_callbacks_list_reports_active_logging_callbacks(self, client: ManagementClient) -> None: listing = unwrap( client.proxy.transport.get( @@ -247,6 +249,7 @@ class TestInventoryRoutes: ) @pytest.mark.covers("mgmt.tool_management.list.happy_path") + @meta(Subject(domain=Domain.MANAGEMENT)) def test_tool_list_returns_catalog_with_consistent_total(self, client: ManagementClient) -> None: listing = unwrap( client.proxy.transport.get( @@ -261,6 +264,7 @@ class TestInventoryRoutes: ) @pytest.mark.covers("mgmt.workflow.list.happy_path") + @meta(Subject(domain=Domain.MANAGEMENT)) def test_workflow_runs_list_returns_consistent_count(self, client: ManagementClient) -> None: listing = unwrap( client.proxy.transport.get( @@ -275,6 +279,7 @@ class TestInventoryRoutes: ) @pytest.mark.covers("mgmt.credential_migration.check.happy_path") + @meta(Subject(domain=Domain.DEPLOY_OPS)) def test_credential_migration_check_reports_residual_scan(self, client: ManagementClient) -> None: report = unwrap( client.proxy.transport.get( @@ -295,6 +300,7 @@ class TestInventoryRoutes: class TestCostEstimate: @pytest.mark.covers("mgmt.cost_tracking.estimate.happy_path") + @meta(Subject(domain=Domain.COST_MAP)) def test_estimate_computes_cost_from_token_counts(self, client: ManagementClient) -> None: estimate = unwrap( client.proxy.transport.post( @@ -325,6 +331,7 @@ class TestCostEstimate: class TestComplianceRoutes: @pytest.mark.covers("mgmt.compliance.gdpr.happy_path") + @meta(Subject(domain=Domain.MANAGEMENT)) def test_gdpr_check_derives_verdict_from_the_request(self, client: ManagementClient) -> None: result = unwrap( client.proxy.transport.post( @@ -356,6 +363,7 @@ class TestComplianceRoutes: class TestFallbackManagement: @pytest.mark.covers("mgmt.fallback_management.update.happy_path") + @meta(Subject(domain=Domain.ROUTING)) def test_create_persists_and_is_read_back(self, client: ManagementClient, resources: ResourceManager) -> None: primary = f"e2e-fallback-primary-{unique_marker()}" secondary = f"e2e-fallback-secondary-{unique_marker()}" @@ -410,6 +418,7 @@ class TestFallbackManagement: class TestJwtKeyMapping: @pytest.mark.covers("mgmt.jwt_key_mapping.new.happy_path") + @meta(Subject(domain=Domain.PROXY_AUTH)) def test_new_persists_mapping_and_is_read_back( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -461,6 +470,7 @@ class TestJwtKeyMapping: class TestRouterSettings: @pytest.mark.covers("mgmt.router_settings.update.happy_path") + @meta(Subject(domain=Domain.ROUTING, route=Route.PROXY_CONFIG)) def test_config_update_persists_router_setting_to_get( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -531,6 +541,7 @@ class TestRouterSettings: class TestMcpServerSubmission: @pytest.mark.covers("mgmt.mcp_server.register.happy_path") + @meta(Subject(domain=Domain.MCP, route=Route.MCP)) def test_register_submits_pending_server(self, client: ManagementClient, resources: ResourceManager) -> None: """A non-admin, team-scoped key submits an MCP server for review; the proxy stores it as pending_review without loading it into the runtime registry.""" @@ -564,6 +575,7 @@ class TestMcpServerSubmission: ) @pytest.mark.covers("mgmt.mcp_server.approve.persists") + @meta(Subject(domain=Domain.MCP, route=Route.MCP)) def test_approve_activates_submission_and_persists( self, client: ManagementClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/management/test_jwt_management_e2e.py b/tests/e2e/management/test_jwt_management_e2e.py index 5898073a4e6..3853ae2c4e6 100644 --- a/tests/e2e/management/test_jwt_management_e2e.py +++ b/tests/e2e/management/test_jwt_management_e2e.py @@ -7,6 +7,7 @@ from typing import Final, Literal import pytest from e2e_config import CHEAP_OPENAI_MODEL, PROXY_BASE_URL, unique_marker from e2e_http import UnauthorizedError, UnknownApiError, unwrap +from e2e_metadata import Domain, Route, Subject, meta from idp import ADMIN_CLIENT_ID, Identity, Keycloak, token_claims from lifecycle import ResourceManager from management.jwt_actors import ActorFactory, ActorRole @@ -32,6 +33,7 @@ class TestJwtManagement: ), ) @pytest.mark.covers("mgmt.user.jwt.database_roles") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.USER_MANAGEMENT)) def test_actor_subject_and_database_role(self, actor_factory: ActorFactory, role: ActorRole) -> None: tenants: Final = ( (actor_factory.tenant(),) if role in ("organization_admin", "team_admin", "team_member") else () @@ -70,6 +72,7 @@ class TestJwtManagement: } == {(actor.identity.user_id, "org_admin" if role == "organization_admin" else "internal_user")} @pytest.mark.covers("mgmt.key.jwt.viewer_denied") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.KEY_MANAGEMENT)) def test_admin_viewer_reads_but_cannot_update(self, actor_factory: ActorFactory) -> None: actor: Final = actor_factory.create("proxy_admin_viewer") viewer: Final = actor_factory.bootstrap.with_caller(actor.mint_caller(actor_factory.idp)) @@ -83,6 +86,7 @@ class TestJwtManagement: assert actor_factory.bootstrap.proxy.key_info(key).key_alias == alias @pytest.mark.covers("mgmt.user.oidc.identity_mapping") + @meta(Subject(domain=Domain.PROXY_AUTH)) def test_oidc_browser_profile_identity_mapping(self, actor_factory: ActorFactory) -> None: actor: Final = actor_factory.create("internal_user") idp: Final = actor_factory.idp.with_strict_cleanup() @@ -100,6 +104,7 @@ class TestJwtManagement: @pytest.mark.covers("mgmt.key.jwt.lifecycle") @pytest.mark.parametrize("credential_kind", ("direct_jwt", "virtual_key")) + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.KEY_MANAGEMENT)) def test_admin_creates_reads_updates_clears_and_deletes_a_key( self, actor_factory: ActorFactory, @@ -142,6 +147,7 @@ class TestJwtManagement: assert unwrap(bound.key_list(updated_alias)).total_count == 0 @pytest.mark.covers("mgmt.team.jwt.tenant_isolation") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.KEY_MANAGEMENT)) def test_two_actor_sets_keep_tenants_and_keys_isolated(self, actor_factory: ActorFactory) -> None: first: Final = actor_factory.tenant() second: Final = actor_factory.tenant() @@ -163,6 +169,7 @@ class TestJwtManagement: assert tuple(actor.identity.groups for actor in actors) == ((first.team_id,), (second.team_id,)) @pytest.mark.covers("mgmt.team.jwt.multiple_memberships") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.TEAM_MANAGEMENT)) def test_multi_group_actor_keeps_exact_memberships(self, actor_factory: ActorFactory) -> None: tenants: Final = (actor_factory.tenant(), actor_factory.tenant()) actor: Final = actor_factory.create("team_member", tenants=tenants, profile="group_scoped") @@ -177,6 +184,7 @@ class TestJwtManagement: } == {(actor.identity.user_id, "user")} @pytest.mark.covers("mgmt.user.jwt.cleanup") + @meta(Subject(domain=Domain.PROXY_AUTH)) def test_successful_actor_cleanup_removes_owned_state(self, actor_factory: ActorFactory) -> None: resources: Final = ResourceManager(client=actor_factory.bootstrap.proxy, strict_cleanup=True) factory: Final = ActorFactory(bootstrap=actor_factory.bootstrap, idp=actor_factory.idp, resources=resources) @@ -197,6 +205,7 @@ class TestJwtManagement: @pytest.mark.parametrize("stage", ("group", "user")) @pytest.mark.covers("mgmt.user.jwt.partial_cleanup") + @meta(Subject(domain=Domain.PROXY_AUTH)) def test_partial_setup_removes_previously_created_identities( self, actor_factory: ActorFactory, @@ -236,6 +245,7 @@ class TestJwtManagement: idp.assert_absent("users", identity.user_id) @pytest.mark.covers("mgmt.key.jwt.member_denied", "mgmt.key.jwt.other_team_denied") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.KEY_MANAGEMENT)) def test_member_cannot_write_and_another_team_cannot_read_the_key( self, client: ManagementClient, idp: Keycloak, jwt_identity: Identity, resources: ResourceManager ) -> None: diff --git a/tests/e2e/management/test_key_lifecycle_e2e.py b/tests/e2e/management/test_key_lifecycle_e2e.py index fb153f2a7f3..956719e0ba3 100644 --- a/tests/e2e/management/test_key_lifecycle_e2e.py +++ b/tests/e2e/management/test_key_lifecycle_e2e.py @@ -23,6 +23,7 @@ import pytest from e2e_config import unique_marker from e2e_http import Result, StreamingResponse, Success, unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from management_client import MODEL_ACCESS_DENIED_MARKER, ManagementClient from models import ( @@ -188,6 +189,7 @@ def _assert_chat_rejected_everywhere(client: ManagementClient, key: str, model: class TestKeyLifecycle: + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.KEY_MANAGEMENT)) def test_create_echoes_every_field_written( self, client: ManagementClient, resources: ResourceManager, mock_deployment: str ) -> None: @@ -206,6 +208,7 @@ class TestKeyLifecycle: ): assert observed == wanted, f"/key/generate echoed {field}={observed!r}, sent {wanted!r}" + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.KEY_MANAGEMENT)) def test_read_reflects_the_create_on_every_replica( self, client: ManagementClient, resources: ResourceManager, mock_deployment: str ) -> None: @@ -221,6 +224,7 @@ class TestKeyLifecycle: ) @pytest.mark.covers("mgmt.key.update.preserves_unrelated_fields") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.KEY_MANAGEMENT)) def test_partial_update_changes_only_the_named_field( self, client: ManagementClient, resources: ResourceManager, mock_deployment: str ) -> None: @@ -238,6 +242,7 @@ class TestKeyLifecycle: ) @pytest.mark.covers("mgmt.key.update.clear_persists") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.KEY_MANAGEMENT)) def test_explicit_null_clears_the_budget_and_its_reset_time( self, client: ManagementClient, resources: ResourceManager, mock_deployment: str ) -> None: @@ -258,6 +263,14 @@ class TestKeyLifecycle: info, created.written.model_copy(update={"max_budget": None, "budget_duration": None}), replica ) + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.OPENAI,), + models=(BACKING_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_key_serves_its_model_and_is_denied_others( self, client: ManagementClient, resources: ResourceManager, mock_deployment: str ) -> None: @@ -274,6 +287,15 @@ class TestKeyLifecycle: f"403 body must be a model-access denial, got: {denied.body[:300]}" ) + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + providers=(Provider.OPENAI,), + models=(BACKING_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_delete_revokes_info_and_chat_on_every_replica( self, client: ManagementClient, resources: ResourceManager, mock_deployment: str ) -> None: diff --git a/tests/e2e/management/test_key_management_e2e.py b/tests/e2e/management/test_key_management_e2e.py index 39a9e657b8c..34f912fe604 100644 --- a/tests/e2e/management/test_key_management_e2e.py +++ b/tests/e2e/management/test_key_management_e2e.py @@ -19,6 +19,7 @@ import pytest from e2e_config import unique_marker from e2e_http import NoBody, StreamingResponse, unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from management_client import ManagementClient from models import ( @@ -31,6 +32,7 @@ pytestmark = pytest.mark.e2e TINY_BUDGET = 3e-6 SPEND_MODEL = "claude-haiku-4-5" +SYNTHETIC_BACKEND: Final = "openai/synthetic-detachment" class KeyToggleBlockBody(BaseModel): @@ -161,13 +163,22 @@ def project_resources(client: ManagementClient) -> Iterator[ResourceManager]: class TestKeyManagementRoutes: @pytest.mark.covers("mgmt.key.update.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + providers=(Provider.OPENAI,), + models=(SYNTHETIC_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_project_detachment_preserves_key_scope_and_refreshes_auth( self, client: ManagementClient, project_resources: ResourceManager ) -> None: resources: Final = project_resources name: Final = f"e2e-detach-{unique_marker()}" model_id: Final = client.proxy.create_model( - name, LiteLLMParamsBody(model="openai/synthetic-detachment", api_key="synthetic", mock_response="orbit") + name, LiteLLMParamsBody(model=SYNTHETIC_BACKEND, api_key="synthetic", mock_response="orbit") ) resources.defer(lambda: client.proxy.delete_model(model_id)) org_id: Final = client.create_org(OrgNewBody(organization_alias=name, models=[name])) @@ -222,6 +233,7 @@ class TestKeyManagementRoutes: assert denied.status_code in (401, 403), denied.body @pytest.mark.covers("mgmt.key.info.persists") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.KEY_MANAGEMENT)) def test_info_reflects_the_fields_the_key_was_created_with( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -246,6 +258,7 @@ class TestKeyManagementRoutes: assert info.rpm_limit == 141414, f"/key/info reports rpm_limit {info.rpm_limit}, configured 141414" @pytest.mark.covers("mgmt.key.unblock.persists") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.KEY_MANAGEMENT)) def test_unblock_flips_key_info_blocked_back( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -266,6 +279,7 @@ class TestKeyManagementRoutes: ) @pytest.mark.covers("mgmt.key.health.happy_path") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.KEY_MANAGEMENT)) def test_health_reports_the_calling_key_healthy( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -285,6 +299,7 @@ class TestKeyManagementRoutes: ) @pytest.mark.covers("mgmt.key.bulk_update.happy_path") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.KEY_MANAGEMENT)) def test_bulk_update_applies_max_budget_to_target_key( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -314,6 +329,15 @@ class TestKeyManagementRoutes: ) @pytest.mark.covers("other.key_mgmt.spend_reset.resets_to_value") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.KEY_MANAGEMENT, + providers=(Provider.ANTHROPIC,), + models=(SPEND_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_reset_spend_zeroes_recorded_spend_and_lifts_the_budget_block( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -340,6 +364,7 @@ class TestKeyManagementRoutes: _ = _poll(client, call_allowed_again, "the key stayed budget-blocked after its spend was reset to 0") @pytest.mark.covers("mgmt.key.generate.admin_only") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.KEY_MANAGEMENT)) def test_generate_forbidden_for_non_admin_key( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -355,6 +380,7 @@ class TestKeyManagementRoutes: ) @pytest.mark.covers("mgmt.key.delete.admin_only") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.KEY_MANAGEMENT)) def test_delete_forbidden_for_non_admin_key( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -374,6 +400,7 @@ class TestKeyManagementRoutes: ) @pytest.mark.covers("mgmt.key.update.admin_only") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.KEY_MANAGEMENT)) def test_update_forbidden_for_non_admin_key( self, client: ManagementClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index 908eb752611..c167a1323cb 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -18,6 +18,7 @@ import pytest from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, UI_PASSWORD, UI_USERNAME, unique_marker from e2e_http import StreamingResponse, Success, unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from management_client import ( DASHBOARD_SESSION_TEAM_ID, @@ -47,6 +48,8 @@ from proxy_client import Converged, await_converged pytestmark = pytest.mark.e2e +GEMINI_MODEL: Final = "gemini-2.5-flash" +OPENAI_MODEL: Final = "gpt-5.5" REGENERATE_GRACE_PERIOD = "15s" REGENERATE_GRACE_SECONDS = 15.0 TEAM_DELETE_POOL_OVERFLOW_MEMBERS = 250 @@ -130,6 +133,15 @@ def _poll_model_access_granted(client: ManagementClient, key: str, model: str) - class TestKeyRoutes: @pytest.mark.covers("mgmt.key.generate.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + providers=(Provider.GEMINI,), + models=(GEMINI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_generate_persists_to_key_info_and_scopes_chat( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -137,12 +149,12 @@ class TestKeyRoutes: key = _generate_key( client, resources, - KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias, tpm_limit=424242, rpm_limit=424243), + KeyGenerateBody(models=[GEMINI_MODEL], key_alias=alias, tpm_limit=424242, rpm_limit=424243), ) info = client.proxy.key_info(key) assert info.key_alias == alias, f"/key/info reports key_alias {info.key_alias!r}, configured {alias!r}" - assert info.models == ["gemini-2.5-flash"], ( + assert info.models == [GEMINI_MODEL], ( f"/key/info reports models {info.models}, configured ['gemini-2.5-flash']" ) assert info.tpm_limit == 424242, ( @@ -152,49 +164,73 @@ class TestKeyRoutes: f"/key/info reports rpm_limit {info.rpm_limit}, configured 424243" ) - _poll_chat_ok(client, key, "gemini-2.5-flash") + _poll_chat_ok(client, key, GEMINI_MODEL) _assert_model_denied( - client.chat_status(key, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5" + client.chat_status(key, OPENAI_MODEL, f"say hi {unique_marker()}"), OPENAI_MODEL ) @pytest.mark.covers("mgmt.key.update.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + providers=(Provider.GEMINI, Provider.OPENAI), + models=(GEMINI_MODEL, OPENAI_MODEL), + mode=Mode.NONSTREAM, + ) + ) def test_update_models_persists_and_flips_enforcement( self, client: ManagementClient, resources: ResourceManager ) -> None: - key = _generate_key(client, resources, KeyGenerateBody(models=["gemini-2.5-flash"])) - _poll_chat_ok(client, key, "gemini-2.5-flash") + key = _generate_key(client, resources, KeyGenerateBody(models=[GEMINI_MODEL])) + _poll_chat_ok(client, key, GEMINI_MODEL) _assert_model_denied( - client.chat_status(key, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5" + client.chat_status(key, OPENAI_MODEL, f"say hi {unique_marker()}"), OPENAI_MODEL ) - client.update_key_models(key, ["gpt-5.5"]) + client.update_key_models(key, [OPENAI_MODEL]) info = client.proxy.key_info(key) - assert info.models == ["gpt-5.5"], ( + assert info.models == [OPENAI_MODEL], ( f"/key/info reports models {info.models} after /key/update to ['gpt-5.5']" ) - _poll_model_access_granted(client, key, "gpt-5.5") - _poll_chat_denied(client, key, "gemini-2.5-flash") + _poll_model_access_granted(client, key, OPENAI_MODEL) + _poll_chat_denied(client, key, GEMINI_MODEL) @pytest.mark.covers("mgmt.key.delete.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + providers=(Provider.GEMINI,), + models=(GEMINI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_delete_revokes_the_key_on_chat(self, client: ManagementClient, resources: ResourceManager) -> None: """The teardown's deferred delete fires again on the already-deleted key by design: the deferred cleanup must survive this test failing before the in-body delete, and a repeat /key/delete is a cheap no-op the warn-only teardown absorbs.""" - key = _generate_key(client, resources, KeyGenerateBody(models=["gemini-2.5-flash"])) - _poll_chat_ok(client, key, "gemini-2.5-flash") + key = _generate_key(client, resources, KeyGenerateBody(models=[GEMINI_MODEL])) + _poll_chat_ok(client, key, GEMINI_MODEL) client.delete_key_strict(key) def rejected() -> bool | None: - outcome = client.chat_status(key, "gemini-2.5-flash", f"say hi {unique_marker()}") + outcome = client.chat_status(key, GEMINI_MODEL, f"say hi {unique_marker()}") return True if outcome.status_code == 401 else None _ = _poll(client, rejected, "deleted key was still accepted on chat (never rejected 401) at the deadline") @pytest.mark.covers("mgmt.key.list.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + ) + ) def test_created_key_appears_in_key_list_inventory( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -214,8 +250,14 @@ class TestKeyRoutes: @pytest.mark.covers("mgmt.key.block.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + ) + ) def test_block_persists_to_key_info(self, client: ManagementClient, resources: ResourceManager) -> None: - key = _generate_key(client, resources, KeyGenerateBody(models=["gemini-2.5-flash"])) + key = _generate_key(client, resources, KeyGenerateBody(models=[GEMINI_MODEL])) assert not client.proxy.key_info(key).blocked, "/key/info reports the key blocked before /key/block ran" client.block_key(key) @@ -233,6 +275,15 @@ class TestDashboardKeyRoutes: are the same routes the API-surface tests cover with a different caller.""" @pytest.mark.covers("mgmt.key.generate.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + providers=(Provider.GEMINI,), + models=(GEMINI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_creating_a_key_from_the_dashboard_persists_and_works( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -260,7 +311,7 @@ class TestDashboardKeyRoutes: def dashboard_creates_the_key() -> str | None: match client.generate_key( - KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias, tpm_limit=100), + KeyGenerateBody(models=[GEMINI_MODEL], key_alias=alias, tpm_limit=100), caller_key=session.session_key, ): case Success(data=created): @@ -280,7 +331,7 @@ class TestDashboardKeyRoutes: f"/key/info reports key_alias {created_info.key_alias!r} for the key the dashboard created, " f"expected {alias!r}" ) - assert created_info.models == ["gemini-2.5-flash"], ( + assert created_info.models == [GEMINI_MODEL], ( f"/key/info reports models {created_info.models} for the key the dashboard created" ) assert created_info.tpm_limit == 100, ( @@ -301,10 +352,19 @@ class TestDashboardKeyRoutes: "would render no keys", ) - _poll_chat_ok(client, created, "gemini-2.5-flash") - _assert_model_denied(client.chat_status(created, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5") + _poll_chat_ok(client, created, GEMINI_MODEL) + _assert_model_denied(client.chat_status(created, OPENAI_MODEL, f"say hi {unique_marker()}"), OPENAI_MODEL) @pytest.mark.covers("mgmt.key.update.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + providers=(Provider.GEMINI, Provider.OPENAI), + models=(GEMINI_MODEL, OPENAI_MODEL), + mode=Mode.NONSTREAM, + ) + ) def test_editing_a_key_from_the_dashboard_persists_and_is_enforced( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -312,17 +372,17 @@ class TestDashboardKeyRoutes: target = _generate_key( client, resources, - KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias, tpm_limit=100, rpm_limit=200), + KeyGenerateBody(models=[GEMINI_MODEL], key_alias=alias, tpm_limit=100, rpm_limit=200), ) - _poll_chat_ok(client, target, "gemini-2.5-flash") - _assert_model_denied(client.chat_status(target, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5") + _poll_chat_ok(client, target, GEMINI_MODEL) + _assert_model_denied(client.chat_status(target, OPENAI_MODEL, f"say hi {unique_marker()}"), OPENAI_MODEL) session = client.dashboard_login(UI_USERNAME, UI_PASSWORD) resources.defer(lambda: client.proxy.delete_key(session.session_key)) def dashboard_saves_the_edit() -> bool | None: match client.update_key( - KeyUpdateBody(key=target, models=["gpt-5.5"], tpm_limit=300, rpm_limit=400), + KeyUpdateBody(key=target, models=[OPENAI_MODEL], tpm_limit=300, rpm_limit=400), caller_key=session.session_key, ): case Success(): @@ -337,7 +397,7 @@ class TestDashboardKeyRoutes: ) info = client.proxy.key_info(target) - assert info.models == ["gpt-5.5"], ( + assert info.models == [OPENAI_MODEL], ( f"/key/info reports models {info.models} after the dashboard edit to ['gpt-5.5']" ) assert info.tpm_limit == 300, f"/key/info reports tpm_limit {info.tpm_limit} after the dashboard edit to 300" @@ -346,29 +406,38 @@ class TestDashboardKeyRoutes: f"the dashboard edit renamed the key to {info.key_alias!r}, it should still be {alias!r}" ) - _poll_model_access_granted(client, target, "gpt-5.5") - _poll_chat_denied(client, target, "gemini-2.5-flash") + _poll_model_access_granted(client, target, OPENAI_MODEL) + _poll_chat_denied(client, target, GEMINI_MODEL) class TestKeyRegeneration: @pytest.mark.covers("mgmt.key.regenerate.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + providers=(Provider.OPENAI,), + models=(OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_regenerate_rotates_to_a_working_new_key( self, client: ManagementClient, resources: ResourceManager ) -> None: - old_key = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"])) + old_key = _generate_key(client, resources, KeyGenerateBody(models=[OPENAI_MODEL])) new_key = client.regenerate_key(old_key) resources.defer(lambda: client.proxy.delete_key(new_key)) assert new_key != old_key, "regenerate returned the same key string, so no rotation happened" def new_accepted() -> bool | None: - outcome = client.chat_status(new_key, "gpt-5.5", f"say hi {unique_marker()}") + outcome = client.chat_status(new_key, OPENAI_MODEL, f"say hi {unique_marker()}") return True if outcome.status_code != 401 else None _ = _poll(client, new_accepted, "regenerated key was never accepted at auth (still 401) at the deadline") def old_rejected() -> bool | None: - outcome = client.chat_status(old_key, "gpt-5.5", f"say hi {unique_marker()}") + outcome = client.chat_status(old_key, OPENAI_MODEL, f"say hi {unique_marker()}") return True if outcome.status_code == 401 else None _ = _poll( @@ -376,10 +445,19 @@ class TestKeyRegeneration: ) @pytest.mark.covers("other.key_mgmt.regenerate.grace_period_honored") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + providers=(Provider.OPENAI,), + models=(OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_regenerate_with_grace_period_keeps_old_key_until_revoked( self, client: ManagementClient, resources: ResourceManager ) -> None: - old_key = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"])) + old_key = _generate_key(client, resources, KeyGenerateBody(models=[OPENAI_MODEL])) new_key = client.regenerate_key(old_key, grace_period=REGENERATE_GRACE_PERIOD) resources.defer(lambda: client.proxy.delete_key(new_key)) @@ -387,7 +465,7 @@ class TestKeyRegeneration: assert new_key != old_key, "regenerate returned the same key string, so no rotation happened" def old_accepted() -> bool | None: - outcome = client.chat_status(old_key, "gpt-5.5", f"say hi {unique_marker()}") + outcome = client.chat_status(old_key, OPENAI_MODEL, f"say hi {unique_marker()}") return True if outcome.ok else None _ = _poll(client, old_accepted, "old key was rejected 401 inside its grace period at the deadline") @@ -396,7 +474,7 @@ class TestKeyRegeneration: ) def old_rejected() -> bool | None: - outcome = client.chat_status(old_key, "gpt-5.5", f"say hi {unique_marker()}") + outcome = client.chat_status(old_key, OPENAI_MODEL, f"say hi {unique_marker()}") return True if outcome.status_code == 401 else None _ = _poll( @@ -408,15 +486,21 @@ class TestKeyRegeneration: class TestTeamRoutes: @pytest.mark.covers("mgmt.team.new.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.TEAM_MANAGEMENT, + ) + ) def test_new_persists_to_team_info_and_binds_keys( self, client: ManagementClient, resources: ResourceManager ) -> None: alias = f"e2e-mgmt-team-{unique_marker()}" - team_id = _create_team(client, resources, alias, ["gemini-2.5-flash"]) + team_id = _create_team(client, resources, alias, [GEMINI_MODEL]) info = client.team_info(team_id) assert info.team_alias == alias, f"/team/info reports team_alias {info.team_alias!r}, configured {alias!r}" - assert info.models == ["gemini-2.5-flash"], ( + assert info.models == [GEMINI_MODEL], ( f"/team/info reports models {info.models}, configured ['gemini-2.5-flash']" ) @@ -427,8 +511,14 @@ class TestTeamRoutes: ) @pytest.mark.covers("mgmt.team.update.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.TEAM_MANAGEMENT, + ) + ) def test_update_persists_to_team_info(self, client: ManagementClient, resources: ResourceManager) -> None: - team_id = _create_team(client, resources, f"e2e-mgmt-team-{unique_marker()}", ["gemini-2.5-flash"]) + team_id = _create_team(client, resources, f"e2e-mgmt-team-{unique_marker()}", [GEMINI_MODEL]) updated_alias = f"e2e-mgmt-team-updated-{unique_marker()}" client.update_team(TeamUpdateBody(team_id=team_id, team_alias=updated_alias)) @@ -438,11 +528,17 @@ class TestTeamRoutes: _ = _poll(client, reflected, f"/team/info never reflected team_alias {updated_alias!r} after /team/update") @pytest.mark.covers("mgmt.team.list.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.TEAM_MANAGEMENT, + ) + ) def test_created_team_appears_in_team_list( self, client: ManagementClient, resources: ResourceManager ) -> None: alias = f"e2e-mgmt-team-{unique_marker()}" - team_id = _create_team(client, resources, alias, ["gemini-2.5-flash"]) + team_id = _create_team(client, resources, alias, [GEMINI_MODEL]) _ = _poll( client, @@ -451,17 +547,26 @@ class TestTeamRoutes: ) @pytest.mark.covers("mgmt.team.delete.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.TEAM_MANAGEMENT, + providers=(Provider.OPENAI,), + models=(OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_delete_persists_and_revokes_team_bound_key( self, client: ManagementClient, resources: ResourceManager ) -> None: """The teardown's deferred delete_team/delete_key fire again on the already- deleted team and key by design: both are warn-only no-ops, and the deferred cleanup must survive this test failing before the in-body delete.""" - team_id = _create_team(client, resources, f"e2e-mgmt-team-{unique_marker()}", ["gpt-5.5"]) + team_id = _create_team(client, resources, f"e2e-mgmt-team-{unique_marker()}", [OPENAI_MODEL]) key = _generate_key(client, resources, KeyGenerateBody(team_id=team_id)) def accepted() -> bool | None: - outcome = client.chat_status(key, "gpt-5.5", f"say hi {unique_marker()}") + outcome = client.chat_status(key, OPENAI_MODEL, f"say hi {unique_marker()}") return True if outcome.status_code != 401 else None _ = _poll(client, accepted, "team-bound key was never accepted at auth before team deletion") @@ -474,7 +579,7 @@ class TestTeamRoutes: ) def rejected() -> bool | None: - outcome = client.chat_status(key, "gpt-5.5", f"say hi {unique_marker()}") + outcome = client.chat_status(key, OPENAI_MODEL, f"say hi {unique_marker()}") return True if outcome.status_code == 401 else None _ = _poll( @@ -482,6 +587,12 @@ class TestTeamRoutes: ) @pytest.mark.covers("mgmt.team.delete.membership_larger_than_db_pool") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.TEAM_MANAGEMENT, + ) + ) def test_team_delete_succeeds_for_team_larger_than_db_pool( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -519,6 +630,12 @@ class TestTeamRoutes: ) @pytest.mark.covers("mgmt.team.member_add.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.TEAM_MANAGEMENT, + ) + ) def test_member_add_and_delete_persist_to_team_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -527,7 +644,7 @@ class TestTeamRoutes: resources, UserNewBody(user_email=f"e2e-mgmt-{unique_marker()}@example.com", user_role="internal_user"), ) - team_id = _create_team(client, resources, f"e2e-mgmt-team-{unique_marker()}", ["gemini-2.5-flash"]) + team_id = _create_team(client, resources, f"e2e-mgmt-team-{unique_marker()}", [GEMINI_MODEL]) client.add_team_member(team_id, user_id) member = next( @@ -545,6 +662,12 @@ class TestTeamRoutes: class TestUserRoutes: @pytest.mark.covers("mgmt.user.new.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.USER_MANAGEMENT, + ) + ) def test_new_persists_to_user_info(self, client: ManagementClient, resources: ResourceManager) -> None: email = f"e2e-mgmt-{unique_marker()}@example.com" user_id = _create_user(client, resources, UserNewBody(user_email=email, user_role="internal_user")) @@ -556,6 +679,12 @@ class TestUserRoutes: ) @pytest.mark.covers("mgmt.user.update.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.USER_MANAGEMENT, + ) + ) def test_update_persists_to_user_info(self, client: ManagementClient, resources: ResourceManager) -> None: email = f"e2e-mgmt-{unique_marker()}@example.com" user_id = _create_user(client, resources, UserNewBody(user_email=email, user_role="internal_user")) @@ -572,6 +701,12 @@ class TestUserRoutes: f"/user/info reports user_role {info.user_role!r} after /user/update to 'internal_user_viewer'" ) @pytest.mark.covers("mgmt.user.delete.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.USER_MANAGEMENT, + ) + ) def test_delete_removes_the_user_from_inventory( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -594,6 +729,12 @@ class TestUserRoutes: _ = _poll(client, removed, f"user {user_id} still present in /user/list after /user/delete at the deadline") @pytest.mark.covers("mgmt.user.list.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.USER_MANAGEMENT, + ) + ) def test_created_users_appear_in_user_list( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -616,22 +757,34 @@ class TestUserRoutes: class TestOrganizationRoutes: @pytest.mark.covers("mgmt.organization.new.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.ORGANIZATION_MANAGEMENT, + ) + ) def test_new_persists_to_organization_info( self, client: ManagementClient, resources: ResourceManager ) -> None: alias = f"e2e-mgmt-org-{unique_marker()}" - org_id = client.create_org(OrgNewBody(organization_alias=alias, models=["gemini-2.5-flash"])) + org_id = client.create_org(OrgNewBody(organization_alias=alias, models=[GEMINI_MODEL])) resources.defer(lambda: client.delete_org(org_id)) info = client.org_info(org_id) assert info.organization_alias == alias, ( f"/organization/info reports alias {info.organization_alias!r}, configured {alias!r}" ) - assert info.models == ["gemini-2.5-flash"], ( + assert info.models == [GEMINI_MODEL], ( f"/organization/info reports models {info.models}, configured ['gemini-2.5-flash']" ) @pytest.mark.covers("mgmt.organization.update.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.ORGANIZATION_MANAGEMENT, + ) + ) def test_update_alias_persists_to_organization_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -650,6 +803,12 @@ class TestOrganizationRoutes: ) @pytest.mark.covers("mgmt.organization.delete.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.ORGANIZATION_MANAGEMENT, + ) + ) def test_delete_removes_from_organization_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -674,6 +833,12 @@ class TestOrganizationRoutes: class TestTagRoutes: @pytest.mark.covers("mgmt.tag.new.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.TAG_MANAGEMENT, + ) + ) def test_new_persists_to_tag_list(self, client: ManagementClient, resources: ResourceManager) -> None: name = f"e2e-mgmt-tag-{unique_marker()}" description = "Tag for spend categorization" @@ -704,6 +869,12 @@ def _model_entry(client: ManagementClient, model_name: str) -> ModelInfoEntry | class TestModelRoutes: @pytest.mark.covers("mgmt.model.update.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.MODEL_MANAGEMENT, + ) + ) def test_update_persists_input_cost_to_model_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -747,6 +918,12 @@ class TestModelRoutes: ) @pytest.mark.covers("mgmt.model.delete.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.MODEL_MANAGEMENT, + ) + ) def test_delete_removes_from_model_info_catalog( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -769,6 +946,12 @@ class TestModelRoutes: _ = _poll(client, absent, f"{model_name} still present in /model/info after /model/delete at the deadline") @pytest.mark.covers("mgmt.model.add.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.MODEL_MANAGEMENT, + ) + ) def test_new_persists_to_model_info_catalog( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -797,6 +980,11 @@ def _assert_route_forbidden(route: str, outcome: StreamingResponse) -> None: class TestManagementRoutePermissions: @pytest.mark.covers("other.auth.virtual_key.route_permission_enforced") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + ) + ) def test_llm_only_key_forbidden_from_management_writes( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -832,6 +1020,12 @@ class TestManagementRoutePermissions: class TestCustomer: @pytest.mark.covers("mgmt.end_user.new.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.CUSTOMER_MANAGEMENT, + ) + ) def test_customer_create_persists_to_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -896,6 +1090,12 @@ def _generate_response( class TestKeyDeletionAuditLog: @pytest.mark.covers("mgmt.key.delete.audit_logged") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + ) + ) def test_key_delete_by_key_writes_audit_row( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -908,6 +1108,12 @@ class TestKeyDeletionAuditLog: _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) @pytest.mark.covers("mgmt.key.delete.audit_logged") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.KEY_MANAGEMENT, + ) + ) def test_key_delete_by_alias_writes_audit_row( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -921,6 +1127,12 @@ class TestKeyDeletionAuditLog: _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) @pytest.mark.covers("mgmt.team.member_delete.audit_logs_keys") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.TEAM_MANAGEMENT, + ) + ) def test_team_member_delete_writes_audit_row_for_member_keys( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -940,6 +1152,12 @@ class TestKeyDeletionAuditLog: _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) @pytest.mark.covers("mgmt.team.delete.audit_logs_keys") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.TEAM_MANAGEMENT, + ) + ) def test_team_delete_writes_audit_row_for_team_keys( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -953,6 +1171,12 @@ class TestKeyDeletionAuditLog: _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) @pytest.mark.covers("mgmt.user.delete.audit_logs_keys") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.USER_MANAGEMENT, + ) + ) def test_user_delete_writes_audit_row_for_user_keys( self, client: ManagementClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/management/test_mcp_lifecycle_e2e.py b/tests/e2e/management/test_mcp_lifecycle_e2e.py index 9257d697647..ca2e99cad8b 100644 --- a/tests/e2e/management/test_mcp_lifecycle_e2e.py +++ b/tests/e2e/management/test_mcp_lifecycle_e2e.py @@ -19,6 +19,7 @@ from typing import Final import pytest from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Domain, Route, Subject, meta from lifecycle import ResourceManager from management_client import ManagementClient from models import ( @@ -88,6 +89,12 @@ def _listed_server_everywhere(client: ManagementClient, server_id: str) -> Mappi class TestMcpServerLifecycle: @pytest.mark.covers("mgmt.mcp_server.new.persists") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_create_persists_every_field_on_every_replica( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -107,6 +114,12 @@ class TestMcpServerLifecycle: ) ) @pytest.mark.covers("mgmt.mcp_server.list.persists") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_created_server_is_listed_with_every_field( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -116,6 +129,12 @@ class TestMcpServerLifecycle: _assert_server_matches(row, body, where=f"GET /v1/mcp/server on {replica}") @pytest.mark.covers("mgmt.mcp_server.update.preserves_unrelated_fields") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_updating_only_the_alias_keeps_every_other_field_on_every_replica( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -133,6 +152,12 @@ class TestMcpServerLifecycle: ) @pytest.mark.covers("mgmt.mcp_server.update.clear_persists") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_clearing_the_description_with_null_reads_back_null( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -149,6 +174,12 @@ class TestMcpServerLifecycle: ) @pytest.mark.covers("mgmt.mcp_server.delete.persists") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_delete_removes_the_server_from_every_replica( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -199,6 +230,12 @@ def _toolset_everywhere( class TestMcpToolsetLifecycle: @pytest.mark.covers("mgmt.mcp_toolset.new.persists") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_create_persists_both_tools_under_the_exact_names_written( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -221,6 +258,12 @@ class TestMcpToolsetLifecycle: ) @pytest.mark.covers("mgmt.mcp_toolset.update.preserves_unrelated_fields") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_updating_only_the_description_keeps_the_tools_and_name( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -238,6 +281,12 @@ class TestMcpToolsetLifecycle: ) @pytest.mark.covers("mgmt.mcp_toolset.update.persists") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_updating_the_tools_to_one_entry_reads_back_exactly_that_entry( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -256,6 +305,12 @@ class TestMcpToolsetLifecycle: ) @pytest.mark.covers("mgmt.mcp_toolset.update.clear_persists") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_clearing_the_description_with_null_reads_back_null( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -273,6 +328,12 @@ class TestMcpToolsetLifecycle: ) @pytest.mark.covers("mgmt.mcp_toolset.delete.persists") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_delete_removes_the_toolset_from_every_replica( self, client: ManagementClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/management/test_model_tag_accessgroup_e2e.py b/tests/e2e/management/test_model_tag_accessgroup_e2e.py index eb3a6093c69..5c104db18c1 100644 --- a/tests/e2e/management/test_model_tag_accessgroup_e2e.py +++ b/tests/e2e/management/test_model_tag_accessgroup_e2e.py @@ -22,6 +22,7 @@ from pydantic import BaseModel, ConfigDict, RootModel from e2e_config import unique_marker from e2e_http import NoBody, unwrap +from e2e_metadata import Domain, Route, Subject, meta from lifecycle import ResourceManager from management_client import ManagementClient from models import KeyGenerateBody, LiteLLMParamsBody, ModelInfoBody, ModelNewBody @@ -216,6 +217,12 @@ def _model_blocked_flag(client: ManagementClient, model_id: str) -> bool | None: class TestModelRoutes: @pytest.mark.covers("mgmt.model.add.admin_only") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.MODEL_MANAGEMENT, + ) + ) def test_non_admin_key_cannot_add_global_model( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -248,6 +255,12 @@ class TestModelRoutes: ) @pytest.mark.covers("mgmt.model.block.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.MODEL_MANAGEMENT, + ) + ) def test_block_then_unblock_persists_to_model_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -281,6 +294,12 @@ class TestModelRoutes: class TestTagRoutes: @pytest.mark.covers("mgmt.tag.list.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.TAG_MANAGEMENT, + ) + ) def test_tag_list_reports_created_tag(self, client: ManagementClient, resources: ResourceManager) -> None: name = f"e2e-mgmt-tag-{unique_marker()}" description = "coverage: tag inventory" @@ -301,6 +320,12 @@ class TestTagRoutes: ) @pytest.mark.covers("mgmt.tag.delete.persists") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.TAG_MANAGEMENT, + ) + ) def test_tag_delete_removes_from_list(self, client: ManagementClient, resources: ResourceManager) -> None: """The teardown's deferred delete fires again on the already-deleted tag by design: it is the safety net if this test fails before the in-body delete, @@ -326,6 +351,12 @@ class TestTagRoutes: class TestModelAccessGroupRoutes: @pytest.mark.covers("mgmt.access_group.new.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.MODEL_MANAGEMENT, + ) + ) def test_new_access_group_tags_the_deployment( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -356,6 +387,12 @@ class TestModelAccessGroupRoutes: ) @pytest.mark.covers("mgmt.access_group.info.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.MODEL_MANAGEMENT, + ) + ) def test_access_group_info_reports_membership( self, client: ManagementClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/management/test_model_test_connection_e2e.py b/tests/e2e/management/test_model_test_connection_e2e.py index 25b0b4f24e6..b0d788d582c 100644 --- a/tests/e2e/management/test_model_test_connection_e2e.py +++ b/tests/e2e/management/test_model_test_connection_e2e.py @@ -23,6 +23,7 @@ import time import pytest from e2e_http import unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from management_client import ManagementClient from models import ConnectionTestBody, ConnectionTestResponse, LiteLLMParamsBody @@ -50,6 +51,15 @@ def _probe_mantle(client: ManagementClient) -> ConnectionTestResponse: class TestModelTestConnection: @pytest.mark.covers("mgmt.model.test_connection.happy_path") + @meta( + Subject( + domain=Domain.MANAGEMENT, + route=Route.HEALTH, + providers=(Provider.BEDROCK_MANTLE,), + models=(MANTLE_RESPONSES_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_mantle_responses_connection_succeeds(self, client: ManagementClient) -> None: for attempt in range(1, PROBE_ATTEMPTS + 1): response = _probe_mantle(client) diff --git a/tests/e2e/management/test_team_management_e2e.py b/tests/e2e/management/test_team_management_e2e.py index f30dc6990a9..68a26838ebd 100644 --- a/tests/e2e/management/test_team_management_e2e.py +++ b/tests/e2e/management/test_team_management_e2e.py @@ -27,6 +27,7 @@ import pytest from pydantic import BaseModel from e2e_config import settle_propagation, unique_marker +from e2e_metadata import Domain, Route, Subject, meta from e2e_http import NoBody, PartialBody, StreamingResponse, unwrap from lifecycle import ResourceManager from management_client import ManagementClient @@ -241,6 +242,7 @@ def _member_delete_status(client: ManagementClient, key: str, team_id: str, user class TestTeamManagementRoutes: @pytest.mark.covers("mgmt.team.info.happy_path") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.TEAM_MANAGEMENT)) def test_info_returns_created_team_fields( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -257,6 +259,7 @@ class TestTeamManagementRoutes: ) @pytest.mark.covers("mgmt.team.block.persists") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.TEAM_MANAGEMENT)) def test_block_then_unblock_persists_to_team_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -278,6 +281,7 @@ class TestTeamManagementRoutes: ) @pytest.mark.covers("mgmt.team.member_update.persists") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.TEAM_MANAGEMENT)) def test_member_update_persists_role_and_budget( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -302,6 +306,7 @@ class TestTeamManagementRoutes: ) @pytest.mark.covers("mgmt.team.member_delete.persists") + @meta(Subject(domain=Domain.MANAGEMENT, route=Route.TEAM_MANAGEMENT)) def test_member_delete_persists_to_team_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -320,6 +325,7 @@ class TestTeamManagementRoutes: ) @pytest.mark.covers("mgmt.team.new.admin_only") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.TEAM_MANAGEMENT)) def test_new_is_denied_to_non_admin_keys( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -334,6 +340,7 @@ class TestTeamManagementRoutes: ) @pytest.mark.covers("mgmt.team.member_add.member_forbidden") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.TEAM_MANAGEMENT)) def test_member_add_forbidden_to_plain_member( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -348,6 +355,7 @@ class TestTeamManagementRoutes: ) @pytest.mark.covers("mgmt.team.member_delete.member_forbidden") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.TEAM_MANAGEMENT)) def test_member_delete_forbidden_to_plain_member( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -458,6 +466,7 @@ class TestTeamAdminWithNoEditableFields: """No proxy admin has enabled a team field for team admins, which is how every proxy starts.""" @pytest.mark.covers("mgmt.team.update.team_admin_forbidden_until_enabled") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.TEAM_MANAGEMENT)) def test_team_admin_cannot_change_any_team_setting( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -483,6 +492,7 @@ class TestTeamAdminWithTpmLimitEnabled: """A proxy admin has enabled tpm_limit, so a team admin may change that setting and no other.""" @pytest.mark.covers("mgmt.team.update.team_admin_limited_to_enabled_fields") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.TEAM_MANAGEMENT)) def test_team_admin_saves_the_settings_form_with_a_new_tpm_limit( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -522,6 +532,7 @@ class TestTeamAdminWithTpmLimitEnabled: pytest.param(TeamSettingsChange(metadata=TeamCustomMetadata(cost_center="team-admin")), id="metadata"), ], ) + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.TEAM_MANAGEMENT)) def test_team_admin_cannot_change_a_setting_that_is_not_enabled( self, client: ManagementClient, resources: ResourceManager, change: TeamSettingsChange ) -> None: @@ -548,6 +559,7 @@ class TestTeamAdminWithTpmLimitEnabled: ) @pytest.mark.covers("mgmt.team.update.team_admin_resend_keeps_budget_reset") + @meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.TEAM_MANAGEMENT)) def test_team_admin_resending_the_budget_settings_keeps_the_next_budget_reset( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -613,6 +625,7 @@ class TestTeamAdminWithRpmLimitAndMaxBudgetEnabled: "current_budget", [pytest.param(_TEAM_MAX_BUDGET, id="lower"), pytest.param(None, id="first-budget")], ) + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.TEAM_MANAGEMENT)) def test_team_admin_saves_a_new_rpm_limit_and_a_tighter_budget( self, client: ManagementClient, resources: ResourceManager, current_budget: float | None ) -> None: @@ -649,6 +662,7 @@ class TestTeamAdminWithRpmLimitAndMaxBudgetEnabled: pytest.param(None, "Only a proxy admin can remove", id="remove"), ], ) + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.TEAM_MANAGEMENT)) def test_team_admin_cannot_raise_or_remove_the_budget( self, client: ManagementClient, resources: ResourceManager, max_budget: float | None, refusal: str ) -> None: @@ -670,6 +684,7 @@ class TestTeamAdminWithRpmLimitAndMaxBudgetEnabled: ) @pytest.mark.covers("mgmt.team.update.team_admin_cannot_grow_budget") + @meta(Subject(domain=Domain.PROXY_AUTH, route=Route.TEAM_MANAGEMENT)) def test_team_admin_cannot_raise_an_org_team_budget_under_the_org_cap( self, client: ManagementClient, resources: ResourceManager ) -> None: From 79209b92a1b87c1f205d87a5d65e23a97a4441b5 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 7 Oct 2026 10:32:25 -0700 Subject: [PATCH 05/35] test(e2e): tag guardrails and logging tests with Subject metadata and record client steps (#44963) * test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata * test(e2e): tag guardrails and logging tests with Subject metadata and record client steps * test(e2e): leave the guardrails and logging harness unit tests untagged * test(e2e): let the inner create_model step name the guardrail backend deployment * docs(e2e): name every markerless harness test file that carries no Subject * test(e2e): keep the step discovery comprehensions to one for clause * test(e2e): declare the default guardrail backend model on the tests that drive it --- tests/e2e/guardrails/guardrails_client.py | 36 +++++- .../guardrails/test_apply_guardrail_e2e.py | 7 ++ .../guardrails/test_bedrock_guardrail_e2e.py | 44 +++++++ ...test_block_code_execution_guardrail_e2e.py | 11 +- .../guardrails/test_guardrail_dispatch_e2e.py | 9 ++ ...test_guardrail_information_response_e2e.py | 23 +++- .../test_key_guardrail_image_edit_e2e.py | 10 ++ .../test_key_guardrail_video_e2e.py | 9 ++ ...t_openai_moderation_category_matrix_e2e.py | 40 ++++++- .../test_openai_moderation_guardrail_e2e.py | 20 ++++ .../test_policy_inherited_guardrail_e2e.py | 17 +++ .../guardrails/test_presidio_masking_e2e.py | 107 +++++++++++++++++ ...est_responses_pre_call_block_stream_e2e.py | 25 +++- .../test_streaming_guardrail_e2e.py | 9 ++ .../test_team_disable_global_guardrail_e2e.py | 17 +++ .../test_tool_permission_guardrail_e2e.py | 19 +++ tests/e2e/logging/datadog_reader.py | 5 + tests/e2e/logging/gcs_reader.py | 5 +- tests/e2e/logging/logging_client.py | 26 ++++ tests/e2e/logging/s3_reader.py | 5 + tests/e2e/logging/test_datadog_log_e2e.py | 79 +++++++++++- tests/e2e/logging/test_gcs_log_e2e.py | 9 ++ .../test_langsmith_batch_serialization_e2e.py | 9 ++ tests/e2e/logging/test_otel_trace_e2e.py | 113 +++++++++++++++++- ..._otel_v2_langfuse_generation_output_e2e.py | 110 +++++++++++++++-- .../test_prometheus_cardinality_e2e.py | 10 ++ .../logging/test_prometheus_queue_time_e2e.py | 10 ++ tests/e2e/logging/test_s3_log_e2e.py | 29 ++++- .../test_team_langfuse_callback_e2e.py | 9 ++ tests/e2e/logging/test_weave_log_e2e.py | 22 +++- tests/e2e/logging/weave_reader.py | 7 +- 31 files changed, 821 insertions(+), 30 deletions(-) diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 7772a6a1e85..b193896bf6c 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -11,6 +11,7 @@ from typing import Final, Literal from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, SLOW_PROVIDER_TIMEOUT_SECONDS, settle_propagation, unique_marker from e2e_http import NoBody, Result, StreamingResponse, Success, unwrap +from e2e_metadata import step from lifecycle import ResourceManager from models import ( AnthropicMessagesBody, @@ -35,7 +36,9 @@ from models import ( VideoCreateResponse, ) from proxy_client import ProxyClient -from pydantic import BaseModel +from pydantic import BaseModel, Field + +GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash" GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"] PiiEntity = Literal["EMAIL_ADDRESS", "PHONE_NUMBER", "PERSON", "CREDIT_CARD", "US_SSN"] @@ -62,14 +65,14 @@ class BedrockGuardrailParamsBody(GuardrailParamsBase): guardrail: Literal["bedrock"] = "bedrock" guardrailIdentifier: str guardrailVersion: str - aws_access_key_id: str | None = None - aws_secret_access_key: str | None = None + aws_access_key_id: str | None = Field(default=None, repr=False) + aws_secret_access_key: str | None = Field(default=None, repr=False) aws_region_name: str | None = None class OpenAIModerationParamsBody(GuardrailParamsBase): guardrail: Literal["openai_moderation"] = "openai_moderation" - api_key: str | None = None + api_key: str | None = Field(default=None, repr=False) model: str | None = None @@ -193,6 +196,7 @@ class _ResponsesGuardrailBody(BaseModel): class GuardrailsClient: proxy: ProxyClient + @step("Register the content filter guardrail {name} that blocks prompts containing {blocked_keyword}") def create_content_filter_guardrail(self, name: str, blocked_keyword: str, *, default_on: bool = True) -> str: return self.register( name, @@ -203,6 +207,7 @@ class GuardrailsClient: ), ) + @step("Register the Bedrock guardrail {name}") def create_bedrock_guardrail( self, name: str, @@ -235,7 +240,7 @@ class GuardrailsClient: resources: ResourceManager, prefix: str = "e2e-guard-backend", *, - backend: str = "gemini/gemini-2.5-flash", + backend: str = GUARDRAIL_BACKEND, api_key: str = "os.environ/GEMINI_API_KEY", ) -> str: """Register a chat deployment for a guardrail test to run against @@ -250,6 +255,7 @@ class GuardrailsClient: resources.defer(lambda: self.proxy.delete_model(model_id)) return model_name + @step("Register the {params.guardrail} guardrail {name} with mode {params.mode}") def register(self, name: str, params: GuardrailParamsBody) -> str: """Register any guardrail via POST /guardrails and return its id, once every replica can be expected to serve it. New built-ins register with @@ -274,6 +280,7 @@ class GuardrailsClient: settle_propagation(time.monotonic()) return guardrail_id + @step("Delete the guardrail") def delete_guardrail(self, guardrail_id: str) -> None: _ = self.proxy.transport.delete( f"/guardrails/{guardrail_id}", @@ -282,6 +289,7 @@ class GuardrailsClient: response_type=NoBody, ) + @step("Create the guardrail policy {body.policy_name} that adds the guardrails {body.guardrails_add}") def create_policy(self, body: PolicyCreateBody) -> str: """Create a policy via POST /policies and return its name once every replica can be expected to serve it (policies reach the data plane on the periodic @@ -297,6 +305,7 @@ class GuardrailsClient: settle_propagation(time.monotonic()) return created.policy_name + @step("Delete every version of the guardrail policy {policy_name}") def delete_policy(self, policy_name: str) -> None: _ = self.proxy.transport.delete( f"/policies/name/{policy_name}/all-versions", @@ -305,6 +314,7 @@ class GuardrailsClient: response_type=NoBody, ) + @step("Attach the guardrail policy {policy_name} to requests tagged {tags}") def attach_policy_to_tags(self, policy_name: str, tags: list[str]) -> str: attachment_id = unwrap( self.proxy.transport.post( @@ -317,6 +327,7 @@ class GuardrailsClient: settle_propagation(time.monotonic()) return attachment_id + @step("Delete the guardrail policy attachment") def delete_policy_attachment(self, attachment_id: str) -> None: _ = self.proxy.transport.delete( f"/policies/attachments/{attachment_id}", @@ -325,6 +336,7 @@ class GuardrailsClient: response_type=NoBody, ) + @step("Create the team {alias} opted out of global guardrails and wait until /team/info returns it") def create_team_opted_out_of_global_guardrails(self, alias: str) -> str: team_id = unwrap( self.proxy.transport.post( @@ -340,6 +352,7 @@ class GuardrailsClient: self._await_team(team_id) return team_id + @step("Delete the team") def delete_team(self, team_id: str) -> None: _ = self.proxy.transport.post( "/team/delete", @@ -348,9 +361,11 @@ class GuardrailsClient: response_type=NoBody, ) + @step("Generate a virtual key in the team") def create_key_in_team(self, team_id: str) -> str: return self.proxy.generate_key(KeyGenerateBody(team_id=team_id, user_id="e2e-guardrails-user")) + @step("Generate a virtual key with the guardrails {guardrails}") def create_key_with_guardrails(self, resources: ResourceManager, guardrails: list[str]) -> str: key = self.proxy.generate_key( KeyGenerateBody(user_id="e2e-guardrails-user", metadata=KeyMetadata(guardrails=guardrails)) @@ -358,6 +373,7 @@ class GuardrailsClient: resources.defer(lambda: self.proxy.delete_key(key)) return key + @step("Send a /v1/videos request to {model}") def create_video(self, key: str, model: str, prompt: str) -> Result[VideoCreateResponse]: return self.proxy.transport.post( "/v1/videos", @@ -366,6 +382,7 @@ class GuardrailsClient: response_type=VideoCreateResponse, ) + @step("Send a /v1/images/edits request to {model}") def edit_image(self, key: str, model: str, prompt: str, image: bytes) -> Result[ImageGenerationResponse]: return self.proxy.transport.upload( "/v1/images/edits", @@ -379,6 +396,7 @@ class GuardrailsClient: timeout=SLOW_PROVIDER_TIMEOUT_SECONDS, ) + @step("Send a /chat/completions request to {model}") def chat( self, key: str, @@ -407,6 +425,7 @@ class GuardrailsClient: ), ) + @step("Send a /chat/completions request to {model}") def chat_raw( self, key: str, @@ -438,6 +457,7 @@ class GuardrailsClient: ), ) + @step("Send a streaming /chat/completions request to {model}") def chat_stream_raw( self, key: str, @@ -462,6 +482,7 @@ class GuardrailsClient: ), ) + @step("Send a /v1/messages request to {model}") def messages( self, key: str, @@ -481,6 +502,7 @@ class GuardrailsClient: ), ) + @step("Send a /v1/messages request to {model}") def messages_raw( self, key: str, @@ -501,6 +523,7 @@ class GuardrailsClient: ), ) + @step("Send a streaming /v1/messages request to {model}") def messages_stream_raw( self, key: str, @@ -521,6 +544,7 @@ class GuardrailsClient: ), ) + @step("Send a /v1/responses request to {model}") def responses( self, key: str, @@ -535,6 +559,7 @@ class GuardrailsClient: json=_ResponsesGuardrailBody(model=model, input=text, guardrails=guardrails), ) + @step("Send a streaming /v1/responses request to {model}") def responses_stream_raw( self, key: str, @@ -553,6 +578,7 @@ class GuardrailsClient: stream=True, ) + @step("Apply the guardrail {name} to a piece of text with /guardrails/apply_guardrail") def apply_guardrail(self, key: str, *, name: str, text: str) -> Result[ApplyGuardrailResponse]: return self.proxy.transport.post( "/guardrails/apply_guardrail", diff --git a/tests/e2e/guardrails/test_apply_guardrail_e2e.py b/tests/e2e/guardrails/test_apply_guardrail_e2e.py index ee691db22da..7a4db80e379 100644 --- a/tests/e2e/guardrails/test_apply_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_apply_guardrail_e2e.py @@ -11,6 +11,7 @@ import pytest from e2e_config import MASTER_KEY, unique_marker from e2e_http import Success, UnauthorizedError, UnknownApiError +from e2e_metadata import Domain, Route, Subject, meta from guardrails_client import GuardrailsClient from lifecycle import ResourceManager @@ -23,6 +24,12 @@ class TestApplyGuardrailEndpoint: "guardrail.litellm_content_filter.apply_endpoint.allows", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.GUARDRAILS, + ) + ) def test_apply_guardrail_blocks_banned_and_allows_clean( self, client: GuardrailsClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py b/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py index aeffec24c61..fac15645f83 100644 --- a/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py @@ -22,6 +22,7 @@ from typing import Final import pytest from e2e_config import unique_marker from e2e_http import StreamingResponse, UnknownApiError +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from guardrails_client import ( BedrockGuardrailParamsBody, GuardrailsClient, @@ -59,6 +60,15 @@ class TestBedrockGuardrail: "guardrail.bedrock.pre_call.blocks", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_pre_call_blocks_harmful_prompt( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -96,6 +106,14 @@ class TestBedrockGuardrail: "guardrail.bedrock.post_call.blocks", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_post_call_blocks_denied_model_output( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -138,6 +156,15 @@ class TestBedrockGuardrail: pytest.fail(f"bedrock post_call guardrail did not block denied model output; got {result}") @pytest.mark.covers("guardrail.bedrock.pre_call.blocks", exercised_on=["messages"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.MESSAGES, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_pre_call_blocks_on_messages( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -149,6 +176,15 @@ class TestBedrockGuardrail: _assert_policy_block(result, "/v1/messages") @pytest.mark.covers("guardrail.bedrock.pre_call.blocks", exercised_on=["responses"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.RESPONSES, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_bedrock_pre_call_blocks_on_responses( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -160,6 +196,14 @@ class TestBedrockGuardrail: _assert_policy_block(result, "/v1/responses") @pytest.mark.covers("guardrail.bedrock.post_call.blocks", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.STREAM, + ) + ) def test_bedrock_post_call_blocks_denied_streamed_output_and_passes_clean_streams( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py b/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py index 7cf4c195424..7cda38d9aa4 100644 --- a/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py @@ -20,7 +20,8 @@ import pytest from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker from e2e_http import unwrap -from guardrails_client import BlockCodeExecutionParamsBody, GuardrailsClient +from e2e_metadata import Domain, Mode, Provider, Subject, meta +from guardrails_client import GUARDRAIL_BACKEND, BlockCodeExecutionParamsBody, GuardrailsClient from lifecycle import ResourceManager from models import ChatResponse @@ -45,6 +46,14 @@ class TestBlockCodeExecutionGuardrail: "guardrail.block_code_execution.pre_call.blocks", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(GUARDRAIL_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_blocks_execution_request_but_allows_explanation( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/guardrails/test_guardrail_dispatch_e2e.py b/tests/e2e/guardrails/test_guardrail_dispatch_e2e.py index 793974ccdb1..c8d295ee459 100644 --- a/tests/e2e/guardrails/test_guardrail_dispatch_e2e.py +++ b/tests/e2e/guardrails/test_guardrail_dispatch_e2e.py @@ -11,6 +11,7 @@ from __future__ import annotations import pytest from e2e_config import unique_marker from e2e_http import UnknownApiError, ValidationError +from e2e_metadata import Domain, Mode, Provider, Subject, meta from guardrails_client import GuardrailsClient pytestmark = pytest.mark.e2e @@ -28,6 +29,14 @@ MODEL = "gemini-2.5-flash" "guardrail.dispatch.pre_call.rejects_unknown_name", exercised_on=["chat_completions"], ) +@meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) +) def test_request_naming_an_unknown_guardrail_fails_closed(client: GuardrailsClient, scoped_key: str) -> None: result = client.chat(scoped_key, MODEL, "say hi", guardrails=[f"e2e-no-such-guardrail-{unique_marker()}"]) diff --git a/tests/e2e/guardrails/test_guardrail_information_response_e2e.py b/tests/e2e/guardrails/test_guardrail_information_response_e2e.py index 9701ea35819..e0475610e8f 100644 --- a/tests/e2e/guardrails/test_guardrail_information_response_e2e.py +++ b/tests/e2e/guardrails/test_guardrail_information_response_e2e.py @@ -9,6 +9,7 @@ import pytest from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Domain, Mode, Provider, Subject, meta from guardrails_client import ( BlockedWordBody, ContentFilterParamsBody, @@ -19,6 +20,8 @@ from models import ChatResponse, GuardrailInformationEntry pytestmark = pytest.mark.e2e +BACKEND_MODEL: Final = "openai/gpt-4.1-mini" + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS: Final = 40.0 GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS: Final = 5.0 @@ -60,6 +63,14 @@ class TestGuardrailInformationResponse: "guardrail.litellm_content_filter.pre_call.returns_guardrail_information", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.OPENAI,), + models=(BACKEND_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_flag_returns_guardrail_information_for_the_guardrail_that_ran( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -68,7 +79,7 @@ class TestGuardrailInformationResponse: model = client.create_backend_model( resources, prefix="e2e-guardrail-info-backend", - backend="openai/gpt-4.1-mini", + backend=BACKEND_MODEL, api_key="os.environ/OPENAI_API_KEY", ) deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS @@ -91,6 +102,14 @@ class TestGuardrailInformationResponse: ) time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.OPENAI,), + models=(BACKEND_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_without_flag_response_has_no_guardrail_information( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -99,7 +118,7 @@ class TestGuardrailInformationResponse: model = client.create_backend_model( resources, prefix="e2e-guardrail-info-backend", - backend="openai/gpt-4.1-mini", + backend=BACKEND_MODEL, api_key="os.environ/OPENAI_API_KEY", ) diff --git a/tests/e2e/guardrails/test_key_guardrail_image_edit_e2e.py b/tests/e2e/guardrails/test_key_guardrail_image_edit_e2e.py index 388a554cde7..153d5fd0aa7 100644 --- a/tests/e2e/guardrails/test_key_guardrail_image_edit_e2e.py +++ b/tests/e2e/guardrails/test_key_guardrail_image_edit_e2e.py @@ -6,6 +6,7 @@ from typing import Final import pytest from e2e_config import unique_marker from e2e_http import Success, UnknownApiError +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from guardrails_client import GuardrailsClient, poll_until_blocked from lifecycle import ResourceManager from models import LiteLLMParamsBody @@ -41,6 +42,15 @@ class TestKeyAttachedGuardrailOnImageEdits: "guardrail.litellm_content_filter.pre_call.blocks_image_edit", exercised_on=["images_edits"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.IMAGES, + providers=(Provider.GEMINI, Provider.OPENAI,), + models=(CHAT_MODEL, IMAGE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_key_attached_content_filter_blocks_banned_image_edit_prompt( self, client: GuardrailsClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/guardrails/test_key_guardrail_video_e2e.py b/tests/e2e/guardrails/test_key_guardrail_video_e2e.py index 5f318e141a5..233cbfb8881 100644 --- a/tests/e2e/guardrails/test_key_guardrail_video_e2e.py +++ b/tests/e2e/guardrails/test_key_guardrail_video_e2e.py @@ -3,6 +3,7 @@ from __future__ import annotations import pytest from e2e_config import unique_marker from e2e_http import Success, UnknownApiError +from e2e_metadata import Domain, Mode, Provider, Subject, meta from guardrails_client import GuardrailsClient, poll_until_blocked from lifecycle import ResourceManager from models import LiteLLMParamsBody @@ -38,6 +39,14 @@ class TestKeyAttachedGuardrailOnVideos: "guardrail.litellm_content_filter.pre_call.blocks_video", exercised_on=["videos"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI, Provider.VERTEX_AI,), + models=(CHAT_MODEL, VIDEO_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_key_attached_content_filter_blocks_banned_video_prompt( self, client: GuardrailsClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/guardrails/test_openai_moderation_category_matrix_e2e.py b/tests/e2e/guardrails/test_openai_moderation_category_matrix_e2e.py index 1f1af818290..10155cf96a5 100644 --- a/tests/e2e/guardrails/test_openai_moderation_category_matrix_e2e.py +++ b/tests/e2e/guardrails/test_openai_moderation_category_matrix_e2e.py @@ -7,15 +7,22 @@ body that names moderation; a refine-wrapper bypass must also be blocked. from __future__ import annotations +from typing import Final + import pytest from e2e_config import unique_marker from e2e_http import Result, UnknownApiError +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from guardrails_client import GuardrailsClient, OpenAIModerationParamsBody from lifecycle import ResourceManager from models import AnthropicMessagesResponse, ChatResponse pytestmark = pytest.mark.e2e +GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash" +ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5" +OPENAI_BACKEND: Final = "openai/gpt-4o-mini" + CATEGORY_PROMPTS: tuple[tuple[str, str], ...] = ( ( "violence", @@ -79,6 +86,15 @@ class TestOpenAIModerationCategoryMatrix: "guardrail.openai_moderations.pre_call.blocks", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(GEMINI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_chat_blocks_category( self, client: GuardrailsClient, @@ -89,7 +105,7 @@ class TestOpenAIModerationCategoryMatrix: client, resources, prefix="e2e-mod-cat-chat", - backend="gemini/gemini-2.5-flash", + backend=GEMINI_BACKEND, api_key="os.environ/GEMINI_API_KEY", ) for category, prompt in CATEGORY_PROMPTS: @@ -99,6 +115,15 @@ class TestOpenAIModerationCategoryMatrix: "guardrail.openai_moderations.pre_call.blocks", exercised_on=["messages"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_messages_blocks_category( self, client: GuardrailsClient, @@ -109,7 +134,7 @@ class TestOpenAIModerationCategoryMatrix: client, resources, prefix="e2e-mod-cat-msg", - backend="anthropic/claude-haiku-4-5", + backend=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY", ) for category, prompt in CATEGORY_PROMPTS: @@ -119,6 +144,15 @@ class TestOpenAIModerationCategoryMatrix: "guardrail.openai_moderations.pre_call.blocks", exercised_on=["responses"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_blocks_category( self, client: GuardrailsClient, @@ -129,7 +163,7 @@ class TestOpenAIModerationCategoryMatrix: client, resources, prefix="e2e-mod-cat-resp", - backend="openai/gpt-4o-mini", + backend=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY", ) for category, prompt in CATEGORY_PROMPTS: diff --git a/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py index 43deb279bc8..b9593feb2ed 100644 --- a/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py @@ -18,7 +18,9 @@ import pytest from e2e_config import unique_marker from e2e_http import UnknownApiError, unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from guardrails_client import ( + GUARDRAIL_BACKEND, GuardrailsClient, OpenAIModerationParamsBody, poll_until_blocked, @@ -37,6 +39,15 @@ class TestOpenAIModerationGuardrail: "guardrail.openai_moderations.pre_call.blocks", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(GUARDRAIL_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_moderation_blocks_flagged_input( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -76,6 +87,15 @@ class TestOpenAIModerationGuardrail: "guardrail.openai_moderations.pre_call.blocks", exercised_on=["messages"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.MESSAGES, + providers=(Provider.GEMINI,), + models=(GUARDRAIL_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_moderation_blocks_flagged_input_on_messages( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/guardrails/test_policy_inherited_guardrail_e2e.py b/tests/e2e/guardrails/test_policy_inherited_guardrail_e2e.py index 6298a1de038..8c8e86ebe7a 100644 --- a/tests/e2e/guardrails/test_policy_inherited_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_policy_inherited_guardrail_e2e.py @@ -16,6 +16,7 @@ from __future__ import annotations import pytest from e2e_config import CHEAP_OPENAI_MODEL, unique_marker from e2e_http import StreamingResponse +from e2e_metadata import Domain, Mode, Provider, Subject, meta from guardrails_client import ( GuardrailsClient, PolicyConditionBody, @@ -73,6 +74,14 @@ def _setup_child_policy_attached_to_tag( class TestPolicyInheritedGuardrail: + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.OPENAI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_child_condition_miss_still_applies_inherited_parent_guardrail( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -101,6 +110,14 @@ class TestPolicyInheritedGuardrail: f"the child's own guardrail must not run when its condition fails; got {outcome.headers}" ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.OPENAI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_child_condition_match_applies_child_and_inherited_parent_guardrails( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/guardrails/test_presidio_masking_e2e.py b/tests/e2e/guardrails/test_presidio_masking_e2e.py index 35eb4884ddf..9fc2c7d3ea2 100644 --- a/tests/e2e/guardrails/test_presidio_masking_e2e.py +++ b/tests/e2e/guardrails/test_presidio_masking_e2e.py @@ -40,6 +40,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter from e2e_config import unique_marker from e2e_http import Result, StreamingResponse, Success +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from guardrails_client import GuardrailMode, GuardrailsClient, PiiAction, PiiEntity, PresidioParamsBody from lifecycle import ResourceManager from models import ( @@ -248,6 +249,15 @@ class TestPresidioPreCallMasking: "guardrail.presidio.pre_call.masks", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_pre_call_masks_pii_on_chat_completions( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -267,6 +277,15 @@ class TestPresidioPreCallMasking: "guardrail.presidio.pre_call.masks", exercised_on=["messages"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.MESSAGES, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_pre_call_masks_pii_on_messages( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -312,6 +331,14 @@ class TestPresidioPostCallMasking: "guardrail.presidio.post_call.masks", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_post_call_masks_pii_in_model_output( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -393,6 +420,15 @@ class TestPresidioCreditCardOutputMasking: "guardrail.presidio.post_call.masks_generated_output", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_ui_default_scope_masks_a_card_number_the_model_generates_on_chat_completions( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -421,6 +457,15 @@ class TestPresidioCreditCardOutputMasking: "guardrail.presidio.post_call.masks_generated_output", exercised_on=["chat_completions_stream"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.STREAM, + ) + ) def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_chat_completions( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -453,6 +498,15 @@ class TestPresidioCreditCardOutputMasking: "guardrail.presidio.post_call.masks_generated_output", exercised_on=["anthropic_messages_stream"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.MESSAGES, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.STREAM, + ) + ) def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_anthropic_messages( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -571,6 +625,15 @@ class TestPresidioSpendLogStoresMaskedOutput: ) @pytest.mark.covers(_CELL, exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_spend_log_stores_masked_output_on_chat_completions( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -585,6 +648,15 @@ class TestPresidioSpendLogStoresMaskedOutput: ) @pytest.mark.covers(_CELL, exercised_on=["chat_completions_stream"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.STREAM, + ) + ) def test_spend_log_stores_masked_output_on_streaming_chat_completions( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -599,6 +671,15 @@ class TestPresidioSpendLogStoresMaskedOutput: ) @pytest.mark.covers(_CELL, exercised_on=["messages"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.MESSAGES, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_spend_log_stores_masked_output_on_anthropic_messages( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -613,6 +694,15 @@ class TestPresidioSpendLogStoresMaskedOutput: ) @pytest.mark.covers(_CELL, exercised_on=["anthropic_messages_stream"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.MESSAGES, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.STREAM, + ) + ) def test_spend_log_stores_masked_output_on_streaming_anthropic_messages( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -627,6 +717,15 @@ class TestPresidioSpendLogStoresMaskedOutput: ) @pytest.mark.covers(_CELL, exercised_on=["responses"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.RESPONSES, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_spend_log_stores_masked_output_on_responses( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -673,6 +772,14 @@ class TestPresidioSpendLogRecord: "guardrail.presidio.pre_call.logs_masked_entities", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_masking_run_is_recorded_on_the_spend_log( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/guardrails/test_responses_pre_call_block_stream_e2e.py b/tests/e2e/guardrails/test_responses_pre_call_block_stream_e2e.py index 93512e2a64c..cdf3098e85d 100644 --- a/tests/e2e/guardrails/test_responses_pre_call_block_stream_e2e.py +++ b/tests/e2e/guardrails/test_responses_pre_call_block_stream_e2e.py @@ -7,12 +7,15 @@ from typing import Final import pytest from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker from e2e_http import StreamingResponse +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from guardrails_client import CustomCodeParamsBody, GuardrailsClient from lifecycle import ResourceManager from pydantic import BaseModel, TypeAdapter pytestmark = pytest.mark.e2e +BACKEND_MODEL: Final = "openai/gpt-4.1-mini" + DENIAL: Final = "This model is not currently available. Please contact support if you think this is a mistake." CUSTOM_CODE: Final = f''' @@ -109,12 +112,21 @@ class TestResponsesPreCallBlock: return name @pytest.mark.covers("guardrail.custom_code.pre_call.blocks", exercised_on=["responses"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(BACKEND_MODEL,), + mode=Mode.STREAM, + ) + ) def test_stream_block_is_sse_with_completed_assistant_message( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: name: Final = self._register_block(client, resources) model: Final = client.create_backend_model( - resources, prefix="e2e-responses-block", backend="openai/gpt-4.1-mini", api_key="os.environ/OPENAI_API_KEY" + resources, prefix="e2e-responses-block", backend=BACKEND_MODEL, api_key="os.environ/OPENAI_API_KEY" ) result: Final = _poll_for_block( @@ -137,12 +149,21 @@ class TestResponsesPreCallBlock: _assert_blocked_response(completed[0].response) @pytest.mark.covers("guardrail.custom_code.pre_call.blocks", exercised_on=["responses"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(BACKEND_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_non_stream_block_is_schema_valid_json( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: name: Final = self._register_block(client, resources) model: Final = client.create_backend_model( - resources, prefix="e2e-responses-block", backend="openai/gpt-4.1-mini", api_key="os.environ/OPENAI_API_KEY" + resources, prefix="e2e-responses-block", backend=BACKEND_MODEL, api_key="os.environ/OPENAI_API_KEY" ) result: Final = _poll_for_block(lambda: client.responses(scoped_key, model, "say hi", guardrails=[name])) diff --git a/tests/e2e/guardrails/test_streaming_guardrail_e2e.py b/tests/e2e/guardrails/test_streaming_guardrail_e2e.py index 911ddf9304b..324dd4107a4 100644 --- a/tests/e2e/guardrails/test_streaming_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_streaming_guardrail_e2e.py @@ -22,6 +22,7 @@ import os import pytest from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Subject, meta from guardrails_client import ( BedrockGuardrailParamsBody, GuardrailsClient, @@ -39,6 +40,14 @@ class TestBedrockDuringCallStreaming: "guardrail.bedrock.during.blocks", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.STREAM, + ) + ) def test_during_call_blocks_stream_before_first_chunk( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py index db917d6ede9..0cc6106abfd 100644 --- a/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py @@ -14,6 +14,7 @@ import pytest from e2e_config import unique_marker from e2e_http import UnknownApiError, unwrap +from e2e_metadata import Domain, Mode, Provider, Subject, meta from guardrails_client import GuardrailsClient from lifecycle import ResourceManager @@ -59,6 +60,14 @@ class TestTeamDisableGlobalGuardrail: "guardrail.litellm_content_filter.pre_call.blocks", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_global_guardrail_blocks_key_without_team_opt_out( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -72,6 +81,14 @@ class TestTeamDisableGlobalGuardrail: "guardrail.litellm_content_filter.pre_call.allows", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_team_with_disable_flag_bypasses_global_guardrail( self, client: GuardrailsClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py b/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py index 8d1047e53c7..d5a6a9d9866 100644 --- a/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py @@ -25,6 +25,7 @@ import pytest from e2e_config import unique_marker from e2e_http import StreamingResponse, UnknownApiError +from e2e_metadata import Capability, Domain, Mode, Provider, Subject, meta from guardrails_client import ( GuardrailsClient, ToolPermissionParamsBody, @@ -101,6 +102,15 @@ def _tool_call_names(response: ChatResponse) -> tuple[str, ...]: class TestToolPermissionPreCall: @pytest.mark.covers("guardrail.tool_permission.pre_call.blocks", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_pre_call_blocks_tool_outside_the_allow_list( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -135,6 +145,15 @@ class TestToolPermissionPreCall: pytest.fail(f"tool_permission let a tool outside the allow-list through; got {result}") @pytest.mark.covers("guardrail.tool_permission.pre_call.allows", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.GUARDRAILS, + providers=(Provider.GEMINI,), + models=(MODEL,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_pre_call_allows_permitted_tool( self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/logging/datadog_reader.py b/tests/e2e/logging/datadog_reader.py index 368c20cb6aa..61b1b5a1536 100644 --- a/tests/e2e/logging/datadog_reader.py +++ b/tests/e2e/logging/datadog_reader.py @@ -32,6 +32,7 @@ from e2e_config import ( POLL_TIMEOUT, ) from e2e_http import URL, Headers, StreamingResponse, send +from e2e_metadata import step type SearchCall = Callable[[str, float], StreamingResponse] @@ -115,6 +116,7 @@ class DdLogsReader: sleep: Callable[[float], None] = field(default=time.sleep, repr=False) jitter: Callable[[], float] = field(default=random.random, repr=False) + @step("Search DataDog for logs carrying the marker {marker}") def events_for_marker(self, marker: str) -> list[DdLogEvent]: """Every ingested event whose attributes carry the marker. DataDog consumes the shipped JSON message into ``attributes`` and leaves the @@ -124,6 +126,7 @@ class DdLogsReader: it).""" return self.events_for_query(f"*:*{marker}*") + @step("Search DataDog for logs matching {query}") def events_for_query(self, query: str) -> list[DdLogEvent]: """Every ingested event the search query matches (failure payloads carry no prompt to mark, so failure scenarios query indexed attributes @@ -156,10 +159,12 @@ class DdLogsReader: timeout=timeout, ) + @step("Wait for DataDog to ingest logs carrying the marker {marker}, then watch for duplicates") def poll_events_for_marker(self, marker: str) -> list[DdLogEvent]: """``poll_events_for_query`` over the every-attribute marker scan.""" return self.poll_events_for_query(f"*:*{marker}*") + @step("Wait for DataDog to ingest logs matching {query}, then watch for duplicates") def poll_events_for_query(self, query: str) -> list[DdLogEvent]: """Poll until at least one matching event is searchable (the callback flushes in periodic batches and DataDog ingestion adds seconds of lag), diff --git a/tests/e2e/logging/gcs_reader.py b/tests/e2e/logging/gcs_reader.py index 60622c121ac..8abde23e221 100644 --- a/tests/e2e/logging/gcs_reader.py +++ b/tests/e2e/logging/gcs_reader.py @@ -30,6 +30,7 @@ from pydantic import BaseModel, ConfigDict, Field from e2e_config import POLL_INTERVAL, POLL_TIMEOUT from e2e_http import URL, Headers, probe +from e2e_metadata import step _GCS_API = "https://storage.googleapis.com" #: Tolerance for clock skew between this host and GCS object timestamps. @@ -44,7 +45,7 @@ class _ServiceAccount(BaseModel): model_config = ConfigDict(extra="ignore") client_email: str - private_key: str + private_key: str = Field(repr=False) class _GcsAuthHeaders(Headers): @@ -146,6 +147,7 @@ class GcsLogReader: ) return result.body + @step("Read the GCS bucket's log objects for the response {response_id}") def records_for_response_id(self, response_id: str, *, since: datetime) -> list[GcsLogRecord]: """Every payload written for ``response_id``: the direct ``{date}/{response_id}`` object plus any hit inside batch NDJSON @@ -168,6 +170,7 @@ class GcsLogReader: ) return records + @step("Wait for the log of the response {response_id} to land in the GCS bucket, then watch for duplicates") def poll_records_for_response_id(self, response_id: str, *, since: datetime) -> list[GcsLogRecord]: """Poll until the payload is readable (the gcs_bucket callback flushes on a ~20s timer), then keep re-reading for GCS_SETTLE_SECONDS - past a diff --git a/tests/e2e/logging/logging_client.py b/tests/e2e/logging/logging_client.py index c2f987ea33d..d522c01c054 100644 --- a/tests/e2e/logging/logging_client.py +++ b/tests/e2e/logging/logging_client.py @@ -24,6 +24,7 @@ import pytest from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, settle_propagation +from e2e_metadata import step from proxy_client import ProxyClient from e2e_http import ( URL, @@ -326,6 +327,7 @@ def observation_has_guardrail(obs: LangfuseObservation, *, guardrail_name: str) class LoggingClient: proxy: ProxyClient + @step("Generate a virtual key named {alias} with models: {models}") def key_with_alias( self, alias: str, @@ -347,9 +349,11 @@ class LoggingClient: ) ) + @step("Delete the virtual key") def delete_key(self, key: str) -> None: self.proxy.delete_key(key) + @step("Create the team {alias} with models: {models}") def create_team( self, alias: str, @@ -370,6 +374,7 @@ class LoggingClient: ) ).team_id + @step("Delete the team") def delete_team(self, team_id: str) -> None: _ = self.proxy.transport.post( "/team/delete", @@ -378,6 +383,7 @@ class LoggingClient: response_type=NoBody, ) + @step("Create the internal user {user_email}") def create_user(self, *, user_email: str, user_id: str | None = None) -> str: return unwrap( self.proxy.transport.post( @@ -392,6 +398,7 @@ class LoggingClient: ) ).user_id + @step("Delete the internal user") def delete_user(self, user_id: str) -> None: _ = self.proxy.transport.post( "/user/delete", @@ -400,6 +407,7 @@ class LoggingClient: response_type=NoBody, ) + @step("Create the organization {alias} with models: {models}") def create_org(self, alias: str, *, models: list[str]) -> str: return unwrap( self.proxy.transport.post( @@ -410,6 +418,7 @@ class LoggingClient: ) ).organization_id + @step("Delete the organization") def delete_org(self, organization_id: str) -> None: _ = self.proxy.transport.delete( "/organization/delete", @@ -418,6 +427,7 @@ class LoggingClient: response_type=NoBody, ) + @step("Add a Langfuse OTel logging callback for {callback_type} events to the team") def add_team_langfuse_callback( self, team_id: str, @@ -441,6 +451,7 @@ class LoggingClient: f"POST /team/{team_id}/callback must return status=success; got {response.status!r}" ) + @step("Create the tool_permission guardrail {name} that allows only the tool {allowed_tool}") def create_tool_permission_guardrail(self, name: str, *, allowed_tool: str) -> str: """Register a tool_permission guardrail that allows one tool and denies the rest.""" response = unwrap( @@ -474,6 +485,7 @@ class LoggingClient: settle_propagation(time.monotonic()) return guardrail_id + @step("Delete the guardrail") def delete_guardrail(self, guardrail_id: str) -> None: _ = self.proxy.transport.delete( f"/guardrails/{guardrail_id}", @@ -482,12 +494,15 @@ class LoggingClient: response_type=NoBody, ) + @step("Add a deployment named {model_name} that calls {litellm_params.model}") def create_model(self, model_name: str, litellm_params: LiteLLMParamsBody) -> str: return self.proxy.create_model(model_name, litellm_params) + @step("Delete the deployment") def delete_model(self, model_id: str) -> None: self.proxy.delete_model(model_id) + @step('Send a /chat/completions request to {model} with the prompt "{text}"') def chat(self, key: str, model: str, text: str) -> ChatResponse: return unwrap( self.proxy.chat( @@ -500,6 +515,7 @@ class LoggingClient: ) ) + @step('Send a /chat/completions request to {model} with stream={stream} and the prompt "{text}"') def chat_raw( self, key: str, @@ -529,6 +545,7 @@ class LoggingClient: json=body, ) + @step('Send a /v1/messages request to {model} with stream={stream} and the prompt "{text}"') def messages_raw( self, key: str, model: str, text: str, *, max_tokens: int = 16, stream: bool = False ) -> StreamingResponse: @@ -545,6 +562,7 @@ class LoggingClient: return self.proxy.transport.stream("/v1/messages", headers=self.proxy.transport.bearer(key), json=body) return self.proxy.transport.send("/v1/messages", headers=self.proxy.transport.bearer(key), json=body) + @step('Send a /v1/responses request to {model} with stream={stream} and the prompt "{text}"') def responses_raw( self, key: str, model: str, text: str, *, max_output_tokens: int = 64, stream: bool = False ) -> StreamingResponse: @@ -560,9 +578,11 @@ class LoggingClient: return self.proxy.transport.stream("/v1/responses", headers=self.proxy.transport.bearer(key), json=body) return self.proxy.transport.send("/v1/responses", headers=self.proxy.transport.bearer(key), json=body) + @step("Scrape the Prometheus metrics from /metrics") def scrape_metrics(self) -> str: return self.proxy.probe("/metrics", params=NoBody()).body + @step("Wait for the key's spend log in /spend/logs") def poll_proxy_spend_for_key( self, key: str, @@ -590,6 +610,7 @@ class LoggingClient: return row return None + @step("List observations from Langfuse") def list_langfuse_observations( self, creds: LangfuseCreds, @@ -616,6 +637,7 @@ class LoggingClient: case _: return [] + @step("Look up the Langfuse generation for the key {key_alias}") def find_langfuse_observation( self, creds: LangfuseCreds, @@ -634,6 +656,7 @@ class LoggingClient: return obs return None + @step("Wait for the Langfuse generation for the key {key_alias}") def poll_langfuse_observation( self, creds: LangfuseCreds, @@ -653,6 +676,7 @@ class LoggingClient: time.sleep(POLL_INTERVAL) return last + @step("Wait for the OTel v2 Langfuse generation for the key {key_alias}") def poll_langfuse_generation( self, creds: LangfuseCreds, *, key_alias: str, from_start_time: str ) -> LangfuseObservation | None: @@ -665,6 +689,7 @@ class LoggingClient: time.sleep(POLL_INTERVAL) return None + @step("Wait for the Langfuse trace of the key {key_alias} and every observation in it") def poll_langfuse_trace_observations( self, creds: LangfuseCreds, @@ -698,6 +723,7 @@ def build_logging_client(proxy: ProxyClient) -> LoggingClient: return LoggingClient(proxy=proxy) +@step("Read the proxy's callback list from /health/readiness/details") def readiness_details_body(client: LoggingClient) -> str: """/health/readiness/details, tolerating the 503 it serves while the ephemeral stack's DB leg blips: the recorded state the logging suites check diff --git a/tests/e2e/logging/s3_reader.py b/tests/e2e/logging/s3_reader.py index d605dec6096..572086c2695 100644 --- a/tests/e2e/logging/s3_reader.py +++ b/tests/e2e/logging/s3_reader.py @@ -24,6 +24,7 @@ import pytest from pydantic import BaseModel, ConfigDict from e2e_config import POLL_INTERVAL, POLL_TIMEOUT +from e2e_metadata import step if TYPE_CHECKING: from types_boto3_s3.client import S3Client @@ -53,17 +54,21 @@ class S3LogReader: bucket: str client: S3Client + @step("List the log objects in the S3 bucket under {prefix}") def list_keys(self, prefix: str) -> list[str]: response = self.client.list_objects_v2(Bucket=self.bucket, Prefix=prefix) return [obj["Key"] for obj in response.get("Contents", []) if "Key" in obj] + @step("Download a log object from the S3 bucket") def read_record(self, key: str) -> S3LogRecord: body = self.client.get_object(Bucket=self.bucket, Key=key)["Body"].read() return S3LogRecord.model_validate_json(body) + @step("Read the log objects in the S3 bucket under {prefix}") def records_matching(self, *, prefix: str, predicate: Callable[[S3LogRecord], bool]) -> list[S3LogRecord]: return [record for record in map(self.read_record, self.list_keys(prefix)) if predicate(record)] + @step("Wait for the request's log object to land in the S3 bucket under {prefix}, then watch for duplicates") def poll_records(self, *, prefix: str, predicate: Callable[[S3LogRecord], bool]) -> list[S3LogRecord]: """Poll until at least one matching object is listed (the s3_v2 callback flushes on a ~10s timer), then keep re-reading for diff --git a/tests/e2e/logging/test_datadog_log_e2e.py b/tests/e2e/logging/test_datadog_log_e2e.py index 2d57181c4bf..108985a3953 100644 --- a/tests/e2e/logging/test_datadog_log_e2e.py +++ b/tests/e2e/logging/test_datadog_log_e2e.py @@ -20,10 +20,12 @@ from __future__ import annotations import math import time +from typing import Final import pytest from datadog_reader import DdLogEvent, DdLogsReader from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from logging_client import INVALID_UPSTREAM_API_KEY, LoggingClient, first_ok, readiness_details_body from models import ChatMessage, LiteLLMParamsBody, ReliabilityChatBody, RouterSettingsOverride @@ -33,6 +35,7 @@ pytestmark = pytest.mark.e2e #: The active DataDog callback's name in /health/readiness/details success_callbacks. DD_LOGGER_NAME = "DataDogLogger" +FAILING_BACKEND_MODEL: Final = "anthropic/claude-haiku-4-5" class _DdMessagePayload(BaseModel): @@ -109,6 +112,15 @@ def _assert_exactly_one_event( class TestDataDogLogDelivery: @pytest.mark.covers("logging.datadog.success.exports_metric", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_chat_completions_emits_one_log_event( self, client: LoggingClient, dd_logs: DdLogsReader, resources: ResourceManager ) -> None: @@ -134,6 +146,15 @@ class TestDataDogLogDelivery: ) @pytest.mark.covers("logging.datadog.success.exports_metric", exercised_on=["messages"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_messages_emits_one_log_event( self, client: LoggingClient, dd_logs: DdLogsReader, resources: ResourceManager ) -> None: @@ -161,6 +182,15 @@ class TestDataDogLogDelivery: ) @pytest.mark.covers("logging.datadog.success.exports_metric", exercised_on=["responses"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_emits_one_log_event( self, client: LoggingClient, dd_logs: DdLogsReader, resources: ResourceManager ) -> None: @@ -186,6 +216,15 @@ class TestDataDogLogDelivery: ) @pytest.mark.covers("logging.datadog.stream.exports_metric", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + mode=Mode.STREAM, + ) + ) def test_chat_completions_stream_emits_one_log_event( self, client: LoggingClient, dd_logs: DdLogsReader, resources: ResourceManager ) -> None: @@ -232,6 +271,15 @@ class TestDataDogLogDelivery: ) @pytest.mark.covers("logging.datadog.stream.exports_metric", exercised_on=["messages"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + mode=Mode.STREAM, + ) + ) def test_messages_stream_emits_one_log_event( self, client: LoggingClient, dd_logs: DdLogsReader, resources: ResourceManager ) -> None: @@ -276,6 +324,15 @@ class TestDataDogLogDelivery: ) @pytest.mark.covers("logging.datadog.stream.exports_metric", exercised_on=["responses"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.STREAM, + ) + ) def test_responses_stream_emits_one_log_event( self, client: LoggingClient, dd_logs: DdLogsReader, resources: ResourceManager ) -> None: @@ -347,6 +404,15 @@ def _assert_exactly_one_failure_event(events: list[DdLogEvent], *, model_group: class TestDataDogFailureDelivery: @pytest.mark.covers("logging.datadog.failure.exports_metric", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(FAILING_BACKEND_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_failed_chat_completions_emits_one_error_event( self, client: LoggingClient, dd_logs: DdLogsReader, resources: ResourceManager ) -> None: @@ -366,7 +432,7 @@ class TestDataDogFailureDelivery: model_name = f"dd-err-{unique_marker()}" model_id = client.create_model( model_name, - LiteLLMParamsBody(model="anthropic/claude-haiku-4-5", api_key=INVALID_UPSTREAM_API_KEY), + LiteLLMParamsBody(model=FAILING_BACKEND_MODEL, api_key=INVALID_UPSTREAM_API_KEY), ) resources.defer(lambda: client.delete_model(model_id)) key = client.key_with_alias(f"dd-err-key-{unique_marker()}", models=[model_name]) @@ -399,6 +465,15 @@ class TestDataDogFailureDelivery: ) @pytest.mark.covers("logging.datadog.stream_failure.exports_metric", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(FAILING_BACKEND_MODEL,), + mode=Mode.STREAM, + ) + ) def test_failed_chat_completions_stream_emits_one_error_event( self, client: LoggingClient, dd_logs: DdLogsReader, resources: ResourceManager ) -> None: @@ -420,7 +495,7 @@ class TestDataDogFailureDelivery: model_id = client.create_model( model_name, LiteLLMParamsBody( - model="anthropic/claude-haiku-4-5", + model=FAILING_BACKEND_MODEL, api_key=INVALID_UPSTREAM_API_KEY, api_base="http://localhost:1", ), diff --git a/tests/e2e/logging/test_gcs_log_e2e.py b/tests/e2e/logging/test_gcs_log_e2e.py index 17ad1507049..259735e6cad 100644 --- a/tests/e2e/logging/test_gcs_log_e2e.py +++ b/tests/e2e/logging/test_gcs_log_e2e.py @@ -22,6 +22,7 @@ import math import pytest from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker +from e2e_metadata import Domain, Mode, Provider, Subject, meta from gcs_reader import GcsLogReader, build_gcs_reader, utc_now from lifecycle import ResourceManager from logging_client import LoggingClient, completion_response_id, first_ok, readiness_details_body @@ -52,6 +53,14 @@ def _assert_gcs_configured(client: LoggingClient) -> None: class TestGcsLogDelivery: @pytest.mark.covers("logging.gcs_bucket.success.writes_object", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_chat_completions_writes_one_success_record( self, client: LoggingClient, gcs_logs: GcsLogReader, resources: ResourceManager ) -> None: diff --git a/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py b/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py index 874b2b6a045..e2dadf9e696 100644 --- a/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py +++ b/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py @@ -21,6 +21,7 @@ from typing import Final import pytest from e2e_config import CHEAP_OPENAI_MODEL, POLL_INTERVAL, POLL_TIMEOUT, unique_marker from e2e_http import Headers, Success, get_external +from e2e_metadata import Domain, Mode, Provider, Subject, meta from pydantic import BaseModel, ConfigDict, Field, JsonValue import litellm @@ -90,6 +91,14 @@ def _poll_run(creds: LangsmithCreds, run_id: uuid.UUID) -> LangsmithRun: class TestLangsmithBatchSerialization: @pytest.mark.asyncio @pytest.mark.covers("logging.langsmith.success.serializes_non_native_metadata") + @meta( + Subject( + domain=Domain.OBSERVABILITY, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) async def test_non_json_native_metadata_reaches_langsmith(self) -> None: creds: Final = load_langsmith_creds() logger: Final = LangsmithLogger( diff --git a/tests/e2e/logging/test_otel_trace_e2e.py b/tests/e2e/logging/test_otel_trace_e2e.py index 4902b0703c3..01eaf5ce80c 100644 --- a/tests/e2e/logging/test_otel_trace_e2e.py +++ b/tests/e2e/logging/test_otel_trace_e2e.py @@ -25,6 +25,7 @@ from typing import Final import pytest from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, OTEL_EXPORTER_ENDPOINT, unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from logging_client import INVALID_UPSTREAM_API_KEY, LoggingClient, first_ok, readiness_details_body from models import LiteLLMParamsBody @@ -34,6 +35,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError pytestmark = pytest.mark.e2e MODEL = CHEAP_ANTHROPIC_MODEL +FAILING_BACKEND_MODEL: Final = "anthropic/claude-haiku-4-5" DB_SPAN_PREFIX = "postgres." #: The active OTEL v2 logger's name in /health/readiness/details success_callbacks. OTEL_V2_LOGGER_NAME = "OpenTelemetryV2" @@ -279,6 +281,15 @@ def _assert_error_span_contract(span: JaegerSpan) -> None: class TestOtelTraceCompleteness: @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_chat_completions_exports_complete_trace( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -312,6 +323,14 @@ class TestOtelTraceCompleteness: _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + providers=(Provider.ANTHROPIC,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) @pytest.mark.otel_tls def test_otel_export_over_tls_with_internal_ca_reaches_destination( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager @@ -337,6 +356,15 @@ class TestOtelTraceCompleteness: _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_messages_exports_complete_trace( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -366,6 +394,15 @@ class TestOtelTraceCompleteness: _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_exports_complete_trace( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -396,6 +433,15 @@ class TestOtelTraceCompleteness: _assert_complete_trace(traces, route=route, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(MODEL,), + mode=Mode.STREAM, + ) + ) def test_chat_completions_stream_exports_complete_trace( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -444,6 +490,15 @@ class TestOtelTraceCompleteness: ) @pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["messages"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(MODEL,), + mode=Mode.STREAM, + ) + ) def test_messages_stream_exports_complete_trace( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -492,6 +547,15 @@ class TestOtelTraceCompleteness: ) @pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["responses"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.STREAM, + ) + ) def test_responses_stream_exports_complete_trace( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -544,6 +608,15 @@ class TestOtelTraceCompleteness: ) @pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(MODEL,), + mode=Mode.STREAM, + ) + ) def test_chat_completions_stream_records_real_ttft( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -582,6 +655,15 @@ class TestOtelTraceCompleteness: _assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(MODEL,), + mode=Mode.STREAM, + ) + ) def test_messages_stream_records_real_ttft( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -620,6 +702,15 @@ class TestOtelTraceCompleteness: _assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.STREAM, + ) + ) def test_responses_stream_records_real_ttft( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -658,6 +749,15 @@ class TestOtelTraceCompleteness: _assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(FAILING_BACKEND_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_failed_chat_completions_error_span_attributes( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -677,7 +777,7 @@ class TestOtelTraceCompleteness: model_name = f"otel-err-{unique_marker()}" model_id = client.create_model( model_name, - LiteLLMParamsBody(model="anthropic/claude-haiku-4-5", api_key=INVALID_UPSTREAM_API_KEY), + LiteLLMParamsBody(model=FAILING_BACKEND_MODEL, api_key=INVALID_UPSTREAM_API_KEY), ) resources.defer(lambda: client.delete_model(model_id)) key = client.key_with_alias(f"otel-err-{unique_marker()}", models=[model_name]) @@ -711,6 +811,15 @@ class TestOtelTraceCompleteness: _assert_error_span_contract(genai) @pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["messages"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(FAILING_BACKEND_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_failed_messages_error_span_attributes( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -729,7 +838,7 @@ class TestOtelTraceCompleteness: model_name = f"otel-err-{unique_marker()}" model_id = client.create_model( model_name, - LiteLLMParamsBody(model="anthropic/claude-haiku-4-5", api_key=INVALID_UPSTREAM_API_KEY), + LiteLLMParamsBody(model=FAILING_BACKEND_MODEL, api_key=INVALID_UPSTREAM_API_KEY), ) resources.defer(lambda: client.delete_model(model_id)) key = client.key_with_alias(f"otel-err-{unique_marker()}", models=[model_name]) diff --git a/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py index 9e08cf21da0..852e10636d0 100644 --- a/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py +++ b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py @@ -24,6 +24,7 @@ from typing import Final import pytest from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from logging_client import LangfuseCreds, LangfuseObservation, LoggingClient, load_langfuse_creds from models import ( @@ -69,6 +70,13 @@ RED_SQUARE_PNG: Final = base64.b64decode( ) BOUNDED_OUTPUT_CHARS: Final = 1024 PLACEHOLDER_INPUT: Final = "default-message-value" +COMPLETION_BACKEND: Final = "openai/gpt-3.5-turbo-instruct" +IMAGE_BACKEND: Final = "openai/gpt-image-1-mini" +SPEECH_BACKEND: Final = "openai/gpt-4o-mini-tts" +TRANSCRIPTION_BACKEND: Final = "openai/gpt-4o-mini-transcribe" +MODERATION_BACKEND: Final = "openai/omni-moderation-latest" +MISTRAL_OCR_BACKEND: Final = "mistral/mistral-ocr-latest" +RERANK_BACKEND: Final = "cohere/rerank-v4.0-fast" class _OutputMessage(BaseModel): @@ -142,7 +150,7 @@ def _openai(model: str) -> LiteLLMParamsBody: def _mistral_ocr() -> LiteLLMParamsBody: - return LiteLLMParamsBody(model="mistral/mistral-ocr-latest", api_key="os.environ/MISTRAL_API_KEY") + return LiteLLMParamsBody(model=MISTRAL_OCR_BACKEND, api_key="os.environ/MISTRAL_API_KEY") def _langfuse_search_tool( @@ -171,10 +179,19 @@ def _langfuse_search_tool( class TestOtelV2LangfuseGenerationOutput: @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.COMPLETIONS, + providers=(Provider.OPENAI,), + models=(COMPLETION_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_completions_output_is_the_completion_text( self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager ) -> None: - model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-3.5-turbo-instruct")) + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai(COMPLETION_BACKEND)) started: Final = datetime.now(timezone.utc) response: Final = unwrap( client.proxy.transport.post( @@ -193,10 +210,19 @@ class TestOtelV2LangfuseGenerationOutput: ) @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["images_generations"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.IMAGES, + providers=(Provider.OPENAI,), + models=(IMAGE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_images_output_is_a_bounded_summary_without_base64( self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager ) -> None: - model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-image-1-mini")) + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai(IMAGE_BACKEND)) started: Final = datetime.now(timezone.utc) response: Final = unwrap( client.proxy.transport.post( @@ -220,10 +246,19 @@ class TestOtelV2LangfuseGenerationOutput: ) @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["audio_speech"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.AUDIO, + providers=(Provider.OPENAI,), + models=(SPEECH_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_speech_output_is_a_bounded_summary_without_audio_bytes( self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager ) -> None: - model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-4o-mini-tts")) + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai(SPEECH_BACKEND)) started: Final = datetime.now(timezone.utc) audio: Final = client.proxy.transport.stream_binary( "/v1/audio/speech", @@ -239,10 +274,19 @@ class TestOtelV2LangfuseGenerationOutput: ) @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["audio_transcriptions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.AUDIO, + providers=(Provider.OPENAI,), + models=(TRANSCRIPTION_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_transcription_output_is_the_transcript( self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager ) -> None: - model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-4o-mini-transcribe")) + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai(TRANSCRIPTION_BACKEND)) started: Final = datetime.now(timezone.utc) response: Final = unwrap( client.proxy.transport.upload( @@ -262,10 +306,19 @@ class TestOtelV2LangfuseGenerationOutput: assert transcript in output, f"generation output lacks the transcript {transcript!r}: {output!r}" @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["moderations"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.MODERATIONS, + providers=(Provider.OPENAI,), + models=(MODERATION_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_moderations_output_is_the_verdict( self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager ) -> None: - model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/omni-moderation-latest")) + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai(MODERATION_BACKEND)) started: Final = datetime.now(timezone.utc) response: Final = unwrap( client.proxy.transport.post( @@ -284,6 +337,15 @@ class TestOtelV2LangfuseGenerationOutput: ) @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["rerank"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.RERANK, + providers=(Provider.COHERE,), + models=(RERANK_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_rerank_output_is_the_ranked_indices_and_scores( self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager ) -> None: @@ -291,7 +353,7 @@ class TestOtelV2LangfuseGenerationOutput: client, langfuse_creds, resources, - LiteLLMParamsBody(model="cohere/rerank-v4.0-fast", api_key="os.environ/COHERE_API_KEY"), + LiteLLMParamsBody(model=RERANK_BACKEND, api_key="os.environ/COHERE_API_KEY"), ) query: Final = f"What is the capital of France? {unique_marker()}" started: Final = datetime.now(timezone.utc) @@ -317,6 +379,15 @@ class TestOtelV2LangfuseGenerationOutput: assert output == "\n\n".join(ranked), f"generation output is not the ranked indices and scores: {output!r}" @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["ocr"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.OCR, + providers=(Provider.MISTRAL,), + models=(MISTRAL_OCR_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_ocr_input_is_the_document_url( self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager ) -> None: @@ -334,6 +405,15 @@ class TestOtelV2LangfuseGenerationOutput: assert response.pages[0].markdown in _output_text(generation) @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["ocr"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.OCR, + providers=(Provider.MISTRAL,), + models=(MISTRAL_OCR_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_ocr_upload_input_is_a_bounded_document_summary_without_base64( self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager ) -> None: @@ -360,10 +440,19 @@ class TestOtelV2LangfuseGenerationOutput: ) @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["images_edits"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.IMAGES, + providers=(Provider.OPENAI,), + models=(IMAGE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_image_edit_input_is_the_edit_prompt( self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager ) -> None: - model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-image-1-mini")) + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai(IMAGE_BACKEND)) prompt: Final = f"make the square blue {unique_marker()}" started: Final = datetime.now(timezone.utc) response: Final = unwrap( @@ -388,6 +477,11 @@ class TestOtelV2LangfuseGenerationOutput: assert _output_text(generation).startswith("b64_json image (") @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["search"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + ) + ) def test_search_input_is_the_query_and_output_the_results( self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager ) -> None: diff --git a/tests/e2e/logging/test_prometheus_cardinality_e2e.py b/tests/e2e/logging/test_prometheus_cardinality_e2e.py index e2d164d8b4c..597e1e814e3 100644 --- a/tests/e2e/logging/test_prometheus_cardinality_e2e.py +++ b/tests/e2e/logging/test_prometheus_cardinality_e2e.py @@ -24,6 +24,7 @@ import pytest from prometheus_client.parser import text_string_to_metric_families from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from logging_client import LoggingClient @@ -47,6 +48,15 @@ def _aliases_in_metric(exposition: str, metric: str, label: str) -> frozenset[st class TestPrometheusPerKeyCardinality: @pytest.mark.covers("logging.prometheus.success.exports_metric", exercised_on=[]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.METRICS, + providers=(Provider.GEMINI,), + models=(DRIVER_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_distinct_key_aliases_produce_distinct_series( self, client: LoggingClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/logging/test_prometheus_queue_time_e2e.py b/tests/e2e/logging/test_prometheus_queue_time_e2e.py index 1f3c111bb65..9668322b362 100644 --- a/tests/e2e/logging/test_prometheus_queue_time_e2e.py +++ b/tests/e2e/logging/test_prometheus_queue_time_e2e.py @@ -6,6 +6,7 @@ import pytest from prometheus_client.parser import text_string_to_metric_families from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from logging_client import LoggingClient @@ -30,6 +31,15 @@ def _observation_count(exposition: str, alias: str) -> float | None: class TestPrometheusRequestQueueTime: @pytest.mark.covers("logging.prometheus.success.records_queue_time") + @meta( + Subject( + domain=Domain.OBSERVABILITY, + route=Route.METRICS, + providers=(Provider.GEMINI,), + models=(DRIVER_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_queue_time_histogram_records_an_observation( self, client: LoggingClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/logging/test_s3_log_e2e.py b/tests/e2e/logging/test_s3_log_e2e.py index 1612fa315a6..e917276edb0 100644 --- a/tests/e2e/logging/test_s3_log_e2e.py +++ b/tests/e2e/logging/test_s3_log_e2e.py @@ -24,10 +24,12 @@ from __future__ import annotations import math import re import time +from typing import Final import pytest from e2e_config import CHEAP_ANTHROPIC_MODEL, S3_PARTITION_GRANULARITY, unique_marker +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from logging_client import ( INVALID_UPSTREAM_API_KEY, @@ -43,6 +45,7 @@ pytestmark = pytest.mark.e2e #: The active s3_v2 callback's name in /health/readiness/details success_callbacks. S3_LOGGER_NAME = "S3Logger" +UNREACHABLE_ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5" @pytest.fixture(scope="session") @@ -64,6 +67,14 @@ def _assert_s3_configured(client: LoggingClient) -> None: class TestS3LogDelivery: @pytest.mark.covers("logging.s3.success.writes_object", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_chat_completions_writes_one_success_object( self, client: LoggingClient, s3_logs: S3LogReader, resources: ResourceManager ) -> None: @@ -108,6 +119,14 @@ class TestS3LogDelivery: ), f"payload response_cost {record.response_cost!r} must equal the header cost {outcome.response_cost}" @pytest.mark.covers("logging.s3.success.partition_layout", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_chat_completions_object_key_follows_the_partition_granularity( self, client: LoggingClient, s3_logs: S3LogReader, resources: ResourceManager ) -> None: @@ -148,6 +167,14 @@ class TestS3LogDelivery: ) @pytest.mark.covers("logging.s3.failure.writes_object", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + providers=(Provider.ANTHROPIC,), + models=(UNREACHABLE_ANTHROPIC_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_chat_completions_failure_writes_one_object( self, client: LoggingClient, s3_logs: S3LogReader, resources: ResourceManager ) -> None: @@ -167,7 +194,7 @@ class TestS3LogDelivery: model_name = f"s3-err-{unique_marker()}" model_id = client.create_model( model_name, - LiteLLMParamsBody(model="anthropic/claude-haiku-4-5", api_key=INVALID_UPSTREAM_API_KEY), + LiteLLMParamsBody(model=UNREACHABLE_ANTHROPIC_BACKEND, api_key=INVALID_UPSTREAM_API_KEY), ) resources.defer(lambda: client.delete_model(model_id)) alias = f"s3-err-key-{unique_marker()}" diff --git a/tests/e2e/logging/test_team_langfuse_callback_e2e.py b/tests/e2e/logging/test_team_langfuse_callback_e2e.py index 89cd45c9f16..f2ece7f5d90 100644 --- a/tests/e2e/logging/test_team_langfuse_callback_e2e.py +++ b/tests/e2e/logging/test_team_langfuse_callback_e2e.py @@ -19,6 +19,7 @@ import time import pytest from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from logging_client import ( LangfuseCreds, @@ -46,6 +47,14 @@ def langfuse_creds() -> LangfuseCreds: class TestTeamLangfuseCallback: @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_team_callback_delivers_and_isolates( self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager ) -> None: diff --git a/tests/e2e/logging/test_weave_log_e2e.py b/tests/e2e/logging/test_weave_log_e2e.py index dab5993c87a..edd8ec7c3c3 100644 --- a/tests/e2e/logging/test_weave_log_e2e.py +++ b/tests/e2e/logging/test_weave_log_e2e.py @@ -21,11 +21,13 @@ query API; nothing is mocked. from __future__ import annotations import time +from typing import Final import pytest from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker from e2e_http import StreamingResponse +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from logging_client import ( INVALID_UPSTREAM_API_KEY, @@ -40,6 +42,8 @@ from weave_reader import WeaveCall, WeaveReader, build_weave_reader pytestmark = pytest.mark.e2e +UNREACHABLE_ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5" + @pytest.fixture(scope="session") def weave_creds() -> WeaveCreds: @@ -79,6 +83,14 @@ WEAVE_STAGE_RED_REASON = ( class TestWeaveLogDelivery: @pytest.mark.skip(reason=WEAVE_STAGE_RED_REASON) @pytest.mark.covers("logging.niche_integrations.success.logs_spend", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_chat_completions_delivers_one_call_with_spend( self, client: LoggingClient, @@ -121,6 +133,14 @@ class TestWeaveLogDelivery: @pytest.mark.skip(reason=WEAVE_STAGE_RED_REASON) @pytest.mark.covers("logging.niche_integrations.failure.logs_spend", exercised_on=["chat_completions"]) + @meta( + Subject( + domain=Domain.OBSERVABILITY, + providers=(Provider.ANTHROPIC,), + models=(UNREACHABLE_ANTHROPIC_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_failed_chat_completions_delivers_one_error_call( self, client: LoggingClient, @@ -135,7 +155,7 @@ class TestWeaveLogDelivery: model_name = f"weave-err-{unique_marker()}" model_id = client.create_model( model_name, - LiteLLMParamsBody(model="anthropic/claude-haiku-4-5", api_key=INVALID_UPSTREAM_API_KEY), + LiteLLMParamsBody(model=UNREACHABLE_ANTHROPIC_BACKEND, api_key=INVALID_UPSTREAM_API_KEY), ) resources.defer(lambda: client.delete_model(model_id)) key = client.key_with_alias( diff --git a/tests/e2e/logging/weave_reader.py b/tests/e2e/logging/weave_reader.py index 2f8f759d299..c43e3280b39 100644 --- a/tests/e2e/logging/weave_reader.py +++ b/tests/e2e/logging/weave_reader.py @@ -28,7 +28,7 @@ import base64 import json import os import time -from dataclasses import dataclass +from dataclasses import dataclass, field from itertools import count, takewhile from typing import Final @@ -37,6 +37,7 @@ from pydantic import BaseModel, ConfigDict, Field from e2e_config import POLL_INTERVAL, POLL_TIMEOUT from e2e_http import URL, AuthHeaders, send +from e2e_metadata import step _WEAVE_TRACE_API: Final = "https://trace.wandb.ai" @@ -201,7 +202,7 @@ class WeaveCall(BaseModel): @dataclass(frozen=True, slots=True) class WeaveReader: project_id: str - api_key: str + api_key: str = field(repr=False) @property def _headers(self) -> AuthHeaders: @@ -232,6 +233,7 @@ class WeaveReader: ) return tuple(WeaveCall.model_validate_json(line) for line in outcome.body.splitlines() if line.strip()) + @step("Read the Weave {op} calls carrying the marker {marker}") def calls_matching(self, marker: str, *, since: float, op: str = LITELLM_REQUEST_OP) -> tuple[WeaveCall, ...]: """Every call under ``op`` started after ``since`` whose inputs carry ``marker``, paging until the window is exhausted. @@ -247,6 +249,7 @@ class WeaveReader: ) return tuple(call for page in pages for call in page if call.mentions(marker)) + @step("Wait for Weave to ingest a {op} call carrying the marker {marker}, then watch for duplicates") def poll_calls_matching(self, marker: str, *, since: float, op: str = LITELLM_REQUEST_OP) -> tuple[WeaveCall, ...]: """Poll until the call is readable, then keep re-reading for WEAVE_SETTLE_SECONDS so a duplicate exported by a later batch flush From 77fc3315e55b5dc22a99b4f3c0a7d76eb7e8c367 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:32:39 +0000 Subject: [PATCH 06/35] fix(caching): skip the cache past max_messages and keep tool_result text in semantic prompts (#43878) * fix(caching): keep tool calls and tool results in semantic cache prompts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(caching): keep semantic tool prompt helpers within lint budgets Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): keep structured function_call_output text in semantic prompts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(caching): split Responses text-field collection to stay within complexity budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): tag each tool result with the position of the call it answers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): encode tool result position and output together so tool text cannot forge result tags Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(caching): expect encoded tool result record in qdrant semantic prompt parity case Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(caching): cover tool result arrangements, SDK clients, concurrency and qdrant outage for semantic cache Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(caching): embed every semantic cache prompt field except volatile ones Replace the per-shape allowlist in the Python and Rust semantic cache prompt walkers with one include-by-default walker. Plain text keeps its old concatenation; any other block or message is embedded as compact JSON with call ids mapped to ordinals, cache_control dropped, and signatures, encrypted content and base64 data replaced with a short sha256 digest. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(code-quality): allow the bounded semantic cache prompt walkers in the recursion check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(rust): expect structured JSON for unknown fields in redis and valkey semantic prompts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * Revert "test(rust): expect structured JSON for unknown fields in redis and valkey semantic prompts" This reverts commit 86c82b949b3da25461e72f317b0f54572ea5359f. * Revert "test(code-quality): allow the bounded semantic cache prompt walkers in the recursion check" This reverts commit 39efb5d9da8292844a9844e01d095702b11ecec4. * Revert "feat(caching): embed every semantic cache prompt field except volatile ones" This reverts commit 5aed3ab3de3597dec22d3afcb33db7196f0a2a24. * refactor(caching): rename get_str_from_messages_with_tools to get_semantic_cache_prompt_from_messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(caching): split semantic cache prompt extraction by API format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): drop TypeIs guard and register Responses prompt walker with the recursion check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): pick the Responses text field without a Final inside a loop Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(caching): walk semantic cache prompts as plain dicts, dumping pydantic items once up front Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(caching): write the semantic cache prompt builders as plain loops Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): skip the cache past max_messages and keep tool_result text in semantic prompts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(caching): drop formatting-only churn from the redis semantic cache tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(integration): drop the caching group wiring that main already carries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(caching): recurse into tool_result content in the semantic cache prompt helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): check the max_messages cap on the shared exact-cache proxy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): read list-form function_call_output text in semantic cache prompts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(caching): extract nested Responses input lookup to keep walker under complexity limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * Revert "refactor(caching): extract nested Responses input lookup to keep walker under complexity limit" This reverts commit 0665296bf1d783583ffc7f494e111ef58fec2b18. * style(caching): suppress C901 on the Responses input walker instead of splitting it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/cache/src/semantic.rs | 37 ++++- litellm-rust/crates/cache/tests/semantic.rs | 34 +++++ litellm/caching/caching.py | 17 +++ litellm/caching/qdrant_semantic_cache.py | 10 +- litellm/caching/redis_semantic_cache.py | 11 +- .../prompt_templates/common_utils.py | 27 ++++ .../caching/test_cache_max_messages.py | 144 ++++++++++++++++++ tests/unit/caching/test_caching.py | 65 +++++++- .../unit/caching/test_redis_semantic_cache.py | 34 +++++ ...ore_utils_prompt_templates_common_utils.py | 92 +++++++++++ 10 files changed, 456 insertions(+), 15 deletions(-) create mode 100644 tests/integration/caching/test_cache_max_messages.py diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index c88706a213b..6200645e555 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -83,17 +83,16 @@ impl Embedder for PreparedEmbedding { } } -/// `get_str_from_messages`: every message's text content followed by its search results. +/// `get_semantic_cache_prompt_from_messages`: every message's text content, including the text of +/// Messages API `tool_result` blocks, followed by its search results. pub fn str_from_messages(messages: &[Value]) -> String { let mut text = String::new(); for message in messages.iter().filter_map(Value::as_object) { match message.get("content") { Some(Value::String(content)) => text.push_str(content), - Some(Value::Array(parts)) => { - for part in parts { - if let Some(part_text) = part.get("text").and_then(Value::as_str) { - text.push_str(part_text); - } + Some(Value::Array(blocks)) => { + for block in blocks { + push_block_text(&mut text, block); } } _ => {} @@ -103,6 +102,28 @@ pub fn str_from_messages(messages: &[Value]) -> String { text } +fn push_block_text(text: &mut String, block: &Value) { + if block.get("type").and_then(Value::as_str) != Some("tool_result") { + push_text_field(text, block); + return; + } + match block.get("content") { + Some(Value::String(result)) => text.push_str(result), + Some(Value::Array(blocks)) => { + for inner in blocks { + push_text_field(text, inner); + } + } + _ => {} + } +} + +fn push_text_field(text: &mut String, block: &Value) { + if let Some(block_text) = block.get("text").and_then(Value::as_str) { + text.push_str(block_text); + } +} + /// The messages prompt Qdrant embeds: `None` when the request carries no messages. pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option { let messages = context.messages.as_ref()?.as_array()?; @@ -163,6 +184,10 @@ fn collect_input_text(value: &Value, parts: &mut Vec) { collect_input_text(content, parts); return; } + if let Some(output) = map.get("output").filter(|output| output.is_array()) { + collect_input_text(output, parts); + return; + } for key in ["text", "output", "input_text", "output_text"] { if let Some(Value::String(text)) = map.get(key) && push_trimmed(text, parts) diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs index 97a552a8010..76af1863e1f 100644 --- a/litellm-rust/crates/cache/tests/semantic.rs +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -30,6 +30,31 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex ]}]), "What is this?", )] +#[case::tool_result_string( + json!([ + {"role": "user", "content": "list the files"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"}, + ]}, + ]), + "list the filescalc.py test_calc.py", +)] +#[case::tool_result_blocks( + json!([{"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": [ + {"type": "text", "text": "x = 1"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}}, + ]}, + ]}]), + "x = 1", +)] +#[case::tool_result_without_content( + json!([{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1"}]}]), + "", +)] #[case::missing_null_and_empty_content( json!([{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]), "", @@ -166,6 +191,15 @@ fn prompt_from_messages_reads_messages_only( ])), Some("model dump prompt\ndict prompt\ninline prompt"), )] +#[case::function_call_output_blocks( + None, + Some(json!([ + {"role": "user", "content": "update the config"}, + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\": \"a\"}"}, + {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "wrote a"}]}, + ])), + Some("update the config\nwrote a"), +)] #[case::object_content( None, Some(json!({"content": [{"text": "object content prompt"}]})), diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 1dc4de04dc1..85ef5a93937 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -71,6 +71,17 @@ class CacheMode(str, Enum): #### LiteLLM.Completion / Embedding Cache #### +def _request_message_count(kwargs: Mapping[str, object]) -> int: + """Chat and Messages API `messages`, else Responses API `input` items; embedding `input` strings count as none""" + messages: Final = kwargs.get("messages") + if isinstance(messages, list): + return len(messages) + input_items: Final = kwargs.get("input") + if not isinstance(input_items, list): + return 0 + return sum(1 for item in input_items if isinstance(item, (Mapping, BaseModel))) + + class Cache: def __init__( self, @@ -119,6 +130,7 @@ class Cache: semantic_cache_embedding_max_input_tokens: int | None = None, semantic_cache_embedding_timeout: float | None = None, semantic_cache_scope: str = SemanticCacheScope.KEY.value, + max_messages: int | None = 4, # GCP IAM authentication parameters gcp_service_account: str | None = None, gcp_ssl_ca_certs: str | None = None, @@ -148,6 +160,7 @@ class Cache: semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens. semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS. semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key". + max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4. # Disk Cache Args disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None. @@ -298,6 +311,7 @@ class Cache: self.ttl = ttl self.mode: CacheMode = mode or CacheMode.default_on self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value + self.max_messages: int | None = max_messages if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None: self.ttl = default_in_memory_ttl @@ -933,7 +947,10 @@ class Cache: If cache is default_on then this is True If cache is default_off then this is only true when user has opted in to use cache + Always False once the request carries more than `max_messages` messages """ + if self.max_messages is not None and _request_message_count(kwargs) > self.max_messages: + return False if self.mode == CacheMode.default_on: return True diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 8dac3f2eef9..eb88df066f6 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -24,7 +24,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_str_from_messages, + get_semantic_cache_prompt_from_messages, ) from litellm.types.utils import EmbeddingResponse @@ -286,7 +286,7 @@ class QdrantSemanticCache(BaseCache): # get the prompt messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages(messages) + prompt: Final = get_semantic_cache_prompt_from_messages(messages) # create an embedding for prompt embedding_response: Final = cast( @@ -325,7 +325,7 @@ class QdrantSemanticCache(BaseCache): # get the messages messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages(messages) + prompt: Final = get_semantic_cache_prompt_from_messages(messages) # convert to embedding embedding_response: Final = cast( @@ -400,7 +400,7 @@ class QdrantSemanticCache(BaseCache): # get the prompt messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages(messages) + prompt: Final = get_semantic_cache_prompt_from_messages(messages) embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding @@ -435,7 +435,7 @@ class QdrantSemanticCache(BaseCache): # get the messages messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages(messages) + prompt: Final = get_semantic_cache_prompt_from_messages(messages) embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata")) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index d4c815e15b7..8f99d76ba46 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -21,7 +21,7 @@ from litellm._logging import print_verbose, verbose_logger from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_str_from_messages, + get_semantic_cache_prompt_from_messages, ) from litellm.types.utils import EmbeddingResponse @@ -263,7 +263,7 @@ class RedisSemanticCache(BaseCache): """ messages: Final = kwargs.get("messages") if messages: - return get_str_from_messages(messages) + return get_semantic_cache_prompt_from_messages(messages) if "input" not in kwargs: return None @@ -274,7 +274,7 @@ class RedisSemanticCache(BaseCache): return prompt or None @classmethod - def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None: + def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None: # noqa: C901 # one branch per Responses input shape value = cls._coerce_response_input_value(value) if value is None: return @@ -296,6 +296,11 @@ class RedisSemanticCache(BaseCache): cls._collect_responses_input_text(content, prompt_parts) return + output = value.get("output") + if isinstance(output, list): + cls._collect_responses_input_text(output, prompt_parts) + return + for text_key in ("text", "output", "input_text", "output_text"): text_value = value.get(text_key) if isinstance(text_value, str): diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 49c4198c939..1f75125df09 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -192,6 +192,33 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str: return text +def get_semantic_cache_prompt_from_messages(messages: Sequence[Mapping[str, object]]) -> str: + """ + The text a semantic cache embeds for a request: `get_str_from_messages` plus the text inside + Messages API `tool_result` blocks, so a tool turn does not embed identically to the turn before it + """ + return "".join( + _semantic_cache_content_text(message.get("content")) + + extract_search_results_text(message.get("search_results")) + for message in messages + ) + + +def _semantic_cache_content_text(content: object) -> str: + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + return "".join(_semantic_cache_block_text(block) for block in content if isinstance(block, Mapping)) + + +def _semantic_cache_block_text(block: Mapping[str, object]) -> str: + if block.get("type") == "tool_result": + return _semantic_cache_content_text(block.get("content")) + text: Final = block.get("text") + return text if isinstance(text, str) else "" + + def is_non_content_values_set(message: AllMessageValues) -> bool: ignore_keys: Final = ["content", "role", "name"] return any(message.get(key, None) is not None for key in message if key not in ignore_keys) diff --git a/tests/integration/caching/test_cache_max_messages.py b/tests/integration/caching/test_cache_max_messages.py new file mode 100644 index 00000000000..89401b7c4ec --- /dev/null +++ b/tests/integration/caching/test_cache_max_messages.py @@ -0,0 +1,144 @@ +import json +import os +import uuid +from collections.abc import Callable +from typing import Final + +import pytest +from integration._support.anthropic_sse import message_json +from integration._support.client import Gateway, eventually, object_value, string_value +from integration._support.openai_wire import chat_reply, responses_reply +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from pydantic import JsonValue +from redis import Redis + +_Turns = Callable[[str], tuple[list[JsonValue], list[JsonValue]]] + + +def _claude_code_turns(task: str) -> tuple[list[JsonValue], list[JsonValue]]: + """Turns 1 and 3 of a Claude Code session on /v1/messages: 1 and 5 messages""" + first: Final[list[JsonValue]] = [{"role": "user", "content": task}] + third: Final[list[JsonValue]] = [ + *first, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"}], + }, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_2", "name": "Read", "input": {"file_path": "calc.py"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_2", "content": "def add(a, b): return a - b"}], + }, + ] + return first, third + + +def _agent_turns(task: str) -> tuple[list[JsonValue], list[JsonValue]]: + """Turns 1 and 3 of an OpenAI tool loop on /v1/chat/completions: 2 and 6 messages""" + + def call(call_id: str, path: str) -> list[JsonValue]: + function: Final[JsonValue] = {"name": "write_file", "arguments": json.dumps({"path": path})} + return [ + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": call_id, "type": "function", "function": function}], + }, + {"role": "tool", "tool_call_id": call_id, "content": f"wrote {path}"}, + ] + + first: Final[list[JsonValue]] = [ + {"role": "system", "content": "You are a coding agent"}, + {"role": "user", "content": task}, + ] + return first, [*first, *call("call_1", "a.yaml"), *call("call_2", "b.yaml")] + + +def _responses_turns(task: str) -> tuple[list[JsonValue], list[JsonValue]]: + """Turns 1 and 3 of an agent on /v1/responses: 1 and 5 input items""" + + def call(call_id: str, path: str) -> list[JsonValue]: + return [ + { + "type": "function_call", + "call_id": call_id, + "name": "write_file", + "arguments": json.dumps({"path": path}), + }, + {"type": "function_call_output", "call_id": call_id, "output": f"wrote {path}"}, + ] + + first: Final[list[JsonValue]] = [{"role": "user", "content": task}] + return first, [*first, *call("call_1", "a.yaml"), *call("call_2", "b.yaml")] + + +def _body(path: str, model: str, conversation: list[JsonValue]) -> dict[str, JsonValue]: + if path == "/v1/responses": + return {"model": model, "input": conversation} + return {"model": model, "max_tokens": 16, "messages": conversation} + + +def _reply(path: str, text: str) -> Reply: + identity: Final = f"id_{uuid.uuid4().hex}" + if path == "/v1/messages": + return Reply(body=message_json(identity, "claude-sonnet-5-5", text)) + if path == "/v1/responses": + return responses_reply(identity, "gpt-5.6-sol", text, stream=False) + return chat_reply(identity, "gpt-5.4", text, stream=False) + + +def _answer(path: str, payload: dict[str, JsonValue]) -> str: + if path == "/v1/messages": + return string_value(_first(payload["content"])["text"]) + if path == "/v1/responses": + return string_value(_first(_first(payload["output"])["content"])["text"]) + return string_value(object_value(_first(payload["choices"])["message"])["content"]) + + +def _first(value: JsonValue) -> dict[str, JsonValue]: + assert isinstance(value, list), value + return object_value(value[0]) + + +def _cached_responses(redis: Redis) -> frozenset[bytes]: + digests: Final = tuple(key for key in redis.scan_iter() if len(key) == 64) + return frozenset(key for key in digests if b'"response"' in (redis.get(key) or b"")) + + +@pytest.mark.parametrize( + ("path", "model", "turns"), + [ + pytest.param("/v1/messages", "anthropic/claude-sonnet-5-5", _claude_code_turns, id="messages"), + pytest.param("/v1/chat/completions", "openai/gpt-5.4", _agent_turns, id="chat-completions"), + pytest.param("/v1/responses", "openai/responses/gpt-5.6-sol", _responses_turns, id="responses"), + ], +) +def test_cache_serves_a_turn_under_max_messages_and_skips_one_past_it( + gateway: Gateway, provider: SharedProvider, path: str, model: str, turns: _Turns +) -> None: + under_cap, past_cap = turns(f"update the config {uuid.uuid4().hex}") + provider.expect(_reply(path, "first answer"), _reply(path, "second answer"), _reply(path, "third answer")) + + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as redis: + cached_before: Final = _cached_responses(redis) + gateway.post(path, _body(path, model, under_cap)) + eventually(lambda: _cached_responses(redis) - cached_before, lambda written: len(written) == 1) + repeated: Final = gateway.post(path, _body(path, model, under_cap)) + past_cap_twice: Final = ( + gateway.post(path, _body(path, model, past_cap)), + gateway.post(path, _body(path, model, past_cap)), + ) + + assert _answer(path, repeated) == "first answer", "a repeated turn under max_messages was not served from the cache" + assert tuple(_answer(path, answer) for answer in past_cap_twice) == ("second answer", "third answer"), ( + "a turn past max_messages was served from the cache" + ) + assert len(provider.received()) == 3 diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index 0a7ac3ecad1..0adb6b9a6f4 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -1,6 +1,7 @@ import asyncio import logging import re +import uuid from typing import Final from unittest.mock import MagicMock @@ -9,7 +10,7 @@ import pytest import litellm import litellm.caching.redis_cache as redis_cache_module from litellm._internal_context import current_service_target -from litellm.caching.caching import Cache, response_cache_phase +from litellm.caching.caching import Cache, CacheMode, response_cache_phase from litellm.caching.caching_handler import _PENDING_CACHE_WRITES from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache, _RedisTimeoutLogThrottle @@ -473,3 +474,65 @@ async def test_a_lookup_already_inside_the_phase_does_not_open_a_second_one(v2_s await cache.async_get_cache(dynamic_cache_object=backend, **_REQUEST) assert backend.seen == [("llm_response", "cache.get llm_response")] assert [s.name for s in v2_span_exporter.get_finished_spans()] == ["cache.get llm_response"] + + +_TOOL_TURN_ITEM: Final = {"role": "user", "content": "hi"} + + +@pytest.mark.parametrize( + ("kwargs", "expected"), + [ + pytest.param({"messages": [_TOOL_TURN_ITEM] * 4}, True, id="four-messages-are-cached"), + pytest.param({"messages": [_TOOL_TURN_ITEM] * 5}, False, id="five-messages-skip-the-cache"), + pytest.param({"input": [_TOOL_TURN_ITEM] * 4}, True, id="four-responses-items-are-cached"), + pytest.param({"input": [_TOOL_TURN_ITEM] * 5}, False, id="five-responses-items-skip-the-cache"), + pytest.param({"input": "one prompt"}, True, id="string-input-is-one-message"), + pytest.param({"input": ["a", "b", "c", "d", "e"]}, True, id="embedding-strings-are-not-messages"), + ], +) +def test_should_use_cache_stops_past_the_default_max_messages(kwargs: dict[str, object], expected: bool) -> None: + assert Cache(type=LiteLLMCacheType.LOCAL).should_use_cache(**kwargs) is expected + + +def test_responses_sdk_items_count_toward_max_messages() -> None: + from openai.types.responses import ResponseFunctionToolCall + + call: Final = ResponseFunctionToolCall(type="function_call", call_id="c1", name="ls", arguments="{}") + + assert Cache(type=LiteLLMCacheType.LOCAL).should_use_cache(input=[_TOOL_TURN_ITEM, call, call, call, call]) is False + + +def test_max_messages_is_configurable_and_none_disables_it() -> None: + three: Final = [_TOOL_TURN_ITEM] * 3 + + assert Cache(type=LiteLLMCacheType.LOCAL, max_messages=2).should_use_cache(messages=three) is False + assert Cache(type=LiteLLMCacheType.LOCAL, max_messages=3).should_use_cache(messages=three) is True + assert Cache(type=LiteLLMCacheType.LOCAL, max_messages=None).should_use_cache(messages=three * 50) is True + + +def test_max_messages_beats_an_explicit_use_cache_opt_in() -> None: + cache: Final = Cache(type=LiteLLMCacheType.LOCAL, mode=CacheMode.default_off) + + assert cache.should_use_cache(messages=[_TOOL_TURN_ITEM] * 4, cache={"use-cache": True}) is True + assert cache.should_use_cache(messages=[_TOOL_TURN_ITEM] * 5, cache={"use-cache": True}) is False + + +def test_completion_past_max_messages_is_neither_served_from_nor_written_to_the_cache( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + tag: Final = uuid.uuid4().hex + four: Final = [{"role": "user", "content": f"{tag} turn {index}"} for index in range(4)] + five: Final = [*four, {"role": "user", "content": f"{tag} turn 4"}] + + def answer(messages: list[dict[str, str]], mock_response: str) -> str: + response: Final = litellm.completion(model="gpt-4o-mini", messages=messages, mock_response=mock_response) + assert isinstance(response, litellm.ModelResponse), response + choice: Final = response.choices[0] + assert isinstance(choice, litellm.Choices), choice + return str(choice.message.content) + + assert answer(four, "four first") == "four first" + assert answer(four, "four second") == "four first", "a 4-message repeat missed the cache" + assert answer(five, "five first") == "five first" + assert answer(five, "five second") == "five second", "a 5-message repeat was served from the cache" diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 99844c695cb..93db26fc9b2 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -568,6 +568,20 @@ def test_redis_semantic_cache_set_cache_flattens_structured_responses_input(): ) +def test_redis_semantic_cache_prompt_extraction_reads_function_call_output_blocks(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + prompt = RedisSemanticCache._get_prompt_from_kwargs( + input=[ + {"role": "user", "content": "update the config"}, + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": '{"path": "a"}'}, + {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "wrote a"}]}, + ] + ) + + assert prompt == "update the config\nwrote a" + + def test_redis_semantic_cache_prompt_extraction_prefers_messages(): from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -1416,3 +1430,23 @@ async def test_redis_async_embedding_truncates_off_the_event_loop(monkeypatch): assert embedding == [0.1, 0.2] assert _token_count("sem-embed", router.aembedding.call_args.kwargs["input"]) == 5 assert_loop_stayed_free(took, lags) + + +def test_redis_semantic_cache_prompt_extraction_keeps_tool_result_text(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + prompt = RedisSemanticCache._get_prompt_from_kwargs( + messages=[ + {"role": "user", "content": "list the files"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"}], + }, + ] + ) + + assert prompt == "list the filescalc.py test_calc.py" diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 7415c74226d..1d19264fb9e 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -16,6 +16,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, get_file_ids_from_messages, get_format_from_file_id, + get_semantic_cache_prompt_from_messages, + get_str_from_messages, handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, is_encrypted_reasoning_block, @@ -2171,3 +2173,93 @@ class TestMergeConsecutiveSystemMessages: ) assert merged == [{"role": "system"}, {"role": "user", "content": "Hi"}] + + +_CLAUDE_CODE_TOOL_TURN: Final = [ + {"role": "user", "content": "list the files"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"}], + }, +] + + +@pytest.mark.parametrize( + ("messages", "expected"), + [ + pytest.param(_CLAUDE_CODE_TOOL_TURN, "list the filescalc.py test_calc.py", id="tool-result-string"), + pytest.param( + [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": [ + {"type": "text", "text": "x = 1"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}}, + ], + } + ], + } + ], + "x = 1", + id="tool-result-blocks", + ), + pytest.param( + [{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1"}]}], + "", + id="tool-result-without-content", + ), + ], +) +def test_get_semantic_cache_prompt_from_messages_keeps_tool_result_text( + messages: list[dict[str, object]], expected: str +) -> None: + assert get_semantic_cache_prompt_from_messages(messages) == expected + + +def test_get_semantic_cache_prompt_from_messages_differs_from_the_turn_before_it() -> None: + assert get_str_from_messages(_CLAUDE_CODE_TOOL_TURN) == get_str_from_messages(_CLAUDE_CODE_TOOL_TURN[:1]) + assert get_semantic_cache_prompt_from_messages(_CLAUDE_CODE_TOOL_TURN) != get_semantic_cache_prompt_from_messages( + _CLAUDE_CODE_TOOL_TURN[:1] + ) + + +@pytest.mark.parametrize( + "messages", + [ + pytest.param([{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}], id="strings"), + pytest.param( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is "}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + {"type": "text", "text": "this?"}, + ], + } + ], + id="text-parts", + ), + pytest.param( + [ + {"role": "assistant"}, + {"role": "assistant", "content": None}, + {"role": "user", "content": ""}, + {"role": "tool", "content": "small", "search_results": [{"source": "s", "title": "t", "content": []}]}, + ], + id="empty-content-and-search-results", + ), + ], +) +def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_without_tool_results( + messages: list[dict[str, object]], +) -> None: + assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages) From c943d650f40a4c32f25d52706f6ef88dd6ee9805 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 7 Oct 2026 10:32:58 -0700 Subject: [PATCH 07/35] test(e2e): tag router, batches and mcp tests with Subject metadata and record client steps (#44964) * test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata * test(e2e): tag router, batches and mcp tests with Subject metadata and record client steps * test(e2e): leave the batches cleanup harness unit tests untagged * docs(e2e): name every markerless harness test file that carries no Subject * test(e2e): keep the step discovery comprehensions to one for clause * test(e2e): name every driven model on the vllm batch, prompt caching and complexity router subjects --- tests/e2e/batches/batch_cleanup.py | 7 +- tests/e2e/batches/batch_client.py | 12 + tests/e2e/batches/capabilities.py | 42 +++- tests/e2e/batches/test_batches_e2e.py | 206 +++++++++++++++++- .../test_managed_files_enforcement_e2e.py | 29 ++- tests/e2e/mcp/conftest.py | 5 +- tests/e2e/mcp/datadog_mcp.py | 7 + tests/e2e/mcp/mcp_client.py | 16 ++ tests/e2e/mcp/oauth_chat_client.py | 9 + tests/e2e/mcp/oauth_gateway.py | 6 + tests/e2e/mcp/test_mcp_access_group_e2e.py | 7 + .../mcp/test_mcp_chat_completion_oauth_e2e.py | 21 ++ tests/e2e/mcp/test_mcp_datadog_e2e.py | 13 +- tests/e2e/mcp/test_mcp_guardrail_e2e.py | 7 + tests/e2e/mcp/test_mcp_key_access_e2e.py | 25 +++ .../e2e/mcp/test_mcp_oauth_happy_path_e2e.py | 7 + .../mcp/test_mcp_toolset_enforcement_e2e.py | 7 + tests/e2e/router/reliability_support.py | 28 +++ .../test_auto_router_regressions_e2e.py | 112 ++++++++++ .../e2e/router/test_complexity_router_e2e.py | 17 +- .../e2e/router/test_reliability_cache_e2e.py | 13 +- ...st_reliability_cancel_on_disconnect_e2e.py | 10 + .../router/test_reliability_cooldowns_e2e.py | 42 ++++ .../router/test_reliability_fallbacks_e2e.py | 46 +++- .../e2e/router/test_reliability_memory_e2e.py | 16 +- .../test_reliability_prompt_caching_e2e.py | 11 + .../router/test_reliability_retries_e2e.py | 43 ++++ ...test_reliability_routing_strategies_e2e.py | 40 ++++ .../router/test_reliability_timeouts_e2e.py | 5 +- 29 files changed, 773 insertions(+), 36 deletions(-) diff --git a/tests/e2e/batches/batch_cleanup.py b/tests/e2e/batches/batch_cleanup.py index f1142a60782..f8fc1bbf01f 100644 --- a/tests/e2e/batches/batch_cleanup.py +++ b/tests/e2e/batches/batch_cleanup.py @@ -8,6 +8,7 @@ from typing import Final, Protocol from batch_client import BatchObject, FileDeleteResponse from capabilities import is_cloud_storage_id, is_managed_id from e2e_http import NetworkError, RateLimitedError, Result, Success, UnknownApiError +from e2e_metadata import STEP_FRAMES, step from pydantic import BaseModel CLEANUP_DELAYS: Final = (1.0, 2.0, 4.0) @@ -52,6 +53,7 @@ def _require_cleanup_success[R: BaseModel](result: Result[R], operation: str) -> raise AssertionError(f"{operation} failed: {result.kind}") +@step("Clean up the uploaded file") def cleanup_file(client: BatchCleanupClient, file_id: str, *, key: str, provider: str | None = None) -> None: delete: Final[Callable[[], Result[FileDeleteResponse]]] = ( (lambda: client.delete_file_as_admin(file_id, provider=provider)) @@ -65,7 +67,7 @@ def cleanup_file(client: BatchCleanupClient, file_id: str, *, key: str, provider warnings.warn( f"Left file {file_id} in place: LiteLLM refused to delete it while a batch still references it", UserWarning, - stacklevel=2, + stacklevel=2 + STEP_FRAMES, ) return deleted: Final = _require_cleanup_success(result, f"Delete file {file_id}") @@ -74,6 +76,7 @@ def cleanup_file(client: BatchCleanupClient, file_id: str, *, key: str, provider ), f"Delete file {file_id} did not confirm deletion" +@step("Cancel the batch if it is still running") def cleanup_batch( client: BatchCleanupClient, batch_id: str, @@ -137,7 +140,7 @@ def cleanup_batch( warnings.warn( f"Left batch {batch_id} cancelling after {BATCH_CANCEL_TIMEOUT_SECONDS}s for the provider to finish", UserWarning, - stacklevel=2, + stacklevel=2 + STEP_FRAMES, ) return wait(BATCH_CANCEL_POLL_SECONDS) diff --git a/tests/e2e/batches/batch_client.py b/tests/e2e/batches/batch_client.py index 8745140a818..b02b09e557b 100644 --- a/tests/e2e/batches/batch_client.py +++ b/tests/e2e/batches/batch_client.py @@ -17,6 +17,7 @@ from typing import Final, Literal from pydantic import BaseModel, Field +from e2e_metadata import step from proxy_client import ProxyClient from e2e_http import ( FileUploadForm, @@ -136,12 +137,15 @@ def is_result_access_denied[R: BaseModel](result: Result[R]) -> bool: class BatchClient: proxy: ProxyClient + @step("Add a batch deployment named {model_name} that calls {litellm_params.model}") def create_model(self, model_name: str, litellm_params: LiteLLMParamsBody) -> str: return self.proxy.create_model(model_name, litellm_params, mode="batch") + @step("Delete the batch deployment") def delete_model(self, model_id: str) -> None: self.proxy.delete_model(model_id) + @step("Upload a batch input file to /v1/files") def upload_file( self, *, @@ -161,6 +165,7 @@ class BatchClient: response_type=FileObject, ) + @step("Retrieve the uploaded file") def retrieve_file( self, file_id: str, *, key: str, provider: str | None = None ) -> Result[FileObject]: @@ -171,6 +176,7 @@ class BatchClient: response_type=FileObject, ) + @step("List the files the key can see from /v1/files") def list_files(self, *, key: str, provider: str | None = None) -> Result[FileList]: return self.proxy.transport.get( _files_path(provider), @@ -179,6 +185,7 @@ class BatchClient: response_type=FileList, ) + @step("Create a batch of {body.endpoint} requests from the uploaded file") def create_batch( self, *, body: BatchCreateBody, key: str, provider: str | None = None ) -> StreamingResponse: @@ -188,6 +195,7 @@ class BatchClient: json=body, ) + @step("Retrieve the batch") def retrieve_batch( self, batch_id: str, *, key: str, provider: str | None = None ) -> Result[BatchObject]: @@ -198,6 +206,7 @@ class BatchClient: response_type=BatchObject, ) + @step("Cancel the batch") def cancel_batch( self, batch_id: str, *, key: str, provider: str | None = None ) -> Result[BatchObject]: @@ -208,6 +217,7 @@ class BatchClient: response_type=BatchObject, ) + @step("List the batches the key can see from /v1/batches") def list_batches( self, *, @@ -223,6 +233,7 @@ class BatchClient: response_type=BatchList, ) + @step("Delete the uploaded file") def delete_file( self, file_id: str, *, key: str, provider: str | None = None ) -> Result[FileDeleteResponse]: @@ -233,6 +244,7 @@ class BatchClient: response_type=FileDeleteResponse, ) + @step("Delete the uploaded file as the proxy admin") def delete_file_as_admin(self, file_id: str, *, provider: str | None = None) -> Result[FileDeleteResponse]: return self.proxy.transport.delete( f"{_files_path(provider)}/{file_id}", diff --git a/tests/e2e/batches/capabilities.py b/tests/e2e/batches/capabilities.py index d510426dee2..c03b481f060 100644 --- a/tests/e2e/batches/capabilities.py +++ b/tests/e2e/batches/capabilities.py @@ -7,7 +7,11 @@ import os from dataclasses import dataclass from typing import Final, Literal +import pytest + from e2e_config import provider_edge_base, unique_marker +from e2e_metadata import Domain, Mode, Route, Subject, meta +from e2e_metadata import Provider as MetaProvider from models import LiteLLMParamsBody _BATCH_RUN = unique_marker() @@ -18,6 +22,9 @@ def batch_model_name(base: str) -> str: OPENAI_BATCH_BACKEND: Final = "gpt-4o-mini" +AZURE_BATCH_BACKEND: Final = "gpt-5.4-mini-batch" +VERTEX_BATCH_BACKEND: Final = "gemini-2.5-flash" +BEDROCK_BATCH_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" def openai_batch_params() -> LiteLLMParamsBody: @@ -65,14 +72,14 @@ class Provider: return openai_batch_params() case "azure": return LiteLLMParamsBody( - model="azure/gpt-5.4-mini-batch", + model=f"azure/{AZURE_BATCH_BACKEND}", api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY", api_version="2025-04-01-preview", ) case "vertex_ai": return LiteLLMParamsBody( - model="vertex_ai/gemini-2.5-flash", + model=f"vertex_ai/{VERTEX_BATCH_BACKEND}", vertex_project="os.environ/VERTEXAI_PROJECT", vertex_location="us-central1", vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", @@ -81,7 +88,7 @@ class Provider: ) case "bedrock": return LiteLLMParamsBody( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + model=BEDROCK_BATCH_BACKEND, aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", aws_region_name="os.environ/AWS_REGION", @@ -132,21 +139,21 @@ PROVIDERS: tuple[Provider, ...] = ( Provider( "azure", batch_model_name("azure-batch"), - "gpt-5.4-mini-batch", + AZURE_BATCH_BACKEND, can_cancel=True, can_list=True, ), Provider( "vertex_ai", batch_model_name("vertex-batch"), - "gemini-2.5-flash", + VERTEX_BATCH_BACKEND, can_cancel=True, can_list=True, ), Provider( "bedrock", batch_model_name("bedrock-batch"), - "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + BEDROCK_BATCH_BACKEND, can_cancel=True, can_list=True, ), @@ -181,6 +188,29 @@ CAPABILITIES: tuple[Capability, ...] = tuple( ) +def lifecycle_meta(cap: Capability) -> pytest.MarkDecorator: + return meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + providers=(MetaProvider(cap.provider),), + models=(cap.raw_model,), + mode=Mode.BATCH, + ) + ) + + +def file_content_meta(provider: Provider) -> pytest.MarkDecorator: + return meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.FILES, + providers=(MetaProvider(provider.name),), + models=(provider.raw_model,), + ) + ) + + def raw_id_matches_provider(provider: str, batch_id: str) -> bool: if provider in ("openai", "azure"): return batch_id.startswith("batch") diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 8da2deb4010..54bcb80ed24 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -44,17 +44,22 @@ from capabilities import ( OPENAI_BATCH_BACKEND, OPENAI_BATCH_MODEL, PROVIDERS, + VERTEX_BATCH_BACKEND, Capability, Provider, batch_model_name, coverage_cells_for_lifecycle, decoded_model_from_id, + file_content_meta, is_managed_id, + lifecycle_meta, matches_id_shape, openai_batch_params, raw_id_matches_provider, ) from e2e_config import MASTER_KEY, PROXY_BASE_URL, unique_marker +from e2e_metadata import Domain, Mode, Route, Subject, meta +from e2e_metadata import Provider as MetaProvider from e2e_http import ( FileUploadForm, Result, @@ -249,7 +254,7 @@ def assert_batch_object(batch: BatchObject) -> None: pytest.param( cap, id=cap.id, - marks=pytest.mark.covers(*coverage_cells_for_lifecycle(cap)), + marks=(pytest.mark.covers(*coverage_cells_for_lifecycle(cap)), lifecycle_meta(cap)), ) for cap in CAPABILITIES ], @@ -350,6 +355,15 @@ def test_batch_lifecycle( @pytest.mark.covers("llm.batches.openai.key_model_access_denied.nonstream.works") +@meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.BATCHES, + providers=(MetaProvider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + mode=Mode.BATCH, + ) +) def test_batch_key_model_access_denied( client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -389,6 +403,14 @@ def test_batch_key_model_access_denied( "llm.files.openai.upload.nonstream.works", "llm.files.openai.delete.nonstream.works", ) +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.FILES, + providers=(MetaProvider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + ) +) def test_file_upload_and_delete_outputs( client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -433,6 +455,15 @@ def unattributed_rows(rows: list[SpendLogRow]) -> list[SpendLogRow]: "once the fetch is bounded." ) ) +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.BATCHES, + providers=(MetaProvider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + mode=Mode.BATCH, + ) +) def test_rate_limited_batch_create_leaves_no_unattributed_spend_row( client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -471,7 +502,7 @@ def test_rate_limited_batch_create_leaves_no_unattributed_spend_row( file = unwrap( client.upload_file( - content=render_jsonl("gpt-4o-mini"), + content=render_jsonl(OPENAI_BATCH_BACKEND), form=FileUploadForm(purpose="batch"), model=OPENAI_BATCH_MODEL, key=key, @@ -520,6 +551,14 @@ class TestBatchFileContent: "llm.files.openai.content.nonstream.works", exercised_on=["files"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.FILES, + providers=(MetaProvider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + ) + ) def test_file_content_matches_upload( self, client: BatchClient, resources: ResourceManager ) -> None: @@ -558,8 +597,9 @@ class TestBatchFileContent: pytest.param( p, id=p.name, - marks=pytest.mark.covers( - FILE_CONTENT_CELLS[p.name], exercised_on=["files"] + marks=( + pytest.mark.covers(FILE_CONTENT_CELLS[p.name], exercised_on=["files"]), + file_content_meta(p), ), ) for p in PROVIDERS @@ -632,6 +672,14 @@ class TestOpenAIFiles: "marker when LIT-4820 is fixed; do not relax the assertion to make it pass." ) ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.FILES, + providers=(MetaProvider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + ) + ) def test_uploaded_file_appears_in_list( self, client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -662,6 +710,7 @@ class TestOpenAIFiles: "llm.files.openai.list_isolation.nonstream.works", exercised_on=["files"], ) + @meta(Subject(domain=Domain.LLM_TRANSLATION, route=Route.FILES, providers=(MetaProvider.OPENAI,))) def test_list_page_cursors_address_only_the_callers_own_files( self, client: BatchClient, resources: ResourceManager ) -> None: @@ -697,6 +746,14 @@ class TestOpenAIFiles: "llm.files.openai.retrieve.nonstream.works", exercised_on=["files"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.FILES, + providers=(MetaProvider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + ) + ) def test_retrieve_round_trips_metadata( self, client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -760,6 +817,15 @@ class TestBatchRateLimitErrorMapping: "quota_management.ratelimit.batch_rpm.blocks_over_limit", exercised_on=["batches"], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.BATCHES, + providers=(MetaProvider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + mode=Mode.BATCH, + ) + ) def test_batch_create_over_rpm_returns_mapped_429( self, client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -773,7 +839,7 @@ class TestBatchRateLimitErrorMapping: file = unwrap( client.upload_file( - content=_multi_request_jsonl("gpt-4o-mini", BATCH_RL_REQUEST_LINES), + content=_multi_request_jsonl(OPENAI_BATCH_BACKEND, BATCH_RL_REQUEST_LINES), form=FileUploadForm(purpose="batch"), model=OPENAI_BATCH_MODEL, key=key, @@ -826,7 +892,7 @@ class TestBatchEnqueuedTokenLimit: ) -> FileObject: file = unwrap( client.upload_file( - content=_multi_request_jsonl("gpt-4o-mini", BATCH_RL_REQUEST_LINES), + content=_multi_request_jsonl(OPENAI_BATCH_BACKEND, BATCH_RL_REQUEST_LINES), form=FileUploadForm(purpose="batch"), model=OPENAI_BATCH_MODEL, key=key, @@ -859,6 +925,15 @@ class TestBatchEnqueuedTokenLimit: "quota_management.ratelimit.batch_enqueued_tokens.accepts_over_rpm", exercised_on=["batches"], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.BATCHES, + providers=(MetaProvider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + mode=Mode.BATCH, + ) + ) def test_enqueued_allowance_accepts_batch_over_key_rpm( self, client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -890,6 +965,15 @@ class TestBatchEnqueuedTokenLimit: "quota_management.ratelimit.batch_enqueued_tokens.refunds_on_cancel", exercised_on=["batches"], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.BATCHES, + providers=(MetaProvider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + mode=Mode.BATCH, + ) + ) def test_exhausted_allowance_blocks_until_cancel_refunds( self, client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -985,6 +1069,15 @@ class TestBedrockBatchAssumeRole: "llm.files.bedrock.upload.nonstream.works", exercised_on=["batches", "files"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + providers=(MetaProvider.BEDROCK,), + models=(ASSUME_ROLE_RAW_MODEL,), + mode=Mode.BATCH, + ) + ) def test_unified_batch_create_with_assume_role( self, client: BatchClient, resources: ResourceManager ) -> None: @@ -1052,6 +1145,14 @@ class TestBedrockBatchSplitS3Credentials: "llm.files.bedrock.split_s3_credentials.nonstream.works", exercised_on=["files"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.FILES, + providers=(MetaProvider.BEDROCK,), + models=(ASSUME_ROLE_RAW_MODEL,), + ) + ) def test_file_lifecycle_signs_s3_with_s3_credentials( self, client: BatchClient, resources: ResourceManager ) -> None: @@ -1123,6 +1224,15 @@ class TestBedrockBatchGovCloud: "llm.files.bedrock.govcloud_partition.nonstream.works", exercised_on=["batches", "files"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + providers=(MetaProvider.BEDROCK,), + models=(GOVCLOUD_RAW_MODEL,), + mode=Mode.BATCH, + ) + ) def test_unified_file_upload_and_batch_create_in_govcloud( self, client: BatchClient, resources: ResourceManager ) -> None: @@ -1193,6 +1303,14 @@ class TestGeminiFiles: "llm.files.gemini.upload.nonstream.works", exercised_on=["files"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.FILES, + providers=(MetaProvider.GEMINI,), + models=(GEMINI_FILES_RAW_MODEL,), + ) + ) def test_gemini_file_upload( self, client: BatchClient, resources: ResourceManager ) -> None: @@ -1227,7 +1345,7 @@ def _vllm_params(api_base: str, api_key: str | None, model_id: str) -> LiteLLMPa ) -HOSTED_VLLM_DEFAULT_MODEL = "Qwen/Qwen2.5-0.5B-Instruct" +HOSTED_VLLM_MODEL: Final = (os.environ.get("HOSTED_VLLM_MODEL") or "Qwen/Qwen2.5-0.5B-Instruct").strip() HOSTED_VLLM_BAD_LINE_CUSTOM_ID = "req-bad" @@ -1236,9 +1354,8 @@ def _hosted_vllm_deployment(client: BatchClient, resources: ResourceManager) -> if api_base is None: pytest.skip("set HOSTED_VLLM_API_BASE (the live vLLM server this deployment targets)") api_key = (os.environ.get("HOSTED_VLLM_API_KEY") or "").strip() or None - model_id = (os.environ.get("HOSTED_VLLM_MODEL") or HOSTED_VLLM_DEFAULT_MODEL).strip() proxy_name = batch_model_name("hosted-vllm-batch") - model_row_id = client.create_model(proxy_name, _vllm_params(api_base, api_key, model_id)) + model_row_id = client.create_model(proxy_name, _vllm_params(api_base, api_key, HOSTED_VLLM_MODEL)) resources.defer(lambda: client.delete_model(model_row_id)) return proxy_name @@ -1290,6 +1407,15 @@ class TestHostedVllmBatch: "llm.files.hosted_vllm.upload.nonstream.works", exercised_on=["batches", "files"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + providers=(MetaProvider.HOSTED_VLLM,), + models=(HOSTED_VLLM_MODEL,), + mode=Mode.BATCH, + ) + ) def test_batch_runs_to_completion_with_a_downloadable_output( self, client: BatchClient, resources: ResourceManager, upload_route: str ) -> None: @@ -1337,6 +1463,15 @@ class TestHostedVllmBatch: ) @pytest.mark.covers("llm.batches.hosted_vllm.basic.nonstream.works", exercised_on=["batches", "files"]) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + providers=(MetaProvider.HOSTED_VLLM,), + models=(HOSTED_VLLM_MODEL,), + mode=Mode.BATCH, + ) + ) def test_failing_line_lands_in_the_error_file_not_the_batch_status( self, client: BatchClient, resources: ResourceManager ) -> None: @@ -1417,6 +1552,7 @@ class TestBatchFailurePaths: "llm.batches.openai.malformed_jsonl.nonstream.works", exercised_on=["files"], ) + @meta(Subject(domain=Domain.LLM_TRANSLATION, route=Route.FILES)) def test_malformed_jsonl_upload_rejected( self, client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -1439,13 +1575,22 @@ class TestBatchFailurePaths: "llm.batches.openai.cancel_terminal.nonstream.works", exercised_on=["batches", "files"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + providers=(MetaProvider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + mode=Mode.BATCH, + ) + ) def test_endpoint_mismatch_fails_batch_and_cancel_conflicts( self, client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: key = resources.key() file = unwrap( client.upload_file( - content=_mismatched_endpoint_jsonl("gpt-4o-mini"), + content=_mismatched_endpoint_jsonl(OPENAI_BATCH_BACKEND), form=FileUploadForm(purpose="batch"), model=OPENAI_BATCH_MODEL, key=key, @@ -1495,6 +1640,15 @@ class TestBatchFailurePaths: "llm.batches.openai.foreign_file_id.nonstream.works", exercised_on=["batches", "files"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + providers=(MetaProvider.AZURE,), + models=(AZURE_BATCH_RAW_MODEL,), + mode=Mode.BATCH, + ) + ) def test_foreign_encoded_file_id_routes_by_file_model( self, client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -1544,6 +1698,15 @@ class TestBatchSecondHop: "llm.batches.openai.second_hop.nonstream.works", exercised_on=["batches", "files"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + providers=(MetaProvider.LITELLM_PROXY, MetaProvider.OPENAI), + models=(OPENAI_BATCH_BACKEND,), + mode=Mode.BATCH, + ) + ) def test_unified_create_and_retrieve_via_chained_gateway( self, client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -1561,7 +1724,7 @@ class TestBatchSecondHop: file = unwrap( client.upload_file( - content=render_jsonl("gpt-4o-mini"), + content=render_jsonl(OPENAI_BATCH_BACKEND), form=FileUploadForm(purpose="batch", target_model_names=hop_name), key=key, ) @@ -1680,13 +1843,22 @@ class TestBatchTerminalState: "llm.batches.openai.terminal_state.nonstream.cost_logged", exercised_on=["batches", "files"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + providers=(MetaProvider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + mode=Mode.BATCH, + ) + ) def test_completed_batch_downloads_output_and_books_cost( self, client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: key = resources.key() file = unwrap( client.upload_file( - content=render_jsonl("gpt-4o-mini"), + content=render_jsonl(OPENAI_BATCH_BACKEND), form=FileUploadForm(purpose="batch"), model=OPENAI_BATCH_MODEL, key=key, @@ -1786,6 +1958,15 @@ class TestVertexNativePassthrough: "llm.batches.vertex.native_passthrough.nonstream.works", exercised_on=["files", "batches"], ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.BATCHES, + providers=(MetaProvider.VERTEX_AI,), + models=(VERTEX_BATCH_BACKEND,), + mode=Mode.BATCH, + ) + ) def test_native_jsonl_round_trips_untouched_and_starts_a_batch( self, client: BatchClient, resources: ResourceManager, batch_deployments: None ) -> None: @@ -1848,6 +2029,7 @@ class TestVertexNativePassthrough: ), ], ) + @meta(Subject(domain=Domain.LLM_TRANSLATION, route=Route.FILES)) def test_passthrough_upload_is_rejected_outside_a_native_vertex_batch( self, content: bytes, diff --git a/tests/e2e/batches/test_managed_files_enforcement_e2e.py b/tests/e2e/batches/test_managed_files_enforcement_e2e.py index 2f5d0588aca..43a2488289c 100644 --- a/tests/e2e/batches/test_managed_files_enforcement_e2e.py +++ b/tests/e2e/batches/test_managed_files_enforcement_e2e.py @@ -22,9 +22,10 @@ import pytest from batch_client import BatchClient, FileObject from batch_cleanup import cleanup_file -from capabilities import batch_model_name, is_managed_id, openai_batch_params +from capabilities import OPENAI_BATCH_BACKEND, batch_model_name, is_managed_id, openai_batch_params from e2e_config import unique_marker from e2e_http import FileUploadForm, Result, UnknownApiError, unwrap +from e2e_metadata import Domain, Provider, Route, Subject, meta from lifecycle import ResourceManager pytestmark = [pytest.mark.e2e, pytest.mark.managed_files] @@ -64,6 +65,12 @@ def managed_model(client: BatchClient) -> Iterator[str]: @pytest.mark.covers(UPLOAD_ROW) +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.FILES, + ) +) def test_upload_without_target_model_names_rejected( client: BatchClient, scoped_key: str, managed_model: str ) -> None: @@ -76,6 +83,12 @@ def test_upload_without_target_model_names_rejected( @pytest.mark.covers(UPLOAD_ROW) +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.FILES, + ) +) def test_upload_with_model_param_rejected( client: BatchClient, scoped_key: str, managed_model: str ) -> None: @@ -89,12 +102,26 @@ def test_upload_with_model_param_rejected( @pytest.mark.covers(ISOLATION_ROW) +@meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.FILES, + ) +) def test_raw_provider_file_id_rejected(client: BatchClient, scoped_key: str) -> None: result = client.retrieve_file("file-e2e-raw-provider-id", key=scoped_key) expect_api_error(result, 400, "Raw provider file ids cannot be used") @pytest.mark.covers(ISOLATION_ROW) +@meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.FILES, + providers=(Provider.OPENAI,), + models=(OPENAI_BATCH_BACKEND,), + ) +) def test_cross_user_managed_id_denied_owner_allowed( client: BatchClient, resources: ResourceManager, managed_model: str ) -> None: diff --git a/tests/e2e/mcp/conftest.py b/tests/e2e/mcp/conftest.py index e6094ab95ea..91f55a528b5 100644 --- a/tests/e2e/mcp/conftest.py +++ b/tests/e2e/mcp/conftest.py @@ -15,14 +15,11 @@ from typing import Protocol, cast import pytest +from datadog_mcp import DdLogsReader from mcp_client import McpClient, build_client from proxy_client import ProxyClient -class DdLogsReader(Protocol): - def poll_events_for_marker(self, marker: str) -> list[object]: ... - - class _DdLogsReaderBuilder(Protocol): def __call__(self) -> DdLogsReader: ... diff --git a/tests/e2e/mcp/datadog_mcp.py b/tests/e2e/mcp/datadog_mcp.py index 352b4446cfd..63194af221e 100644 --- a/tests/e2e/mcp/datadog_mcp.py +++ b/tests/e2e/mcp/datadog_mcp.py @@ -4,14 +4,20 @@ from __future__ import annotations import os from collections.abc import Sequence +from typing import Protocol from e2e_config import datadog_mcp_url, unique_marker +from e2e_metadata import step from lifecycle import ResourceManager from mcp_client import McpClient SEARCH_LOGS_TOOL = "search_datadog_logs" +class DdLogsReader(Protocol): + def poll_events_for_marker(self, marker: str) -> list[object]: ... + + def _dd_api_key() -> str: return os.environ.get("DD_API_KEY", "").strip() @@ -31,6 +37,7 @@ def assert_dd_mcp_creds() -> None: ) +@step("Register the Datadog remote MCP server with its credentials from the environment") def register_datadog_mcp( client: McpClient, resources: ResourceManager, diff --git a/tests/e2e/mcp/mcp_client.py b/tests/e2e/mcp/mcp_client.py index 56f7fffba29..e373a8e31ae 100644 --- a/tests/e2e/mcp/mcp_client.py +++ b/tests/e2e/mcp/mcp_client.py @@ -21,6 +21,7 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel from e2e_config import settle_propagation from e2e_http import Headers, NoBody, Result, Success, UnknownApiError, unwrap +from e2e_metadata import step from models import KeyGenerateBody, McpServerListResponse, McpServerRow, ObjectPermission from proxy_client import ProxyClient @@ -153,6 +154,7 @@ class McpCallToolResponse(BaseModel): class McpClient: proxy: ProxyClient + @step("Register the MCP server {server_name} with the alias {alias}") def register_server( self, *, @@ -183,6 +185,7 @@ class McpClient: ) ).server_id + @step("Delete the MCP server") def delete_server(self, server_id: str) -> None: _ = self.proxy.transport.delete( f"/v1/mcp/server/{server_id}", @@ -191,6 +194,7 @@ class McpClient: response_type=NoBody, ) + @step("List the MCP servers from /v1/mcp/server") def registered_servers(self) -> list[McpServerRow]: return unwrap( self.proxy.transport.get( @@ -201,6 +205,7 @@ class McpClient: ) ).root + @step("List the MCP servers the key can see from /v1/mcp/server") def list_servers(self, key: str) -> Result[McpServerListResponse]: return self.proxy.transport.get( "/v1/mcp/server", @@ -209,6 +214,7 @@ class McpClient: response_type=McpServerListResponse, ) + @step("Check the health of the MCP servers the key can see from /v1/mcp/server/health") def server_health(self, key: str, server_ids: list[str] | None = None) -> Result[McpHealthResponse]: return self.proxy.transport.get( "/v1/mcp/server/health", @@ -217,6 +223,7 @@ class McpClient: response_type=McpHealthResponse, ) + @step("Wait for every proxy replica to list the MCP server in /v1/mcp/server") def await_registered(self, server_id: str) -> McpServerRow: """Wait for every configured replica to list the server and return its row.""" registered = self.proxy.read_body_back_everywhere( @@ -228,6 +235,7 @@ class McpClient: row for response in registered.values() for row in response.root if row.server_id == server_id ) + @step("Generate a virtual key for the user {user_id}") def generate_key( self, *, @@ -254,6 +262,7 @@ class McpClient: ) ) + @step("List the MCP tools the key can see from /mcp-rest/tools/list") def list_tools(self, key: str) -> Result[McpToolsListResponse]: return self.proxy.transport.get( "/mcp-rest/tools/list", @@ -262,6 +271,7 @@ class McpClient: response_type=McpToolsListResponse, ) + @step('Wait for /mcp-rest/tools/list to show the MCP server\'s tool matching "{needle}"') def await_tool(self, key: str, server_id: str, needle: str) -> str: """Poll tools/list until `server_id` serves a tool matching `needle`, and return its fully-qualified name. Fails at poll_timeout. @@ -287,6 +297,7 @@ class McpClient: ) time.sleep(self.proxy.poll_interval) + @step("Wait for /mcp-rest/tools/list to show the key exactly the expected tools on the MCP server") def await_tools(self, key: str, server_id: str, *, expected: frozenset[str]) -> frozenset[str]: """Poll tools/list until `server_id`'s tools as `key` sees them are exactly `expected`, and return the last listing either way, so the caller's equality @@ -301,6 +312,7 @@ class McpClient: return unwrap(result).tool_names_for_server(server_id) time.sleep(self.proxy.poll_interval) + @step("Call the MCP tool {name} through /mcp-rest/tools/call") def await_call_tool( self, key: str, @@ -329,6 +341,7 @@ class McpClient: ) time.sleep(self.proxy.poll_interval) + @step("Call the MCP tool {name} through /mcp-rest/tools/call and wait for a 403") def await_call_tool_denied( self, key: str, @@ -355,6 +368,7 @@ class McpClient: ) time.sleep(self.proxy.poll_interval) + @step('Create the guardrail {name} that blocks MCP tool calls containing "{blocked_keyword}"') def register_mcp_content_filter(self, *, name: str, blocked_keyword: str) -> str: """Register a default-on content-filter guardrail that runs on the MCP tool-call hook (pre_mcp_call) and blocks a single keyword. The keyword is @@ -378,6 +392,7 @@ class McpClient: settle_propagation(time.monotonic()) return guardrail_id + @step("Delete the guardrail") def delete_guardrail(self, guardrail_id: str) -> None: _ = self.proxy.transport.delete( f"/guardrails/{guardrail_id}", @@ -386,6 +401,7 @@ class McpClient: response_type=NoBody, ) + @step("Call the MCP tool {name} through /mcp-rest/tools/call with {arguments}") def call_tool( self, key: str, diff --git a/tests/e2e/mcp/oauth_chat_client.py b/tests/e2e/mcp/oauth_chat_client.py index 0c5c6106259..68e838d8745 100644 --- a/tests/e2e/mcp/oauth_chat_client.py +++ b/tests/e2e/mcp/oauth_chat_client.py @@ -26,6 +26,7 @@ import httpx2 import pytest from e2e_config import PROXY_BASE_URL, REQUEST_TIMEOUT from e2e_http import AuthHeaders, NoBody, unwrap +from e2e_metadata import step from idp import Identity from mcp import ClientSession from mcp.client.auth import OAuthClientProvider @@ -318,6 +319,7 @@ async def _list_and_call( class ChatMcpClient: proxy: ProxyClient + @step("Register the MCP server with the alias {body.alias}") def create_server(self, body: McpServerCreateBody) -> McpServerInfo: return unwrap( self.proxy.transport.post( @@ -328,6 +330,7 @@ class ChatMcpClient: ) ) + @step("Read the MCP server back from /v1/mcp/server") def server_info(self, server_id: str) -> McpServerInfo: return unwrap( self.proxy.transport.get( @@ -338,6 +341,7 @@ class ChatMcpClient: ) ) + @step("Delete the MCP server") def delete_server(self, server_id: str) -> None: _ = self.proxy.transport.delete( f"/v1/mcp/server/{server_id}", @@ -346,6 +350,7 @@ class ChatMcpClient: response_type=NoBody, ) + @step("Sign the key's user in to the MCP server {alias} through the OAuth consent flow") def seed_user_token(self, alias: str, key: str, storage_state_path: str) -> tuple[str, ...]: """Drive the interactive authorize dance for `key`'s user so the gateway stores their upstream token, retried to the shared deadline since the @@ -367,6 +372,7 @@ class ChatMcpClient: f"last error: {last_error!r}" ) + @step("List the tools on the MCP server {alias} and call {tool} over the MCP protocol") def list_and_call( self, alias: str, @@ -394,6 +400,7 @@ class ChatMcpClient: ) ) + @step("List the users with a stored OAuth token for the MCP server") def server_user_credentials(self, server_id: str) -> tuple[McpServerUserCredentialRow, ...]: return unwrap( self.proxy.transport.get( @@ -404,6 +411,7 @@ class ChatMcpClient: ) ).root + @step("Revoke the user's stored OAuth token for the MCP server") def revoke_user_token(self, server_id: str, headers: AuthHeaders) -> None: _ = unwrap( self.proxy.transport.delete( @@ -414,6 +422,7 @@ class ChatMcpClient: ) ) + @step("Send a /chat/completions request to {body.model} with an MCP server attached as a tool") def chat_with_mcp(self, headers: AuthHeaders, body: ChatBody) -> ChatResponse: """POST /chat/completions carrying the LiteLLM key in `headers` (either ingress form) with an MCP server attached in `body.tools`. The gateway diff --git a/tests/e2e/mcp/oauth_gateway.py b/tests/e2e/mcp/oauth_gateway.py index 029b0135900..2abe8262b6e 100644 --- a/tests/e2e/mcp/oauth_gateway.py +++ b/tests/e2e/mcp/oauth_gateway.py @@ -21,6 +21,7 @@ from typing import Final import psycopg from e2e_config import INHERITED_ENV_PREFIXES, available_port from e2e_http import NoBody +from e2e_metadata import step from idp import Keycloak, stop_process_group from proxy_client import ProxyClient, build_proxy_client from psycopg.rows import class_row @@ -37,6 +38,7 @@ class CredentialRow: credential_b64: str = field(repr=False) +@step("Read the user's stored OAuth credential for the MCP server from the database and decrypt it") def stored_oauth(user_id: str, server_id: str) -> StoredOAuth: """Read the encrypted credential because management APIs omit the plaintext token.""" from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper @@ -108,6 +110,7 @@ class OAuthGateway: _log_path: Path _child: subprocess.Popen[bytes] | None = field(default=None, init=False, repr=False) + @step("Start the separate LiteLLM proxy and wait for /health/liveliness") def start(self) -> None: with self._log_path.open("ab") as log: self._child = subprocess.Popen( @@ -126,11 +129,13 @@ class OAuthGateway: time.sleep(0.5) raise AssertionError("owned OAuth gateway did not become ready") + @step("Stop the separate LiteLLM proxy") def stop(self) -> None: if self._child is not None: stop_process_group(self._child) assert self._child.poll() is not None, "old gateway process is still alive" + @step("Restart the separate LiteLLM proxy process so its in-memory caches start empty") def restart(self) -> None: assert self._child is not None previous: Final = self._child.pid @@ -139,6 +144,7 @@ class OAuthGateway: assert self._child.pid != previous, "gateway restart did not create a new process" +@step("Start a separate LiteLLM proxy from source with JWT auth against Keycloak") def owned_gateway(idp: Keycloak, directory: Path, cleanup: ExitStack) -> OAuthGateway: for name in ("DATABASE_URL", "LITELLM_LICENSE", "LITELLM_SALT_KEY", "LITELLM_MASTER_KEY"): assert os.environ.get(name), f"{name} is required for the owned OAuth gateway" diff --git a/tests/e2e/mcp/test_mcp_access_group_e2e.py b/tests/e2e/mcp/test_mcp_access_group_e2e.py index f72b75fd43d..744acb78aea 100644 --- a/tests/e2e/mcp/test_mcp_access_group_e2e.py +++ b/tests/e2e/mcp/test_mcp_access_group_e2e.py @@ -16,6 +16,7 @@ import pytest from datadog_mcp import SEARCH_LOGS_TOOL, register_datadog_mcp from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Domain, Route, Subject, meta from lifecycle import ResourceManager from mcp_client import McpClient @@ -24,6 +25,12 @@ pytestmark = pytest.mark.e2e class TestMcpAccessGroupToolSelection: @pytest.mark.covers("mcp.list_tools.api_key.access_group_scoped") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_access_group_scopes_tool_selection( self, client: McpClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/mcp/test_mcp_chat_completion_oauth_e2e.py b/tests/e2e/mcp/test_mcp_chat_completion_oauth_e2e.py index 01e94f7b86f..9d7e4713963 100644 --- a/tests/e2e/mcp/test_mcp_chat_completion_oauth_e2e.py +++ b/tests/e2e/mcp/test_mcp_chat_completion_oauth_e2e.py @@ -30,6 +30,7 @@ import pytest from e2e_config import CHEAP_ANTHROPIC_MODEL, LINEAR_MCP_URL, LINEAR_STORAGE_STATE, unique_marker from e2e_http import AuthHeaders +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, KeyGenerateBody, McpChatTool, McpServerCreateBody, ObjectPermission from proxy_client import ProxyClient @@ -70,6 +71,16 @@ class TestMcpChatCompletionOauth: @pytest.mark.covers("mcp.list_tools.oauth.succeeds") @pytest.mark.covers("mcp.call_tool.oauth.succeeds") + @meta( + Subject( + domain=Domain.MCP, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_chat_completion_uses_linear_with_x_litellm_api_key_header( self, chat_client: ChatMcpClient, resources: ResourceManager ) -> None: @@ -134,6 +145,16 @@ class TestMcpChatCompletionOauth: @pytest.mark.covers("mcp.list_tools.oauth.succeeds") @pytest.mark.covers("mcp.call_tool.oauth.succeeds") + @meta( + Subject( + domain=Domain.MCP, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) def test_chat_completion_uses_linear_with_authorization_bearer_header( self, chat_client: ChatMcpClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/mcp/test_mcp_datadog_e2e.py b/tests/e2e/mcp/test_mcp_datadog_e2e.py index 031fbf6d936..e383e9ab766 100644 --- a/tests/e2e/mcp/test_mcp_datadog_e2e.py +++ b/tests/e2e/mcp/test_mcp_datadog_e2e.py @@ -13,10 +13,10 @@ from __future__ import annotations import pytest -from conftest import DdLogsReader -from datadog_mcp import SEARCH_LOGS_TOOL, assert_dd_mcp_creds, register_datadog_mcp +from datadog_mcp import SEARCH_LOGS_TOOL, DdLogsReader, assert_dd_mcp_creds, register_datadog_mcp from e2e_config import CHEAP_ANTHROPIC_MODEL, DD_SEARCH_FROM, unique_marker from e2e_http import NoBody, unwrap +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from mcp_client import McpClient from models import ChatBody, ChatMessage @@ -50,6 +50,15 @@ def _seed_completion(proxy: ProxyClient, *, key: str, marker: str) -> None: class TestDatadogMcpRoundTrip: @pytest.mark.covers("mcp.list_tools.api_key.succeeds", "mcp.call_tool.api_key.succeeds") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_ANTHROPIC_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_search_logs_finds_seeded_completion( self, client: McpClient, diff --git a/tests/e2e/mcp/test_mcp_guardrail_e2e.py b/tests/e2e/mcp/test_mcp_guardrail_e2e.py index 92c632cb316..d34bd80e564 100644 --- a/tests/e2e/mcp/test_mcp_guardrail_e2e.py +++ b/tests/e2e/mcp/test_mcp_guardrail_e2e.py @@ -23,6 +23,7 @@ import pytest from datadog_mcp import SEARCH_LOGS_TOOL, assert_dd_mcp_creds, register_datadog_mcp from e2e_config import DD_SEARCH_FROM, unique_marker from e2e_http import Result, Success, UnknownApiError +from e2e_metadata import Domain, Route, Subject, meta from lifecycle import ResourceManager from mcp_client import McpCallToolResponse, McpClient, McpToolArguments @@ -82,6 +83,12 @@ class TestMcpToolCallGuardrail: "guardrail.litellm_content_filter.pre_mcp_call.blocks", exercised_on=["mcp_operations"], ) + @meta( + Subject( + domain=Domain.GUARDRAILS, + route=Route.MCP, + ) + ) def test_content_filter_blocks_banned_keyword_in_tool_args( self, client: McpClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/mcp/test_mcp_key_access_e2e.py b/tests/e2e/mcp/test_mcp_key_access_e2e.py index c00d67bc9cf..9f5e04ccb9b 100644 --- a/tests/e2e/mcp/test_mcp_key_access_e2e.py +++ b/tests/e2e/mcp/test_mcp_key_access_e2e.py @@ -18,6 +18,7 @@ from typing import Final from datadog_mcp import SEARCH_LOGS_TOOL, register_datadog_mcp from e2e_config import DD_SEARCH_FROM, unique_marker from e2e_http import unwrap +from e2e_metadata import Domain, Route, Subject, meta from lifecycle import ResourceManager from mcp_client import McpClient from models import KeyGenerateBody, ObjectPermission @@ -33,6 +34,12 @@ def _key(client: McpClient, resources: ResourceManager, *, mcp_servers: list[str class TestMcpKeyGrantByAlias: + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_alias_grant_persists_verbatim_and_lists_tools( self, client: McpClient, @@ -62,6 +69,12 @@ class TestMcpKeyGrantByAlias: class TestMcpKeyWithoutAccessIsDenied: @pytest.mark.covers("mcp.list_tools.api_key.denied_without_permission") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_list_tools_denied_without_permission( self, client: McpClient, @@ -82,6 +95,12 @@ class TestMcpKeyWithoutAccessIsDenied: ) @pytest.mark.covers("mcp.call_tool.api_key.denied_without_permission") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_call_tool_denied_without_permission( self, client: McpClient, @@ -113,6 +132,12 @@ class TestMcpKeyWithoutAccessIsDenied: class TestMcpHealthVisibility: + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_route_restricted_health_matches_server_grants( self, client: McpClient, diff --git a/tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py b/tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py index 81470c21d51..b57c941cb18 100644 --- a/tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py +++ b/tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py @@ -18,6 +18,7 @@ from typing import Final, Literal import pytest from e2e_config import LINEAR_MCP_URL, LINEAR_READONLY_TOOL, LINEAR_STORAGE_STATE, unique_marker from e2e_http import AuthHeaders, NoBody, get_external, unwrap +from e2e_metadata import Domain, Route, Subject, meta from idp import Identity, Keycloak from lifecycle import ResourceManager from models import ( @@ -88,6 +89,12 @@ class TestMcpOauthHappyPath: @pytest.mark.covers("mcp.list_tools.oauth.succeeds") @pytest.mark.covers("mcp.call_tool.oauth.succeeds") @pytest.mark.covers("mcp.call_tool.oauth.persists_across_processes") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) @pytest.mark.parametrize("route", ("aggregate_sso", "explicit_header_jwt")) @pytest.mark.parametrize("observed", (False, True), ids=("direct", "observed")) def test_consent_list_call_and_cold_restart( diff --git a/tests/e2e/mcp/test_mcp_toolset_enforcement_e2e.py b/tests/e2e/mcp/test_mcp_toolset_enforcement_e2e.py index 6b901145eb1..cff72f3c95d 100644 --- a/tests/e2e/mcp/test_mcp_toolset_enforcement_e2e.py +++ b/tests/e2e/mcp/test_mcp_toolset_enforcement_e2e.py @@ -18,6 +18,7 @@ import pytest from datadog_mcp import SEARCH_LOGS_TOOL, register_datadog_mcp from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Domain, Route, Subject, meta from lifecycle import ResourceManager from mcp_client import McpClient from models import ToolsetCreateBody, ToolsetTool @@ -60,6 +61,12 @@ def _wire_prefix(wire_name: str, tool_name: str, catalog: frozenset[str]) -> str class TestMcpToolsetEnforcement: @pytest.mark.covers("mcp.list_tools.api_key.toolset_scoped") + @meta( + Subject( + domain=Domain.MCP, + route=Route.MCP, + ) + ) def test_key_granted_a_toolset_lists_exactly_its_tools(self, client: McpClient, resources: ResourceManager) -> None: server_id: Final = register_datadog_mcp(client, resources, allowed_tools=None) client.await_registered(server_id) diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index 04fa30a0d14..03e585273c0 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -23,6 +23,7 @@ from pydantic import BaseModel, ValidationError from proxy_client import ProxyClient from e2e_config import CHEAP_OPENAI_MODEL, PROXY_BASE_URL, unique_marker from e2e_http import NetworkError, StreamHead, StreamingResponse +from e2e_metadata import step from models import ( CacheControl, ChatMessage, @@ -79,6 +80,7 @@ def cached_system_turn(marker: str) -> ChatMessage: return ChatMessage(role="system", content=[TextContentPart(text=filler, cache_control=CacheControl())]) +@step(f"Add a deployment named {{name}} that calls {REAL_MODEL} at an unreachable address") def create_bad_base_deployment(proxy: ProxyClient, name: str) -> str: """Register a deployment pointing at an unreachable base, so every call to it fails with a real connection error the fallback can reroute around.""" @@ -87,6 +89,7 @@ def create_bad_base_deployment(proxy: ProxyClient, name: str) -> str: ) +@step(f"Add a deployment named {{name}} that calls {REAL_MODEL} at an unreachable address and is never benched") def create_never_benched_refusing_deployment(proxy: ProxyClient, name: str) -> str: return proxy.create_model( name, @@ -94,6 +97,7 @@ def create_never_benched_refusing_deployment(proxy: ProxyClient, name: str) -> s ) +@step(f"Add a deployment named {{name}} that calls {REAL_MODEL} with a 1ms timeout") def create_timeout_deployment(proxy: ProxyClient, name: str) -> str: """Register a deployment with a 1ms deadline the real backend always exceeds.""" return proxy.create_model( @@ -101,12 +105,14 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str: ) +@step(f"Add a deployment named {{name}} that calls the small-context model {SMALL_CONTEXT_MODEL}") def create_small_context_deployment(proxy: ProxyClient, name: str) -> str: """Register a deployment on the smallest-context model OpenAI still serves, so an oversized prompt earns a real context-window refusal from the provider.""" return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY)) +@step(f"Add a deployment named {{name}} that calls {AZURE_MODEL} behind Azure's content filter") def create_content_filtered_deployment(proxy: ProxyClient, name: str) -> str: """Register the Azure OpenAI deployment whose content filter refuses CONTENT_POLICY_PROMPT with a real policy-violation 400 (the one live trigger @@ -124,6 +130,10 @@ def create_content_filtered_deployment(proxy: ProxyClient, name: str) -> str: ) +@step( + f"Add a deployment named {{name}} that calls {AZURE_MODEL} and is benched for {{cooldown_time}}s" + " on its first failure" +) def create_azure_benched_on_first_failure_deployment(proxy: ProxyClient, name: str, cooldown_time: float) -> str: """The live Azure OpenAI deployment holding all of the group's shuffle weight, benched on its first failure of any class, with the client's own retries off.""" @@ -144,6 +154,7 @@ def create_azure_benched_on_first_failure_deployment(proxy: ProxyClient, name: s ) +@step(f"Add a deployment named {{name}} that calls {CACHING_MODEL} with prompt caching") def create_caching_deployment(proxy: ProxyClient, name: str) -> str: """Register the Anthropic deployment whose prompt cache the affinity check pins to.""" return proxy.create_model(name, LiteLLMParamsBody(model=CACHING_MODEL, api_key=CACHING_KEY, weight=1)) @@ -165,6 +176,7 @@ def _register_benched_on_first_failure( ) +@step(f"Add a deployment named {{name}} that calls {REAL_MODEL} with a 1ms timeout and is benched on its first timeout") def create_always_timing_out_deployment(proxy: ProxyClient, name: str, cooldown_time: float | None = None) -> str: """A 1ms deadline the real backend always exceeds, benched on its first Timeout.""" return _register_benched_on_first_failure( @@ -176,6 +188,7 @@ def create_always_timing_out_deployment(proxy: ProxyClient, name: str, cooldown_ ) +@step(f"Add a deployment named {{name}} that calls {REAL_MODEL} with an invalid key and is benched on its first 401") def create_always_unauthorized_deployment(proxy: ProxyClient, name: str, cooldown_time: float | None = None) -> str: """A key the real backend rejects with a 401, benched on its first AuthenticationError.""" return _register_benched_on_first_failure( @@ -204,6 +217,10 @@ def _nested_proxy_params(upstream_group: str, upstream_key: str, cooldown_time: ) +@step( + "Add a deployment named {name} that fronts {upstream_group} on this proxy, so it always gets a 500" + " and is benched on the first one" +) def create_always_5xx_deployment( proxy: ProxyClient, name: str, upstream_group: str, upstream_key: str, cooldown_time: float | None = None ) -> str: @@ -217,6 +234,10 @@ def create_always_5xx_deployment( ) +@step( + "Add a deployment named {name} that fronts {upstream_group} on this proxy with a key out of rpm," + " so it always gets a 429 and is benched on the first one" +) def create_always_rate_limited_deployment( proxy: ProxyClient, name: str, upstream_group: str, upstream_key: str, cooldown_time: float | None = None ) -> str: @@ -227,6 +248,7 @@ def create_always_rate_limited_deployment( ) +@step(f"Use up the rpm-limited key's one allowed request with a /chat/completions call to {CHEAP_OPENAI_MODEL}") def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None: """Uses up the one request an rpm_limit=1 key allows. The proxy's rate limiter opens the key's 60s window on this call, so it goes right before the calls that @@ -239,6 +261,7 @@ def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None: ) +@step(f"Add a deployment named {{name}} that calls {SMALL_CONTEXT_MODEL} and takes all of its group's traffic") def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) -> str: """The always-picked half of a retry pair on the smallest-context model OpenAI still serves: it holds all of the model group's shuffle weight, so an oversized @@ -253,12 +276,14 @@ def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) ) +@step(f"Add a deployment named {{name}} for {REAL_MODEL} that answers with a canned reply") def create_canned_deployment(proxy: ProxyClient, name: str) -> str: """A deployment that answers from a canned reply, so a call to it goes through the router's deployment pick like any other but never reaches a provider.""" return proxy.create_model(name, LiteLLMParamsBody(model=REAL_MODEL, mock_response="ok")) +@step(f"Add a zero-weight backup deployment named {{name}} that calls {REAL_MODEL}") def create_zero_weight_backup_deployment(proxy: ProxyClient, name: str) -> str: """The other half of a retry pair: healthy, but weight 0, so the weighted shuffle never opens on it. It is reachable only once its sibling is out of the running, @@ -273,6 +298,7 @@ def create_zero_weight_backup_deployment(proxy: ProxyClient, name: str) -> str: ) +@step("Send a /chat/completions request to {model} with a full message history and stream set to {stream}") def chat_turns_override( proxy: ProxyClient, key: str, @@ -300,6 +326,7 @@ def chat_turns_override( ) +@step("Send a /chat/completions request to {model} with stream set to {stream}") def chat_override( proxy: ProxyClient, key: str, @@ -322,6 +349,7 @@ def chat_override( ) +@step('Open a streaming /chat/completions request to {model} with the prompt "{content}" and leave it in flight') def open_chat_stream( proxy: ProxyClient, key: str, diff --git a/tests/e2e/router/test_auto_router_regressions_e2e.py b/tests/e2e/router/test_auto_router_regressions_e2e.py index 374badcf5fc..898b164e713 100644 --- a/tests/e2e/router/test_auto_router_regressions_e2e.py +++ b/tests/e2e/router/test_auto_router_regressions_e2e.py @@ -49,6 +49,7 @@ import pytest from pydantic import BaseModel, ConfigDict, Field from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from e2e_http import AnthropicHeaders, AuthHeaders, UnauthorizedError, unwrap from lifecycle import ResourceManager from models import ( @@ -341,6 +342,15 @@ def credentialed_alias(proxy: ProxyClient, router_stack: ExitStack) -> Credentia class TestTagSplitRouting: @pytest.mark.covers("reliability.routing.tagged_marker.request_tag_selects_marker") + @meta( + Subject( + domain=Domain.ROUTING, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_body_tagged_chat_routes_through_the_marker_to_its_tier( self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: @@ -355,6 +365,15 @@ class TestTagSplitRouting: _assert_served_only_by(rows, CHEAP_SERVED | {plain_first_split.tier}, "body-tagged chat on the shared name") @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") + @meta( + Subject( + domain=Domain.ROUTING, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(PLAIN_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_untagged_chat_is_always_served_by_the_plain_deployment( self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: @@ -370,6 +389,15 @@ class TestTagSplitRouting: _assert_served_only_by(rows, PLAIN_SERVED | {plain_first_split.shared}, "untagged chat on the shared name") @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") + @meta( + Subject( + domain=Domain.ROUTING, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(PLAIN_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_untagged_messages_is_served_by_the_plain_deployment( self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: @@ -387,6 +415,15 @@ class TestTagSplitRouting: class TestUntaggedTierDeployments: @pytest.mark.covers("reliability.routing.tagged_marker.header_tag_selects_marker") + @meta( + Subject( + domain=Domain.ROUTING, + route=Route.MESSAGES, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_header_tagged_messages_routes_through_the_marker_to_an_untagged_tier( self, proxy: ProxyClient, resources: ResourceManager, marker_first_split: TagSplitDeployment ) -> None: @@ -413,6 +450,15 @@ class TestUntaggedTierDeployments: ) @pytest.mark.covers("reliability.routing.tagged_marker.untagged_tier_deployments_still_served") + @meta( + Subject( + domain=Domain.ROUTING, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_body_tagged_chat_reaches_the_untagged_tier_after_marker_rewrite( self, proxy: ProxyClient, resources: ResourceManager, marker_first_split: TagSplitDeployment ) -> None: @@ -431,6 +477,12 @@ class TestUntaggedTierDeployments: _assert_served_only_by(rows, CHEAP_SERVED | {marker_first_split.tier}, "body-tagged chat with untagged tier") @pytest.mark.covers("reliability.routing.tagged_marker.tag_semantics_stay_strict") + @meta( + Subject( + domain=Domain.ROUTING, + route=Route.CHAT_COMPLETIONS, + ) + ) def test_tagged_call_straight_at_an_untagged_deployment_stays_denied( self, proxy: ProxyClient, resources: ResourceManager, marker_first_split: TagSplitDeployment ) -> None: @@ -449,6 +501,15 @@ class TestUntaggedTierDeployments: class TestResponsesApiTagRouting: @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker") + @meta( + Subject( + domain=Domain.ROUTING, + route=Route.RESPONSES, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_header_tagged_responses_with_string_input_routes_to_the_tier( self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: @@ -471,6 +532,15 @@ class TestResponsesApiTagRouting: ) @pytest.mark.covers("reliability.routing.tagged_marker.responses_input_routes_through_marker") + @meta( + Subject( + domain=Domain.ROUTING, + route=Route.RESPONSES, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_body_tagged_responses_with_list_input_routes_to_the_tier( self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: @@ -497,6 +567,15 @@ class TestResponsesApiTagRouting: _assert_served_only_by(rows, CHEAP_SERVED | {plain_first_split.tier}, "body-tagged /v1/responses list input") @pytest.mark.covers("reliability.routing.tagged_marker.untagged_request_served_by_plain_deployment") + @meta( + Subject( + domain=Domain.ROUTING, + route=Route.RESPONSES, + providers=(Provider.ANTHROPIC,), + models=(PLAIN_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_untagged_responses_is_served_by_the_plain_deployment( self, proxy: ProxyClient, resources: ResourceManager, plain_first_split: TagSplitDeployment ) -> None: @@ -524,6 +603,14 @@ class TestResponsesApiTagRouting: class TestStrategyAliasPricing: @pytest.mark.covers("reliability.routing.strategy_alias.custom_pricing_ignored") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_zero_priced_alias_still_logs_spend_at_the_tier_rate( self, proxy: ProxyClient, resources: ResourceManager, zero_priced_alias: ZeroPricedAlias ) -> None: @@ -546,6 +633,14 @@ class TestStrategyAliasPricing: class TestComplexityHeuristicScope: @pytest.mark.covers("reliability.routing.complexity_heuristic.scores_current_ask_only") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_trivial_ask_behind_keyword_heavy_system_prompt_stays_on_the_cheap_tier( self, proxy: ProxyClient, resources: ResourceManager, heuristic_split: HeuristicSplit ) -> None: @@ -573,6 +668,15 @@ class TestComplexityHeuristicScope: class TestSemanticAutoRouterResponses: @pytest.mark.covers("reliability.routing.semantic_auto_router.responses_input_routed") + @meta( + Subject( + domain=Domain.ROUTING, + route=Route.RESPONSES, + providers=(Provider.ANTHROPIC, Provider.OPENAI,), + models=(CHEAP_MODEL, EMBEDDING_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_responses_input_reaches_the_semantic_auto_router( self, proxy: ProxyClient, resources: ResourceManager, semantic_auto_router: SemanticAutoRouter ) -> None: @@ -617,6 +721,14 @@ class TestSemanticAutoRouterResponses: class TestAliasParamForwarding: @pytest.mark.covers("reliability.routing.tagged_marker.alias_connection_params_stay_with_tier") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.ANTHROPIC,), + models=(CHEAP_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_alias_api_key_never_overrides_the_tier_credential( self, proxy: ProxyClient, resources: ResourceManager, credentialed_alias: CredentialedAlias ) -> None: diff --git a/tests/e2e/router/test_complexity_router_e2e.py b/tests/e2e/router/test_complexity_router_e2e.py index e8508c963b8..c8f20102223 100644 --- a/tests/e2e/router/test_complexity_router_e2e.py +++ b/tests/e2e/router/test_complexity_router_e2e.py @@ -19,9 +19,12 @@ anthropic proves the classifier ran and openai proves it silently fell back - th exact failure before the fix. """ +from typing import Final + import pytest from complexity_router_client import ComplexityRouterClient +from e2e_metadata import Domain, Mode, Provider, Subject, meta from e2e_http import unwrap from models import ChatBody, ChatMessage @@ -33,9 +36,11 @@ LEXICALLY_SIMPLE_HARD_PROMPT = "Should I pay off my mortgage early or invest the # SIMPLE tier backend; served only when the classifier silently falls back to heuristic. # Spend logs may store the alias (gpt-5.5) or the provider-prefixed form depending on # how the deployment is registered (compose vs /model/new). -HEURISTIC_TIER_MODELS = frozenset({"openai/gpt-5.5", "gpt-5.5"}) +HEURISTIC_TIER_BACKEND: Final = "openai/gpt-5.5" +HEURISTIC_TIER_MODELS = frozenset({HEURISTIC_TIER_BACKEND, "gpt-5.5"}) # MEDIUM/COMPLEX/REASONING tier backend; served only when the LLM classifier runs. -LLM_TIER_MODELS = frozenset({"anthropic/claude-haiku-4-5", "claude-haiku-4-5"}) +LLM_TIER_BACKEND: Final = "anthropic/claude-haiku-4-5" +LLM_TIER_MODELS = frozenset({LLM_TIER_BACKEND, "claude-haiku-4-5"}) @pytest.mark.usefixtures("_ensure_complexity_smart_router") @@ -45,6 +50,14 @@ class TestComplexityRouterLlmClassifier: "(e.g. Is P equal to NP?); re-enable when classifier tier quality is fixed" ) @pytest.mark.covers("reliability.routing.complexity_llm_classifier.routes_by_llm_tier") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI, Provider.ANTHROPIC), + models=(HEURISTIC_TIER_BACKEND, LLM_TIER_BACKEND), + mode=Mode.NONSTREAM, + ) + ) def test_llm_classifier_runs_and_routes_by_semantic_tier( self, client: ComplexityRouterClient, complexity_key: str ) -> None: diff --git a/tests/e2e/router/test_reliability_cache_e2e.py b/tests/e2e/router/test_reliability_cache_e2e.py index f7a2f2ffeb7..5452853a57e 100644 --- a/tests/e2e/router/test_reliability_cache_e2e.py +++ b/tests/e2e/router/test_reliability_cache_e2e.py @@ -18,6 +18,7 @@ from e2e_config import ( REQUEST_TIMEOUT, unique_marker, ) +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody from provider_edge import ProviderRequestObservation, observed_provider_edge @@ -25,6 +26,8 @@ from pydantic import BaseModel, JsonValue pytestmark = [pytest.mark.e2e, pytest.mark.replayable] +CACHE_MODEL: Final = "openai/gpt-5.6" + class _CacheChatBody(ChatBody): ttl: int = 600 @@ -38,6 +41,14 @@ class _CachedAnswer(BaseModel): class TestReliabilityCache: @pytest.mark.covers("reliability.cache.exact.returns_cached") + @meta( + Subject( + domain=Domain.CACHING, + providers=(Provider.OPENAI,), + models=(CACHE_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_exact_cache_returns_cached( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -57,7 +68,7 @@ class TestReliabilityCache: model_id: Final = client.proxy.create_model( model, LiteLLMParamsBody( - model="openai/gpt-5.6", + model=CACHE_MODEL, api_key="os.environ/OPENAI_API_KEY", api_base=f"{edge.api_base('openai')}/v1", ), diff --git a/tests/e2e/router/test_reliability_cancel_on_disconnect_e2e.py b/tests/e2e/router/test_reliability_cancel_on_disconnect_e2e.py index 06174e97d20..e39f1ddcb90 100644 --- a/tests/e2e/router/test_reliability_cancel_on_disconnect_e2e.py +++ b/tests/e2e/router/test_reliability_cancel_on_disconnect_e2e.py @@ -29,9 +29,11 @@ import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import unique_marker from e2e_http import AbandonedRequest, StreamingResponse +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from models import ChatMessage, ReliabilityChatBody, RouterSettingsOverride from reliability_support import ( + AZURE_MODEL, REPLICA_PROPAGATION_SECONDS, chat_override, create_azure_benched_on_first_failure_deployment, @@ -102,6 +104,14 @@ def _hang_up_mid_answer(client: ComplexityRouterClient, key: str, group: str) -> class TestReliabilityCancelOnDisconnect: @pytest.mark.covers("reliability.cooldown.client_disconnect.stays_healthy") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.AZURE,), + models=(AZURE_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_client_hanging_up_never_benches_the_deployment( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/router/test_reliability_cooldowns_e2e.py b/tests/e2e/router/test_reliability_cooldowns_e2e.py index 2456bfb5f85..bdb02256976 100644 --- a/tests/e2e/router/test_reliability_cooldowns_e2e.py +++ b/tests/e2e/router/test_reliability_cooldowns_e2e.py @@ -71,10 +71,12 @@ import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import CHEAP_OPENAI_MODEL, unique_marker from e2e_http import StreamingResponse +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from models import KeyGenerateBody, RouterSettingsOverride from reliability_support import ( COOLDOWN_SECONDS, + REAL_MODEL, REPLICA_PROPAGATION_SECONDS, chat_override, create_always_5xx_deployment, @@ -212,6 +214,14 @@ def _assert_trips_then_recovers( class TestReliabilityCooldowns: @pytest.mark.covers("reliability.cooldown.5xx.trips_then_recovers") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_5xx_trips_cooldown_then_recovers( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -230,6 +240,14 @@ class TestReliabilityCooldowns: _assert_trips_then_recovers(client, scoped_key, group, failing, backup, failure_status=500) @pytest.mark.covers("reliability.cooldown.sibling_replica.serves_backup_within_read_interval") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_sibling_replica_serves_backup_within_redis_read_interval( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -272,6 +290,14 @@ class TestReliabilityCooldowns: ) @pytest.mark.covers("reliability.cooldown.429.trips_then_recovers") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL, REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_429_trips_cooldown_then_recovers( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -292,6 +318,14 @@ class TestReliabilityCooldowns: _assert_trips_then_recovers(client, scoped_key, group, failing, backup, failure_status=429) @pytest.mark.covers("reliability.cooldown.auth.trips_then_recovers") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_auth_failure_trips_cooldown_then_recovers( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -304,6 +338,14 @@ class TestReliabilityCooldowns: _assert_trips_then_recovers(client, scoped_key, group, failing, backup, failure_status=401) @pytest.mark.covers("reliability.cooldown.timeout.trips_then_recovers") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_timeout_trips_cooldown_then_recovers( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/router/test_reliability_fallbacks_e2e.py b/tests/e2e/router/test_reliability_fallbacks_e2e.py index 54c11b163d0..61115e00d07 100644 --- a/tests/e2e/router/test_reliability_fallbacks_e2e.py +++ b/tests/e2e/router/test_reliability_fallbacks_e2e.py @@ -31,10 +31,14 @@ import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import unique_marker from e2e_http import StreamingResponse +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from models import RouterSettingsOverride from reliability_support import ( + AZURE_MODEL, CONTENT_POLICY_PROMPT, + REAL_MODEL, + SMALL_CONTEXT_MODEL, azure_prompt_filter_skipped, chat_override, completion_tokens_of, @@ -50,6 +54,8 @@ from reliability_support import ( pytestmark = pytest.mark.e2e +FALLBACK_MODEL: Final = "gpt-5.5" + def _assert_served_by_fallback(resp: StreamingResponse) -> None: assert resp.status_code == 200, f"expected 200 after fallback, got {resp.status_code}: {resp.body[:300]}" @@ -95,6 +101,14 @@ def _filter_verdict(resp: StreamingResponse) -> str: class TestReliabilityFallbacks: @pytest.mark.covers("reliability.fallback.5xx.routes_to_fallback") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(FALLBACK_MODEL, REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_5xx_routes_to_fallback( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -107,11 +121,19 @@ class TestReliabilityFallbacks: scoped_key, primary, f"say hi {unique_marker()}", - override=RouterSettingsOverride(fallbacks=[{primary: ["gpt-5.5"]}]), + override=RouterSettingsOverride(fallbacks=[{primary: [FALLBACK_MODEL]}]), ) _assert_served_by_fallback(resp) @pytest.mark.covers("reliability.fallback.timeout.routes_to_fallback") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(FALLBACK_MODEL, REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_timeout_routes_to_fallback( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -124,11 +146,19 @@ class TestReliabilityFallbacks: scoped_key, primary, f"say hi {unique_marker()}", - override=RouterSettingsOverride(fallbacks=[{primary: ["gpt-5.5"]}]), + override=RouterSettingsOverride(fallbacks=[{primary: [FALLBACK_MODEL]}]), ) _assert_served_by_fallback(resp) @pytest.mark.covers("reliability.fallback.context_window.routes_to_fallback") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(FALLBACK_MODEL, SMALL_CONTEXT_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_context_window_routes_to_fallback( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -141,11 +171,19 @@ class TestReliabilityFallbacks: scoped_key, primary, oversized_prompt(unique_marker()), - override=RouterSettingsOverride(context_window_fallbacks=[{primary: ["gpt-5.5"]}]), + override=RouterSettingsOverride(context_window_fallbacks=[{primary: [FALLBACK_MODEL]}]), ) _assert_served_by_fallback(resp) @pytest.mark.covers("reliability.fallback.content_policy.routes_to_fallback") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.AZURE, Provider.OPENAI,), + models=(AZURE_MODEL, FALLBACK_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_content_policy_routes_to_fallback( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -167,7 +205,7 @@ class TestReliabilityFallbacks: scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}", - override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]), + override=RouterSettingsOverride(content_policy_fallbacks=[{primary: [FALLBACK_MODEL]}]), ) ) _assert_served_by_fallback(resp) diff --git a/tests/e2e/router/test_reliability_memory_e2e.py b/tests/e2e/router/test_reliability_memory_e2e.py index 77d3a68cae5..9ae6edb3eb2 100644 --- a/tests/e2e/router/test_reliability_memory_e2e.py +++ b/tests/e2e/router/test_reliability_memory_e2e.py @@ -80,11 +80,12 @@ from e2e_config import ( PROXY_REPLICA_URLS, unique_marker, ) +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from memory_readings import RssCapture, RssReading, WorkerKey, read_rss_everywhere from models import ChatMessage, RouterSettingsOverride, SpendLogRow from proxy_client import ProxyClient -from reliability_support import chat_override, create_never_benched_refusing_deployment +from reliability_support import REAL_MODEL, chat_override, create_never_benched_refusing_deployment pytestmark = [pytest.mark.e2e, pytest.mark.quiet_stack] @@ -222,6 +223,11 @@ def _stored_request_kb(proxy: ProxyClient, call: FailedCall) -> float: class TestReliabilityMemory: @pytest.mark.covers("reliability.perf.idle_memory.under_slo") + @meta( + Subject( + domain=Domain.DEPLOY_OPS, + ) + ) def test_workers_idle_under_rss_budget_before_traffic(self, idle_rss: RssCapture) -> None: assert not idle_rss.failures, ( f"{len(idle_rss.failures)} replica(s) gave no RSS reading when the session started, so their idle " @@ -242,6 +248,14 @@ class TestReliabilityMemory: ) @pytest.mark.covers("reliability.perf.memory.under_slo") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_failing_requests_do_not_grow_rss_or_stored_request( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/router/test_reliability_prompt_caching_e2e.py b/tests/e2e/router/test_reliability_prompt_caching_e2e.py index 667b398cd16..7bf23bf967f 100644 --- a/tests/e2e/router/test_reliability_prompt_caching_e2e.py +++ b/tests/e2e/router/test_reliability_prompt_caching_e2e.py @@ -23,9 +23,11 @@ import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from models import ChatMessage, LiteLLMParamsBody, ModelInfoBody, ModelNewBody from reliability_support import ( + CACHING_MODEL, REAL_KEY, REAL_MODEL, cached_system_turn, @@ -42,6 +44,15 @@ FOLLOW_UPS = 3 class TestReliabilityPromptCachingAffinity: @pytest.mark.covers("reliability.cache.prompt_caching_model_select.returns_cached") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.ANTHROPIC, Provider.OPENAI), + models=(CACHING_MODEL, REAL_MODEL), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_cached_conversation_stays_on_deployment_holding_its_cache( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/router/test_reliability_retries_e2e.py b/tests/e2e/router/test_reliability_retries_e2e.py index a90efa52b5f..c6ae8c457e4 100644 --- a/tests/e2e/router/test_reliability_retries_e2e.py +++ b/tests/e2e/router/test_reliability_retries_e2e.py @@ -30,9 +30,12 @@ import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import CHEAP_OPENAI_MODEL, unique_marker from e2e_http import StreamingResponse +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from models import KeyGenerateBody, RouterSettingsOverride from reliability_support import ( + REAL_MODEL, + SMALL_CONTEXT_MODEL, chat_override, completion_tokens_of, content_of, @@ -84,6 +87,14 @@ def _retry_once(client: ComplexityRouterClient, key: str, group: str) -> Streami class TestReliabilityRetries: @pytest.mark.covers("reliability.retry.timeout.succeeds_within_retries") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_timeout_on_first_deployment_succeeds_on_retry( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -96,6 +107,14 @@ class TestReliabilityRetries: _assert_served_after_retry(_retry_once(client, scoped_key, group)) @pytest.mark.covers("reliability.retry.5xx.succeeds_within_retries") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_5xx_on_first_deployment_succeeds_on_retry( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -112,6 +131,14 @@ class TestReliabilityRetries: _assert_served_after_retry(_retry_once(client, scoped_key, group)) @pytest.mark.covers("reliability.retry.429.succeeds_within_retries") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL, REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_429_on_first_deployment_succeeds_on_retry( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -130,6 +157,14 @@ class TestReliabilityRetries: _assert_served_after_retry(_retry_once(client, scoped_key, group)) @pytest.mark.covers("reliability.retry.auth.succeeds_within_retries") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_auth_failure_on_first_deployment_succeeds_on_retry( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -142,6 +177,14 @@ class TestReliabilityRetries: _assert_served_after_retry(_retry_once(client, scoped_key, group)) @pytest.mark.covers("reliability.retry.context_window.succeeds_within_retries") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL, SMALL_CONTEXT_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_context_window_refusal_on_first_deployment_succeeds_on_retry( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/router/test_reliability_routing_strategies_e2e.py b/tests/e2e/router/test_reliability_routing_strategies_e2e.py index 2abc2ee5f54..ee0334f989d 100644 --- a/tests/e2e/router/test_reliability_routing_strategies_e2e.py +++ b/tests/e2e/router/test_reliability_routing_strategies_e2e.py @@ -63,6 +63,7 @@ import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import unique_marker from e2e_http import StreamChunk, StreamHead, StreamStep, StreamTruncation +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody, ModelInfoBody, ModelNewBody, RouterSettingsOverride, RoutingStrategy from reliability_support import REAL_KEY, REAL_MODEL, chat_override, model_id_of, open_chat_stream @@ -173,6 +174,14 @@ def _assert_shuffle_control_lands_on(client: ComplexityRouterClient, key: str, g class TestReliabilityRoutingStrategies: @pytest.mark.covers("reliability.routing.simple_shuffle.picks_healthy_deployment") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_simple_shuffle_honors_weights( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -191,6 +200,14 @@ class TestReliabilityRoutingStrategies: ) @pytest.mark.covers("reliability.routing.cost_based.picks_lowest_cost") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_cost_based_picks_cheapest_deployment( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -206,6 +223,14 @@ class TestReliabilityRoutingStrategies: _assert_shuffle_control_lands_on(client, scoped_key, group, pricey) @pytest.mark.covers("reliability.routing.usage_based.picks_under_tpm") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_usage_based_picks_deployment_with_tpm_headroom( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -223,6 +248,14 @@ class TestReliabilityRoutingStrategies: "so latency-based has no signal to route on" ) @pytest.mark.covers("reliability.routing.latency_based.picks_lowest_latency") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_latency_based_routes_around_deployment_that_times_out( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -253,6 +286,13 @@ class TestReliabilityRoutingStrategies: "so least-busy has no signal to route on" ) @pytest.mark.covers("reliability.routing.least_busy.picks_lowest_traffic") + @meta( + Subject( + domain=Domain.ROUTING, + providers=(Provider.OPENAI,), + models=(REAL_MODEL,), + ) + ) def test_least_busy_avoids_deployment_with_request_in_flight( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/router/test_reliability_timeouts_e2e.py b/tests/e2e/router/test_reliability_timeouts_e2e.py index f24d5139e66..926d8539d6d 100644 --- a/tests/e2e/router/test_reliability_timeouts_e2e.py +++ b/tests/e2e/router/test_reliability_timeouts_e2e.py @@ -13,14 +13,16 @@ import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager -from reliability_support import chat_override, create_timeout_deployment +from reliability_support import REAL_MODEL, chat_override, create_timeout_deployment pytestmark = pytest.mark.e2e class TestReliabilityTimeouts: @pytest.mark.covers("reliability.timeout.request_timeout.exceeds_deadline") + @meta(Subject(domain=Domain.ROUTING, providers=(Provider.OPENAI,), models=(REAL_MODEL,), mode=Mode.NONSTREAM)) def test_request_timeout_exceeds_deadline( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -35,6 +37,7 @@ class TestReliabilityTimeouts: assert "timeout" in resp.body.lower(), f"the 408 body should name the timeout, got: {resp.body[:300]}" @pytest.mark.covers("reliability.timeout.stream_timeout.exceeds_deadline") + @meta(Subject(domain=Domain.ROUTING, providers=(Provider.OPENAI,), models=(REAL_MODEL,), mode=Mode.STREAM)) def test_stream_timeout_exceeds_deadline( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str ) -> None: From 3538e87e45d147102b0f1dee034bb9424f966874 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 7 Oct 2026 10:33:12 -0700 Subject: [PATCH 08/35] test(e2e): tag the remaining quota_management tests and record budget and spend client steps (#44966) * test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata * test(e2e): tag quota_management tests with Subject metadata and record budget client steps * docs(e2e): name every markerless harness test file that carries no Subject * test(e2e): keep the step discovery comprehensions to one for clause --- .../quota_management/budgets/budget_client.py | 27 +++++++++++++ .../test_model_group_alias_rate_limit_e2e.py | 17 +++++++++ .../spend_tracking/cost_rows.py | 4 ++ .../spend_tracking/spend_e2e_client.py | 38 +++++++++++++++++++ .../spend_tracking/spend_reconciliation.py | 6 +++ .../test_service_tier_pricing_e2e.py | 38 ++++++++++++++++++- .../spend_tracking/test_spend_routes.py | 6 +++ .../test_spend_surface_consistency_e2e.py | 12 +++++- .../spend_tracking/test_spend_tracking_e2e.py | 28 +++++++++++++- ...test_websearch_interception_session_e2e.py | 11 ++++++ 10 files changed, 184 insertions(+), 3 deletions(-) diff --git a/tests/e2e/quota_management/budgets/budget_client.py b/tests/e2e/quota_management/budgets/budget_client.py index 087dc8ca522..926dae0d954 100644 --- a/tests/e2e/quota_management/budgets/budget_client.py +++ b/tests/e2e/quota_management/budgets/budget_client.py @@ -17,6 +17,7 @@ from datetime import datetime from pydantic import AliasPath, BaseModel, Field, RootModel from e2e_http import NoBody, Result, StreamingResponse, Success, unwrap +from e2e_metadata import step from proxy_client import ProxyClient from models import ( AnthropicMessagesBody, @@ -253,15 +254,18 @@ class BudgetClient: ) ) + @step("Delete the virtual key") def delete_key(self, key: str) -> None: self.proxy.delete_key(key) + @step("Read the key's budget windows from /key/info") def key_budget_windows(self, key: str) -> list[BudgetWindowState]: """A key's budget_limits windows as /key/info stores them. Each window's reset_at is advanced by the reset job in the same pass that zeroes the window's spend counter, so a strictly-later value proves the wipe ran.""" return self.proxy.key_info(key).budget_limits or [] + @step("Read the team's budget windows from /team/info") def team_budget_windows(self, team_id: str) -> list[BudgetWindowState]: """Team analog of key_budget_windows, read from /team/info.""" match self._team_info(team_id): @@ -270,11 +274,13 @@ class BudgetClient: case _: return [] + @step("Delete the end users {user_ids}") def delete_customers(self, user_ids: list[str]) -> None: self.proxy.delete_customers(user_ids) # ---- chat (raw HTTP outcome: a budget block surfaces as a non-2xx) -- + @step('Send a /chat/completions request to {model} with the prompt "{content}"') def chat( self, key: str, @@ -297,6 +303,7 @@ class BudgetClient: ), ) + @step('Send a /v1/messages request to {model} with the prompt "{content}"') def messages( self, key: str, @@ -317,6 +324,7 @@ class BudgetClient: # ---- internal user -------------------------------------------------- + @step("Create an internal user with max budget: {max_budget}") def create_user(self, *, max_budget: float, budget_duration: str | None = None) -> str: return unwrap( self.proxy.transport.post( @@ -327,6 +335,7 @@ class BudgetClient: ) ).user_id + @step("Delete the internal user") def delete_user(self, user_id: str) -> None: _ = self.proxy.transport.post( "/user/delete", @@ -335,6 +344,7 @@ class BudgetClient: response_type=NoBody, ) + @step("Read the internal user's spend and budget from /user/info") def user_info(self, user_id: str) -> UserInfoRow | None: result = self.proxy.transport.get( "/user/info", @@ -350,6 +360,7 @@ class BudgetClient: # ---- customer / end-user ------------------------------------------- + @step("Create the end user {customer_id}") def create_customer( self, customer_id: str, @@ -369,6 +380,7 @@ class BudgetClient: # ---- organization --------------------------------------------------- + @step("Create the organization {alias} with max budget: {max_budget}") def create_org(self, *, max_budget: float, alias: str, budget_duration: str | None = None) -> str: return unwrap( self.proxy.transport.post( @@ -383,6 +395,7 @@ class BudgetClient: ) ).organization_id + @step("Read the organization's budget id from /organization/info") def org_budget_id(self, org_id: str) -> str | None: """The id of the budget row backing an org; its budget_reset_at is read via budget_info (LIT-4570: /organization/new stores budget_duration without @@ -399,6 +412,7 @@ class BudgetClient: case _: return None + @step("Delete the organization") def delete_org(self, org_id: str) -> None: _ = self.proxy.transport.delete( "/organization/delete", @@ -409,6 +423,7 @@ class BudgetClient: # ---- team ----------------------------------------------------------- + @step("Create the team {alias} and wait until /team/info returns it") def create_team( self, *, @@ -435,6 +450,7 @@ class BudgetClient: self._wait_for_team(team_id) return team_id + @step("Delete the team") def delete_team(self, team_id: str) -> None: _ = self.proxy.transport.post( "/team/delete", @@ -463,6 +479,7 @@ class BudgetClient: assert last is not None raise AssertionError(last) + @step("Add the internal user to the team") def add_team_member(self, team_id: str, user_id: str, *, max_budget_in_team: float | None = None) -> None: last_body = "" for attempt in range(_TEAM_READY_ATTEMPTS): @@ -484,6 +501,7 @@ class BudgetClient: break raise AssertionError(last_body) + @step("Update the team member's budget with /team/member_update") def update_team_member( self, team_id: str, @@ -504,6 +522,7 @@ class BudgetClient: ) assert resp.ok, resp.body + @step("Read the team member's budget reset time from /team/info") def member_budget_reset_at(self, team_id: str, user_id: str) -> str | None: """The member's per-team budget_reset_at as /team/info reports it, or None if no reset is scheduled. The reset job advances this each time the window @@ -519,6 +538,7 @@ class BudgetClient: # ---- tag ------------------------------------------------------------ + @step("Create the tag {name} with max budget: {max_budget}") def create_tag(self, name: str, *, max_budget: float) -> str: resp = self.proxy.transport.send( "/tag/new", @@ -528,6 +548,7 @@ class BudgetClient: assert resp.ok, resp.body return name + @step("Delete the tag {name}") def delete_tag(self, name: str) -> None: _ = self.proxy.transport.post( "/tag/delete", @@ -538,6 +559,7 @@ class BudgetClient: # ---- model access group --------------------------------------------- + @step("Set a shared budget on the model access group {access_group}") def set_access_group_budget( self, access_group: str, @@ -561,6 +583,7 @@ class BudgetClient: ) ) + @step("Read the budget and spend of the model access group {access_group}") def access_group_budget(self, access_group: str) -> AccessGroupBudgetResponse: return unwrap( self.proxy.transport.get( @@ -571,6 +594,7 @@ class BudgetClient: ) ) + @step("Delete the budget on the model access group {access_group}") def delete_access_group_budget(self, access_group: str) -> None: _ = self.proxy.transport.delete( f"/access_group/{access_group}/budget", @@ -581,6 +605,7 @@ class BudgetClient: # ---- budget table --------------------------------------------------- + @step("Create a budget with /budget/new") def create_budget( self, *, @@ -603,6 +628,7 @@ class BudgetClient: ) ).budget_id + @step("Delete the budget") def delete_budget(self, budget_id: str) -> None: _ = self.proxy.transport.post( "/budget/delete", @@ -611,6 +637,7 @@ class BudgetClient: response_type=NoBody, ) + @step("Read the budget from /budget/info") def budget_info(self, budget_id: str) -> tuple[BudgetRow, ...]: result = self.proxy.transport.post( "/budget/info", diff --git a/tests/e2e/quota_management/ratelimit/test_model_group_alias_rate_limit_e2e.py b/tests/e2e/quota_management/ratelimit/test_model_group_alias_rate_limit_e2e.py index 1c3fff47b78..523436f8d1e 100644 --- a/tests/e2e/quota_management/ratelimit/test_model_group_alias_rate_limit_e2e.py +++ b/tests/e2e/quota_management/ratelimit/test_model_group_alias_rate_limit_e2e.py @@ -21,6 +21,7 @@ import time import pytest from e2e_config import unique_marker from e2e_http import StreamingResponse, require_successful_call +from e2e_metadata import Domain, Mode, Provider, Subject, meta from quota_client import QuotaClient pytestmark = pytest.mark.e2e @@ -75,11 +76,27 @@ def _assert_blocked_inside_window( class TestModelGroupAliasRateLimit: @pytest.mark.covers("quota_management.ratelimit.model_group_alias.shares_bucket") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + providers=(Provider.ANTHROPIC,), + models=(MODEL_GROUP, MODEL_ALIAS), + mode=Mode.NONSTREAM, + ) + ) def test_alias_shares_rpm_bucket_with_model_group(self, client: QuotaClient, scoped_key: str) -> None: opened_at = _exhaust_rpm(client, scoped_key, MODEL_GROUP) _assert_blocked_inside_window(client, scoped_key, MODEL_ALIAS, opened_at) @pytest.mark.covers("quota_management.ratelimit.model_group_alias.shares_bucket") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + providers=(Provider.ANTHROPIC,), + models=(MODEL_GROUP, MODEL_ALIAS), + mode=Mode.NONSTREAM, + ) + ) def test_model_group_shares_rpm_bucket_with_alias(self, client: QuotaClient, scoped_key: str) -> None: opened_at = _exhaust_rpm(client, scoped_key, MODEL_ALIAS) _assert_blocked_inside_window(client, scoped_key, MODEL_GROUP, opened_at) diff --git a/tests/e2e/quota_management/spend_tracking/cost_rows.py b/tests/e2e/quota_management/spend_tracking/cost_rows.py index 87af54fe83f..8b15c555c66 100644 --- a/tests/e2e/quota_management/spend_tracking/cost_rows.py +++ b/tests/e2e/quota_management/spend_tracking/cost_rows.py @@ -36,6 +36,7 @@ from pydantic import BaseModel, RootModel from e2e_config import unique_marker from e2e_http import Success +from e2e_metadata import step from lifecycle import ResourceManager from models import LiteLLMParamsBody, SpendLogsParams from proxy_client import ProxyClient @@ -133,6 +134,7 @@ def assert_fresh_tokens_billed_at(row: CostRow, input_rate: float) -> None: ) +@step("Wait for the request's cost breakdown in /spend/logs") def poll_cost_row(proxy: ProxyClient, request_id: str) -> CostRow | None: """Poll /spend/logs for the call's row until it lands with a cost breakdown (rows flush ~60s behind the call via proxy_batch_write_at); None on timeout.""" @@ -156,6 +158,7 @@ def poll_cost_row(proxy: ProxyClient, request_id: str) -> CostRow | None: return None +@step("Wait for a matching cost breakdown in the key's /spend/logs") def poll_cost_row_where( proxy: ProxyClient, api_key: str, predicate: Callable[[CostRow], bool] ) -> CostRow | None: @@ -182,6 +185,7 @@ def poll_cost_row_where( return None +@step("Add a deployment with custom rates that calls {litellm_params.model}") def register_priced_model( proxy: ProxyClient, resources: ResourceManager, diff --git a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py index e607c12b731..60372a35afe 100644 --- a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py +++ b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py @@ -31,6 +31,7 @@ from e2e_http import ( is_ok, unwrap, ) +from e2e_metadata import step from models import ( AnthropicMessagesBody, ChatBody, @@ -264,6 +265,7 @@ def _chat_body( class SpendClient: proxy: ProxyClient + @step('Send a /chat/completions request to {model} with the prompt "{content}"') def chat( self, key: str, @@ -280,6 +282,7 @@ class SpendClient: _chat_body(model, content, max_tokens=max_tokens, tags=tags, user=user, cache=cache), ) + @step('Send a streaming /chat/completions request to {model} with the prompt "{content}"') def chat_stream( self, key: str, model: str, content: str, *, max_tokens: int | None = None ) -> StreamingResponse: @@ -287,6 +290,7 @@ class SpendClient: key, _chat_body(model, content, max_tokens=max_tokens, stream=True) ) + @step('Send a streaming /v1/messages request to {model} with the prompt "{content}"') def messages_stream( self, key: str, model: str, content: str, *, max_tokens: int ) -> StreamingResponse: @@ -300,9 +304,11 @@ class SpendClient: ), ) + @step('Send an /embeddings request to {model} for "{content}"') def embed(self, key: str, model: str, content: str) -> Result[EmbedResponse]: return self.proxy.embed(key, EmbedBody(model=model, input=content)) + @step("Wait for at least {min_rows} of the key's spend logs in /spend/logs") def poll_logs_for_key( self, key: str, @@ -314,6 +320,7 @@ class SpendClient: key, min_rows=min_rows, predicate=predicate ) + @step('Estimate the cost of sending "{content}" to {model} with /spend/calculate') def calculate_spend(self, model: str, content: str) -> float: return unwrap( self.proxy.transport.post( @@ -326,6 +333,7 @@ class SpendClient: ) ).cost + @step("Read the spend per tag from /spend/tags") def spend_by_tags(self) -> list[TagSpend]: result = self.proxy.transport.get( "/spend/tags", @@ -339,6 +347,7 @@ class SpendClient: case _: return [] + @step("Wait for the tag {tag} to reach the expected spend in /spend/tags") def poll_tag_spend(self, tag: str, *, minimum: float = 0.0) -> TagSpend | None: """Poll /spend/tags until the tag's aggregate reaches `minimum`; last seen.""" deadline = time.monotonic() + self.proxy.poll_timeout @@ -354,6 +363,7 @@ class SpendClient: time.sleep(self.proxy.poll_interval) return entry + @step("Wait for the key's spend in /key/info to reach the expected minimum") def poll_key_spend(self, key: str, *, minimum: float = 0.0) -> float: deadline = time.monotonic() + self.proxy.poll_timeout spend = 0.0 @@ -364,6 +374,7 @@ class SpendClient: time.sleep(self.proxy.poll_interval) return spend + @step("Read the team's spend from /team/info") def team_spend(self, team_id: str) -> float: return ( unwrap( @@ -377,6 +388,7 @@ class SpendClient: or 0.0 ) + @step("Wait for the team's spend in /team/info to reach the expected minimum") def poll_team_spend(self, team_id: str, *, minimum: float = 0.0) -> float: outcome: Final = await_converged( lambda: self.team_spend(team_id), @@ -388,6 +400,7 @@ class SpendClient: ) return outcome.result if isinstance(outcome, Converged) else outcome.last_result + @step("Read the end user's spend from /customer/info") def customer_spend(self, customer_id: str) -> float: """0.0 until the spend writer has upserted the end-user row, which /customer/info 404s before.""" looked_up: Final = self.proxy.transport.get( @@ -402,6 +415,7 @@ class SpendClient: case _: return 0.0 + @step("Wait for the end user's spend in /customer/info to go above {minimum}") def poll_customer_spend(self, customer_id: str, *, minimum: float = 0.0) -> float: outcome: Final = await_converged( lambda: self.customer_spend(customer_id), @@ -413,6 +427,7 @@ class SpendClient: ) return outcome.result if isinstance(outcome, Converged) else outcome.last_result + @step("Scrape /metrics/ on every proxy replica") def scrape_metrics(self) -> Mapping[str, ProbeResult]: """GET /metrics/ on every replica in PROXY_REPLICA_URLS, keyed by replica. The counter is per pod, so the union of the replicas is the fleet's exposition; the @@ -425,6 +440,7 @@ class SpendClient: } ) + @step("Read page {page} of /spend/logs/v2 at a page size of {page_size}") def spend_logs_page( self, *, api_key: str | None, page: int, page_size: int ) -> SpendLogsPage: @@ -447,9 +463,11 @@ class SpendClient: ) ) + @step("Call the management route {path}") def probe(self, path: str, *, params: DateRangeParams) -> ProbeResult: return self.proxy.transport.probe(path, params=params) + @step("Call the management route {path} until it answers successfully") def probe_until_healthy(self, path: str, *, params: DateRangeParams) -> ProbeResult: outcome: Final = await_converged( lambda: self.probe(path, params=params), @@ -461,6 +479,7 @@ class SpendClient: ) return outcome.result if isinstance(outcome, Converged) else outcome.last_result + @step("Create an internal user with the role {role}") def create_user(self, *, email: str, role: UserRole, user_id: str) -> str: return unwrap( self.proxy.transport.post( @@ -471,6 +490,7 @@ class SpendClient: ) ).user_id + @step("Delete the internal user") def delete_user(self, user_id: str) -> None: _ = unwrap( self.proxy.transport.post( @@ -481,6 +501,7 @@ class SpendClient: ) ) + @step("Generate a virtual key with {body}") def generate_key_record(self, body: KeyGenerateBody) -> KeyGenerateResponse: return unwrap( self.proxy.transport.post( @@ -491,6 +512,7 @@ class SpendClient: ) ) + @step('Send a /chat/completions request to {model} with the prompt "{content}"') def send_chat(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: return self.proxy.transport.send( "/chat/completions", @@ -498,6 +520,7 @@ class SpendClient: json=_chat_body(model, content, max_tokens=max_tokens), ) + @step('Send a /queue/chat/completions request to {model} with the prompt "{content}"') def send_queued_chat(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: return self.proxy.transport.send( "/queue/chat/completions", @@ -509,6 +532,7 @@ class SpendClient: ), ) + @step('Send a /v1/messages request to {model} with the prompt "{content}"') def send_messages(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: return self.proxy.transport.send( "/v1/messages", @@ -520,9 +544,11 @@ class SpendClient: ), ) + @step('Send a /v1/responses request to {model} with the prompt "{content}"') def send_responses(self, key: str, model: str, content: str) -> StreamingResponse: return self.send_responses_with_headers(self.proxy.transport.bearer(key), model, content) + @step('Send a /v1/responses request to {model} with custom headers and the prompt "{content}"') def send_responses_with_headers(self, headers: AuthHeaders, model: str, content: str) -> StreamingResponse: return self.proxy.transport.send( "/v1/responses", @@ -530,6 +556,7 @@ class SpendClient: json=ResponsesBody(model=model, input=content), ) + @step('Send an /embeddings request to {model} for "{content}"') def send_embed(self, key: str, model: str, content: str) -> StreamingResponse: return self.proxy.transport.send( "/embeddings", @@ -537,6 +564,7 @@ class SpendClient: json=EmbedBody(model=model, input=content), ) + @step('Send a Gemini generateContent request to {model} through /gemini with the prompt "{content}"') def send_gemini_generate(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: return self.proxy.transport.send( f"/gemini/v1beta/models/{model}:generateContent", @@ -547,6 +575,7 @@ class SpendClient: ), ) + @step("Upload a batch input file for {model} to /v1/files") def upload_batch_file(self, key: str, model: str, content: bytes) -> FileObject: return unwrap( self.proxy.transport.upload( @@ -560,6 +589,7 @@ class SpendClient: ) ) + @step("Create a batch for {body.model} on /v1/batches") def create_batch(self, key: str, body: BatchCreateBody) -> BatchObject: return unwrap( self.proxy.transport.post( @@ -570,6 +600,7 @@ class SpendClient: ) ) + @step("Retrieve the {provider} batch from /v1/batches") def retrieve_batch(self, key: str, batch_id: str, *, provider: str) -> BatchObject: return unwrap( self.proxy.transport.get( @@ -580,6 +611,7 @@ class SpendClient: ) ) + @step("Post a callback log for {payload.model} to /v1/rust_control_plane/logs") def replay_callback_log(self, key: str, payload: CallbackLogPayload) -> CallbackLogsResponse: return unwrap( self.proxy.transport.post( @@ -590,12 +622,15 @@ class SpendClient: ) ) + @step("Run a health check on {model} with /health") def health(self, model: str) -> ProbeResult: return self.proxy.transport.probe("/health", params=HealthParams(model=model)) + @step("Read the key's daily activity from /user/daily/activity") def daily_activity_for_key(self, token: str, *, start: datetime, end: datetime) -> DailyActivityKeyBreakdown | None: return self._key_breakdown("/user/daily/activity", token, start=start, end=end) + @step("Read the key's usage export row from /user/daily/activity/aggregated") def usage_export_row_for_key( self, token: str, *, start: datetime, end: datetime ) -> DailyActivityKeyBreakdown | None: @@ -623,11 +658,13 @@ class SpendClient: None, ) + @step("Wait for at least {min_requests} of the key's requests in /user/daily/activity") def poll_daily_activity_for_key( self, token: str, *, start: datetime, end: datetime, min_requests: int ) -> DailyActivityKeyBreakdown | None: return self._poll_key_breakdown(lambda: self.daily_activity_for_key(token, start=start, end=end), min_requests) + @step("Wait for at least {min_requests} of the key's requests in /user/daily/activity/aggregated") def poll_usage_export_row_for_key( self, token: str, *, start: datetime, end: datetime, min_requests: int ) -> DailyActivityKeyBreakdown | None: @@ -648,6 +685,7 @@ class SpendClient: ) return outcome.result if isinstance(outcome, Converged) else outcome.last_result + @step("Read the OpenAPI schema from /openapi.json") def openapi(self) -> OpenAPISchema: return unwrap( self.proxy.transport.get( diff --git a/tests/e2e/quota_management/spend_tracking/spend_reconciliation.py b/tests/e2e/quota_management/spend_tracking/spend_reconciliation.py index f313325dbda..96f1fbd4a7a 100644 --- a/tests/e2e/quota_management/spend_tracking/spend_reconciliation.py +++ b/tests/e2e/quota_management/spend_tracking/spend_reconciliation.py @@ -5,6 +5,7 @@ from typing import Final from e2e_config import provider_edge_base, unique_marker from e2e_http import unwrap +from e2e_metadata import step from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody, LiteLLMParamsBody, TeamNewBody from spend_e2e_client import SpendClient @@ -33,6 +34,10 @@ class TeamTraffic: return self.prompt_tokens * INPUT_RATE + self.completion_tokens * OUTPUT_RATE +@step( + "Add a priced deployment, then create two teams with one key each" + " and send 7 /chat/completions requests per key, 6 of them at once" +) def create_traffic(client: SpendClient, resources: ResourceManager) -> tuple[TeamTraffic, ...]: base: Final = provider_edge_base("openai") model: Final = f"e2e-reconciliation-{unique_marker()}" @@ -85,6 +90,7 @@ def create_traffic(client: SpendClient, resources: ResourceManager) -> tuple[Tea return tuple(team_traffic() for _ in range(2)) +@step("Check that the /spend/logs rows of team {traffic.team_id} match each response's tokens and cost") def assert_logs_match(client: SpendClient, traffic: TeamTraffic) -> None: expected_ids: Final = frozenset(response.id for response in traffic.responses) assert len(expected_ids) == len(traffic.responses), "responses must have distinct IDs" diff --git a/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py b/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py index 76d80b1aab8..31c9b90d209 100644 --- a/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py @@ -36,7 +36,7 @@ from cost_rows import ( ) from e2e_config import CHEAP_OPENAI_MODEL, unique_marker from e2e_http import unwrap -from e2e_metadata import Capability, Domain, Mode, Provider, Subject, meta +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ( AnthropicMessagesBody, @@ -184,6 +184,15 @@ class TestServiceTierPricing: assert_total_is_sum_of_components(row) @pytest.mark.covers("quota_management.spend_tracking.service_tier_stream.records_served_tier") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(BACKEND,), + mode=Mode.STREAM, + ) + ) def test_streamed_call_records_and_bills_the_served_tier( self, client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -232,6 +241,15 @@ class TestServiceTierPricing: assert_total_is_sum_of_components(row) @pytest.mark.covers("llm.chat_completions.openai.service_tier.stream.echoes_served_tier") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(STREAM_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_every_streamed_chunk_carries_the_served_tier( self, client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -262,6 +280,15 @@ class TestServiceTierPricing: ) @pytest.mark.covers("quota_management.spend_tracking.service_tier_stream.responses_records_served_tier") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(STREAM_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_responses_stream_records_the_served_tier( self, client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -308,6 +335,15 @@ class TestServiceTierPricing: assert_fresh_tokens_billed_at(row, INPUT_RATE_FOR_PRICING_BASIS[pricing_basis]) @pytest.mark.covers("quota_management.spend_tracking.service_tier_stream.messages_records_served_tier") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.MESSAGES, + providers=(Provider.OPENAI,), + models=(STREAM_BACKEND,), + mode=Mode.STREAM, + ) + ) def test_messages_stream_records_the_served_tier( self, client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_routes.py b/tests/e2e/quota_management/spend_tracking/test_spend_routes.py index 7b5db9ccd27..1b2069abdc6 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_routes.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_routes.py @@ -163,6 +163,12 @@ def test_schema_listed_spend_routes_are_responsive(client: SpendClient) -> None: assert not offenders, "non-responsive schema spend routes:\n" + "\n".join(offenders) +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.SPEND_REPORTING, + ) +) def test_capture_rate_reports_or_names_the_missing_billing_key(client: SpendClient) -> None: result: Final = client.probe(_CAPTURE_RATE_ROUTE, params=_date_range()) print(f"{_CAPTURE_RATE_ROUTE} -> {result.status_code}\n{result.body[:600]}") diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_surface_consistency_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_surface_consistency_e2e.py index 9033b9d75c4..63ba785a2ee 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_surface_consistency_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_surface_consistency_e2e.py @@ -32,6 +32,7 @@ from typing import Final import pytest from e2e_config import provider_edge_base, unique_marker from e2e_http import ProbeResult +from e2e_metadata import Domain, Mode, Provider, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, KeyGenerateBody, LiteLLMParamsBody, TeamNewBody from prometheus_client.parser import text_string_to_metric_families @@ -41,6 +42,7 @@ from spend_reconciliation import INPUT_RATE, OUTPUT_RATE pytestmark = pytest.mark.e2e +BACKEND: Final = "openai/gpt-5.6-luna" SPEND_METRIC: Final = "litellm_spend_metric_total" KEY_HASH_LABEL: Final = "hashed_api_key" TEAM_LABEL: Final = "team" @@ -86,6 +88,14 @@ def _same_spend(actual: float | None, expected: float) -> bool: class TestSpendSurfaceConsistency: @pytest.mark.replayable @pytest.mark.covers("quota_management.spend_tracking.surface_consistency.matches_every_surface") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + providers=(Provider.OPENAI,), + models=(BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_one_request_lands_the_same_spend_on_every_surface( self, client: SpendClient, resources: ResourceManager ) -> None: @@ -96,7 +106,7 @@ class TestSpendSurfaceConsistency: model_id: Final = client.proxy.create_model( model, LiteLLMParamsBody( - model="openai/gpt-5.6-luna", + model=BACKEND, api_key="os.environ/OPENAI_API_KEY", api_base=None if base is None else f"{base}/v1", input_cost_per_token=INPUT_RATE, diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index a4c37c2df94..c7ef81f826a 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -42,6 +42,7 @@ CLAUDE_MODEL = "claude-haiku-4-5" CODEX_MODEL = "openai-responses-codex" EMBEDDING_MODEL = "openai-text-embedding-3-small" OPENAI_BACKEND = "openai/gpt-5.5" +ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5" def _approx_equal(actual: float, expected: float) -> bool: @@ -534,6 +535,15 @@ def test_end_user_spend_attributed_on_row( @pytest.mark.covers("quota_management.spend_tracking.end_user.attributes_responses_header") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(CODEX_MODEL,), + mode=Mode.NONSTREAM, + ) +) @pytest.mark.parametrize("header", ["x-litellm-customer-id", "x-litellm-end-user-id"]) def test_end_user_header_attributes_responses_row( client: SpendClient, scoped_key: str, resources: ResourceManager, header: str @@ -659,6 +669,14 @@ def test_failure_call_writes_failure_status_row( @pytest.mark.covers("quota_management.spend_tracking.failure.writes_normalized_error") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + providers=(Provider.ANTHROPIC, Provider.OPENAI), + models=(OPENAI_BACKEND, ANTHROPIC_BACKEND), + mode=Mode.NONSTREAM, + ) +) def test_failure_rows_share_normalized_error_across_provider_wording( client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -668,7 +686,7 @@ def test_failure_rows_share_normalized_error_across_provider_wording( marker = unique_marker() deployments: Final = ( (f"e2e-norm-openai-{marker}", OPENAI_BACKEND), - (f"e2e-norm-anthropic-{marker}", "anthropic/claude-haiku-4-5"), + (f"e2e-norm-anthropic-{marker}", ANTHROPIC_BACKEND), ) for name, provider_model in deployments: model_id = client.proxy.create_model( @@ -700,6 +718,14 @@ def test_failure_rows_share_normalized_error_across_provider_wording( @pytest.mark.covers("quota_management.spend_tracking.failure.attributes_provider") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) +) def test_pre_call_rejection_row_attributes_provider_and_model_id( client: SpendClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py b/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py index 6352ab67c3c..ab7c2370365 100644 --- a/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py @@ -17,6 +17,7 @@ from typing import Final, Literal import pytest from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ( AnthropicContentBlock, @@ -56,6 +57,16 @@ class TestWebSearchInterceptionSession: "quota_management.spend_tracking.websearch_interception.bills_under_request_session", exercised_on=("messages",), ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.MESSAGES, + providers=(Provider.BEDROCK, Provider.PERPLEXITY), + models=(BEDROCK_INVOKE_BACKEND,), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.NONSTREAM, + ) + ) def test_intercepted_search_is_billed_under_the_request_session( self, proxy: ProxyClient, resources: ResourceManager ) -> None: From 0338498067c60720cac27f5df176def143173bff Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 7 Oct 2026 11:00:31 -0700 Subject: [PATCH 09/35] fix(router): preserve native baseline identity and accounting (#44960) * fix(router): preserve native baseline identity and accounting * fix(router): preserve injected system caches in native baselines * fix(router): prepare native baselines through shared request owners * fix(router): capture native baseline fields from the provider schema * refactor(router): reuse native provider parameter discovery * fix(router): keep long native baselines and abstain after compaction Message history no longer counts against the settings snapshot budget, so long and non-ASCII native sessions keep modeled baselines. Selected-tier compaction now abstains because the baseline would otherwise inherit the compacted history. Co-Authored-By: Claude Opus 5.5 * fix(router): judge implicit caching against the selected request The implicit-cache guard compared the selected response's cache usage with the projected baseline's breakpoints, so selected-tier cache markers made a usable unmarked baseline plan look like unexplained caching. The guard now checks the selected wire request. Also removes a stamp-reuse branch that could never run because routing clears the stamp first; every pass already captures caller settings from fresh kwargs. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- litellm/integrations/custom_logger.py | 18 +- litellm/litellm_core_utils/logging_worker.py | 27 +- litellm/litellm_core_utils/redact_messages.py | 27 +- .../pass_through/messages/handler.py | 63 +- .../anthropic/pass_through/messages/utils.py | 52 +- .../llms/anthropic/prompt_cache_prediction.py | 64 +- litellm/proxy/db/baseline_accounting.py | 3 +- .../proxy/hooks/autorouter_baseline_cache.py | 229 +++-- .../spend_tracking/baseline_accounting.py | 18 +- litellm/router.py | 6 +- .../complexity_router/context_compaction.py | 5 + litellm/router_utils/baseline_request.py | 153 ++++ litellm/types/router.py | 5 +- .../code_coverage_tests/recursive_detector.py | 1 + .../spend/test_baseline_accounting.py | 7 +- .../litellm_core_utils/test_logging_worker.py | 60 +- .../test_redact_messages.py | 176 ++-- ...erimental_pass_through_messages_handler.py | 18 +- .../test_request_optional_param_utils.py | 10 +- .../hooks/test_autorouter_baseline_cache.py | 804 +++++++++++++++++- .../test_baseline_accounting.py | 80 +- .../router_utils/test_baseline_request.py | 59 ++ 22 files changed, 1608 insertions(+), 277 deletions(-) create mode 100644 litellm/router_utils/baseline_request.py create mode 100644 tests/unit/router_utils/test_baseline_request.py diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 6a4987298c4..45fce665bf0 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -909,7 +909,8 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac """ import litellm from litellm import Choices, Message, ModelResponse - from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS, without_classifier_audit + from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS + from litellm.litellm_core_utils.redact_messages import redacted_litellm_params turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False) excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None) @@ -918,9 +919,15 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac if turn_off_message_logging is False and not excluded_fields: return model_call_details + params: Final = model_call_details.get("litellm_params") + redacted_params: Final = ( + MappingProxyType({"litellm_params": redacted_litellm_params(params)}) + if turn_off_message_logging and isinstance(params, Mapping) + else EMPTY_MAPPING + ) standard_logging_object: Final = model_call_details.get("standard_logging_object") if standard_logging_object is None: - return model_call_details.copy() + return {**model_call_details, **redacted_params} # Make a copy of just the standard_logging_object to avoid modifying the original standard_logging_object_copy: Final = { @@ -960,13 +967,6 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac model_response_dict: Final = model_response.model_dump() standard_logging_object_copy["response"] = model_response_dict - params: Final = model_call_details.get("litellm_params") - request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None - redacted_params: Final = ( - MappingProxyType({"litellm_params": {**params, "proxy_server_request": without_classifier_audit(request)}}) - if turn_off_message_logging and isinstance(params, dict) and isinstance(request, dict) - else EMPTY_MAPPING - ) return { **model_call_details, **redacted_params, diff --git a/litellm/litellm_core_utils/logging_worker.py b/litellm/litellm_core_utils/logging_worker.py index 03420b84c22..57da3f8dabe 100644 --- a/litellm/litellm_core_utils/logging_worker.py +++ b/litellm/litellm_core_utils/logging_worker.py @@ -23,6 +23,19 @@ from litellm.constants import ( MAX_TIME_TO_CLEAR_QUEUE, ) +_CALLBACK_DEADLINE: Final[contextvars.ContextVar[float | None]] = contextvars.ContextVar( + "logging_callback_deadline", default=None +) + + +def optional_callback_budget(maximum: float, *, fraction: float = 0.25) -> float: + deadline: Final = _CALLBACK_DEADLINE.get() + return ( + maximum + if deadline is None + else max(0.0, min(maximum, (deadline - asyncio.get_running_loop().time()) * fraction)) + ) + def _coroutine_name(coroutine: Coroutine) -> str: return getattr(coroutine, "__qualname__", None) or getattr(coroutine, "__name__", None) or type(coroutine).__name__ @@ -100,12 +113,20 @@ class LoggingWorker: return len(revived) def _run_coroutine_silently(self, loop: asyncio.AbstractEventLoop, coroutine: Coroutine) -> bool: + token: Final = _CALLBACK_DEADLINE.set(loop.time() + self.timeout) try: loop.run_until_complete(asyncio.wait_for(coroutine, timeout=self.timeout)) except (Exception, asyncio.CancelledError): # noqa: BLE001 # atexit flush must never break the user's program return False + finally: + _CALLBACK_DEADLINE.reset(token) return True + def _create_callback_task(self, task: LoggingTask) -> asyncio.Task[object]: + context: Final = task["context"].copy() + context.run(_CALLBACK_DEADLINE.set, asyncio.get_running_loop().time() + self.timeout) + return context.run(asyncio.create_task, task["coroutine"]) + @staticmethod def _drain_pending(queue: "asyncio.Queue[LoggingTask]") -> tuple[LoggingTask, ...]: """Pop every task still queued, without awaiting them, so they can be moved to another queue.""" @@ -172,7 +193,7 @@ class LoggingWorker: try: if self._queue is not None: # Run the coroutine in its original context - callback_task: Final = task["context"].run(asyncio.create_task, task["coroutine"]) + callback_task: Final = self._create_callback_task(task) try: await asyncio.wait_for(callback_task, timeout=self.timeout) except asyncio.TimeoutError as e: @@ -424,7 +445,7 @@ class LoggingWorker: try: await asyncio.wait_for( - task["context"].run(asyncio.create_task, task["coroutine"]), + self._create_callback_task(task), timeout=self.timeout, ) except Exception: @@ -517,7 +538,7 @@ class LoggingWorker: # Await the coroutine to properly execute and avoid "never awaited" warnings try: await asyncio.wait_for( - task["context"].run(asyncio.create_task, task["coroutine"]), + self._create_callback_task(task), timeout=self.timeout, ) except Exception: diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 85ed0a40687..7c6c39abb76 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -11,6 +11,7 @@ import asyncio import copy import inspect from collections.abc import Mapping +from dataclasses import replace from typing import TYPE_CHECKING, Any, Final import litellm @@ -26,6 +27,7 @@ from litellm.llms.vertex_ai.common_utils import ( redact_vertex_ai_metadata_from_logged_object, ) from litellm.secret_managers.main import str_to_bool +from litellm.types.router import BaselineRouteStamp from litellm.types.utils import StandardCallbackDynamicParams if TYPE_CHECKING: @@ -252,6 +254,26 @@ def _redact_model_response_dict_choices(choices, redacted_str: str): _redact_choice_content(choice) +def _redacted_baseline_metadata(metadata: Mapping[str, object]) -> Mapping[str, object]: + route: Final = metadata.get("_autorouter_baseline_route") + if not isinstance(route, BaselineRouteStamp): + return metadata + return {**metadata, "_autorouter_baseline_route": replace(route, request_parameters=None)} + + +def redacted_litellm_params(params: Mapping[str, object]) -> dict[str, object]: + request: Final = params.get("proxy_server_request") + return { + **params, + **{ + key: _redacted_baseline_metadata(value) + for key, value in params.items() + if key in ("metadata", "litellm_metadata") and isinstance(value, Mapping) + }, + **({"proxy_server_request": without_classifier_audit(request)} if isinstance(request, Mapping) else {}), + } + + def perform_redaction(model_call_details: dict, result, redact_streaming_responses: bool = True): """ Performs the actual redaction on the logging object and result. @@ -262,9 +284,8 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons """ # Redact model_call_details params: Final = model_call_details.get("litellm_params") - request: Final = params.get("proxy_server_request") if isinstance(params, dict) else None - if isinstance(params, dict) and isinstance(request, Mapping): - model_call_details["litellm_params"] = {**params, "proxy_server_request": without_classifier_audit(request)} + if isinstance(params, Mapping): + model_call_details["litellm_params"] = redacted_litellm_params(params) model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] model_call_details["prompt"] = "" model_call_details["input"] = "" diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index 6d13e38aa45..4e7a154be67 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -15,9 +15,6 @@ import litellm from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.common_utils import ( - flatten_unencrypted_web_search_results_in_anthropic_messages, - sanitize_tool_use_ids_in_anthropic_messages, - strip_empty_content_blocks_from_anthropic_messages, strip_provider_specific_fields_from_anthropic_messages, ) from litellm.llms.base_llm.anthropic_messages.transformation import ( @@ -36,9 +33,8 @@ from litellm.utils import ProviderConfigManager, client from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler -from ..utils import is_reasoning_auto_summary_enabled from .interceptors import get_messages_interceptors -from .utils import AnthropicMessagesRequestUtils, mock_response +from .utils import AnthropicMessagesRequestUtils, mock_response, prepare_native_messages __all__ = ("anthropic_messages", "anthropic_messages_handler") @@ -251,28 +247,7 @@ async def anthropic_messages( Runs the empty-content-block sanitizer before any backend dispatch. """ - # Anthropic's API rejects requests containing empty / whitespace-only - # text content blocks ("messages: text content blocks must be - # non-empty") and empty thinking blocks ("each thinking block must - # contain thinking"). Multi-turn tool-use clients (e.g. Claude Code) - # routinely loop assistant responses that contain such blocks — an empty - # text block alongside tool_use, or an empty thinking block from a turn - # a non-Anthropic reasoning model served through the bridge — back as - # conversation history, which then causes the next /v1/messages call to - # 400. /v1/chat/completions already handles this in - # anthropic_messages_pt; sanitize the native Anthropic Messages path - # here for the same guarantee. See #22930. - messages = strip_empty_content_blocks_from_anthropic_messages(messages) - # Replay of cross-provider tool history (e.g. kimi -> Anthropic) may carry - # ids like ``functions.Bash:0`` that violate Anthropic's id pattern. - messages = sanitize_tool_use_ids_in_anthropic_messages(messages) - messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages) - - from litellm.integrations.anthropic_cache_control_hook import ( - AnthropicCacheControlHook, - ) - - messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( + messages, system = prepare_native_messages( messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base ) @@ -454,23 +429,15 @@ def anthropic_messages_handler( """ from litellm.types.utils import LlmProviders - # Sanitize empty text blocks so the sync entry point - # (litellm.messages.create -> anthropic_messages_handler) gets the same - # protection as the async wrapper. The async wrapper already sanitized and - # does not reassign messages before dispatch, so it sets - # ``_litellm_messages_presanitized`` to skip this redundant second - # full-messages scan. Pop it so it never leaks into provider params. - if not kwargs.pop("_litellm_messages_presanitized", False): - messages = strip_empty_content_blocks_from_anthropic_messages(messages) - messages = sanitize_tool_use_ids_in_anthropic_messages(messages) - messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages) - - from litellm.integrations.anthropic_cache_control_hook import ( - AnthropicCacheControlHook, - ) - - messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( - messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base + messages, system = prepare_native_messages( + messages, + system, + kwargs, + model=model, + custom_llm_provider=custom_llm_provider, + tools=tools, + api_base=api_base, + presanitized=bool(kwargs.pop("_litellm_messages_presanitized", False)), ) metadata = validate_anthropic_api_metadata(metadata) @@ -645,14 +612,6 @@ def anthropic_messages_handler( custom_llm_provider=custom_llm_provider, ) ) - if is_reasoning_auto_summary_enabled(): - thinking_param: Final = anthropic_messages_optional_request_params.get("thinking") - if isinstance(thinking_param, dict) and thinking_param.get("type") != "disabled": - anthropic_messages_optional_request_params["thinking"] = { - **thinking_param, - "display": "summarized", - } - resolved_api_base: Final = ( dynamic_api_base if dynamic_api_base is not None and anthropic_messages_provider_config.uses_get_llm_provider_api_base() diff --git a/litellm/llms/anthropic/pass_through/messages/utils.py b/litellm/llms/anthropic/pass_through/messages/utils.py index dfab0af8eaa..8371615baea 100644 --- a/litellm/llms/anthropic/pass_through/messages/utils.py +++ b/litellm/llms/anthropic/pass_through/messages/utils.py @@ -2,6 +2,15 @@ from collections.abc import Iterable, Mapping, Sequence from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints +from pydantic import JsonValue + +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.llms.anthropic.common_utils import ( + flatten_unencrypted_web_search_results_in_anthropic_messages, + sanitize_tool_use_ids_in_anthropic_messages, + strip_empty_content_blocks_from_anthropic_messages, +) +from litellm.llms.anthropic.pass_through.utils import is_reasoning_auto_summary_enabled from litellm.types.llms.anthropic import ( AnthropicMessagesRequestOptionalParams, AnthropicStopDetails, @@ -119,8 +128,40 @@ def anthropic_system_to_openai_message(system: object) -> ChatCompletionSystemMe return ChatCompletionSystemMessage(role="system", content=system) +def prepare_native_messages( + messages: list[dict[str, JsonValue]], + system: str | list[dict[str, JsonValue]] | None, + kwargs: dict[str, object], + *, + model: str, + custom_llm_provider: str | None = None, + tools: list[dict[str, JsonValue]] | None = None, + api_base: str | None = None, + presanitized: bool = False, +) -> tuple[list[dict[str, JsonValue]], str | list[dict[str, JsonValue]] | None]: + normalized: Final = ( + messages + if presanitized + else flatten_unencrypted_web_search_results_in_anthropic_messages( + sanitize_tool_use_ids_in_anthropic_messages(strip_empty_content_blocks_from_anthropic_messages(messages)) + ) + ) + return cast( # cast-ok: legacy normalizers and injection preserve the JSON message and system shapes + tuple[list[dict[str, JsonValue]], str | list[dict[str, JsonValue]] | None], + AnthropicCacheControlHook.maybe_inject_cache_control( + normalized, + system, + kwargs, + model=model, + custom_llm_provider=custom_llm_provider, + tools=tools, + api_base=api_base, + ), + ) + + @lru_cache(maxsize=1) -def _anthropic_messages_optional_param_keys() -> frozenset[str]: +def anthropic_messages_optional_param_keys() -> frozenset[str]: """ Valid AnthropicMessagesRequestOptionalParams keys. @@ -152,7 +193,7 @@ class AnthropicMessagesRequestUtils: Returns: AnthropicMessagesRequestOptionalParams instance with only the valid parameters """ - valid_keys: Final = _anthropic_messages_optional_param_keys() + valid_keys: Final = anthropic_messages_optional_param_keys() filtered_params: Final = {k: v for k, v in params.items() if k in valid_keys and v is not None} if model is not None: from litellm.llms.anthropic.chat.transformation import AnthropicConfig @@ -174,6 +215,13 @@ class AnthropicMessagesRequestUtils: drop_params=drop_params, output_key=param, ) + if is_reasoning_auto_summary_enabled(): + thinking_param: Final = filtered_params.get("thinking") + if isinstance(thinking_param, dict) and thinking_param.get("type") != "disabled": + return cast( + AnthropicMessagesRequestOptionalParams, + {**filtered_params, "thinking": {**thinking_param, "display": "summarized"}}, + ) return cast(AnthropicMessagesRequestOptionalParams, filtered_params) diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 00fe56a5e39..634eb264b38 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -5,6 +5,7 @@ import hashlib import json from collections.abc import Mapping, Sequence from dataclasses import dataclass, field +from functools import reduce from itertools import accumulate, groupby from types import MappingProxyType from typing import Annotated, Final, Literal, Protocol, TypeAlias @@ -13,20 +14,32 @@ import httpx from pydantic import ConfigDict, Field, JsonValue, StrictInt, TypeAdapter, ValidationError import litellm -from litellm.llms.anthropic.common_utils import AnthropicModelInfo, is_anthropic_oauth_key +from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value +from litellm.llms.anthropic.common_utils import ( + AnthropicModelInfo, + is_anthropic_oauth_key, + strip_provider_specific_fields_from_anthropic_messages, +) from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES from litellm.llms.anthropic.pass_through.messages.transformation import ( DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig, ) +from litellm.llms.anthropic.pass_through.messages.utils import AnthropicMessagesRequestUtils, prepare_native_messages +from litellm.router_utils.baseline_request import ( + BASELINE_PARAMETERS, + capture_baseline_parameters, +) from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.router import LiteLLM_Params +from litellm.types.router import GenericLiteLLMParams, LiteLLM_Params from litellm.types.utils import ModelResponse from litellm.utils import supports_thinking_cache_preservation _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _HEADERS: Final = TypeAdapter(dict[str, str]) +_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) +_SYSTEM: Final = TypeAdapter(str | list[dict[str, JsonValue]] | None) _counter: Final = AnthropicCountTokensHandler() @@ -325,7 +338,7 @@ def _entry_fingerprint(fingerprint: str, ttl_seconds: int) -> str: def parse_cache_plan(body: Mapping[str, JsonValue]) -> PromptCachePlan | UnsupportedCachePlan: try: - request: Final = _PlanRequest.model_validate(body) + request: Final = _PlanRequest.model_validate(dict(body)) positions: Final = _positions(body) except ValidationError: return UnsupportedCachePlan("unsupported_prompt_shape") @@ -618,13 +631,56 @@ def resolve_baseline_prediction_target(params: LiteLLM_Params) -> NativePredicti return _resolve_prediction_target(params, allow_configured_endpoint=True) +def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> Mapping[str, JsonValue] | None: + parameters: Final = capture_baseline_parameters(request) + if parameters is None: + return None + source: Final = {**parameters, "messages": request.get("messages"), "stream": request.get("stream", False)} + try: + owned: Final = _JSON_OBJECT.validate_python(source) + context: Final = {**{k: v for k, v in request.items() if k not in ("metadata", "litellm_metadata")}, **owned} + resolved_model: Final = litellm.get_llm_provider(model=model, custom_llm_provider="anthropic")[0] + messages, system = prepare_native_messages( + _MESSAGES.validate_python(owned.get("messages")), + _SYSTEM.validate_python(owned.get("system")), + context, + model=resolved_model, + custom_llm_provider="anthropic", + tools=_MESSAGES.validate_python(owned.get("tools") or []), + ) + options: Final = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + {**owned, "system": system}, + model=resolved_model, + custom_llm_provider="anthropic", + drop_params=owned.get("drop_params") is True, + ) + filtered: Final = reduce( + delete_nested_value, + TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()), + dict(options), + ) + body: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + model=resolved_model, + messages=strip_provider_specific_fields_from_anthropic_messages(messages), + anthropic_messages_optional_request_params=filtered, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + return MappingProxyType(_JSON_OBJECT.validate_python(body)) + except Exception: # noqa: BLE001 # an unsupported hypothetical request is unavailable, never an inference failure + return None + + def _resolve_prediction_target( params: LiteLLM_Params, *, allow_configured_endpoint: bool, ) -> NativePredictionTarget | UnsupportedPredictionTarget: configured_options: Final = frozenset(params.model_dump(exclude_defaults=True, exclude_none=True)) - if configured_options - _DEPLOYMENT_OPTIONS: + allowed: Final = ( + _DEPLOYMENT_OPTIONS | frozenset(BASELINE_PARAMETERS) if allow_configured_endpoint else _DEPLOYMENT_OPTIONS + ) + if configured_options - allowed: return UnsupportedPredictionTarget("unsupported_deployment_configuration") api_base: Final = AnthropicModelInfo.get_api_base(params.api_base) if not allow_configured_endpoint and api_base not in ( diff --git a/litellm/proxy/db/baseline_accounting.py b/litellm/proxy/db/baseline_accounting.py index b21b7c8a2b9..486717f9b93 100644 --- a/litellm/proxy/db/baseline_accounting.py +++ b/litellm/proxy/db/baseline_accounting.py @@ -223,7 +223,8 @@ ON CONFLICT (request_id) DO NOTHING _MARK_CONFLICT: Final = """ UPDATE "LiteLLM_AutoRouterBaselineObservation" SET conflicted = TRUE, revision = $4::bigint -WHERE request_id = $1 AND scope = $2 AND data <> $3 AND NOT conflicted +WHERE request_id = $1 AND scope = $2 AND NOT conflicted + AND (data::jsonb #- '{turn,turn_at}') <> ($3::jsonb #- '{turn,turn_at}') """ _READ_PAGE: Final = """ WITH times AS ( diff --git a/litellm/proxy/hooks/autorouter_baseline_cache.py b/litellm/proxy/hooks/autorouter_baseline_cache.py index e3cd6c67aa0..c4232609124 100644 --- a/litellm/proxy/hooks/autorouter_baseline_cache.py +++ b/litellm/proxy/hooks/autorouter_baseline_cache.py @@ -5,7 +5,7 @@ import hashlib import json import time from collections.abc import Callable, Mapping -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Final @@ -16,9 +16,8 @@ from pydantic import ConfigDict, Field, JsonValue, TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.core_helpers import ( - get_litellm_metadata_from_kwargs, # pyright: ignore[reportUnknownVariableType] # legacy metadata boundary validated below -) +from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs +from litellm.litellm_core_utils.logging_worker import optional_callback_budget from litellm.llms.anthropic.prompt_cache_prediction import ( CountedPromptCachePlan, NativePredictionTarget, @@ -28,6 +27,7 @@ from litellm.llms.anthropic.prompt_cache_prediction import ( count_cache_plan, count_prompt_tokens, parse_cache_plan, + prepare_native_baseline_body, resolve_baseline_prediction_target, supported_baseline_recipient, supported_prediction_headers, @@ -37,6 +37,8 @@ from litellm.proxy.spend_tracking.savings import ( _effective_model_info, # pyright: ignore[reportPrivateUsage] # existing deployment-price owner _proxy_llm_router, # pyright: ignore[reportPrivateUsage] # existing optional proxy-router owner ) +from litellm.router_strategy.complexity_router.context_compaction import compaction_applied +from litellm.router_utils.baseline_request import baseline_request from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.router import BaselineRouteStamp from litellm.types.utils import CallTypes, ModelInfo, Usage @@ -66,6 +68,9 @@ class CapturedBaselineObservation(LiteLLMBaseModel): prices: ModelInfo | None observation: BaselineObservation + def with_observation(self, observation: BaselineObservation) -> CapturedBaselineObservation: + return self.model_copy(update={"observation": observation}) + @dataclass(frozen=True, slots=True) class BaselineCacheContext: @@ -73,7 +78,10 @@ class BaselineCacheContext: capture: CapturedBaselineObservation target: NativePredictionTarget | UnsupportedPredictionTarget baseline_deployment_id: str + baseline_body: Mapping[str, JsonValue] | None = field(default=None, repr=False) + selected_body_digest: str | None = field(default=None, repr=False) invalidated: str | None = None + finalization: asyncio.Task[CapturedBaselineObservation] | None = field(default=None, repr=False, compare=False) class _Metadata(LiteLLMBaseModel): @@ -102,6 +110,10 @@ def _digest(value: object) -> str: return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() +def _native_body_digest(body: Mapping[str, JsonValue]) -> str: + return _digest({key: value for key, value in body.items() if key not in ("metadata", "stream")}) + + class AutoRouterBaselineCache(CustomLogger): def __init__( self, @@ -124,12 +136,15 @@ class AutoRouterBaselineCache(CustomLogger): if not isinstance(logging_obj, Logging) or call_type != CallTypes.anthropic_messages: return try: - metadata: Final = _METADATA.validate_python(get_litellm_metadata_from_kwargs({"litellm_params": kwargs})) + raw_metadata: Final = kwargs.get(get_metadata_variable_name_from_kwargs(kwargs)) + metadata: Final = _METADATA.validate_python(raw_metadata) if isinstance(raw_metadata, Mapping) else {} if metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY): return if logging_obj.baseline_cache_context is not None: await invalidate_baseline_cache(logging_obj, "retried_request") return + if not isinstance(metadata.get("_autorouter_baseline_route"), BaselineRouteStamp): + return request: Final = _Metadata.model_validate(metadata) session: Final = kwargs.get("litellm_session_id") or request.session_id or logging_obj.litellm_session_id if not isinstance(session, str) or not session or len(session) > 256: @@ -142,13 +157,27 @@ class AutoRouterBaselineCache(CustomLogger): prices: Final = _PRICES.validate_python( _effective_model_info(router, request.route.baseline_deployment_id, request.route.baseline_model) ) + params: Final = ( + _METADATA.validate_python(deployment.litellm_params.model_dump(mode="json")) if deployment else {} + ) + projected: Final = ( + baseline_request( + kwargs, + request.route.request_parameters, + params, + include_extra_body=False, + ) + if request.route.request_parameters is not None + else None + ) scope: Final = "autorouter-baseline:v3:" + _digest( ( + "baseline_request_v4", request.user_api_key_hash, session, request.route.router_name, request.route.baseline_deployment_id, - deployment.litellm_params.model_dump(mode="json"), + params, prices, ) ) @@ -170,8 +199,21 @@ class AutoRouterBaselineCache(CustomLogger): reason="incomplete_response", ), ) + selected_model: Final = kwargs.get("model") + selected_body: Final = prepare_native_baseline_body( + kwargs, selected_model if isinstance(selected_model, str) else logging_obj.model + ) logging_obj.baseline_cache_context = BaselineCacheContext( - self, capture, target, request.route.baseline_deployment_id + self, + capture, + target, + request.route.baseline_deployment_id, + prepare_native_baseline_body(projected, target.model) + if projected is not None and isinstance(target, NativePredictionTarget) + else None, + _native_body_digest(selected_body) + if selected_body is not None and not compaction_applied(kwargs) + else None, ) except Exception: # noqa: BLE001 # optional observation cannot fail inference verbose_proxy_logger.warning("Auto-router baseline observation could not be initialized") @@ -197,14 +239,16 @@ class AutoRouterBaselineCache(CustomLogger): async def plan( self, target: NativePredictionTarget, wire: httpx.Request, body: Mapping[str, JsonValue], usage: Usage | None ) -> tuple[CountedPromptCachePlan | None, str | None]: + deadline: Final = asyncio.get_running_loop().time() + optional_callback_budget(_COUNT_TIMEOUT, fraction=0.75) if not supported_prediction_headers(wire.headers): return None, "unsupported_request_headers" plan: Final = parse_cache_plan(body) if isinstance(plan, UnsupportedCachePlan): return None, plan.reason details: Final = usage.prompt_tokens_details if usage is not None else None + selected: Final = parse_cache_plan(_JSON_BODY.validate_json(wire.content)) if ( - not plan.breakpoints + (isinstance(selected, UnsupportedCachePlan) or not selected.breakpoints) and details is not None and ((details.cached_tokens or 0) + (details.cache_creation_tokens or 0)) ): @@ -215,7 +259,8 @@ class AutoRouterBaselineCache(CustomLogger): try: counted: Final = await asyncio.wait_for( - count_cache_plan(target.model, target.api_key, plan, token_counter=count), timeout=_COUNT_TIMEOUT + count_cache_plan(target.model, target.api_key, plan, token_counter=count), + timeout=max(0.0, deadline - asyncio.get_running_loop().time()), ) return (None, counted.reason) if isinstance(counted, UnsupportedCachePlan) else (counted, None) except TimeoutError: @@ -229,111 +274,121 @@ async def invalidate_baseline_cache(logging_obj: Logging, reason: str, *, comple if context is not None: logging_obj.baseline_cache_context = replace(context, invalidated=reason) logging_obj.baseline_observation = context.capture.model_copy( - update=MappingProxyType( - { - "observation": context.capture.observation.model_copy( - update=MappingProxyType( - { - "available_at": max(context.capture.observation.started_at, context.collector.clock()), - "reason": reason, - } - ) - ), - } - ) + update={ + "observation": context.capture.observation.model_copy( + update={ + "available_at": max(context.capture.observation.started_at, context.collector.clock()), + "reason": reason, + } + ), + } ) async def finalize_baseline_cache(logging_obj: Logging, response_obj: object) -> None: context: Final = logging_obj.baseline_cache_context - if context is None: + if context is None or logging_obj.baseline_observation is not None: return + task: Final = context.finalization or asyncio.create_task(_capture(context, logging_obj, response_obj)) + active: Final = context if context.finalization is not None else replace(context, finalization=task) + if context.finalization is None: + task.add_done_callback(_consume_finalization) + logging_obj.baseline_cache_context = active try: - capture: Final = await _capture(context, logging_obj, response_obj) - if logging_obj.baseline_cache_context is context: - logging_obj.baseline_observation = capture # rebind-ok: attach only to the captured request owner - except Exception: # noqa: BLE001 # observation failures must preserve inference and billing + capture: Final = await asyncio.shield(task) + if logging_obj.baseline_cache_context is active: + logging_obj.baseline_observation = capture # rebind-ok: publish only for the current attempt + except Exception: # noqa: BLE001 # estimation must preserve inference and billing await invalidate_baseline_cache(logging_obj, "observation_unavailable") -async def _capture( +def _consume_finalization(task: asyncio.Task[CapturedBaselineObservation]) -> None: + if not task.cancelled(): + task.exception() + + +async def _capture_native( context: BaselineCacheContext, logging_obj: Logging, response_obj: object ) -> CapturedBaselineObservation: - original: Final = context.capture.observation - details: Final = _METADATA.validate_python(logging_obj.model_call_details) - if details.get("cache_hit") is True: - return context.capture.model_copy( - update=MappingProxyType( - { - "observation": original.model_copy( - update=MappingProxyType({"outcome": "response_cache", "reason": "response_cache_hit"}) - ) - } - ) - ) - event: Final = _WireEvent.model_validate(details) + capture: Final = context.capture + original: Final = capture.observation + event: Final = _WireEvent.model_validate(logging_obj.model_call_details) wire: Final = event.httpx_response.request usage: Final = _ResponseUsage.model_validate(response_obj).usage + available: Final = event.completion_start_time.timestamp() complete: Final = ( event.custom_llm_provider == "anthropic" and event.httpx_response.status_code == 200 and (not event.stream or event.prompt_cache_response_complete) ) - started: Final = original.started_at - available: Final = event.completion_start_time.timestamp() - if context.invalidated or not complete or not started <= available <= context.collector.clock(): - return context.capture.model_copy( - update=MappingProxyType( - { - "observation": original.model_copy( - update=MappingProxyType( - { - "available_at": max(started, context.collector.clock()), - "reason": context.invalidated or "incomplete_response", - } - ) - ) + if context.invalidated or not complete or not original.started_at <= available <= context.collector.clock(): + return capture.with_observation( + original.model_copy( + update={ + "available_at": max(original.started_at, context.collector.clock()), + "reason": context.invalidated or "incomplete_response", } ) ) target: Final = context.target if isinstance(target, UnsupportedPredictionTarget) or not supported_baseline_recipient(target, wire): - return context.capture.model_copy( - update=MappingProxyType( - { - "observation": original.model_copy( - update=MappingProxyType( - { - "available_at": available, - "reason": target.reason - if isinstance(target, UnsupportedPredictionTarget) - else "unsupported_baseline_recipient", - } - ) - ) + return capture.with_observation( + original.model_copy( + update={ + "available_at": available, + "reason": target.reason + if isinstance(target, UnsupportedPredictionTarget) + else "unsupported_baseline_recipient", } ) ) body: Final = _JSON_BODY.validate_json(wire.content) - same: Final = ( - logging_obj.get_router_model_id() == context.baseline_deployment_id and body.get("model") == target.model - ) - plan, reason = await context.collector.plan(target, wire, body, usage) - minimum: Final = get_prompt_cache_min_tokens(target.model) - return context.capture.model_copy( - update=MappingProxyType( - { - "observation": BaselineObservation( - request_id=original.request_id, - started_at=started, - available_at=available, - outcome="complete", - baseline_equivalent=same, - usage=usage, - plan=plan, - minimum_cache_tokens=minimum, - reason=reason, - ) - } + projected: Final = context.baseline_body + if projected is None or context.selected_body_digest != _native_body_digest(body): + return capture.with_observation( + original.model_copy( + update={ + "available_at": available, + "usage": usage, + "reason": "unsupported_baseline_settings" + if projected is None + else "unsupported_request_transformation", + } + ) + ) + same: Final = logging_obj.get_router_model_id() == context.baseline_deployment_id and _native_body_digest( + projected + ) == _native_body_digest(body) + plan, reason = await context.collector.plan(target, wire, projected, usage) + return capture.with_observation( + BaselineObservation( + request_id=original.request_id, + started_at=original.started_at, + available_at=available, + outcome="complete", + baseline_equivalent=same, + usage=usage.model_copy(update={key: projected.get(key) for key in ("speed", "inference_geo")}) + if usage is not None and not same + else usage, + plan=plan, + reason=reason, + minimum_cache_tokens=get_prompt_cache_min_tokens(target.model), ) ) + + +async def _capture( + context: BaselineCacheContext, logging_obj: Logging, response_obj: object +) -> CapturedBaselineObservation: + if _METADATA.validate_python(logging_obj.model_call_details).get("cache_hit") is True: + return context.capture.model_copy( + update={ + "observation": context.capture.observation.model_copy( + update={ + "outcome": "response_cache", + "reason": "response_cache_hit", + } + ), + } + ) + return await _capture_native(context, logging_obj, response_obj) diff --git a/litellm/proxy/spend_tracking/baseline_accounting.py b/litellm/proxy/spend_tracking/baseline_accounting.py index 46a38f71260..e344c0d7169 100644 --- a/litellm/proxy/spend_tracking/baseline_accounting.py +++ b/litellm/proxy/spend_tracking/baseline_accounting.py @@ -71,12 +71,12 @@ def _complete_usage(usage: Usage | None) -> bool: if usage is None or usage.prompt_tokens < 0 or usage.completion_tokens < 0: return False details: Final = usage.prompt_tokens_details - if details is None: + if details is None or not hasattr(details, "cache_creation_tokens"): return False values: Final = (details.text_tokens, details.cached_tokens, details.cache_creation_tokens) if any(value is None or value < 0 for value in values): return False - split: Final = details.cache_creation_token_details + split: Final = details.cache_creation_token_details if hasattr(details, "cache_creation_token_details") else None writes: Final = details.cache_creation_tokens or 0 return ( usage.total_tokens == usage.prompt_tokens + usage.completion_tokens @@ -133,10 +133,13 @@ def _matches(entry: CacheEntry, markers: tuple[CountedBreakpoint, ...], started: def _ambiguous(entry: CacheEntry, markers: tuple[CountedBreakpoint, ...], started: float) -> bool: - return entry.available_at <= started < entry.expires_at and any( - entry.content_fingerprint in marker.lookback_content_fingerprints - and (entry.uncertain or entry.ttl_seconds != marker.ttl_seconds) - for marker in markers + matching: Final = tuple( + marker for marker in markers if entry.content_fingerprint in marker.lookback_content_fingerprints + ) + return ( + entry.available_at <= started < entry.expires_at + and bool(matching) + and (entry.uncertain or all(entry.ttl_seconds != marker.ttl_seconds for marker in matching)) ) @@ -261,7 +264,7 @@ def _writes(history: BaselineHistory, observation: BaselineObservation) -> tuple observation.started_at + hit.ttl_seconds, ), ) - if hit is not None and all(marker.fingerprint != hit.fingerprint for marker in markers) + if hit is not None else () ) return ( @@ -277,6 +280,7 @@ def _writes(history: BaselineHistory, observation: BaselineObservation) -> tuple uncertain=bool(ambiguous), ) for marker in markers + if hit is None or marker.prefix_tokens > hit.tokens ), ) diff --git a/litellm/router.py b/litellm/router.py index 77cff6254ea..ab9d8884a1d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14582,17 +14582,21 @@ class Router: to the deployment that actually served the request. Every attempt therefore writes or clears, never just writes. """ + from litellm.router_utils.baseline_request import capture_baseline_parameters from litellm.types.router import BaselineRouteStamp phase_attributes(routing_decision_attributes(routing_decision)) baseline_model: Final = routing_decision.get("savings_baseline_model") if routing_decision else None baseline_id: Final = routing_decision.get("savings_baseline_deployment_id") if routing_decision else None router_name: Final = routing_decision.get("router_model_name") if routing_decision else None + caller_parameters: Final = ( + capture_baseline_parameters(request_kwargs) if router_name and baseline_model else None + ) Router._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key="_autorouter_baseline_route", value=( - BaselineRouteStamp(router_name, baseline_model, baseline_id) + BaselineRouteStamp(router_name, baseline_model, baseline_id, caller_parameters) if router_name and baseline_model and baseline_id else None ), diff --git a/litellm/router_strategy/complexity_router/context_compaction.py b/litellm/router_strategy/complexity_router/context_compaction.py index d82090f3b99..6a1a63ea4b6 100644 --- a/litellm/router_strategy/complexity_router/context_compaction.py +++ b/litellm/router_strategy/complexity_router/context_compaction.py @@ -164,6 +164,11 @@ def compaction_pending(kwargs: Mapping[str, object] | None) -> bool: return isinstance(state, CompactionState) and state.config is not None and not _client_managed(kwargs or _EMPTY) +def compaction_applied(kwargs: Mapping[str, object]) -> bool: + state: Final = kwargs.get(_STATE_KEY) + return isinstance(state, CompactionState) and state.summary is not None + + def _reject(model: str, reason: str) -> NoReturn: from litellm.exceptions import BadRequestError diff --git a/litellm/router_utils/baseline_request.py b/litellm/router_utils/baseline_request.py new file mode 100644 index 00000000000..2f6262197a1 --- /dev/null +++ b/litellm/router_utils/baseline_request.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from itertools import accumulate +from types import MappingProxyType +from typing import Final, cast + +from pydantic import JsonValue, TypeAdapter, ValidationError + +from litellm.llms.anthropic.pass_through.messages.utils import anthropic_messages_optional_param_keys + +CACHE_SETTINGS: Final = ( + "system", + "instructions", + "tools", + "tool_choice", + "parallel_tool_calls", + "response_format", + "text", + "reasoning", + "reasoning_effort", + "thinking", + "verbosity", + "output_config", + "output_format", + "speed", + "prompt_cache_key", + "cache_key", + "cached_content", + "previous_response_id", + "conversation", + "context_management", + "compaction", +) +_GENERIC_PARAMETERS: Final = ( + *CACHE_SETTINGS, + "prompt_cache_options", + "prompt_cache_retention", + "cache_control", + "max_tokens", + "max_completion_tokens", + "max_output_tokens", + "temperature", + "top_p", + "top_k", + "stop_sequences", + "enable_prompt_caching", + "cache_control_injection_points", + "drop_params", + "additional_drop_params", +) +NATIVE_ONLY_PARAMETERS: Final = tuple( + key + for key in sorted(anthropic_messages_optional_param_keys()) + if key not in (*_GENERIC_PARAMETERS, "metadata", "stream") +) +BASELINE_PARAMETERS: Final = (*_GENERIC_PARAMETERS, *NATIVE_ONLY_PARAMETERS) +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MAX_BYTES: Final = 4 * 1024 * 1024 +_MAX_NODES: Final = 32768 +_MAX_DEPTH: Final = 32 + + +def _json_cost(value: object, depth: int = 0) -> Iterator[int]: + if depth > _MAX_DEPTH: + yield _MAX_BYTES + 1 + elif isinstance(value, str): + yield (6 if value.isascii() else 12) * len(value) + 2 + elif isinstance(value, dict): + yield 2 + for key, item in cast(dict[object, object], value).items(): + yield from _json_cost(key, depth + 1) + yield from _json_cost(item, depth + 1) + yield 2 + elif isinstance(value, (list, tuple)): + yield 2 + for item in cast(list[object] | tuple[object, ...], value): + yield from _json_cost(item, depth + 1) + yield 1 + elif isinstance(value, int) and value.bit_length() > 64: + yield _MAX_BYTES + 1 + elif value is None or isinstance(value, (bool, int, float)): + yield 32 + else: + yield _MAX_BYTES + 1 + + +def within_baseline_budget(value: object) -> bool: + return all( + size <= _MAX_BYTES and nodes <= _MAX_NODES for nodes, size in enumerate(accumulate(_json_cost(value)), 1) + ) + + +def _parameters(value: object, *, envelope: bool = False) -> dict[str, object]: + if not isinstance(value, Mapping): + return {} + mapping: Final = cast(Mapping[str, object], value) + keys: Final = (*BASELINE_PARAMETERS, "messages") if envelope else BASELINE_PARAMETERS + return {key: mapping[key] for key in keys if key in mapping} + + +def capture_baseline_parameters( + kwargs: Mapping[str, object], *, include_extra_body: bool = True +) -> Mapping[str, JsonValue] | None: + extra: Final = ( + {"extra_body": _parameters(kwargs.get("extra_body"), envelope=True)} + if include_extra_body and "extra_body" in kwargs + else {} + ) + parameters: Final = {**_parameters(kwargs), **extra} + if not within_baseline_budget(parameters): + return None + try: + return MappingProxyType(_JSON_OBJECT.validate_python(parameters)) + except ValidationError: + return None + + +def baseline_request( + kwargs: Mapping[str, object], + caller: Mapping[str, JsonValue], + deployment: Mapping[str, object], + *, + include_extra_body: bool = True, +) -> Mapping[str, object] | None: + snapshot: Final = capture_baseline_parameters(deployment) + if snapshot is None: + return None + configured: Final = { + **_parameters(snapshot), + **(_parameters(snapshot.get("extra_body")) if include_extra_body else {}), + } + requested: Final = {**_parameters(caller), **(_parameters(caller.get("extra_body")) if include_extra_body else {})} + configured_tools: Final = configured.get("tools") or [] + caller_tools: Final = requested.get("tools") or [] + merged_tools: Final = ( + {"tools": [*configured_tools, *caller_tools]} + if (configured_tools or caller_tools) and isinstance(configured_tools, list) and isinstance(caller_tools, list) + else {} + ) + return MappingProxyType( + { + **{key: value for key, value in kwargs.items() if key not in (*BASELINE_PARAMETERS, "extra_body")}, + **configured, + **requested, + **merged_tools, + **( + {"extra_body": caller.get("extra_body", snapshot.get("extra_body"))} + if not include_extra_body and ("extra_body" in caller or "extra_body" in snapshot) + else {} + ), + } + ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 66a5b3540f9..a66c4571b39 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -5,7 +5,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum from collections.abc import Container, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import ( TYPE_CHECKING, Annotated, @@ -21,7 +21,7 @@ from typing import ( from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx -from pydantic import ConfigDict, Field, field_validator, model_validator +from pydantic import ConfigDict, Field, JsonValue, field_validator, model_validator from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._logging import verbose_logger @@ -1223,6 +1223,7 @@ class BaselineRouteStamp: router_name: str baseline_model: str baseline_deployment_id: str + request_parameters: Mapping[str, JsonValue] | None = field(default=None, repr=False) @dataclass(frozen=True, slots=True) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 0c53137cfab..7166891bb1a 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -2,6 +2,7 @@ import ast import os IGNORE_FUNCTIONS = [ + "_json_cost", # bounded at depth 32 and consumed under byte/node limits. "_format_type", "remove_additional_properties", "remove_strict_from_schema", diff --git a/tests/proxy_behavior/spend/test_baseline_accounting.py b/tests/proxy_behavior/spend/test_baseline_accounting.py index dbaf32d579f..4298b85cab4 100644 --- a/tests/proxy_behavior/spend/test_baseline_accounting.py +++ b/tests/proxy_behavior/spend/test_baseline_accounting.py @@ -3,7 +3,8 @@ import json import uuid from collections.abc import AsyncIterator, Callable from contextlib import asynccontextmanager -from datetime import datetime, timezone +from dataclasses import replace +from datetime import datetime, timedelta, timezone from typing import Final import pytest @@ -174,6 +175,10 @@ async def test_commit_ack_loss_and_concurrent_duplicate_delivery_are_idempotent( assert await _store(db, after_commit=True).append(event) == "unavailable" store: Final = _store(db) assert set(await asyncio.gather(*(store.append(event) for _ in range(4)))) == {"recorded"} + duplicate: Final = event.model_copy(update={ + "turn": replace(event.turn, turn_at=event.turn.turn_at + timedelta(seconds=1)) + }) + assert await store.append(duplicate) == "recorded" await _log(db, other) assert await store.append(other) == "recorded" if not attributed: diff --git a/tests/unit/litellm_core_utils/test_logging_worker.py b/tests/unit/litellm_core_utils/test_logging_worker.py index 5d4c9e65d9b..f48495d1062 100644 --- a/tests/unit/litellm_core_utils/test_logging_worker.py +++ b/tests/unit/litellm_core_utils/test_logging_worker.py @@ -6,6 +6,7 @@ import asyncio import contextvars import io import logging +from typing import Final from unittest.mock import AsyncMock, patch import pytest @@ -14,6 +15,61 @@ from litellm.constants import LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS from litellm.litellm_core_utils.logging_worker import LoggingWorker +@pytest.mark.asyncio +@pytest.mark.parametrize("dispatch", ("worker", "flush", "extracted")) +async def test_optional_work_budget_preserves_callback_context_and_reserves_logging_time(dispatch: str) -> None: + from litellm.litellm_core_utils.logging_worker import optional_callback_budget + + worker: Final = LoggingWorker(timeout=1.0) + identity: Final = contextvars.ContextVar("test_callback_identity", default="outside") + results: Final[asyncio.Queue[tuple[str, float]]] = asyncio.Queue() + + async def callback() -> None: + results.put_nowait((identity.get(), optional_callback_budget(3.0))) + + token: Final = identity.set("request") + worker._ensure_queue() + worker.enqueue(callback()) + identity.reset(token) + try: + if dispatch == "worker": + worker.start() + elif dispatch == "flush": + await worker.flush() + else: + assert worker._queue is not None + await worker._process_single_task(worker._queue.get_nowait()) + restored_identity, budget = await asyncio.wait_for(results.get(), timeout=2) + assert restored_identity == "request" + assert 0 < budget <= worker.timeout / 4 + assert identity.get() == "outside" + assert optional_callback_budget(3.0) == 3.0 + finally: + await worker.stop() + + +def test_exit_flush_bounds_optional_work_and_restores_callers_budget() -> None: + from queue import SimpleQueue + + from litellm.litellm_core_utils.logging_worker import optional_callback_budget + + worker: Final = LoggingWorker(timeout=1.0) + observed: Final[SimpleQueue[float]] = SimpleQueue() + + async def callback() -> None: + observed.put(optional_callback_budget(3.0)) + + async def enqueue() -> None: + worker._ensure_queue() + worker.enqueue(callback()) + + asyncio.run(enqueue()) + worker._flush_on_exit() + assert observed.qsize() == 1 + assert 0 < observed.get_nowait() <= worker.timeout / 4 + assert optional_callback_budget(3.0) == 3.0 + + class _RecordCollector(logging.Handler): """Captures emitted log records so a test can assert on real logging output (level, message args, traceback) instead of patching the logger object.""" @@ -205,7 +261,9 @@ class TestLoggingWorker: asyncio.run(log_on_second_loop()) first_loop.run_until_complete(asyncio.sleep(0.1)) failures = [ - task.exception() for task in first_loop_tasks if task.done() and not task.cancelled() and task.exception() + task.exception() + for task in first_loop_tasks + if task.done() and not task.cancelled() and task.exception() ] finally: first_loop.close() diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 3bb6b379873..6fbcba170f1 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -5,14 +5,26 @@ Covers the proxy flow where headers arrive in litellm_params["metadata"]["header but litellm_params["litellm_metadata"] is None. """ -import asyncio, httpx, importlib, json, os, pytest_asyncio, threading +import asyncio +import importlib +import json +import os +import threading +from collections.abc import AsyncIterator, Mapping +from datetime import datetime +from types import MappingProxyType, SimpleNamespace from typing import Final, Optional, Union -from types import SimpleNamespace +from unittest.mock import patch +import httpx import pytest +import pytest_asyncio +from pydantic import JsonValue import litellm +from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.redact_messages import ( _redact_responses_api_output, perform_redaction, @@ -21,18 +33,14 @@ from litellm.litellm_core_utils.redact_messages import ( should_redact_message_logging, ) from litellm.responses.main import mock_responses_api_response -from collections.abc import AsyncIterator -from datetime import datetime -from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.types.utils import( +from litellm.types.router import BaselineRouteStamp +from litellm.types.utils import ( ModelResponse, ResponsesAPIResponse, StandardLoggingPayload, TextCompletionResponse, ) from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from unittest.mock import patch @pytest.fixture(autouse=True) @@ -113,9 +121,7 @@ class TestShouldRedactMessageLogging: def test_enable_redaction_via_header_in_litellm_metadata(self): """Headers inside litellm_metadata (SDK direct call) should work.""" details = _make_model_call_details( - litellm_metadata={ - "headers": {"x-litellm-enable-message-redaction": "true"} - }, + litellm_metadata={"headers": {"x-litellm-enable-message-redaction": "true"}}, ) assert should_redact_message_logging(details) is True @@ -217,21 +223,15 @@ class TestPerformRedaction: redacted = perform_redaction(details, result) - assert details["messages"] == [ - {"role": "user", "content": "redacted-by-litellm"} - ] + assert details["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] assert details["prompt"] == "" assert details["input"] == "" logged_response = details["standard_logging_object"]["response"] assert logged_response["usage"] == {"total_tokens": 1} assert logged_response["output"][0]["text"] == "redacted-by-litellm" - assert logged_response["output"][1]["content"][0]["text"] == ( - "redacted-by-litellm" - ) - assert logged_response["output"][2]["summary"][0]["text"] == ( - "redacted-by-litellm" - ) + assert logged_response["output"][1]["content"][0]["text"] == ("redacted-by-litellm") + assert logged_response["output"][2]["summary"][0]["text"] == ("redacted-by-litellm") assert redacted["usage"] == {"total_tokens": 1} assert redacted["output"][0]["text"] == "redacted-by-litellm" @@ -444,9 +444,7 @@ class TestPerformRedaction: tool_call = redacted.choices[0].message.tool_calls[0] assert tool_call.function.arguments == "redacted-by-litellm" assert tool_call.function.name == "get_weather" - assert result.choices[0].message.tool_calls[0].function.arguments == ( - '{"city": "sensitive-city"}' - ) + assert result.choices[0].message.tool_calls[0].function.arguments == ('{"city": "sensitive-city"}') def test_redacts_tool_call_arguments_on_streaming_response_object(self): """Reproduces the Stream=True path where tool calls arrive as deltas.""" @@ -714,12 +712,8 @@ class TestPerformRedaction: } } ], - "vertex_ai_grounding_metadata": [ - {"webSearchQueries": ["sensitive search term"]} - ], - "vertex_ai_url_context_metadata": [ - {"urlMetadata": [{"retrievedUrl": "https://example.com"}]} - ], + "vertex_ai_grounding_metadata": [{"webSearchQueries": ["sensitive search term"]}], + "vertex_ai_url_context_metadata": [{"urlMetadata": [{"retrievedUrl": "https://example.com"}]}], }, } } @@ -749,9 +743,7 @@ class TestPerformRedaction: "vertex_ai_grounding_metadata", [{"webSearchQueries": ["sensitive search term"]}], ) - response._hidden_params["vertex_ai_grounding_metadata"] = [ - {"webSearchQueries": ["sensitive search term"]} - ] + response._hidden_params["vertex_ai_grounding_metadata"] = [{"webSearchQueries": ["sensitive search term"]}] details = { "stream": True, @@ -772,12 +764,8 @@ class TestPerformRedaction: "metadata": { "hidden_params": { "response_cost": 0.01, - "vertex_ai_grounding_metadata": [ - {"webSearchQueries": ["sensitive search term"]} - ], - "vertex_ai_url_context_metadata": [ - {"urlMetadata": [{"retrievedUrl": "https://example.com"}]} - ], + "vertex_ai_grounding_metadata": [{"webSearchQueries": ["sensitive search term"]}], + "vertex_ai_url_context_metadata": [{"urlMetadata": [{"retrievedUrl": "https://example.com"}]}], "vertex_ai_safety_ratings": [{"category": "HARM"}], "vertex_ai_citation_metadata": [{"citations": ["source"]}], } @@ -797,11 +785,7 @@ class TestPerformRedaction: def test_redact_async_complete_streaming_response(self): """Test that async_complete_streaming_response is properly redacted.""" response_obj = litellm.ModelResponse( - choices=[ - litellm.Choices( - message=litellm.Message(content="secret content", role="assistant") - ) - ] + choices=[litellm.Choices(message=litellm.Message(content="secret content", role="assistant"))] ) model_call_details = { @@ -820,11 +804,7 @@ class TestPerformRedaction: def test_redact_complete_streaming_response(self): """Test that complete_streaming_response is properly redacted.""" response_obj = litellm.ModelResponse( - choices=[ - litellm.Choices( - message=litellm.Message(content="secret content", role="assistant") - ) - ] + choices=[litellm.Choices(message=litellm.Message(content="secret content", role="assistant"))] ) model_call_details = { @@ -842,11 +822,7 @@ class TestPerformRedaction: def test_streaming_responses_untouched_when_disabled(self): response_obj = litellm.ModelResponse( - choices=[ - litellm.Choices( - message=litellm.Message(content="secret content", role="assistant") - ) - ] + choices=[litellm.Choices(message=litellm.Message(content="secret content", role="assistant"))] ) model_call_details = { @@ -909,11 +885,7 @@ class TestPerformRedaction: class TestRedactStreamingResponsesForCustomLogger: def _model_call_details(self): response_obj = litellm.ModelResponse( - choices=[ - litellm.Choices( - message=litellm.Message(content="secret content", role="assistant") - ) - ] + choices=[litellm.Choices(message=litellm.Message(content="secret content", role="assistant"))] ) return { "stream": True, @@ -947,7 +919,10 @@ class TestRedactStreamingResponsesForCustomLogger: @pytest.mark.parametrize("callback_only", [False, True]) def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callback_only: bool) -> None: - audit: Final = {"classifier_input": {"system": "private rubric"}, "originating_request_masked": {"input": "private source"}} + audit: Final = { + "classifier_input": {"system": "private rubric"}, + "originating_request_masked": {"input": "private source"}, + } standard_payload: Final = { **audit, "messages": [{"role": "user", "content": "private prompt"}], @@ -956,7 +931,9 @@ def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callb } details: Final = { "standard_logging_object": standard_payload, - "litellm_params": {"proxy_server_request": {"body": {}, "originating_request_masked": audit["originating_request_masked"]}}, + "litellm_params": { + "proxy_server_request": {"body": {}, "originating_request_masked": audit["originating_request_masked"]} + }, } logger: Final = CustomLogger() logger.turn_off_message_logging = True @@ -966,7 +943,10 @@ def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callb assert "originating_request_masked" not in redacted["standard_logging_object"] assert "originating_request_masked" not in redacted["litellm_params"]["proxy_server_request"] assert details["standard_logging_object"]["classifier_input"] == audit["classifier_input"] - assert details["litellm_params"]["proxy_server_request"]["originating_request_masked"] == audit["originating_request_masked"] + assert ( + details["litellm_params"]["proxy_server_request"]["originating_request_masked"] + == audit["originating_request_masked"] + ) else: perform_redaction(details, result=None) assert "classifier_input" not in details["standard_logging_object"] @@ -981,7 +961,9 @@ def test_classifier_audit_redaction_removes_both_fields_and_source_carrier(callb @pytest.mark.parametrize("excluded", [False, True]) def test_classifier_callback_redaction_preserves_exclusions(monkeypatch: pytest.MonkeyPatch, excluded: bool) -> None: - monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["messages", "response"] if excluded else []) + monkeypatch.setattr( + litellm, "standard_logging_payload_excluded_fields", ["messages", "response"] if excluded else [] + ) payload: Final = { "classifier_input": {"system": "private rubric"}, "originating_request_masked": {"input": "private source"}, @@ -991,7 +973,9 @@ def test_classifier_callback_redaction_preserves_exclusions(monkeypatch: pytest. } logger: Final = CustomLogger() logger.turn_off_message_logging = True - redacted: Final = logger.redact_standard_logging_payload_from_model_call_details({"standard_logging_object": payload}) + redacted: Final = logger.redact_standard_logging_payload_from_model_call_details( + {"standard_logging_object": payload} + ) stored: Final = redacted["standard_logging_object"] assert "classifier_input" not in stored assert "originating_request_masked" not in stored @@ -1017,16 +1001,18 @@ class _SelfRedactingLogger(CustomLogger): @pytest.mark.parametrize("logger", [CustomLogger(), _SelfRedactingLogger()], ids=["default", "redacts_itself"]) -def test_field_exclusion_alone_leaves_messages_and_responses_intact(monkeypatch: pytest.MonkeyPatch, logger: CustomLogger) -> None: +def test_field_exclusion_alone_leaves_messages_and_responses_intact( + monkeypatch: pytest.MonkeyPatch, logger: CustomLogger +) -> None: monkeypatch.setattr(litellm, "standard_logging_payload_excluded_fields", ["model"]) payload: Final = { "messages": [{"role": "user", "content": "private prompt"}], "response": {"choices": [{"message": {"content": "private answer"}}]}, "model": "classifier", } - stored: Final = logger.redact_standard_logging_payload_from_model_call_details({"standard_logging_object": payload})[ - "standard_logging_object" - ] + stored: Final = logger.redact_standard_logging_payload_from_model_call_details( + {"standard_logging_object": payload} + )["standard_logging_object"] assert stored == {"messages": payload["messages"], "response": payload["response"]} @@ -1038,9 +1024,9 @@ def test_a_callback_that_redacts_itself_keeps_its_messages_but_not_the_classifie } logger: Final = _SelfRedactingLogger() logger.turn_off_message_logging = True - stored: Final = logger.redact_standard_logging_payload_from_model_call_details({"standard_logging_object": payload})[ - "standard_logging_object" - ] + stored: Final = logger.redact_standard_logging_payload_from_model_call_details( + {"standard_logging_object": payload} + )["standard_logging_object"] assert "classifier_input" not in stored assert stored["messages"] == payload["messages"] assert stored["response"] == payload["response"] @@ -1054,19 +1040,54 @@ def test_perform_redaction_drops_the_served_output_texts_from_the_callback_kwarg assert SERVED_OUTPUT_TEXTS_KEY not in details +@pytest.mark.parametrize("callback_only", (False, True)) +@pytest.mark.parametrize("with_standard_payload", (False, True)) +def test_baseline_snapshots_are_redacted_without_mutating_request_state( + callback_only: bool, with_standard_payload: bool +) -> None: + snapshot: Final[Mapping[str, JsonValue]] = MappingProxyType({"system": "private system"}) + route: Final = BaselineRouteStamp("router", "baseline", "deployment", snapshot) + metadata: Final = {"_autorouter_baseline_route": route, "session_id": "session"} + params: Final = {"metadata": metadata, "litellm_metadata": metadata} + details: Final = { + "litellm_params": params, + **({"standard_logging_object": {"model": "model"}} if with_standard_payload else {}), + } + logger: Final = CustomLogger() + logger.turn_off_message_logging = True + if not callback_only: + perform_redaction(details, None) + redacted: Final = ( + logger.redact_standard_logging_payload_from_model_call_details(details) if callback_only else details + ) + expected: Final = BaselineRouteStamp(route.router_name, route.baseline_model, route.baseline_deployment_id) + assert redacted["litellm_params"] == { + key: {"_autorouter_baseline_route": expected, "session_id": "session"} + for key in ("metadata", "litellm_metadata") + } + assert route.request_parameters is snapshot + assert params["metadata"]["_autorouter_baseline_route"] is route + assert params["litellm_metadata"]["_autorouter_baseline_route"] is route + if callback_only: + assert details["litellm_params"] is params + + @pytest.fixture() def _vcr_outcome_gate(request, vcr): install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) + @pytest_asyncio.fixture(loop_scope="function") async def drain_logging_worker(isolate_litellm_state: None) -> AsyncIterator[None]: yield await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS) + LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS: Final = LOGGING_WORKER_MAX_TIME_PER_COROUTINE + 5.0 + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -1104,6 +1125,7 @@ def isolate_litellm_state(): if attr in _DEFAULTS: setattr(litellm, attr, _DEFAULTS[attr]) + _LIST_ATTRS = ( "callbacks", "success_callback", @@ -1131,6 +1153,7 @@ _SCALAR_ATTRS = ( _DEFAULTS: dict = {} + @pytest.fixture(scope="module") def setup_and_teardown(): """ @@ -1153,6 +1176,7 @@ def setup_and_teardown(): litellm.in_memory_llm_clients_cache.flush_cache() yield + class TestCustomLogger(CustomLogger): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -1164,6 +1188,7 @@ class TestCustomLogger(CustomLogger): self.logged_standard_logging_payload = standard_logging_payload self.response_obj = response_obj + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_global_redaction_on(): @@ -1187,6 +1212,7 @@ async def test_global_redaction_on(): json.dumps(standard_logging_payload, indent=2), ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "dynamic_turn_off, expect_redacted", @@ -1213,6 +1239,7 @@ async def test_dynamic_turn_off_message_logging_overrides_global_on(dynamic_turn assert standard_logging_payload["response"]["choices"][0]["message"]["content"] == expected_response_content assert standard_logging_payload["messages"][0]["content"] == expected_message_content + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize( "dynamic_turn_off, expect_redacted", @@ -1239,6 +1266,7 @@ async def test_dynamic_turn_off_message_logging_overrides_global_off(dynamic_tur assert standard_logging_payload["response"]["choices"][0]["message"]["content"] == expected_response_content assert standard_logging_payload["messages"][0]["content"] == expected_message_content + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_redaction_with_custom_logger_streaming(): @@ -1284,6 +1312,7 @@ async def test_redaction_with_custom_logger_streaming(): finally: litellm.turn_off_message_logging = False + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_streaming_redaction_scoped_to_opted_out_logger(): @@ -1311,6 +1340,7 @@ async def test_streaming_redaction_scoped_to_opted_out_logger(): finally: litellm.callbacks = [] + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_redaction_responses_api(): @@ -1355,6 +1385,7 @@ async def test_redaction_responses_api(): json.dumps(standard_logging_payload, indent=2), ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_redaction_responses_api_stream(): @@ -1430,6 +1461,7 @@ async def test_redaction_responses_api_stream(): json.dumps(standard_logging_payload, indent=2), ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_redaction_responses_api_with_reasoning_summary(): @@ -1490,6 +1522,7 @@ async def test_redaction_responses_api_with_reasoning_summary(): assert model_call_details["messages"][0]["content"] == "redacted-by-litellm", "Input messages should be redacted" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_redaction_with_coroutine_objects(): @@ -1535,6 +1568,7 @@ async def test_redaction_with_coroutine_objects(): result = perform_redaction({}, mock_iter) assert result == {"text": "redacted-by-litellm"} + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_redaction_with_streaming_response(): @@ -1570,6 +1604,7 @@ async def test_redaction_with_streaming_response(): json.dumps(standard_logging_payload, indent=2), ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_disable_redaction_header_responses_api(): @@ -1605,6 +1640,7 @@ async def test_disable_redaction_header_responses_api(): assert response["output"][0]["content"][0]["text"] == "This is a test response" assert standard_logging_payload["messages"][0]["content"] == "hi" + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_redaction_with_metadata_completion_api(): diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index c3d4dba7376..ac55e8e9350 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -684,12 +684,12 @@ def _empty_block_msgs(): def test_handler_strips_when_no_presanitized_flag(): """Sync entry point (no async wrapper): handler must still sanitize.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.pass_through.messages import handler, utils with patch.object( - handler, + utils, "strip_empty_content_blocks_from_anthropic_messages", - wraps=handler.strip_empty_content_blocks_from_anthropic_messages, + wraps=utils.strip_empty_content_blocks_from_anthropic_messages, ) as spy: result = handler.anthropic_messages_handler( max_tokens=10, @@ -704,12 +704,12 @@ def test_handler_strips_when_no_presanitized_flag(): def test_handler_skips_strip_when_presanitized(): """Async wrapper already sanitized -> handler must NOT rescan.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.pass_through.messages import handler, utils with patch.object( - handler, + utils, "strip_empty_content_blocks_from_anthropic_messages", - wraps=handler.strip_empty_content_blocks_from_anthropic_messages, + wraps=utils.strip_empty_content_blocks_from_anthropic_messages, ) as spy: result = handler.anthropic_messages_handler( max_tokens=10, @@ -809,7 +809,7 @@ def test_presanitized_flag_not_leaked_to_provider_params(): @pytest.mark.asyncio async def test_async_wrapper_sets_presanitized_and_sanitizes_once(): """End-to-end: wrapper sanitizes (once) AND signals the handler to skip.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.pass_through.messages import handler, utils captured = {} @@ -825,9 +825,9 @@ async def test_async_wrapper_sets_presanitized_and_sanitizes_once(): patch.object(handler, "anthropic_messages_handler", side_effect=fake_handler), patch("asyncio.get_event_loop", return_value=fake_loop), patch.object( - handler, + utils, "strip_empty_content_blocks_from_anthropic_messages", - wraps=handler.strip_empty_content_blocks_from_anthropic_messages, + wraps=utils.strip_empty_content_blocks_from_anthropic_messages, ) as spy, ): await handler.anthropic_messages( diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py b/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py index dd744ca66a1..7a4f3aa5f90 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py @@ -11,7 +11,7 @@ import pytest import litellm from litellm.llms.anthropic.pass_through.messages.utils import ( AnthropicMessagesRequestUtils, - _anthropic_messages_optional_param_keys, + anthropic_messages_optional_param_keys, ) @@ -30,16 +30,16 @@ def test_optional_param_filtering_unchanged(): def test_valid_keys_are_memoized(): - _anthropic_messages_optional_param_keys.cache_clear() - first = _anthropic_messages_optional_param_keys() + anthropic_messages_optional_param_keys.cache_clear() + first = anthropic_messages_optional_param_keys() for _ in range(50): AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param({"temperature": 0.1}) - info = _anthropic_messages_optional_param_keys.cache_info() + info = anthropic_messages_optional_param_keys.cache_info() # Resolved exactly once despite many calls. assert info.misses == 1 assert info.hits >= 50 # Stable identity (frozenset) returned each call. - assert _anthropic_messages_optional_param_keys() is first + assert anthropic_messages_optional_param_keys() is first assert isinstance(first, frozenset) assert "temperature" in first and "tools" in first diff --git a/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py b/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py index 0f4fd2ff5cb..1be02d2f269 100644 --- a/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py +++ b/tests/unit/proxy/hooks/test_autorouter_baseline_cache.py @@ -3,6 +3,7 @@ import json from collections.abc import AsyncIterator, Callable, Generator, Mapping from contextlib import contextmanager from datetime import datetime +from itertools import product from types import MappingProxyType from typing import Final, cast from uuid import uuid4 @@ -42,7 +43,7 @@ _MESSAGES_JSON: Final = """[{"role":"user","content":[ _MODELS: Final = _MESSAGES.validate_json("""[ {"model_name":"test-router","litellm_params":{"model":"auto_router/complexity_router", "complexity_router_config":{"tiers":{"SIMPLE":"sonnet","MEDIUM":"sonnet","COMPLEX":"sonnet", - "REASONING":"opus"},"session_affinity":false, + "REASONING":{"model_name":"opus","litellm_params":{"max_tokens":16}}},"session_affinity":false, "keyword_tier_rules":[{"keywords":["USE_OPUS"],"tier":"REASONING"}]}}}, {"model_name":"sonnet","litellm_params":{"model":"anthropic/claude-sonnet-5","api_key":"test-selected"}, "model_info":{"id":"selected"}}, @@ -82,7 +83,9 @@ class _CallContext(TypedDict): def _kwargs(logging_obj: Logging, trusted: bool = True, *, explicit_logging: bool = True) -> _CallContext: - context: Final = _OBJECTS.validate_json('{"litellm_metadata":{"user_api_key_hash":"test-caller-hash"}}') + context: Final = _OBJECTS.validate_json( + '{"max_tokens":16,"litellm_metadata":{"user_api_key_hash":"test-caller-hash"}}' + ) Router._record_routing_decision( # pyright: ignore[reportUnknownMemberType, reportPrivateUsage] # production trusted stamp owner context, StandardLoggingRoutingDecision( @@ -133,7 +136,10 @@ def _upstream(request: httpx.Request) -> httpx.Response: assert isinstance(model, str) stream: Final = body.get("stream") is True content: Final = b"".join(_sse(model=model)) if stream else json.dumps(_message(True, model)).encode() - return httpx.Response(200, content=content, request=request, + return httpx.Response( + 200, + content=content, + request=request, headers=MappingProxyType({"content-type": "text/event-stream" if stream else "application/json"}), ) @@ -182,6 +188,7 @@ async def _call( stream: Final = cast(AsyncIterator[object], response) # cast-ok: iterator checked; all items satisfy object assert tuple([chunk async for chunk in stream]) + class _Capture(CustomLogger): def __init__(self, call_id: str) -> None: self.call_id: Final = call_id @@ -199,9 +206,20 @@ class _Capture(CustomLogger): class _Rig: - def __init__(self, monkeypatch: pytest.MonkeyPatch, *, retries: int = 0, count: TokenCounter = _count) -> None: - self.router: Final = Router(model_list=_MODELS, num_retries=retries, - retry_policy=RetryPolicy(RateLimitErrorRetries=retries), disable_cooldowns=True) + def __init__( + self, + monkeypatch: pytest.MonkeyPatch, + *, + retries: int = 0, + count: TokenCounter = _count, + models: list[dict[str, JsonValue]] = _MODELS, + ) -> None: + self.router: Final = Router( + model_list=models, + num_retries=retries, + retry_policy=RetryPolicy(RateLimitErrorRetries=retries), + disable_cooldowns=True, + ) def router() -> Router: return self.router @@ -218,9 +236,16 @@ class _Rig: monkeypatch.setattr(litellm, "_async_success_callback", [self.capture]) def logging(self, stream: bool = False) -> Logging: - return Logging(model="anthropic/claude-sonnet-5", messages=_MESSAGES.validate_json(_MESSAGES_JSON), - stream=stream, call_type=CallTypes.anthropic_messages.value, start_time=datetime.now(), - litellm_call_id=self.call_id, function_id=self.call_id, kwargs={"litellm_session_id":"baseline-session"}) + return Logging( + model="anthropic/claude-sonnet-5", + messages=_MESSAGES.validate_json(_MESSAGES_JSON), + stream=stream, + call_type=CallTypes.anthropic_messages.value, + start_time=datetime.now(), + litellm_call_id=self.call_id, + function_id=self.call_id, + kwargs={"litellm_session_id": "baseline-session"}, + ) def _observation(payload: Mapping[str, object]) -> CapturedBaselineObservation: @@ -232,7 +257,9 @@ def _observation(payload: Mapping[str, object]) -> CapturedBaselineObservation: @pytest.mark.parametrize("stream,baseline", ((False, False), (True, False), (False, True), (True, True))) async def test_native_logging_captures_usage_without_publishing_hypothetical_savings( - monkeypatch: pytest.MonkeyPatch, stream: bool, baseline: bool, + monkeypatch: pytest.MonkeyPatch, + stream: bool, + baseline: bool, ) -> None: rig: Final = _Rig(monkeypatch) messages: Final = _MESSAGES_JSON.replace("question", "question USE_OPUS") if baseline else _MESSAGES_JSON @@ -283,11 +310,14 @@ async def test_caller_cannot_forge_an_observation_scope(monkeypatch: pytest.Monk assert payload["autorouter_savings"] is None -@pytest.mark.parametrize("model,key,endpoint", ( - ("claude-sonnet-5", "test-first", None), - ("claude-opus-5", "test-second", None), - ("claude-opus-5", "test-first", "https://example.test"), -)) +@pytest.mark.parametrize( + "model,key,endpoint", + ( + ("claude-sonnet-5", "test-first", None), + ("claude-opus-5", "test-second", None), + ("claude-opus-5", "test-first", "https://example.test"), + ), +) async def test_count_memo_is_scoped_to_provider_recipient(model: str, key: str, endpoint: str | None) -> None: counts: Final = iter((5000, 6000)) @@ -304,7 +334,8 @@ async def test_count_memo_is_scoped_to_provider_recipient(model: str, key: str, @pytest.mark.parametrize("stream", (False, True)) async def test_provider_counting_does_not_hold_the_inference_response( - monkeypatch: pytest.MonkeyPatch, stream: bool, + monkeypatch: pytest.MonkeyPatch, + stream: bool, ) -> None: counting: Final = asyncio.Event() release: Final = asyncio.Event() @@ -324,3 +355,744 @@ async def test_provider_counting_does_not_hold_the_inference_response( assert _observation(await rig.capture.payload()).observation.plan is not None finally: release.set() + + +@pytest.mark.parametrize("baseline_effort", (None, "medium")) +@pytest.mark.parametrize( + "automatic_system, caching", + ( + (None, "explicit"), + ("stable system", "request"), + ([{"type": "text", "text": "stable system"}], "request"), + ("stable system", "global"), + ([{"type": "text", "text": "stable system"}], "configured"), + ), +) +async def test_native_tier_switch_uses_baseline_settings_and_preserves_history( + monkeypatch: pytest.MonkeyPatch, + baseline_effort: str | None, + automatic_system: str | list[dict[str, str]] | None, + caching: str, +) -> None: + from litellm.proxy.spend_tracking.baseline_accounting import BaselineHistory, advance_baseline_history + + models: Final = _MESSAGES.validate_python( + [ + { + "model_name": "test-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": { + "SIMPLE": {"model_name": "sonnet", "litellm_params": {"reasoning_effort": "low"}}, + "MEDIUM": {"model_name": "sonnet", "litellm_params": {"reasoning_effort": "low"}}, + "COMPLEX": {"model_name": "sonnet", "litellm_params": {"reasoning_effort": "high"}}, + "REASONING": "opus", + }, + "session_affinity": False, + "keyword_tier_rules": [{"keywords": ["ESCALATE"], "tier": "COMPLEX"}], + }, + }, + }, + _MODELS[1], + { + "model_name": "opus", + "model_info": {"id": "baseline"}, + "litellm_params": { + "model": "anthropic/claude-opus-5", + "api_key": "test-selected", + **({"reasoning_effort": baseline_effort} if baseline_effort else {}), + }, + }, + ] + ) + rig: Final = _Rig(monkeypatch, models=models) + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", caching == "global") + controls: Final = ( + { + "cache_control_injection_points": [ + {"location": "message", "role": "system", "control": {"type": "ephemeral", "ttl": "1h"}}, + {"location": "message", "index": -1, "control": {"type": "ephemeral", "ttl": "1h"}}, + ] + } + if caching == "configured" + else {"enable_prompt_caching": caching == "request"} + ) + captures: Final[asyncio.Queue[CapturedBaselineObservation]] = asyncio.Queue() + with _transport(_upstream) as route: + for suffix in ("", " ESCALATE"): + log: Final = rig.logging() + await rig.router.anthropic_messages( + model="test-router", + max_tokens=4096, + messages=( + [{"role": "user", "content": "question" + suffix}] + if automatic_system is not None + else _MESSAGES.validate_json(_MESSAGES_JSON.replace("question", "question" + suffix)) + ), + system=automatic_system, + **controls, + litellm_logging_obj=log, + litellm_call_id=rig.call_id, + litellm_metadata={"user_api_key_hash": "test-caller-hash"}, + litellm_session_id="native-tiers", + ) + captures.put_nowait(_observation(await rig.capture.payload())) + first_wire, second_wire = (_JSON_OBJECT.validate_json(call.request.content) for call in route.calls) + assert (first_wire.get("thinking"), first_wire.get("output_config")) != ( + second_wire.get("thinking"), + second_wire.get("output_config"), + ) + first, second = (captures.get_nowait() for _ in range(2)) + assert first.scope == second.scope + assert first.observation.plan is not None and second.observation.plan is not None + assert first.observation.plan.breakpoints[0] == second.observation.plan.breakpoints[0] + assert len(first.observation.plan.breakpoints) == (2 if automatic_system is not None else 1) + history, _ = advance_baseline_history( + BaselineHistory(first_at=0.0), + (first.observation.model_copy(update={"request_id": "first", "started_at": 10000.0, "available_at": 10001.0}),), + ) + _, result = advance_baseline_history( + history, + ( + second.observation.model_copy( + update={"request_id": "second", "started_at": 10020.0, "available_at": 10021.0} + ), + ), + ) + assert result[0].usage is not None and result[0].usage.prompt_tokens_details.cached_tokens == 5000 + + +@pytest.mark.parametrize("call_type", (CallTypes.acompletion, CallTypes.aresponses, CallTypes.anthropic_messages)) +async def test_plain_requests_do_not_initialize_or_warn( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + call_type: CallTypes, +) -> None: + rig: Final = _Rig(monkeypatch) + logging: Final = rig.logging() + await rig.hook.async_pre_call_deployment_hook( + { + "litellm_logging_obj": logging, + "litellm_metadata": {"session_id": "ordinary"}, + }, + call_type, + ) + assert logging.baseline_cache_context is None + assert "baseline observation could not be initialized" not in caplog.text + assert not rig.hook.counts + + +async def test_plain_fallback_invalidates_existing_autorouter_capture(monkeypatch: pytest.MonkeyPatch) -> None: + rig: Final = _Rig(monkeypatch) + logging: Final = rig.logging() + await rig.hook.async_pre_call_deployment_hook(_kwargs(logging), CallTypes.anthropic_messages) + assert logging.baseline_cache_context is not None + await rig.hook.async_pre_call_deployment_hook({"litellm_logging_obj": logging}, CallTypes.anthropic_messages) + assert logging.baseline_observation is not None + assert logging.baseline_observation.observation.reason == "retried_request" + + +async def test_native_count_finishing_after_quarter_worker_budget_keeps_plan_and_spend( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.litellm_core_utils import logging_worker + from litellm.litellm_core_utils.logging_worker import LoggingWorker + + release: Final = asyncio.Event() + + async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int: + if not release.is_set(): + asyncio.get_running_loop().call_later(2.3, release.set) + await release.wait() + return await _count(model, api_key, body) + + worker: Final = LoggingWorker(timeout=8.0) + monkeypatch.setattr(logging_worker, "GLOBAL_LOGGING_WORKER", worker) + rig: Final = _Rig(monkeypatch, count=count) + try: + with _transport(_upstream): + await _call(rig.router, rig.logging()) + payload: Final = await rig.capture.payload() + observed: Final = _observation(payload).observation + assert observed.outcome == "complete" and observed.reason is None + assert observed.plan is not None and observed.plan.breakpoints[0].prefix_tokens == 5000 + assert payload["response_cost"] is not None and worker._timeout_total == 0 + finally: + release.set() + await worker.stop() + + +@pytest.mark.parametrize( + "options,on_deployment", + ( + ({"thinking": {"type": "enabled", "budget_tokens": 2048}}, False), + ({"extra_body": {"speed": "fast", "output_config": {"effort": "high"}}}, False), + *product( + ( + {"container": {"id": "container_test"}}, + {"mcp_servers": [{"type": "url", "name": "test", "url": "https://example.com/mcp"}]}, + {"inference_geo": "us"}, + {"safeguards": [{"type": "default"}]}, + ), + (False, True), + ), + ), +) +async def test_native_baseline_identity_keeps_the_actual_transformed_body( + monkeypatch: pytest.MonkeyPatch, options: dict[str, JsonValue], on_deployment: bool +) -> None: + models: Final = _MESSAGES.validate_python( + [ + *_MODELS[:2], + { + **_MODELS[2], + "litellm_params": { + **_JSON_OBJECT.validate_python(_MODELS[2]["litellm_params"]), + **(options if on_deployment else {}), + }, + }, + ] + ) + rig: Final = _Rig(monkeypatch, models=models) + log: Final = rig.logging() + with _transport(_upstream): + await rig.router.anthropic_messages( + model="test-router", + max_tokens=16, + messages=_MESSAGES.validate_json(_MESSAGES_JSON.replace("question", "question USE_OPUS")), + litellm_logging_obj=log, + litellm_call_id=rig.call_id, + litellm_metadata={"user_api_key_hash": "test-caller-hash"}, + litellm_session_id="native-identical", + **({} if on_deployment else options), + ) + observed: Final = _observation(await rig.capture.payload()).observation + assert observed.outcome == "complete" + assert log.baseline_cache_context is not None + assert observed.baseline_equivalent and observed.usage is not None, ( + log.baseline_cache_context.baseline_body, + log.baseline_cache_context.selected_body_digest, + ) + + from litellm.proxy.spend_tracking.baseline_accounting import BaselineHistory, advance_baseline_history + + _, estimates = advance_baseline_history(BaselineHistory(), (observed,)) + assert estimates[0].provenance == "observed_identical" and estimates[0].usage == observed.usage + + +@pytest.mark.parametrize("tier_limit", (8, 16)) +@pytest.mark.parametrize("extra", ({}, {"max_tokens": 8})) +async def test_native_baseline_identity_respects_caller_limit_and_tier_override( + monkeypatch: pytest.MonkeyPatch, tier_limit: int, extra: dict[str, int] +) -> None: + models: Final = _MESSAGES.validate_python( + [ + { + "model_name": "test-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": { + "SIMPLE": {"model_name": "opus", "litellm_params": {"max_tokens": tier_limit}}, + "MEDIUM": {"model_name": "opus", "litellm_params": {"max_tokens": tier_limit}}, + "COMPLEX": "opus", + "REASONING": "opus", + }, + "session_affinity": False, + }, + }, + }, + {**_MODELS[2], "litellm_params": {**_MODELS[2]["litellm_params"], "max_tokens": 64}}, + ] + ) + rig: Final = _Rig(monkeypatch, models=models) + with _transport(_upstream) as route: + await rig.router.anthropic_messages( + model="test-router", + max_tokens=8, + messages=_MESSAGES.validate_json(_MESSAGES_JSON), + litellm_logging_obj=rig.logging(), + litellm_call_id=rig.call_id, + litellm_metadata={"user_api_key_hash": "test-caller-hash"}, + litellm_session_id="native-limits", + extra_body=extra, + ) + observed: Final = _observation(await rig.capture.payload()).observation + wire: Final = _JSON_OBJECT.validate_json(route.calls.last.request.content) + assert wire["max_tokens"] == tier_limit + assert observed.baseline_equivalent == (tier_limit == 8) + + +@pytest.mark.parametrize("nested", (False, True)) +async def test_native_baseline_projection_matches_wire_parameter_placement( + monkeypatch: pytest.MonkeyPatch, + nested: bool, +) -> None: + counted: Final[asyncio.Queue[Mapping[str, JsonValue]]] = asyncio.Queue() + + async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int: + counted.put_nowait(body) + return await _count(model, api_key, body) + + models: Final = _MESSAGES.validate_python( + [ + { + **entry, + "litellm_params": { + **_JSON_OBJECT.validate_python(entry["litellm_params"]), + "model": "anthropic/claude-opus-5", + }, + } + if entry["model_name"] == "sonnet" + else entry + for entry in _MODELS + ] + ) + rig: Final = _Rig(monkeypatch, count=count, models=models) + settings: Final = {"speed": "standard", "thinking": {"type": "adaptive"}, "output_config": {"effort": "medium"}} + with _transport(_upstream) as route: + await rig.router.anthropic_messages( + model="test-router", + max_tokens=4096, + messages=_MESSAGES.validate_json(_MESSAGES_JSON), + litellm_logging_obj=rig.logging(), + litellm_call_id=rig.call_id, + litellm_metadata={"user_api_key_hash": "test-caller-hash"}, + litellm_session_id="native-placement", + **({"extra_body": settings} if nested else settings), + ) + captured: Final = _observation(await rig.capture.payload()) + wire: Final = _JSON_OBJECT.validate_json(route.calls.last.request.content) + assert captured.observation.plan is not None and not captured.observation.baseline_equivalent + projected: Final = counted.get_nowait() + assert {key: projected[key] for key in settings if key in projected} == { + key: wire[key] for key in settings if key in wire + } + assert {key: wire[key] for key in settings if key in wire} == ({} if nested else settings) + + +_PARITY_TOOL: Final = {"name": "custom", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}} +_PARITY_SYSTEM: Final = [{"type": "text", "text": "stable system", "cache_control": {"type": "ephemeral"}}] +_PARITY_POINTS: Final = [{"location": "message", "role": "system"}, {"location": "message", "index": -1}] + + +@pytest.mark.parametrize( + "caller,selected,baseline,summary", + ( + pytest.param({"extra_body": {"cache_control": {"type": "ephemeral"}}}, {}, {}, False, id="envelope-control"), + pytest.param({"extra_body": {"system": _PARITY_SYSTEM}}, {}, {}, False, id="envelope-system"), + pytest.param( + {"extra_body": {"messages": _MESSAGES.validate_json(_MESSAGES_JSON)}}, {}, {}, False, id="envelope-messages" + ), + pytest.param({}, {"tools": [_PARITY_TOOL]}, {}, False, id="selected-tool-mark"), + pytest.param({}, {}, {"tools": [_PARITY_TOOL]}, False, id="baseline-tool-mark"), + pytest.param({}, {"system": _PARITY_SYSTEM}, {"system": "baseline system"}, False, id="selected-system-mark"), + pytest.param({}, {"system": "selected system"}, {"system": _PARITY_SYSTEM}, False, id="baseline-system-mark"), + pytest.param({"system": None}, {}, {"system": "configured system"}, False, id="null-system"), + pytest.param({"thinking": None}, {}, {"thinking": {"type": "adaptive"}}, False, id="null-thinking"), + pytest.param({"tools": None}, {}, {"tools": [_PARITY_TOOL]}, False, id="null-tools"), + pytest.param({"verbosity": "low", "instructions": "ignored"}, {}, {}, False, id="ignored-native-options"), + pytest.param( + {"messages": _MESSAGES.validate_json(_MESSAGES_JSON.replace("stable", " "))}, + {}, + {}, + False, + id="empty-marked-block", + ), + pytest.param({"thinking": {"type": "adaptive"}}, {}, {}, True, id="reasoning-summary"), + pytest.param( + {"thinking": {"type": "adaptive"}, "additional_drop_params": ["thinking.display"]}, + {}, + {}, + True, + id="drop-nested-option", + ), + pytest.param( + {"cache_control_injection_points": _PARITY_POINTS}, + {"tools": [{**_PARITY_TOOL, "name": f"custom_{index}"} for index in range(4)]}, + {}, + False, + id="configured-cap", + ), + ), +) +async def test_native_baseline_projection_matches_direct_baseline_request( + monkeypatch: pytest.MonkeyPatch, + caller: dict[str, JsonValue], + selected: dict[str, JsonValue], + baseline: dict[str, JsonValue], + summary: bool, +) -> None: + counted: Final[asyncio.Queue[Mapping[str, JsonValue]]] = asyncio.Queue() + + async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int: + counted.put_nowait(body) + return await _count(model, api_key, body) + + def upstream(request: httpx.Request) -> httpx.Response: + body: Final = _JSON_OBJECT.validate_json(request.content) + model: Final = body.get("model") + assert isinstance(model, str) + return httpx.Response( + 200, + request=request, + json={ + **_message(True, model), + "usage": {"input_tokens": 6000, "output_tokens": 10}, + }, + ) + + models: Final = _MESSAGES.validate_python( + [ + _MODELS[0], + { + **_MODELS[1], + "litellm_params": {**_JSON_OBJECT.validate_python(_MODELS[1]["litellm_params"]), **selected}, + }, + { + **_MODELS[2], + "litellm_params": {**_JSON_OBJECT.validate_python(_MODELS[2]["litellm_params"]), **baseline}, + }, + ] + ) + rig: Final = _Rig(monkeypatch, models=models, count=count) + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False) + monkeypatch.setattr(litellm, "reasoning_auto_summary", summary) + monkeypatch.delenv("LITELLM_REASONING_AUTO_SUMMARY", raising=False) + request: Final = { + "messages": [{"role": "user", "content": "question"}], + **({"system": "stable system"} if "system" not in selected and "system" not in baseline else {}), + "max_tokens": 4096, + "enable_prompt_caching": True, + **caller, + } + with _transport(upstream) as route: + await rig.router.anthropic_messages(model="opus", **_JSON_OBJECT.validate_python(request)) + direct: Final = _JSON_OBJECT.validate_json(route.calls.last.request.content) + await rig.router.anthropic_messages( + model="test-router", + litellm_logging_obj=rig.logging(), + litellm_call_id=rig.call_id, + litellm_metadata={"user_api_key_hash": "test-caller-hash"}, + litellm_session_id="native-projection-parity", + **_JSON_OBJECT.validate_python(request), + ) + captured: Final = _observation(await rig.capture.payload()) + assert captured.observation.plan is not None, captured.observation.reason + projected: Final = counted.get_nowait() + assert {key: value for key, value in projected.items() if key not in ("metadata", "stream")} == { + key: value for key, value in direct.items() if key not in ("metadata", "stream") + } + + +@pytest.mark.parametrize( + "selected,baseline,usage_field,observed_value,multiplier", + ( + ({}, {"speed": "fast"}, "speed", "standard", 3.0), + ({"inference_geo": "us"}, {}, "inference_geo", "us", 1.0), + ), +) +async def test_native_baseline_prices_projected_settings_without_changing_actual_spend( + monkeypatch: pytest.MonkeyPatch, + selected: dict[str, JsonValue], + baseline: dict[str, JsonValue], + usage_field: str, + observed_value: str, + multiplier: float, +) -> None: + from litellm.proxy.spend_tracking.baseline_accounting import BaselineHistory, advance_baseline_history + from litellm.proxy.spend_tracking.savings import baseline_cost_snapshot, price_baseline_comparison + from litellm.types.utils import ModelInfo + + def upstream(request: httpx.Request) -> httpx.Response: + body: Final = _JSON_OBJECT.validate_json(request.content) + model: Final = body.get("model") + assert isinstance(model, str) + assert body.get(usage_field) == selected.get(usage_field) + return httpx.Response( + 200, + request=request, + json={ + **_message(True, model), + "usage": { + "input_tokens": 6000, + "output_tokens": 10, + usage_field: observed_value, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + }, + ) + + rig: Final = _Rig( + monkeypatch, + models=_MESSAGES.validate_python( + [ + _MODELS[0], + { + **_MODELS[1], + "litellm_params": { + **_JSON_OBJECT.validate_python(_MODELS[1]["litellm_params"]), + **selected, + }, + }, + { + **_MODELS[2], + "litellm_params": { + **_JSON_OBJECT.validate_python(_MODELS[2]["litellm_params"]), + **baseline, + }, + }, + ] + ), + ) + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False) + with _transport(upstream): + response: Final = _JSON_OBJECT.validate_python( + await rig.router.anthropic_messages( + model="test-router", + max_tokens=16, + messages=[{"role": "user", "content": "question"}], + litellm_logging_obj=rig.logging(), + litellm_call_id=rig.call_id, + litellm_metadata={"user_api_key_hash": "test-caller-hash"}, + litellm_session_id="baseline-speed", + ) + ) + payload: Final = await rig.capture.payload() + captured: Final = _observation(payload) + _, estimates = advance_baseline_history(BaselineHistory(), (captured.observation,)) + estimate: Final = estimates[0] + assert captured.prices is not None + prices: Final[ModelInfo] = { + **captured.prices, + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "provider_specific_entry": {"fast": 3.0, "us": 2.0}, + } + actual: Final = payload["response_cost"] + assert isinstance(actual, float) + snapshot: Final = baseline_cost_snapshot( + captured.model, + prices, + actual, + _OBJECTS.validate_python(payload["cost_breakdown"]), + None, + ) + comparison: Final = price_baseline_comparison(snapshot, estimate.usage, estimate.provenance) + assert comparison is not None and snapshot.actual_token_cost is not None, estimate.reason + assert comparison.baseline == pytest.approx( + actual + (6000 * 1e-6 + 10 * 2e-6) * multiplier - snapshot.actual_token_cost + ) + assert comparison.actual == actual + assert _JSON_OBJECT.validate_python(response["usage"])[usage_field] == observed_value + + +async def test_native_request_rewritten_after_capture_preserves_spend_without_guessing_baseline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class RewriteSystem(CustomLogger): + async def async_pre_call_deployment_hook( + self, kwargs: Mapping[str, object], call_type: CallTypes | None + ) -> dict[str, object]: + return {**kwargs, "system": "hook system"} + + rig: Final = _Rig(monkeypatch) + monkeypatch.setattr(litellm, "callbacks", [rig.hook, RewriteSystem()]) + with _transport(_upstream) as route: + await _call(rig.router, rig.logging()) + payload: Final = await rig.capture.payload() + wire: Final = _JSON_OBJECT.validate_json(route.calls.last.request.content) + observed: Final = _observation(payload).observation + assert wire["system"] == "hook system" + assert observed.reason == "unsupported_request_transformation" and observed.plan is None + assert observed.usage is not None + actual: Final = payload["response_cost"] + assert isinstance(actual, float) and actual > 0 + + +@pytest.mark.parametrize("history", ("long_session", "non_ascii")) +async def test_native_baseline_models_long_and_non_ascii_history(monkeypatch: pytest.MonkeyPatch, history: str) -> None: + rounds: Final = tuple( + message + for index in range(1200) + for message in ( + {"role": "assistant", "content": [{"type": "tool_use", "id": f"t{index}", "name": "Read", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": f"t{index}", "content": "ok"}]}, + ) + ) + prefix: Final = ( + [{"role": "user", "content": "start"}, *rounds] + if history == "long_session" + else [{"role": "user", "content": "a" * 400_000 + "é"}, {"role": "assistant", "content": "ok"}] + ) + messages: Final = json.dumps([*prefix, *_MESSAGES.validate_json(_MESSAGES_JSON)]) + rig: Final = _Rig(monkeypatch) + with _transport(_upstream): + await _call(rig.router, rig.logging(), messages=messages) + observed: Final = _observation(await rig.capture.payload()).observation + assert observed.outcome == "complete" and observed.plan is not None + + +async def test_native_baseline_abstains_after_selected_tier_compaction(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.router_strategy.complexity_router.context_compaction import compaction_executor + + monkeypatch.setitem( + litellm.model_cost, + "summary-fixture", + { + "litellm_provider": "anthropic", + "mode": "chat", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "supports_anthropic_compaction": True, + }, + ) + models: Final = _MESSAGES.validate_python( + [ + { + "model_name": "test-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": {"SIMPLE": "sonnet", "MEDIUM": "opus", "COMPLEX": "opus", "REASONING": "opus"}, + "keyword_tier_rules": [{"keywords": ["answer"], "tier": "SIMPLE"}], + "session_affinity": False, + "enable_context_window_escalation": False, + "max_tokens_from_tier_model": False, + "context_compaction": {"model": "compactor", "max_tokens": 512}, + }, + }, + }, + { + "model_name": "sonnet", + "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "test-selected"}, + "model_info": {"id": "selected", "max_input_tokens": 512, "max_output_tokens": 64}, + }, + { + "model_name": "opus", + "litellm_params": {"model": "anthropic/claude-opus-5", "api_key": "test-selected"}, + "model_info": {"id": "baseline", "max_input_tokens": 200000, "max_output_tokens": 4096}, + }, + { + "model_name": "compactor", + "litellm_params": {"model": "anthropic/summary-fixture", "api_key": "test-compactor"}, + "model_info": {"id": "compactor"}, + }, + ] + ) + + async def summarize(protocol: object, request: object, parent_model: object = None) -> Mapping[str, object]: + return { + "stop_reason": "compaction", + "content": [{"type": "compaction", "content": "compacted", "signature": "s"}], + "usage": {"input_tokens": 0, "output_tokens": 0}, + } + + messages: Final = json.dumps( + [ + {"role": "user", "content": "Background detail. " * 300}, + {"role": "assistant", "content": "Recorded"}, + {"role": "user", "content": "Answer briefly"}, + ] + ) + rig: Final = _Rig(monkeypatch, models=models) + token: Final = compaction_executor.set(summarize) + try: + with _transport(_upstream) as route: + await _call(rig.router, rig.logging(), messages=messages) + observed: Final = _observation(await rig.capture.payload()).observation + wire: Final = route.calls.last.request.content.decode() + finally: + compaction_executor.reset(token) + assert "compacted" in wire and "Background detail" not in wire + assert observed.reason == "unsupported_request_transformation" and observed.plan is None + assert observed.usage is not None + + +async def test_selected_tier_cache_markers_do_not_hide_an_unmarked_baseline_plan( + monkeypatch: pytest.MonkeyPatch, +) -> None: + models: Final = _MESSAGES.validate_python( + [ + _MODELS[0], + { + "model_name": "sonnet", + "litellm_params": { + "model": "anthropic/claude-sonnet-5", + "api_key": "test-selected", + "cache_control_injection_points": [{"location": "message", "role": "user", "index": -1}], + }, + "model_info": {"id": "selected"}, + }, + _MODELS[2], + ] + ) + rig: Final = _Rig(monkeypatch, models=models) + with _transport(_upstream) as route: + await _call( + rig.router, + rig.logging(), + messages='[{"role":"user","content":[{"type":"text","text":"stable"},{"type":"text","text":"question"}]}]', + ) + observed: Final = _observation(await rig.capture.payload()).observation + wire: Final = route.calls.last.request.content.decode() + assert "cache_control" in wire + assert observed.outcome == "complete" and not observed.baseline_equivalent + assert observed.reason is None and observed.plan is not None and not observed.plan.breakpoints + + +@pytest.mark.parametrize("recovery", ("retry", "fallback")) +async def test_tier_pins_never_enter_the_caller_snapshot_on_later_routing_passes( + monkeypatch: pytest.MonkeyPatch, recovery: str +) -> None: + pinned: Final = {"model_name": "first", "litellm_params": {"reasoning_effort": "high", "max_tokens": 777}} + models: Final = _MESSAGES.validate_python( + [ + { + "model_name": "test-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": {"SIMPLE": pinned, "MEDIUM": pinned, "COMPLEX": pinned, "REASONING": "opus"}, + "session_affinity": False, + }, + }, + }, + { + "model_name": "fallback-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": {"SIMPLE": "sonnet", "MEDIUM": "sonnet", "COMPLEX": "sonnet", "REASONING": "opus"}, + "session_affinity": False, + }, + }, + }, + { + "model_name": "first", + "litellm_params": { + "model": "anthropic/claude-sonnet-5" if recovery == "retry" else "anthropic/claude-haiku-5", + "api_key": "test-selected", + }, + "model_info": {"id": "first"}, + }, + *_MODELS[1:], + ] + ) + rig: Final = _Rig(monkeypatch, models=models, retries=1 if recovery == "retry" else 0) + rig.router.fallbacks = [{"test-router": ["fallback-router"]}] + + def upstream(request: httpx.Request) -> httpx.Response: + return _upstream(request) if route.call_count else _error(request, 429, "first attempt") + + log: Final = rig.logging() + with _transport(upstream) as route: + await _call(rig.router, log) + await rig.capture.payload() + first_wire: Final = _JSON_OBJECT.validate_json(route.calls[0].request.content) + assert first_wire.get("output_config") == {"effort": "high"} and first_wire.get("max_tokens") == 777 + context: Final = log.baseline_cache_context + assert context is not None and context.baseline_body is not None + assert context.baseline_body.get("max_tokens") == 16 + assert "output_config" not in context.baseline_body and "thinking" not in context.baseline_body diff --git a/tests/unit/proxy/spend_tracking/test_baseline_accounting.py b/tests/unit/proxy/spend_tracking/test_baseline_accounting.py index a188d65502d..368704e7a75 100644 --- a/tests/unit/proxy/spend_tracking/test_baseline_accounting.py +++ b/tests/unit/proxy/spend_tracking/test_baseline_accounting.py @@ -111,7 +111,13 @@ def test_prefix_match_expiry_and_usage_pricing_fields(ttl: int) -> None: assert warm.usage.prompt_tokens_details.cached_tokens == 6000 assert cold.usage.prompt_tokens_details.cached_tokens == 0 assert cold.usage.prompt_tokens_details.cache_creation_tokens == 6000 - unaffected: Final = {"prompt_tokens", "total_tokens", "prompt_tokens_details", "cache_read_input_tokens", "cache_creation_input_tokens"} + unaffected: Final = { + "prompt_tokens", + "total_tokens", + "prompt_tokens_details", + "cache_read_input_tokens", + "cache_creation_input_tokens", + } assert warm.usage.model_dump(exclude=unaffected) == first.usage.model_dump(exclude=unaffected) assert cold.usage.model_dump(exclude=unaffected) == first.usage.model_dump(exclude=unaffected) @@ -124,7 +130,10 @@ def test_growth_lookback_and_mixed_ttl_keep_distinct_read_write_buckets(warm_tai second: Final = _replay(first, _observation("second", 10001.0, plan=grown))[-1] assert second.reason == "history_unavailable" history: Final = BaselineHistory( - first_at=1.0, last_at=10000.0, equivalent=False, uncertain_before=1.0, + first_at=1.0, + last_at=10000.0, + equivalent=False, + uncertain_before=1.0, entries=(CacheEntry("tail:300", "tail", 7000, 300, 10000.0, 10300.0),) if warm_tail else (), ) _, estimates = advance_baseline_history(history, (_observation("mixed", 10001.0, plan=grown),)) @@ -134,8 +143,12 @@ def test_growth_lookback_and_mixed_ttl_keep_distinct_read_write_buckets(warm_tai # Anthropic billing locations: B is the highest 1h breakpoint AFTER the highest hit A. # https://platform.claude.com/docs/en/build-with-claude/prompt-caching#mixing-different-ttls (2026-09-15) assert usage.prompt_tokens_details.cached_tokens == (7000 if warm_tail else 0) - assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_1h_input_tokens == (0 if warm_tail else 6500) - assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_5m_input_tokens == (0 if warm_tail else 500) + assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_1h_input_tokens == ( + 0 if warm_tail else 6500 + ) + assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_5m_input_tokens == ( + 0 if warm_tail else 500 + ) @pytest.mark.parametrize("change", ["prefix", "ttl", "unavailable", "failed", "response_cache"]) @@ -197,3 +210,62 @@ def test_modeled_read_cannot_recharge_the_original_private_write_count() -> None } input_cost, output_cost = cost_per_token("claude-opus-5", warm.usage, model_info=prices) assert input_cost + output_cost == pytest.approx((200 * 1e-6 + 6000 * 1e-7 + 30 * 2e-6) * 2.0 * 1.1) + + +def test_mixed_lifetime_lookback_preserves_a_compatible_native_hit() -> None: + marker: Final = _marker("prefix", 3600, 6000) + history: Final = BaselineHistory( + first_at=1.0, + last_at=10000.0, + equivalent=False, + uncertain_before=1.0, + entries=(CacheEntry(marker.fingerprint, marker.content_fingerprint, 6000, 3600, 10000.0, 13600.0),), + ) + plan: Final = CountedPromptCachePlan( + 7100, (_marker("grown", 3600, 6500, ("prefix",)), _marker("tail", 300, 7000, ("prefix",))) + ) + _, estimates = advance_baseline_history(history, (_observation("next", 10001.0, plan=plan),)) + usage: Final = estimates[0].usage + assert usage is not None, estimates[0].reason + assert usage.prompt_tokens_details.cached_tokens == 6000 + assert usage.prompt_tokens_details.text_tokens == 100 + assert usage.prompt_tokens_details.cache_creation_token_details == CacheCreationTokenDetails( + ephemeral_5m_input_tokens=500, ephemeral_1h_input_tokens=500 + ) + + +def test_short_lifetime_hit_cannot_seed_an_unpaid_long_lifetime_entry() -> None: + first: Final = _observation("initial", plan=CountedPromptCachePlan(6200, (_marker("5", 3600, 6000),))) + short: Final = _observation("short", 13700.0, plan=CountedPromptCachePlan(6200, (_marker("3", 300, 4600),))) + mixed: Final = _observation( + "mixed", + 13710.0, + plan=CountedPromptCachePlan(6200, (_marker("3", 3600, 4600), _marker("4", 300, 5500, ("3",)))), + ) + later: Final = _observation("later", 14710.0, plan=CountedPromptCachePlan(6200, (_marker("3", 3600, 4600),))) + _, _, upgrade, after_expiry = _replay(first, short, mixed, later) + assert upgrade.usage is not None and after_expiry.usage is not None + assert upgrade.usage.prompt_tokens_details.cached_tokens == 4600 + assert upgrade.usage.prompt_tokens_details.cache_creation_token_details.ephemeral_1h_input_tokens == 0 + assert after_expiry.usage.prompt_tokens_details.cached_tokens == 0 + assert after_expiry.usage.prompt_tokens_details.cache_creation_token_details.ephemeral_1h_input_tokens == 4600 + + +@pytest.mark.parametrize("writes", (0, 50, None)) +def test_cache_creation_split_is_optional_only_without_writes(writes: int | None) -> None: + usage: Final = Usage( + prompt_tokens=100, + completion_tokens=10, + total_tokens=110, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=100 - (writes or 0), + cached_tokens=0, + cache_creation_tokens=writes, + ), + ) + observed: Final = _observation("no-split", usage=usage, plan=CountedPromptCachePlan(100, ())) + restored: Final = BaselineObservation.model_validate_json(observed.model_dump_json()) + estimate: Final = _replay(restored)[0] + assert (estimate.usage is not None) is (writes == 0) + if estimate.usage is not None: + assert estimate.usage.prompt_tokens == usage.prompt_tokens diff --git a/tests/unit/router_utils/test_baseline_request.py b/tests/unit/router_utils/test_baseline_request.py new file mode 100644 index 00000000000..37d21597752 --- /dev/null +++ b/tests/unit/router_utils/test_baseline_request.py @@ -0,0 +1,59 @@ +from typing import Final + +from litellm.router_utils.baseline_request import baseline_request, capture_baseline_parameters + + +def test_baseline_snapshot_owns_nested_caller_settings_and_overrides_routed_settings() -> None: + reasoning: Final = {"effort": "medium"} + snapshot: Final = capture_baseline_parameters({"reasoning": reasoning, "verbosity": "low"}) + assert snapshot is not None + reasoning["effort"] = "high" + projected: Final = baseline_request( + {"messages": [{"role": "user", "content": "hello"}], "reasoning": reasoning, "verbosity": "high"}, + snapshot, + {"verbosity": "medium"}, + ) + assert projected == { + "messages": [{"role": "user", "content": "hello"}], + "reasoning": {"effort": "medium"}, + "verbosity": "low", + } + + +def test_oversized_snapshot_fails_closed_before_json_validation() -> None: + assert capture_baseline_parameters({"output_config": {"format": "x" * 5_000_000}}) is None + + +def test_snapshot_retains_extra_body_settings_but_no_credentials() -> None: + assert capture_baseline_parameters({"api_key": "private", "extra_body": {"verbosity": "low"}}) == { + "extra_body": {"verbosity": "low"} + } + + +def test_chat_projection_applies_extra_body_after_top_level_parameters() -> None: + snapshot: Final = capture_baseline_parameters({"verbosity": "high", "extra_body": {"verbosity": "low"}}) + assert snapshot is not None + assert baseline_request({}, snapshot, {}) == {"verbosity": "low"} + + +def test_baseline_projection_keeps_caller_tools_and_request_parameter_precedence() -> None: + from litellm.router import Router + + deployment: Final = { + "tools": [{"type": "function", "function": {"name": "configured"}}], + "tool_choice": "required", + "max_tokens": 64, + } + caller: Final = { + "tools": [{"type": "function", "function": {"name": "caller"}}], + "tool_choice": "auto", + "max_tokens": 128, + } + actual_request: Final = dict(caller) + Router._merge_tools_from_deployment({"litellm_params": deployment}, actual_request) + snapshot: Final = capture_baseline_parameters(caller) + assert snapshot is not None + assert baseline_request({"tools": [{"name": "routed-only"}], "max_tokens": 4}, snapshot, deployment) == { + **deployment, + **actual_request, + } From 740d0435a806eec44e7aa359a9c633a5f9d3531b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:09:56 -0700 Subject: [PATCH 10/35] test(integration): scope the team-scoped models upstream check to its own model (#45106) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../authorization/test_team_scoped_models.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/integration/authorization/test_team_scoped_models.py b/tests/integration/authorization/test_team_scoped_models.py index aea6b9e4ba4..1323c1ee004 100644 --- a/tests/integration/authorization/test_team_scoped_models.py +++ b/tests/integration/authorization/test_team_scoped_models.py @@ -15,10 +15,16 @@ def upstream(gateway: Gateway) -> Iterator[httpx.Client]: yield client -def _observed_models(upstream: httpx.Client) -> list[JsonValue]: +def _observed_requests(upstream: httpx.Client) -> list[JsonValue]: observed: Final = upstream.get("/__observations") observed.raise_for_status() - return [request["body"]["model"] for request in observed.json()["requests"]] + requests: Final = object_value(observed.json())["requests"] + assert isinstance(requests, list) + return requests + + +def _calls_to(observed: list[JsonValue], provider_model: str) -> int: + return sum(object_value(object_value(request)["body"]).get("model") == provider_model for request in observed) def _chat(gateway: Gateway, model: str, key: str) -> httpx.Response: @@ -62,7 +68,8 @@ def test_a_team_model_is_listed_and_served_only_for_keys_of_its_team(gateway: Ga assert served.status_code == 200, served.text refused: Final = _chat(gateway, model, other_key) assert refused.status_code == 400, refused.text - assert _observed_models(upstream) == [provider_model] + observed: Final = _observed_requests(upstream) + assert _calls_to(observed, provider_model) == 1, observed def _v2_team_public_names(gateway: Gateway, key: str, model: str) -> list[JsonValue]: @@ -98,8 +105,10 @@ def test_team_model_alias_routes_a_team_key_to_its_target( response: Final = _chat(gateway, alias, key) assert response.status_code == 200, response.text assert string_value(response.json()["model"]) == alias - assert _observed_models(upstream) == [provider_model] + observed: Final = _observed_requests(upstream) + assert _calls_to(observed, provider_model) == 1, observed unaliased: Final = _chat(gateway, f"alias-{uuid.uuid4().hex}", key) assert unaliased.status_code == 403, unaliased.text assert unaliased.json()["error"]["type"] == "key_model_access_denied" - assert _observed_models(upstream) == [] + after_refusal: Final = _observed_requests(upstream) + assert _calls_to(after_refusal, provider_model) == 0, after_refusal From 2c1847f8a262a1967de3db2dfc9c341286eb71e6 Mon Sep 17 00:00:00 2001 From: ahamedshaik16 Date: Wed, 7 Oct 2026 23:48:45 +0530 Subject: [PATCH 11/35] fix(prometheus): add model_group label to end-to-end latency metrics (#44860) litellm_llm_api_latency_metric, litellm_llm_api_time_to_first_token_metric, litellm_request_total_latency_metric, and litellm_deployment_latency_per_output_token previously carried requested_model/litellm_model_name/model_id but not model_group, so pooled-deployment latency couldn't be grouped by model pool on dashboards -- only the proxy-overhead-only metrics (litellm_overhead_latency_metric and friends) had model_group. All four metrics read enum_values.model_group through the existing prometheus_label_factory plumbing, so no new parameter threading was needed, just the label-list addition. Co-authored-by: ahamedshaik16 <24526479+ahamedshaik16@users.noreply.github.com> --- litellm/types/integrations/prometheus.py | 4 + .../test_prometheus_logging_callbacks.py | 4 + .../integrations/test_prometheus_labels.py | 187 ++++++++++++++++++ 3 files changed, 195 insertions(+) diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index dc552a730ea..a951381a2c9 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -416,6 +416,7 @@ def _resolve_deployment_and_latency_caller_identity_labels( class PrometheusMetricLabels: litellm_llm_api_latency_metric = [ + UserAPIKeyLabelNames.MODEL_GROUP.value, UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, UserAPIKeyLabelNames.API_KEY_HASH.value, UserAPIKeyLabelNames.API_KEY_ALIAS.value, @@ -430,6 +431,7 @@ class PrometheusMetricLabels: ] litellm_llm_api_time_to_first_token_metric = [ + UserAPIKeyLabelNames.MODEL_GROUP.value, UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, UserAPIKeyLabelNames.API_KEY_HASH.value, UserAPIKeyLabelNames.API_KEY_ALIAS.value, @@ -444,6 +446,7 @@ class PrometheusMetricLabels: ] litellm_request_total_latency_metric = [ + UserAPIKeyLabelNames.MODEL_GROUP.value, UserAPIKeyLabelNames.END_USER.value, UserAPIKeyLabelNames.API_KEY_HASH.value, UserAPIKeyLabelNames.API_KEY_ALIAS.value, @@ -516,6 +519,7 @@ class PrometheusMetricLabels: ] litellm_deployment_latency_per_output_token = [ + UserAPIKeyLabelNames.MODEL_GROUP.value, UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_BASE.value, diff --git a/tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index f1c80bb11ea..78545d3fb62 100644 --- a/tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -433,6 +433,7 @@ def test_set_latency_metrics(prometheus_logger): team_alias="test_team_alias", org_id=None, org_alias=None, + model_group="openai-gpt", requested_model="openai-gpt", model="gpt-5-mini", model_id="model-123", @@ -453,6 +454,7 @@ def test_set_latency_metrics(prometheus_logger): team_alias="test_team_alias", org_id=None, org_alias=None, + model_group="openai-gpt", requested_model="openai-gpt", model="gpt-5-mini", model_id="model-123", @@ -473,6 +475,7 @@ def test_set_latency_metrics(prometheus_logger): team_alias="test_team_alias", org_id=None, org_alias=None, + model_group="openai-gpt", requested_model="openai-gpt", model="gpt-5-mini", model_id="model-123", @@ -1054,6 +1057,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): # Verify latency per output token metric prometheus_logger.litellm_deployment_latency_per_output_token.labels.assert_called_once_with( + model_group="my_custom_model_group", litellm_model_name="gpt-5-mini", model_id="model-123", api_base="https://api.openai.com", diff --git a/tests/unit/integrations/test_prometheus_labels.py b/tests/unit/integrations/test_prometheus_labels.py index 8a1f5f5a0a8..ead0ce8ecff 100644 --- a/tests/unit/integrations/test_prometheus_labels.py +++ b/tests/unit/integrations/test_prometheus_labels.py @@ -980,6 +980,193 @@ def test_deployment_tpm_rpm_limit_metrics_emit_model_group_from_enum_values(): _clear_prometheus_registry() +def test_model_group_in_latency_metrics(): + """ + Test that model_group label is present on the end-to-end / per-call + latency metrics needed to build model-group latency dashboards. These + metrics previously only carried requested_model, litellm_model_name and + model_id, none of which identify the model_group a pooled deployment + belongs to -- only the proxy-overhead-only latency metrics + (litellm_overhead_latency_metric and friends) carried model_group. + """ + model_group_label = UserAPIKeyLabelNames.MODEL_GROUP.value + + metrics_with_model_group = [ + "litellm_llm_api_latency_metric", + "litellm_llm_api_time_to_first_token_metric", + "litellm_request_total_latency_metric", + "litellm_deployment_latency_per_output_token", + ] + + for metric_name in metrics_with_model_group: + labels = PrometheusMetricLabels.get_labels(metric_name) + assert ( + model_group_label in labels + ), f"Metric {metric_name} should contain model_group label" + print(f"✅ {metric_name} contains model_group label") + + +def test_model_group_value_flows_through_latency_metrics_label_factory(): + """ + The label being in the allow-list is necessary but not sufficient: the + factory must also carry the value from the enum through to the emitted + label. This would fail if the label were dropped from a metric's list or + if the value plumbing regressed, which the allow-list assertion above + cannot catch on its own. + """ + from unittest.mock import MagicMock + + from litellm.integrations.prometheus import ( + PrometheusLogger, + UserAPIKeyLabelValues, + prometheus_label_factory, + ) + + prometheus_logger = MagicMock() + prometheus_logger._cached_metric_labels = {} + prometheus_logger.label_filters = {} + prometheus_logger.get_labels_for_metric = ( + PrometheusLogger.get_labels_for_metric.__get__(prometheus_logger) + ) + + enum_values = UserAPIKeyLabelValues( + model_group="example-model-group", + litellm_model_name="gpt-4o-mini", + requested_model="example-model-group", + status_code="200", + ) + + for metric_name in [ + "litellm_llm_api_latency_metric", + "litellm_llm_api_time_to_first_token_metric", + "litellm_request_total_latency_metric", + "litellm_deployment_latency_per_output_token", + ]: + labels = prometheus_label_factory( + supported_enum_labels=prometheus_logger.get_labels_for_metric( + metric_name=metric_name + ), + enum_values=enum_values, + ) + assert ( + labels.get("model_group") == "example-model-group" + ), f"{metric_name} should emit model_group=example-model-group, got {labels.get('model_group')!r}" + + +def test_latency_metrics_emit_model_group_from_set_latency_metrics(): + """ + End-to-end emit wiring for _set_latency_metrics. + + The label-list and factory tests above prove the label exists and that + the factory carries a value handed to it, but neither drives the real + _set_latency_metrics code path, so deleting the production model_group + plumbing there would still pass them. This calls it directly with a + streaming request (so the time-to-first-token branch also fires) and + asserts the real litellm_llm_api_latency_metric, + litellm_llm_api_time_to_first_token_metric and + litellm_request_total_latency_metric Histogram series actually carry it. + """ + import datetime + + from litellm.integrations.prometheus import PrometheusLogger, UserAPIKeyLabelValues + + _clear_prometheus_registry() + try: + logger = PrometheusLogger() + start_time = datetime.datetime(2024, 1, 1, 0, 0, 0) + api_call_start_time = datetime.datetime(2024, 1, 1, 0, 0, 1) + completion_start_time = datetime.datetime(2024, 1, 1, 0, 0, 2) + end_time = datetime.datetime(2024, 1, 1, 0, 0, 3) + + enum_values = UserAPIKeyLabelValues( + model_group="example-model-group", + litellm_model_name="gpt-4o-mini", + requested_model="example-model-group", + status_code="200", + ) + + logger._set_latency_metrics( + kwargs={ + "start_time": start_time, + "end_time": end_time, + "api_call_start_time": api_call_start_time, + "completion_start_time": completion_start_time, + "stream": True, + "litellm_params": {"metadata": {}}, + }, + model="gpt-4o-mini", + user_api_key=None, + user_api_key_alias=None, + user_api_team=None, + user_api_team_alias=None, + enum_values=enum_values, + ) + + for metric in ( + logger.litellm_llm_api_latency_metric, + logger.litellm_llm_api_time_to_first_token_metric, + logger.litellm_request_total_latency_metric, + ): + index = metric._labelnames.index("model_group") + values = {sample_key[index] for sample_key in metric._metrics} + assert values == {"example-model-group"}, ( + f"expected model_group=example-model-group on {metric._name}, got {values}" + ) + finally: + _clear_prometheus_registry() + + +def test_deployment_latency_per_output_token_emits_model_group_from_enum_values(): + """ + End-to-end emit wiring for litellm_deployment_latency_per_output_token. + + Drives set_llm_deployment_success_metrics directly (its only caller) with + output_tokens > 0 so the latency-per-token branch fires, and asserts the + real Histogram series carries model_group; fails if that label-list + addition or the enum_values plumbing is removed. + """ + import datetime + + from litellm.integrations.prometheus import PrometheusLogger, UserAPIKeyLabelValues + + _clear_prometheus_registry() + try: + logger = PrometheusLogger() + start_time = datetime.datetime(2024, 1, 1, 0, 0, 0) + end_time = datetime.datetime(2024, 1, 1, 0, 0, 2) + enum_values = UserAPIKeyLabelValues( + model_group="example-model-group", + litellm_model_name="gpt-4o-mini", + requested_model="example-model-group", + status_code="200", + ) + logger.set_llm_deployment_success_metrics( + request_kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {"model_info": {"id": "model-123"}}}, + "standard_logging_object": { + "model_group": "example-model-group", + "model_id": "model-123", + "api_base": "https://api.openai.com", + "hidden_params": {"additional_headers": None, "litellm_overhead_time_ms": None}, + }, + }, + start_time=start_time, + end_time=end_time, + enum_values=enum_values, + output_tokens=10.0, + ) + + metric = logger.litellm_deployment_latency_per_output_token + index = metric._labelnames.index("model_group") + values = {sample_key[index] for sample_key in metric._metrics} + assert values == {"example-model-group"}, ( + f"expected model_group=example-model-group on {metric._name}, got {values}" + ) + finally: + _clear_prometheus_registry() + + if __name__ == "__main__": test_user_email_in_required_metrics() test_user_email_label_exists() From 0586289817b01610250b3e294db3cd14d24cb819 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:28:14 -0700 Subject: [PATCH 12/35] test(mcp): fix stale bridge-hook, applied-guardrails and pagination-revoke integration tests (#45008) * test(mcp): fix stale bridge-hook, applied-guardrails and pagination-revoke integration tests * test(mcp): clear the spare direct grant when restoring the access-group policy --------- Co-authored-by: yuneng --- .../mcp/test_mcp_accounting_guardrails.py | 2 +- tests/integration/mcp/test_mcp_llm_endpoints.py | 4 ++-- tests/integration/mcp/test_pagination.py | 15 ++++++++------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/integration/mcp/test_mcp_accounting_guardrails.py b/tests/integration/mcp/test_mcp_accounting_guardrails.py index a276718c280..0d1c707d1ff 100644 --- a/tests/integration/mcp/test_mcp_accounting_guardrails.py +++ b/tests/integration/mcp/test_mcp_accounting_guardrails.py @@ -468,7 +468,7 @@ def test_generic_sink_and_native_hooks_receive_listed_metadata_on_typed_keys_wit assert row["status"] == "success", row assert _tool_metadata(row)["name"] == "lookup", row metadata: Final = row["metadata"] - assert isinstance(metadata, dict) and metadata["applied_guardrails"] == [hooks_rig.guardrail], metadata + assert isinstance(metadata, dict) and metadata["applied_guardrails"].count(hooks_rig.guardrail) == 1, metadata def test_pre_call_mask_reaches_the_peer_and_post_call_mask_reaches_the_caller_on_one_call_id( diff --git a/tests/integration/mcp/test_mcp_llm_endpoints.py b/tests/integration/mcp/test_mcp_llm_endpoints.py index af3bff1fd8b..9e946dfc253 100644 --- a/tests/integration/mcp/test_mcp_llm_endpoints.py +++ b/tests/integration/mcp/test_mcp_llm_endpoints.py @@ -934,7 +934,7 @@ def test_identical_nonstream_repeat_is_a_cache_hit_without_new_model_peer_or_hoo assert _spend_row(key, repeat.call_id)["cache_hit"] == "True" -def test_messages_bridge_hook_keeps_the_base_shape_without_request_local_metadata(hooked: Hooked) -> None: +def test_messages_bridge_hook_sees_the_definition_the_request_served(hooked: Hooked) -> None: with _bridge_rig(hooked, "messages") as rig: key: Final = _bridge_key(rig) marker: Final = "m" + uuid.uuid4().hex @@ -943,6 +943,6 @@ def test_messages_bridge_hook_keeps_the_base_shape_without_request_local_metadat assert found.status_code == 200, found.text blocked: Final = rig.post(key, probe, [rig.mcp("lookup")]) assert blocked.status_code == 200, blocked.text - assert _echoed(JSON_VALUE.validate_json(blocked.content)) == COLD, blocked.text + assert _echoed(JSON_VALUE.validate_json(blocked.content)) == _served(LOOKUP), blocked.text assert rig.peer_calls() == (("lookup", {"query": marker}),) assert rig.hook_messages(marker) == (f"Tool: lookup\nArguments: {dict(query=marker)}",) diff --git a/tests/integration/mcp/test_pagination.py b/tests/integration/mcp/test_pagination.py index 84689232038..b8b28bd39ce 100644 --- a/tests/integration/mcp/test_pagination.py +++ b/tests/integration/mcp/test_pagination.py @@ -1,7 +1,7 @@ import asyncio from contextlib import asynccontextmanager from pathlib import Path -from typing import Literal +from typing import Final, Literal import httpx import pytest @@ -138,7 +138,7 @@ def test_continuations_reauthorize_and_reject_registry_changes(tmp_path: Path, m assert os.environ.get("DATABASE_URL"), "This integration case requires disposable-database access" - async def exercise(a, b, peer, identity, owner, stranger, policy): + async def exercise(a, b, peer, identity, owner, stranger, policy, spare): owner_a = Gateway(a.client, owner, peer.url) owner_b = Gateway(b.client, owner, peer.url) stranger_b = Gateway(b.client, stranger, peer.url) @@ -165,9 +165,9 @@ def test_continuations_reauthorize_and_reject_registry_changes(tmp_path: Path, m { "key": owner, **( - {"access_group_ids": []} + {"access_group_ids": [], "object_permission": {"mcp_servers": [spare]}} if grant == "access_group" - else {"object_permission": {"mcp_servers": ["no-mcp-servers"]}} + else {"object_permission": {"mcp_servers": [spare]}} ), }, ) @@ -177,7 +177,7 @@ def test_continuations_reauthorize_and_reject_registry_changes(tmp_path: Path, m with pytest.raises(MCPError, match="fresh listing"): await getattr(session, method)(params=PaginatedRequestParams(cursor=first.next_cursor)) assert not any(call["body"].get("method", "").endswith("/list") for call in peer.drain()) - a.post("/key/update", {"key": owner, **policy}) + a.post("/key/update", {"key": owner, "object_permission": {"mcp_servers": []}, **policy}) changed = a.request("PUT", "/v1/mcp/server", {"server_id": identity, "description": "new catalog generation"}) assert changed.status_code == 202, changed.text for method, first in first_pages.items(): @@ -207,7 +207,7 @@ def test_continuations_reauthorize_and_reject_registry_changes(tmp_path: Path, m capture_output=True, text=True, ) - with paginated_mcp_peer() as peer, httpx.Client() as client: + with paginated_mcp_peer() as peer, paginated_mcp_peer() as spare_peer, httpx.Client() as client: seed = Gateway(client, "sk-pagination-test", peer.url) config = tmp_path / "database-proxy.yaml" config.write_text( @@ -231,6 +231,7 @@ def test_continuations_reauthorize_and_reject_registry_changes(tmp_path: Path, m a.scenario() as scenario, ): identity = register_mcp(scenario, peer, "pages") + spare: Final = register_mcp(scenario, spare_peer, "spare") group = a.request( "POST", "/v1/access_group", @@ -248,7 +249,7 @@ def test_continuations_reauthorize_and_reject_registry_changes(tmp_path: Path, m owner = scenario.key(**policy) stranger = scenario.key(object_permission={"mcp_servers": [identity]}) assert owner != stranger - asyncio.run(exercise(a, b, peer, identity, owner, stranger, policy)) + asyncio.run(exercise(a, b, peer, identity, owner, stranger, policy, spare)) @pytest.mark.parametrize("changed", ["key", "snapshot"]) From 31b90ebb47d3b47d40e2c01f416adba2ecb19540 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:29:05 -0700 Subject: [PATCH 13/35] test(observability): wait for warm-up spend rows in capped and count only the post-wipe half in X4 (#45006) Co-authored-by: yuneng --- tests/integration/observability/conftest.py | 8 +++++++- .../test_cache_hit_guardrail_metrics_chaos.py | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/integration/observability/conftest.py b/tests/integration/observability/conftest.py index c5150877857..fcac8eaba3c 100644 --- a/tests/integration/observability/conftest.py +++ b/tests/integration/observability/conftest.py @@ -8,8 +8,9 @@ from urllib.parse import urlparse import pytest import yaml +from integration._support.client import eventually from integration._support.otlp_sink import SpanSinks, owned_sinks -from integration._support.prometheus_series import CapRig, series_cap_rig +from integration._support.prometheus_series import CapRig, series_cap_rig, spend_rows from pydantic import JsonValue AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path] @@ -63,4 +64,9 @@ def capped(tmp_path_factory: pytest.TempPathFactory) -> Iterator[CapRig]: workers=2, warm_keys=3, ) as rig: + eventually( + lambda: tuple(len(spend_rows(key.alias)) for key in rig.warm), + lambda counts: all(count == 1 for count in counts), + seconds=70, + ) yield rig diff --git a/tests/integration/observability/test_cache_hit_guardrail_metrics_chaos.py b/tests/integration/observability/test_cache_hit_guardrail_metrics_chaos.py index c77b8eebe33..7e3b479e5f0 100644 --- a/tests/integration/observability/test_cache_hit_guardrail_metrics_chaos.py +++ b/tests/integration/observability/test_cache_hit_guardrail_metrics_chaos.py @@ -263,7 +263,7 @@ def test_worker_kill_mid_burst_keeps_counting(gateway: Gateway, tmp_path: Path) def test_proxy_restart_mid_burst_keeps_counting(gateway: Gateway, tmp_path: Path) -> None: - """X4: restart the owned proxy between the two halves; pre-restart count asserted, then recounted.""" + """X4: the boot wipes the kept directory, so the second proxy counts only the second half.""" marker: Final = uuid.uuid4().hex prom_dir: Final = tmp_path / "prom" prom_dir.mkdir() @@ -321,8 +321,8 @@ def test_proxy_restart_mid_burst_keeps_counting(gateway: Gateway, tmp_path: Path _populated(_samples(owned_two.gateway, (model,)), deployment), _blank(_samples(owned_two.gateway, (model,))), ), - lambda observed: observed[0] == len(named) and observed[1] == 0, + lambda observed: observed[0] == len(second_half) and observed[1] == 0, seconds=70, ) - assert post[0] == len(named), (pre, post, outcomes_two) + assert post[0] == len(second_half), (pre, post, outcomes_two) owned_two.gateway.post("/model/delete", {"id": deployment}) From 2121322983b2b40ca830edef67bb46972b329d66 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:39:28 -0700 Subject: [PATCH 14/35] fix(cost-map): lower anthropic claude-sonnet-5-5 cache read price (#45113) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dbc1bb0de46..8a20bbd3430 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15229,8 +15229,8 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_creation_input_token_cost_batches": 1.25e-06, - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "anthropic", @@ -15267,7 +15267,7 @@ "supports_forced_tool_use": false, "thinking_always_on": true, "prompt_cache_min_tokens": 512, - "source": "https://platform.claude.com/docs/en/models/sonnet-5-5/overview", + "source": "https://platform.claude.com/docs/en/about-claude/pricing", "supports_web_search": true }, "claude-sonnet-4-6": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dbc1bb0de46..8a20bbd3430 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15229,8 +15229,8 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_creation_input_token_cost_batches": 1.25e-06, - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "anthropic", @@ -15267,7 +15267,7 @@ "supports_forced_tool_use": false, "thinking_always_on": true, "prompt_cache_min_tokens": 512, - "source": "https://platform.claude.com/docs/en/models/sonnet-5-5/overview", + "source": "https://platform.claude.com/docs/en/about-claude/pricing", "supports_web_search": true }, "claude-sonnet-4-6": { From 44bc1656e7ada372eef39f12ad2491e39c160cde Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 7 Oct 2026 11:42:33 -0700 Subject: [PATCH 15/35] fix(prompt-caching): default injected /v1/messages breakpoints to implicit lookup (#45121) --- .../anthropic_cache_control_hook.py | 2 +- .../test_anthropic_cache_control_hook.py | 32 ++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 893cb0d84d3..22d76242f2f 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -1029,7 +1029,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) AnthropicCacheControlHook.record_gateway_injection(kwargs, breakpoints_added) if openai_dialect and breakpoints_added > 0: - kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit")) + kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="implicit")) if remaining: kwargs["cache_control_injection_points"] = remaining return messages, system diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 23fa2e36bce..8392cef8a7b 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -3088,7 +3088,7 @@ class TestOpenAIPromptCacheBreakpoint: messages, system = self._inject([{"role": "user", "content": "hi"}], "sys", kwargs) assert system == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] assert messages == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] - assert kwargs == {"prompt_cache_options": self.EXPLICIT} + assert kwargs == {"prompt_cache_options": {"mode": "implicit"}} assert not _contains_key(system, "cache_control") def test_v1_messages_list_system_marks_last_block_only(self): @@ -3099,7 +3099,7 @@ class TestOpenAIPromptCacheBreakpoint: {"type": "text", "text": "a"}, {"type": "text", "text": "b", "prompt_cache_breakpoint": self.EXPLICIT}, ] - assert kwargs["prompt_cache_options"] == self.EXPLICIT + assert kwargs["prompt_cache_options"] == {"mode": "implicit"} def test_v1_messages_targets_by_role(self): messages = [ @@ -3115,7 +3115,7 @@ class TestOpenAIPromptCacheBreakpoint: ] assert result[1] == messages[1] assert result[2]["content"] == [{"type": "text", "text": "last", "prompt_cache_breakpoint": self.EXPLICIT}] - assert kwargs["prompt_cache_options"] == self.EXPLICIT + assert kwargs["prompt_cache_options"] == {"mode": "implicit"} def test_v1_messages_targets_by_index(self): messages = [ @@ -3142,8 +3142,9 @@ class TestOpenAIPromptCacheBreakpoint: assert not _contains_key(system, "cache_control") assert not _contains_key(messages, "cache_control") - def test_v1_messages_keeps_caller_prompt_cache_options(self): - caller_options = {"mode": "explicit", "ttl": "30m"} + @pytest.mark.parametrize("mode", ["explicit", "implicit"]) + def test_v1_messages_keeps_caller_prompt_cache_options(self, mode): + caller_options = {"mode": mode, "ttl": "30m"} kwargs = { "cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT), "prompt_cache_options": dict(caller_options), @@ -3152,6 +3153,15 @@ class TestOpenAIPromptCacheBreakpoint: assert system[0]["prompt_cache_breakpoint"] == self.EXPLICIT assert kwargs["prompt_cache_options"] == caller_options + def test_v1_messages_and_chat_paths_default_to_the_same_implicit_mode(self): + messages_kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} + self._inject([{"role": "user", "content": "hi"}], "sys", messages_kwargs) + _, _, chat_params = self._chat( + [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}, + ) + assert messages_kwargs["prompt_cache_options"] == chat_params["prompt_cache_options"] == {"mode": "implicit"} + def test_v1_messages_no_prompt_cache_options_when_nothing_injected(self): kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} messages, system = self._inject([{"role": "user", "content": "hi"}], None, kwargs) @@ -3185,7 +3195,7 @@ class TestOpenAIPromptCacheBreakpoint: result, system = self._inject(messages, "sys", kwargs) assert result == messages assert system == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] - assert kwargs == {"prompt_cache_options": self.EXPLICIT} + assert kwargs == {"prompt_cache_options": {"mode": "implicit"}} def test_v1_messages_tail_point_applies_beside_client_system_breakpoint(self): system = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] @@ -3196,7 +3206,7 @@ class TestOpenAIPromptCacheBreakpoint: {"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]} ] assert result_system == system - assert kwargs == {"prompt_cache_options": self.EXPLICIT} + assert kwargs == {"prompt_cache_options": {"mode": "implicit"}} def test_chat_system_string_wrapped_with_block_breakpoint(self): params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} @@ -3375,7 +3385,7 @@ class TestOpenAIPromptCacheBreakpointPlacementRules: {"type": "tool_result", "tool_use_id": "t1", "content": "sunny"}, {"type": "text", "text": "thanks", "prompt_cache_breakpoint": self.EXPLICIT}, ] - assert kwargs["prompt_cache_options"] == self.EXPLICIT + assert kwargs["prompt_cache_options"] == {"mode": "implicit"} def test_marker_walks_back_to_last_eligible_block(self): messages = [ @@ -3656,12 +3666,12 @@ class TestMessagesPathApiBaseGate: def test_regional_openai_api_base_uses_openai_dialect(self): block, kwargs = self._inject("gpt-5.6", api_base="https://eu.api.openai.com/v1") assert block == self.BREAKPOINT_BLOCK - assert kwargs["prompt_cache_options"] == self.EXPLICIT + assert kwargs["prompt_cache_options"] == {"mode": "implicit"} def test_default_api_base_uses_openai_dialect(self): block, kwargs = self._inject("openai/gpt-5.6") assert block == self.BREAKPOINT_BLOCK - assert kwargs["prompt_cache_options"] == self.EXPLICIT + assert kwargs["prompt_cache_options"] == {"mode": "implicit"} class TestToolConfigSlotInOpenAIDialect: @@ -3752,7 +3762,7 @@ class TestPromptCacheBreakpointCapability: [{"role": "user", "content": "hi"}], "sys", kwargs, model="gpt-5.6", custom_llm_provider="openai" ) assert system == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] - assert kwargs == {"prompt_cache_options": {"mode": "explicit"}} + assert kwargs == {"prompt_cache_options": {"mode": "implicit"}} @pytest.mark.parametrize("model,expected", [("gpt-5.6-2026-01-01", True), ("gpt-5.5-preview-unlisted", False)]) def test_unlisted_model_falls_back_to_the_version_rule(self, model, expected): From 8283033c0ea0c1eb87e8a24e08b0aa4a999d44ac Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:46:34 -0700 Subject: [PATCH 16/35] test(integration): turn the model-info refresh off in every integration proxy config (#45104) The shared integration proxy configs left the five-minute /v1/models refresh on, so every proxy booted from them sent a GET /v1/models to each openai-compatible deployment at boot and every 300 s after, including the per-test deployments other cells register against their own scripted wires. A scripted provider that asserts on the exact requests it receives then saw a GET it never scripted, mid-test. Set disable_model_info_refresh in proxy_config.yaml, coordination_redis_proxy_config.yaml, and oci_proxy_test_config.yaml, and drop the three per-test copies of that setting from the observability cells that each loaded the stock config and set it themselves. Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/integration/coordination_redis_proxy_config.yaml | 1 + tests/integration/observability/test_langtrace_delivery.py | 3 +-- tests/integration/observability/test_otel_conversation_id.py | 2 +- tests/integration/observability/test_signoz_delivery.py | 1 - tests/integration/oci_proxy_test_config.yaml | 1 + tests/integration/proxy_config.yaml | 1 + 6 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/integration/coordination_redis_proxy_config.yaml b/tests/integration/coordination_redis_proxy_config.yaml index 30294c291bf..e28e0f5f2ad 100644 --- a/tests/integration/coordination_redis_proxy_config.yaml +++ b/tests/integration/coordination_redis_proxy_config.yaml @@ -3,6 +3,7 @@ general_settings: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL store_model_in_db: true + disable_model_info_refresh: true disable_spend_logs: false proxy_batch_write_at: 1 coordination_redis: diff --git a/tests/integration/observability/test_langtrace_delivery.py b/tests/integration/observability/test_langtrace_delivery.py index 84c9ed5bec0..03e42087dc7 100644 --- a/tests/integration/observability/test_langtrace_delivery.py +++ b/tests/integration/observability/test_langtrace_delivery.py @@ -132,9 +132,8 @@ def _upstream(request: Request) -> Reply: def _config(tmp_path: Path, **litellm_settings: object) -> Path: config: Final = _PROXY_CONFIG.validate_python(yaml.safe_load(STOCK_CONFIG.read_text())) settings: Final = {**_SETTINGS.validate_python(config["litellm_settings"]), **litellm_settings} - general: Final = {**_SETTINGS.validate_python(config["general_settings"]), "disable_model_info_refresh": True} path: Final = tmp_path / "langtrace.yaml" - path.write_text(yaml.safe_dump({**config, "litellm_settings": settings, "general_settings": general})) + path.write_text(yaml.safe_dump({**config, "litellm_settings": settings})) return path diff --git a/tests/integration/observability/test_otel_conversation_id.py b/tests/integration/observability/test_otel_conversation_id.py index 78ff40927e5..e77e5603148 100644 --- a/tests/integration/observability/test_otel_conversation_id.py +++ b/tests/integration/observability/test_otel_conversation_id.py @@ -289,7 +289,7 @@ class RigFactory: def start(self) -> Iterator[Rig]: config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) config["litellm_settings"].update({"callbacks": ["otel"]}) - config["general_settings"].update({"disable_model_info_refresh": True, **self.settings}) + config["general_settings"].update(self.settings) config["callback_settings"] = { "otel": {"exporter": "http/json", "endpoint": self.sink.wire.url, "mapper_names": ["genai"]}, } diff --git a/tests/integration/observability/test_signoz_delivery.py b/tests/integration/observability/test_signoz_delivery.py index f3d715fe5cf..56a98b3f17c 100644 --- a/tests/integration/observability/test_signoz_delivery.py +++ b/tests/integration/observability/test_signoz_delivery.py @@ -394,7 +394,6 @@ class RigFactory: "callbacks": ["signoz"], "provider_url_destination_allowed_hosts": [self.tenant_sink.wire.url], }, - "general_settings": {**object_value(loaded["general_settings"]), "disable_model_info_refresh": True}, } path: Final = self.directory / f"signoz-{uuid.uuid4().hex}.yaml" path.write_text(yaml.safe_dump(config)) diff --git a/tests/integration/oci_proxy_test_config.yaml b/tests/integration/oci_proxy_test_config.yaml index 95e74963cd2..f09216c4f94 100644 --- a/tests/integration/oci_proxy_test_config.yaml +++ b/tests/integration/oci_proxy_test_config.yaml @@ -19,6 +19,7 @@ model_list: general_settings: master_key: os.environ/LITELLM_MASTER_KEY + disable_model_info_refresh: true litellm_settings: drop_params: True diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 9e4a90fc608..6924b641f13 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -326,6 +326,7 @@ general_settings: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL store_model_in_db: true + disable_model_info_refresh: true disable_spend_logs: false proxy_batch_write_at: 1 proxy_batch_polling_interval: 1 From 6352613d1b32a462e15097ebdf515d5f6ec0607a Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 7 Oct 2026 11:51:20 -0700 Subject: [PATCH 17/35] test(e2e): let the realtime send step accept input audio buffer frames (#45127) --- tests/e2e/llm_translation/realtime/realtime_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index 6eae4fdf54d..c9f06e787f3 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -308,7 +308,7 @@ class RealtimeSession: connection: Connection @step("Send the realtime event {event.type} over the websocket") - def send(self, event: SessionUpdate | ConversationItemCreate | ResponseCreate) -> None: + def send(self, event: SessionUpdate | ConversationItemCreate | ResponseCreate | InputAudioBufferAppend) -> None: self.connection.send(event.model_dump_json(by_alias=True, exclude_none=True)) @step("Wait for a {stop_type} event on the realtime websocket") From 5cd112687087b01f84e443386c3e27d1f16b62bc Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 7 Oct 2026 11:52:35 -0700 Subject: [PATCH 18/35] feat(model_prices): add claude-haiku-5-5 model pricing (#45108) * feat(model_prices): add claude-haiku-5-5 model pricing and capabilities Source: https://docs.anthropic.com/en/docs/about-claude/models * fix(model_prices): allow above_100k_tokens tier keys in the cost map schemas Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 643 ++++++++++++++++++ model_prices_and_context_window.json | 643 ++++++++++++++++++ model_prices_and_context_window.schema.json | 25 + tests/unit/test_utils.py | 5 + 4 files changed, 1316 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8a20bbd3430..13503b28544 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -80768,5 +80768,648 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true + }, +"claude-haiku-5-5": { + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 5e-09, + "input_cost_per_token": 1e-07, + "input_cost_per_token_batches": 5e-08, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_batches": 2.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/models/haiku-5-5/overview", + "supports_web_search": true, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08 + }, + "bedrock_mantle/anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.375e-07, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_read_input_token_cost": 1.1e-08, + "input_cost_per_token": 1.1e-07, + "litellm_provider": "bedrock_mantle", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html" + }, + "bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 1.5e-07, + "cache_creation_input_token_cost_above_1hr": 2.4e-07, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock_mantle", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html" + }, + "anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + }, + "apac.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_creation_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.1e-07, + "output_cost_per_token": 5.5e-07, + "cache_read_input_token_cost": 1.1e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + }, + "au.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.375e-07, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_read_input_token_cost": 1.1e-08, + "input_cost_per_token": 1.1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "azure_ai/claude-haiku-5-5": { + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_read_input_token_cost": 1e-08, + "deprecation_date": "2027-09-29", + "input_cost_per_token": 1e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" + }, + "bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 1.5e-07, + "cache_creation_input_token_cost_above_1hr": 2.4e-07, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-west-1/anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 1.5e-07, + "cache_creation_input_token_cost_above_1hr": 2.4e-07, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "eu.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.375e-07, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_read_input_token_cost": 1.1e-08, + "input_cost_per_token": 1.1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "global.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "jp.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.375e-07, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_read_input_token_cost": 1.1e-08, + "input_cost_per_token": 1.1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "perplexity/anthropic/claude-haiku-5-5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_adaptive_thinking": true, + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 1e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "us-gov.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 1.5e-07, + "cache_creation_input_token_cost_above_1hr": 2.4e-07, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.375e-07, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_read_input_token_cost": 1.1e-08, + "input_cost_per_token": 1.1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "vertex_ai/claude-haiku-5-5": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 5e-09, + "input_cost_per_token": 1e-07, + "input_cost_per_token_batches": 5e-08, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_batches": 2.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + }, + "vertex_ai/claude-haiku-5-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 5e-09, + "input_cost_per_token": 1e-07, + "input_cost_per_token_batches": 5e-08, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_batches": 2.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8a20bbd3430..13503b28544 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -80768,5 +80768,648 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true + }, +"claude-haiku-5-5": { + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 5e-09, + "input_cost_per_token": 1e-07, + "input_cost_per_token_batches": 5e-08, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_batches": 2.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/models/haiku-5-5/overview", + "supports_web_search": true, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08 + }, + "bedrock_mantle/anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.375e-07, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_read_input_token_cost": 1.1e-08, + "input_cost_per_token": 1.1e-07, + "litellm_provider": "bedrock_mantle", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html" + }, + "bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 1.5e-07, + "cache_creation_input_token_cost_above_1hr": 2.4e-07, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock_mantle", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html" + }, + "anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + }, + "apac.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_creation_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.1e-07, + "output_cost_per_token": 5.5e-07, + "cache_read_input_token_cost": 1.1e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + }, + "au.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.375e-07, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_read_input_token_cost": 1.1e-08, + "input_cost_per_token": 1.1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "azure_ai/claude-haiku-5-5": { + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_read_input_token_cost": 1e-08, + "deprecation_date": "2027-09-29", + "input_cost_per_token": 1e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" + }, + "bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 1.5e-07, + "cache_creation_input_token_cost_above_1hr": 2.4e-07, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-west-1/anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 1.5e-07, + "cache_creation_input_token_cost_above_1hr": 2.4e-07, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "eu.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.375e-07, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_read_input_token_cost": 1.1e-08, + "input_cost_per_token": 1.1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "global.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "jp.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.375e-07, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_read_input_token_cost": 1.1e-08, + "input_cost_per_token": 1.1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "perplexity/anthropic/claude-haiku-5-5": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_adaptive_thinking": true, + "supports_web_search": true, + "supports_function_calling": true, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 1e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "us-gov.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 1.5e-07, + "cache_creation_input_token_cost_above_1hr": 2.4e-07, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 1.2e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us.anthropic.claude-haiku-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 1.375e-07, + "cache_creation_input_token_cost_above_1hr": 2.2e-07, + "cache_read_input_token_cost": 1.1e-08, + "input_cost_per_token": 1.1e-07, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh", + "supports_parallel_tool_use_config": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "vertex_ai/claude-haiku-5-5": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 5e-09, + "input_cost_per_token": 1e-07, + "input_cost_per_token_batches": 5e-08, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_batches": 2.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + }, + "vertex_ai/claude-haiku-5-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 5e-09, + "input_cost_per_token": 1e-07, + "input_cost_per_token_batches": 5e-08, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_batches": 2.5e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "supports_forced_tool_use": false, + "thinking_always_on": true, + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" } } diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index de3d760d57a..41b616a6b1f 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -83,6 +83,11 @@ "minimum": 0, "description": "USD per token written to the provider's prompt cache." }, + "cache_creation_input_token_cost_above_100k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_creation_input_token_cost_above_128k_tokens": { "type": "number", "minimum": 0, @@ -93,6 +98,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": { "type": "number", "minimum": 0, @@ -174,6 +184,11 @@ "minimum": 0, "description": "USD per prompt token served from the provider's prompt cache." }, + "cache_read_input_token_cost_above_100k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_read_input_token_cost_above_128k_tokens": { "type": "number", "minimum": 0, @@ -377,6 +392,11 @@ "minimum": 0, "description": "USD per prompt token." }, + "input_cost_per_token_above_100k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "input_cost_per_token_above_128k_tokens": { "type": "number", "minimum": 0, @@ -756,6 +776,11 @@ "minimum": 0, "description": "USD per generated token." }, + "output_cost_per_token_above_100k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "output_cost_per_token_above_128k_tokens": { "type": "number", "minimum": 0, diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index c899083ca80..1f420d0e8dc 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -777,6 +777,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_token_cost": {"type": "number"}, "cache_creation_input_token_cost_above_1hr": {"type": "number"}, "cache_creation_input_token_cost_above_32k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_100k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_128k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_200k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_256k_tokens": {"type": "number"}, @@ -792,6 +793,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_token_cost_ultrafast": {"type": "number"}, "cache_read_input_token_cost": {"type": "number"}, "cache_read_input_token_cost_above_32k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_100k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_128k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_200k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_200k_tokens_batches": {"type": "number"}, @@ -803,6 +805,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"}, "cache_read_input_token_cost_batches": {"type": "number"}, "cache_read_input_token_cost_above_272k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, "cache_read_input_image_token_cost": {"type": "number"}, @@ -819,6 +822,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_image_above_128k_tokens": {"type": "number"}, "input_cost_per_video_token": {"type": "number"}, "input_cost_per_token_above_32k_tokens": {"type": "number"}, + "input_cost_per_token_above_100k_tokens": {"type": "number"}, "input_cost_per_token_above_200k_tokens": {"type": "number"}, "input_cost_per_token_above_200k_tokens_batches": {"type": "number"}, "input_cost_per_token_above_256k_tokens": {"type": "number"}, @@ -928,6 +932,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "output_cost_per_second_4k": {"type": "number"}, "output_cost_per_token": {"type": "number"}, "output_cost_per_token_above_32k_tokens": {"type": "number"}, + "output_cost_per_token_above_100k_tokens": {"type": "number"}, "output_cost_per_token_above_128k_tokens": {"type": "number"}, "output_cost_per_token_above_200k_tokens": {"type": "number"}, "output_cost_per_token_above_200k_tokens_batches": {"type": "number"}, From 1c6f714187c7446a7588157ec28e2759e9cad478 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:59:34 -0700 Subject: [PATCH 19/35] fix(realtime): run transcript guardrails on raw-path transcription sessions with a transcription-safe block (#44844) * fix(realtime): skip guardrail VAD session.update injection for transcription sessions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(realtime): run transcript guardrails on raw-path transcription sessions with a transcription-safe block Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(realtime): assert transcription session keeps transcribing after a guardrail block Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(realtime): only expect a follow-up transcript when the block keeps the session open Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(realtime): type the transcription block regression test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(realtime): flag transcription sessions from the route intent and backend events only A client session.update declaring session.type transcription on a voice session no longer sets the transcription flag, so it cannot switch off the guardrail's create_response gate or skip the transcript guardrail * fix(realtime): flag transcription sessions from provider-transformed session events * test(realtime): cover transcript guardrail blocks on transcription sessions --------- Co-authored-by: gabriele Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../litellm_core_utils/realtime_streaming.py | 47 +- ..._transcription_guardrail_session_update.py | 1047 ++++++++++++++++- .../test_realtime_streaming.py | 115 +- 3 files changed, 1152 insertions(+), 57 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 62b8ce95b22..a781be610a6 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -896,8 +896,8 @@ class RealTimeStreaming: # clientContent / cancel messages are sent. if pre_block_backend_message is not None: await self._send_to_backend(pre_block_backend_message) - # Cancel any in-progress LLM response (e.g. VAD auto-response). - await self._send_to_backend(json.dumps({"type": "response.cancel"})) + if not self._is_transcription_session: + await self._send_to_backend(json.dumps({"type": "response.cancel"})) # Send the policy violation hint (shows as small gray status text in UI). await self.websocket.send_text( json.dumps( @@ -911,25 +911,26 @@ class RealTimeStreaming: } ) ) - # Ask the LLM to voice the exact guardrail message so the - # user hears it as audio in voice sessions (not just text). - guardrail_prompt = ( - f"Say exactly the following message to the user, word for word, " - f"do not add anything else: {error_msg}" - ) - await self._send_to_backend( - json.dumps( - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": guardrail_prompt}], - }, - } + if not self._is_transcription_session: + # Ask the LLM to voice the exact guardrail message so the + # user hears it as audio in voice sessions (not just text). + guardrail_prompt = ( + f"Say exactly the following message to the user, word for word, " + f"do not add anything else: {error_msg}" ) - ) - await self._send_to_backend(json.dumps({"type": "response.create"})) + await self._send_to_backend( + json.dumps( + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": guardrail_prompt}], + }, + } + ) + ) + await self._send_to_backend(json.dumps({"type": "response.create"})) self._violation_count += 1 end_session_after: int | None = getattr(callback, "end_session_after_n_fails", None) @@ -1070,18 +1071,14 @@ class RealTimeStreaming: self.store_message(event_obj) await self.websocket.send_text(self._event_to_client_json(event_obj)) - # Transcription-only sessions (e.g. gpt-realtime-whisper) have no - # assistant turn: capture audio-duration usage for cost and never - # trigger response.create. if self._is_transcription_session: self._capture_transcription_usage(event_obj) - return True blocked: Final = await self.run_realtime_guardrails( transcript, item_id=event_obj.get("item_id"), ) - if not blocked: + if not blocked and not self._is_transcription_session: await self._send_to_backend(json.dumps({"type": "response.create"})) return True return False diff --git a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py index b60fbcdbdeb..fd8879e02d4 100644 --- a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py +++ b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py @@ -6,6 +6,12 @@ every assistant turn. A transcription session has no assistant turn and the vend injection is skipped when the route intent or a backend session event says the session is transcription-only, while a client frame alone never flips a voice session into one. Every row runs against the scripted upstream, which answers the injected and rewritten updates the way the vendors do. + +A blocked transcript on a transcription session reaches the client as a ``guardrail_violation`` error and +nothing else: the ``response.cancel``, the voiced block prompt and the ``response.create`` a voice session gets +never go to a backend that cannot speak, while ``on_violation: end_session`` and ``end_session_after_n_fails`` +still close the session. A guardrail that raises anything but a block closes the session the way it does on a +voice session. """ from __future__ import annotations @@ -130,6 +136,69 @@ APPEND: Final[dict[str, JsonValue]] = {"type": "input_audio_buffer.append", "aud PUSH_TO_TALK: Final = "PUSH_TO_TALK" ENDPOINTING: Final = "ENDPOINTING" END_STREAM: Final[dict[str, JsonValue]] = {"type": "endStream"} +RESPONSE_CREATE: Final[dict[str, JsonValue]] = {"type": "response.create"} +TRANSCRIPTION_UPDATE_FRAME: Final[dict[str, JsonValue]] = {"type": "session.update", "session": GA_TRANSCRIPTION_UPDATE} +VOICE_UPDATE_FRAME: Final[dict[str, JsonValue]] = {"type": "session.update", "session": VOICE_UPDATE} +REALTIME_HOOK: Final = "realtime_input_transcription" +ENDER_GUARDRAIL: Final = "transcript-ender" +ENDER_WORD: Final = "anchovy" +ENDER_MESSAGE: Final = "The session was ended by the transcript policy." +TWO_STRIKES_GUARDRAIL: Final = "transcript-two-strikes" +TWO_STRIKES_WORD: Final = "olives" +ONE_STRIKE_GUARDRAIL: Final = "transcript-one-strike" +ONE_STRIKE_WORD: Final = "radish" +WARN_GUARDRAIL: Final = "transcript-warn" +WARN_WORD: Final = "capers" +VALUE_ERROR_GUARDRAIL: Final = "transcript-value-error" +VALUE_ERROR_WORD: Final = "durian" +RUNTIME_ERROR_GUARDRAIL: Final = "transcript-runtime-error" +RUNTIME_ERROR_WORD: Final = "lychee" +RAISING_MODULE: Final = "raising_guardrails" +CONFIGURED_GUARDRAILS: Final = frozenset( + { + TRANSCRIPT_GUARDRAIL, + OPTIN_GUARDRAIL, + PROMPT_GUARDRAIL, + ENDER_GUARDRAIL, + TWO_STRIKES_GUARDRAIL, + ONE_STRIKE_GUARDRAIL, + WARN_GUARDRAIL, + VALUE_ERROR_GUARDRAIL, + RUNTIME_ERROR_GUARDRAIL, + } +) +RELAYED_CLOSE: Final = ("server_error", "") +GUARDRAIL_END_CLOSE: Final = 1000 +PROXY_FAILURE_CLOSE: Final = 1011 +PROXY_FAILURE_REASON: Final = "proxy failed while relaying the upstream websocket" +TOOL_OUTPUT_BLOCKED: Final = json.dumps({"error": "Tool output blocked by content policy"}) +VOICE_BLOCK_FRAMES: Final = ("response.cancel", "conversation.item.create", "response.create") +CREATED_SECONDS: Final = 60.0 +STEP_SECONDS: Final = 15.0 +SDK_SECONDS: Final = 45.0 +RAISING_GUARDRAILS_SOURCE: Final = f"""\ +from litellm.integrations.custom_guardrail import CustomGuardrail + + +class WordRaiser(CustomGuardrail): + word = "" + error = Exception + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + if any(self.word in str(text) for text in inputs["texts"]): + raise self.error(f"{{self.word}} is not allowed") + return inputs + + +class ValueErrorGuardrail(WordRaiser): + word = "{VALUE_ERROR_WORD}" + error = ValueError + + +class RuntimeErrorGuardrail(WordRaiser): + word = "{RUNTIME_ERROR_WORD}" + error = RuntimeError +""" @dataclass(frozen=True, slots=True) @@ -183,7 +252,15 @@ def _owned_url(owned: OwnedProxy) -> str: return str(owned.gateway.client.base_url).rstrip("/") -def _transcript_event(transcript: str) -> dict[str, JsonValue]: +def _blocked(word: str) -> str: + return f"please add {word} to the order" + + +def _optin(guardrail: str) -> str: + return f"guardrails={guardrail}" + + +def _transcript_event(transcript: JsonValue) -> dict[str, JsonValue]: return { "type": TRANSCRIPT_COMPLETED, "event_id": "evt_$UNIQUE_ID", @@ -208,15 +285,20 @@ def _done_event() -> dict[str, JsonValue]: } -def _transcription_scenario(transcript: str, *, repeats: int = 1) -> RealtimeResponse: +def _transcript_event_without_the_field() -> dict[str, JsonValue]: + return {key: value for key, value in _transcript_event("").items() if key != "transcript"} + + +def _transcription_events(events: tuple[dict[str, JsonValue], ...], *, repeats: int = 1) -> RealtimeResponse: return RealtimeResponse( - content_type="application/x-realtime", - events=(_transcript_event(transcript),), - session_type=TRANSCRIPTION, - created_repeats=repeats, + content_type="application/x-realtime", events=events, session_type=TRANSCRIPTION, created_repeats=repeats ) +def _transcription_scenario(*transcripts: JsonValue, repeats: int = 1) -> RealtimeResponse: + return _transcription_events(tuple(_transcript_event(transcript) for transcript in transcripts), repeats=repeats) + + def _older_transcription_scenario(transcript: str) -> RealtimeResponse: return RealtimeResponse( content_type="application/x-realtime", @@ -226,10 +308,14 @@ def _older_transcription_scenario(transcript: str) -> RealtimeResponse: ) -def _voice_scenario(transcript: str) -> RealtimeResponse: - return RealtimeResponse( - content_type="application/x-realtime", events=(_transcript_event(transcript), _done_event()) - ) +def _voice_turns(transcripts: tuple[JsonValue, ...]) -> Iterator[dict[str, JsonValue]]: + for transcript in transcripts: + yield _transcript_event(transcript) + yield _done_event() + + +def _voice_scenario(*transcripts: JsonValue) -> RealtimeResponse: + return RealtimeResponse(content_type="application/x-realtime", events=tuple(_voice_turns(transcripts))) def _muse_scenario(transcript: str) -> RealtimeResponse: @@ -345,6 +431,97 @@ async def _session( return Session((), refusal.response.status_code) +@dataclass(frozen=True, slots=True) +class Step: + frames: tuple[dict[str, JsonValue], ...] + until: str + + +def _step(*frames: dict[str, JsonValue], until: str) -> Step: + return Step(frames, until) + + +def _update_frame(update: JsonValue) -> dict[str, JsonValue]: + return {"type": "session.update", "session": update} + + +def _probe(update: JsonValue = GA_TRANSCRIPTION_UPDATE) -> Step: + return Step((_update_frame(update),), "session.updated") + + +async def _until(socket: ClientConnection, until: str, seconds: float) -> AsyncIterator[dict[str, JsonValue]]: + deadline: Final = asyncio.get_running_loop().time() + seconds + while True: + event: Final = await _next_event(socket, deadline) + if event is None: + yield {"type": "timeout"} + return + yield event + if event.get("type") == until: + return + + +async def _stepped( + socket: ClientConnection, steps: tuple[Step, ...], seconds: float +) -> AsyncIterator[dict[str, JsonValue]]: + try: + first: Final = JSON_OBJECT.validate_json(await asyncio.wait_for(socket.recv(), CREATED_SECONDS)) + yield first + if first.get("type") not in CREATED_TYPES: + async for message in socket: + yield JSON_OBJECT.validate_json(message) + return + for step in steps: + for frame in step.frames: + await socket.send(json.dumps(frame)) + async for event in _until(socket, step.until, seconds): + yield event + if event["type"] == "timeout": + return + except ConnectionClosed as closed: + yield {"type": "closed", "code": _close_code(closed)} + + +async def _driven( + ws_base: str, + path: str, + query: str, + key: str, + steps: tuple[Step, ...], + headers: Mapping[str, str] | None, + seconds: float, +) -> Session: + request_headers: Final = {"Authorization": f"Bearer {key}", **(headers or {})} + async with websockets.connect(f"{ws_base}{path}?{query}", additional_headers=request_headers) as socket: + return Session(tuple([event async for event in _stepped(socket, steps, seconds)]), None) + + +def _drive( + ws_base: str, + query: str, + key: str, + steps: tuple[Step, ...], + *, + path: str = "/v1/realtime", + headers: Mapping[str, str] | None = None, + seconds: float = STEP_SECONDS, +) -> Session: + return asyncio.run(_driven(ws_base, path, query, key, steps, headers, seconds)) + + +def _transcribe_blocked( + ws_base: str, + query: str, + key: str, + *, + path: str = "/v1/realtime", + update: JsonValue = GA_TRANSCRIPTION_UPDATE, + headers: Mapping[str, str] | None = None, +) -> Session: + steps: Final = (_step(_update_frame(update), COMMIT, until="error"), _probe(update)) + return _drive(ws_base, query, key, steps, path=path, headers=headers) + + def _transcribe( ws_base: str, query: str, @@ -439,27 +616,70 @@ def _spend_rows(key: str, count: int) -> list[dict[str, JsonValue]]: ) -def _content_filter(name: str, mode: str, *, default_on: bool) -> dict[str, JsonValue]: +def _content_filter( + name: str, mode: str, *, default_on: bool, word: str = BLOCKED_WORD, **settings: JsonValue +) -> dict[str, JsonValue]: return { "guardrail_name": name, "litellm_params": { "guardrail": "litellm_content_filter", "mode": mode, "default_on": default_on, - "blocked_words": [{"keyword": BLOCKED_WORD, "action": "BLOCK"}], + "blocked_words": [{"keyword": word, "action": "BLOCK"}], + **settings, }, } +def _raising_guardrail(name: str, class_name: str) -> dict[str, JsonValue]: + return { + "guardrail_name": name, + "litellm_params": {"guardrail": f"{RAISING_MODULE}.{class_name}", "mode": REALTIME_HOOK, "default_on": False}, + } + + def _write_config(directory: Path, ca_bundle: Path) -> Path: config: Final = directory / f"realtime_guardrails_{uuid.uuid4().hex[:8]}.yaml" + (directory / f"{RAISING_MODULE}.py").write_text(RAISING_GUARDRAILS_SOURCE) config.write_text( json.dumps( { "guardrails": [ - _content_filter(TRANSCRIPT_GUARDRAIL, "realtime_input_transcription", default_on=True), - _content_filter(OPTIN_GUARDRAIL, "realtime_input_transcription", default_on=False), + _content_filter(TRANSCRIPT_GUARDRAIL, REALTIME_HOOK, default_on=True), + _content_filter(OPTIN_GUARDRAIL, REALTIME_HOOK, default_on=False), _content_filter(PROMPT_GUARDRAIL, "pre_call", default_on=True), + _content_filter( + ENDER_GUARDRAIL, + REALTIME_HOOK, + default_on=False, + word=ENDER_WORD, + on_violation="end_session", + realtime_violation_message=ENDER_MESSAGE, + ), + _content_filter( + TWO_STRIKES_GUARDRAIL, + REALTIME_HOOK, + default_on=False, + word=TWO_STRIKES_WORD, + end_session_after_n_fails=2, + ), + _content_filter( + ONE_STRIKE_GUARDRAIL, + REALTIME_HOOK, + default_on=False, + word=ONE_STRIKE_WORD, + end_session_after_n_fails=1, + ), + _content_filter( + WARN_GUARDRAIL, + REALTIME_HOOK, + default_on=False, + word=WARN_WORD, + on_violation="warn", + end_session_after_n_fails=None, + ), + _raising_guardrail(VALUE_ERROR_GUARDRAIL, "ValueErrorGuardrail"), + _raising_guardrail(RUNTIME_ERROR_GUARDRAIL, "RuntimeErrorGuardrail"), ], "general_settings": { "master_key": "os.environ/LITELLM_MASTER_KEY", @@ -524,6 +744,35 @@ def _assert_transcription_left_alone( assert _session_updates(observed) == (update,), _session_updates(observed) +def _assert_transcription_blocked( + session: Session, observed: tuple[dict[str, JsonValue], ...], transcript: str, *, update: JsonValue +) -> None: + assert session.types == ("session.created", "session.updated", TRANSCRIPT_COMPLETED, "error", "session.updated"), ( + session + ) + assert session.session_type == TRANSCRIPTION, session + assert session.transcripts == (transcript,), session + assert session.errors == (GUARDRAIL_VIOLATION,), session + assert _sent_types(observed) == ("session.update", "input_audio_buffer.commit", "session.update"), _sent(observed) + assert _session_updates(observed) == (update, update), _session_updates(observed) + + +def _relayed_close(session: Session) -> str: + return f"upstream websocket closed with code {session.close_code}" + + +def _assert_ended_by_the_guardrail(session: Session, *, violations: int) -> None: + assert session.errors == (GUARDRAIL_VIOLATION,) * violations, session + assert session.types[-1] == "closed", session + assert session.close_code == GUARDRAIL_END_CLOSE, session + + +def _assert_closed_by_a_proxy_failure(session: Session) -> None: + assert session.errors[-1] == RELAYED_CLOSE, session + assert session.close_code == PROXY_FAILURE_CLOSE, session + assert session.error_messages[-1] == f"{_relayed_close(session)}: {PROXY_FAILURE_REASON}", session + + @pytest.mark.parametrize("path", ["/v1/realtime", "/realtime", "/openai/v1/realtime"]) def test_transcription_session_update_reaches_the_upstream_verbatim(guardrail_proxy: OwnedProxy, path: str) -> None: with guardrail_proxy.gateway.scenario() as scenario: @@ -542,17 +791,21 @@ def test_transcription_session_update_reaches_the_upstream_verbatim(guardrail_pr assert rows[0]["call_type"] == "_arealtime", rows -async def _sdk_async_events(connection: AsyncRealtimeConnection) -> AsyncIterator[dict[str, JsonValue]]: +async def _sdk_async_events( + connection: AsyncRealtimeConnection, until: str = TRANSCRIPT_COMPLETED +) -> AsyncIterator[dict[str, JsonValue]]: async for event in connection: yield JSON_OBJECT.validate_python(event.model_dump()) - if event.type == TRANSCRIPT_COMPLETED: + if event.type == until: return -def _sdk_sync_events(connection: RealtimeConnection) -> Iterator[dict[str, JsonValue]]: +def _sdk_sync_events( + connection: RealtimeConnection, until: str = TRANSCRIPT_COMPLETED +) -> Iterator[dict[str, JsonValue]]: for event in connection: yield JSON_OBJECT.validate_python(event.model_dump()) - if event.type == TRANSCRIPT_COMPLETED: + if event.type == until: return @@ -750,6 +1003,14 @@ def test_backend_session_created_typed_transcription_skips_the_injection_on_the_ assert [_query(upgrade) for upgrade in _upgrades(observed)] == [[["model", TRANSCRIBE_MODEL]]] +MUSE_PUSH_TO_TALK_FRAMES: Final[tuple[dict[str, JsonValue], ...]] = ( + {"binary_bytes": MUSE_PACKET_BYTES}, + {"binary_bytes": MUSE_PACKET_BYTES}, + {"binary_bytes": MUSE_REMAINDER_BYTES}, + END_STREAM, +) + + def _muse_handshake(observed: tuple[dict[str, JsonValue], ...]) -> dict[str, JsonValue]: upgrades: Final = _upgrades(observed) assert len(upgrades) == 1, upgrades @@ -767,13 +1028,19 @@ def _muse_session( transcript: str, query: str, turn_detection: JsonValue, + *, + until: str | None = None, ) -> tuple[Session, tuple[dict[str, JsonValue], ...]]: handle: Final = _scripted(scenario, _muse_scenario(transcript), control_url=tls_upstream) key: Final = scenario.key() model: Final = _muse_deployment(scenario, handle.scenario_id, tls_upstream) - until: Final = TRANSCRIPT_COMPLETED if BLOCKED_WORD not in transcript else "error" + verdict: Final = TRANSCRIPT_COMPLETED if BLOCKED_WORD not in transcript else "error" session: Final = _talk( - _ws_base(_owned_url(guardrail_proxy)), f"model={model}{query}", key, _muse_frames(turn_detection), until=until + _ws_base(_owned_url(guardrail_proxy)), + f"model={model}{query}", + key, + _muse_frames(turn_detection), + until=verdict if until is None else until, ) return session, _observed(tls_upstream, handle.scenario_id) @@ -789,12 +1056,7 @@ def test_muse_push_to_talk_transcription_session_keeps_push_to_talk( assert session.transcripts == (CLEAN_TRANSCRIPT,), session assert session.errors == (), session assert _muse_handshake(observed)["mode"] == PUSH_TO_TALK, _muse_handshake(observed) - assert _muse_audio_frames(observed) == ( - {"binary_bytes": MUSE_PACKET_BYTES}, - {"binary_bytes": MUSE_PACKET_BYTES}, - {"binary_bytes": MUSE_REMAINDER_BYTES}, - END_STREAM, - ), _muse_audio_frames(observed) + assert _muse_audio_frames(observed) == MUSE_PUSH_TO_TALK_FRAMES, _muse_audio_frames(observed) def test_muse_transcription_session_blocked_transcript_reaches_the_client_as_a_violation( @@ -807,7 +1069,27 @@ def test_muse_transcription_session_blocked_transcript_reaches_the_client_as_a_v assert session.transcripts == (BLOCKED_TRANSCRIPT,), session assert session.errors == (GUARDRAIL_VIOLATION,), session assert _muse_handshake(observed)["mode"] == PUSH_TO_TALK, _muse_handshake(observed) - assert _muse_audio_frames(observed)[-1] == END_STREAM, _muse_audio_frames(observed) + assert _muse_audio_frames(observed) == MUSE_PUSH_TO_TALK_FRAMES, _muse_audio_frames(observed) + + +def test_muse_transcription_session_on_violation_end_session_closes_the_session( + guardrail_proxy: OwnedProxy, tls_upstream: str +) -> None: + blocked: Final = _blocked(ENDER_WORD) + with guardrail_proxy.gateway.scenario() as scenario: + session, observed = _muse_session( + guardrail_proxy, + scenario, + tls_upstream, + blocked, + f"&{TRANSCRIPTION_QUERY}&{_optin(ENDER_GUARDRAIL)}", + None, + until="closed", + ) + assert session.transcripts == (blocked,), session + _assert_ended_by_the_guardrail(session, violations=1) + assert session.error_messages[0] == ENDER_MESSAGE, session + assert _muse_audio_frames(observed) == MUSE_PUSH_TO_TALK_FRAMES, _muse_audio_frames(observed) def test_muse_server_vad_transcription_session_keeps_endpointing( @@ -949,10 +1231,10 @@ def test_opt_in_guardrail_leaves_a_transcription_session_alone_on_an_opted_out_k _assert_transcription_left_alone(session, observed, CLEAN_TRANSCRIPT, update=GA_TRANSCRIPTION_UPDATE) -def test_guardrails_list_names_the_three_configured_guardrails(guardrail_proxy: OwnedProxy) -> None: +def test_guardrails_list_names_every_configured_guardrail(guardrail_proxy: OwnedProxy) -> None: listed: Final = guardrail_proxy.gateway.get("/guardrails/list") names: Final = {string_value(object_value(entry)["guardrail_name"]) for entry in _list(listed["guardrails"])} - assert names == {TRANSCRIPT_GUARDRAIL, OPTIN_GUARDRAIL, PROMPT_GUARDRAIL}, listed + assert names == CONFIGURED_GUARDRAILS, listed def _list(value: JsonValue) -> list[JsonValue]: @@ -1089,6 +1371,583 @@ def test_repeated_transcription_sessions_write_one_spend_row_each(guardrail_prox assert {str(row["call_type"]) for row in rows} == {"_arealtime"}, rows +@pytest.mark.parametrize("path", ["/v1/realtime", "/realtime", "/openai/v1/realtime"]) +def test_blocked_transcript_on_a_transcription_session_reports_a_violation_and_sends_nothing_upstream( + guardrail_proxy: OwnedProxy, path: str +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(BLOCKED_TRANSCRIPT)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + session: Final = _transcribe_blocked( + _ws_base(_owned_url(guardrail_proxy)), f"model={model}&{TRANSCRIPTION_QUERY}", key, path=path + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + _assert_transcription_blocked(session, observed, BLOCKED_TRANSCRIPT, update=GA_TRANSCRIPTION_UPDATE) + assert [_query(upgrade) for upgrade in _upgrades(observed)] == [ + [["model", TRANSCRIBE_MODEL], ["intent", TRANSCRIPTION]] + ] + rows: Final = _spend_rows(key, 1) + assert rows[0]["call_type"] == "_arealtime", rows + + +async def _sdk_async_blocked_transcription(proxy_url: str, key: str, model: str) -> Session: + client: Final = AsyncOpenAI(api_key=key, base_url=f"{proxy_url}/v1", websocket_base_url=f"{_ws_base(proxy_url)}/v1") + async with client.realtime.connect(model=model, extra_query={"intent": TRANSCRIPTION}) as connection: + await connection.session.update(session=SDK_TRANSCRIPTION_SESSION) + await connection.input_audio_buffer.commit() + verdict: Final = tuple([event async for event in _sdk_async_events(connection, until="error")]) + await connection.session.update(session=SDK_TRANSCRIPTION_SESSION) + probe: Final = tuple([event async for event in _sdk_async_events(connection, until="session.updated")]) + return Session((*verdict, *probe), None) + + +def _sdk_sync_blocked_transcription(proxy_url: str, key: str, model: str) -> Session: + client: Final = OpenAI(api_key=key, base_url=f"{proxy_url}/v1", websocket_base_url=f"{_ws_base(proxy_url)}/v1") + with client.realtime.connect(model=model, extra_query={"intent": TRANSCRIPTION}) as connection: + connection.session.update(session=SDK_TRANSCRIPTION_SESSION) + connection.input_audio_buffer.commit() + verdict: Final = tuple(_sdk_sync_events(connection, until="error")) + connection.session.update(session=SDK_TRANSCRIPTION_SESSION) + probe: Final = tuple(_sdk_sync_events(connection, until="session.updated")) + return Session((*verdict, *probe), None) + + +@pytest.mark.parametrize("client", ["async", pytest.param("sync", marks=pytest.mark.timeout(90))]) +def test_openai_sdk_transcription_session_gets_the_violation_and_stays_open( + guardrail_proxy: OwnedProxy, client: str +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(BLOCKED_TRANSCRIPT)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + proxy_url: Final = _owned_url(guardrail_proxy) + session: Final = ( + asyncio.run(asyncio.wait_for(_sdk_async_blocked_transcription(proxy_url, key, model), SDK_SECONDS)) + if client == "async" + else _sdk_sync_blocked_transcription(proxy_url, key, model) + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + _assert_transcription_blocked(session, observed, BLOCKED_TRANSCRIPT, update=GA_TRANSCRIPTION_UPDATE) + + +def test_beta_protocol_transcription_session_blocked_transcript_reports_a_violation( + guardrail_proxy: OwnedProxy, +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(BLOCKED_TRANSCRIPT)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + session: Final = _transcribe_blocked( + _ws_base(_owned_url(guardrail_proxy)), + f"model={model}&{TRANSCRIPTION_QUERY}", + key, + update=BETA_TRANSCRIPTION_UPDATE, + headers=BETA_HEADERS, + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + _assert_transcription_blocked(session, observed, BLOCKED_TRANSCRIPT, update=BETA_TRANSCRIPTION_UPDATE) + + +def test_intent_without_model_blocked_transcript_reports_a_violation_on_the_whisper_default( + guardrail_proxy: OwnedProxy, +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(BLOCKED_TRANSCRIPT)) + key: Final = scenario.key() + _named_deployment( + guardrail_proxy.gateway, scenario, WHISPER_DEFAULT, f"openai/{WHISPER_DEFAULT}", handle.scenario_id + ) + session: Final = _transcribe_blocked(_ws_base(_owned_url(guardrail_proxy)), TRANSCRIPTION_QUERY, key) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + forwarded: Final = _with_transcription_model(GA_TRANSCRIPTION_UPDATE, WHISPER_DEFAULT) + _assert_transcription_blocked(session, observed, BLOCKED_TRANSCRIPT, update=forwarded) + assert [_query(upgrade) for upgrade in _upgrades(observed)] == [[["intent", TRANSCRIPTION]]] + + +def test_azure_transcription_session_blocked_transcript_reports_a_violation_and_sends_nothing_upstream( + guardrail_proxy: OwnedProxy, tls_upstream: str +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(BLOCKED_TRANSCRIPT), control_url=tls_upstream) + key: Final = scenario.key() + model: Final = _azure_deployment(scenario, handle.scenario_id, tls_upstream) + session: Final = _transcribe_blocked( + _ws_base(_owned_url(guardrail_proxy)), f"model={model}&{TRANSCRIPTION_QUERY}", key + ) + observed: Final = _observed(tls_upstream, handle.scenario_id) + _assert_transcription_blocked(session, observed, BLOCKED_TRANSCRIPT, update=GA_TRANSCRIPTION_UPDATE) + upgrades: Final = tuple(request for request in observed if request["method"] == "WEBSOCKET") + assert [upgrade["path"] for upgrade in upgrades] == ["/openai/v1/realtime"], upgrades + assert [_query(object_value(upgrade["body"])) for upgrade in upgrades] == [[["intent", TRANSCRIPTION]]], ( + upgrades + ) + + +def test_second_violation_under_end_session_after_n_fails_closes_the_transcription_session( + guardrail_proxy: OwnedProxy, +) -> None: + blocked: Final = _blocked(TWO_STRIKES_WORD) + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(blocked, blocked)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + steps: Final = (_step(TRANSCRIPTION_UPDATE_FRAME, COMMIT, until="error"), _step(COMMIT, until="closed")) + session: Final = _drive( + _ws_base(_owned_url(guardrail_proxy)), + f"model={model}&{TRANSCRIPTION_QUERY}&{_optin(TWO_STRIKES_GUARDRAIL)}", + key, + steps, + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + assert session.types == ( + "session.created", + "session.updated", + TRANSCRIPT_COMPLETED, + "error", + TRANSCRIPT_COMPLETED, + "error", + "closed", + ), session + assert session.transcripts == (blocked, blocked), session + _assert_ended_by_the_guardrail(session, violations=2) + assert _sent_types(observed) == ( + "session.update", + "input_audio_buffer.commit", + "input_audio_buffer.commit", + ), _sent(observed) + rows: Final = _spend_rows(key, 1) + assert rows[0]["call_type"] == "_arealtime", rows + + +def test_on_violation_end_session_closes_the_transcription_session_with_the_configured_message( + guardrail_proxy: OwnedProxy, +) -> None: + blocked: Final = _blocked(ENDER_WORD) + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(blocked)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + session: Final = _drive( + _ws_base(_owned_url(guardrail_proxy)), + f"model={model}&{TRANSCRIPTION_QUERY}&{_optin(ENDER_GUARDRAIL)}", + key, + (_step(TRANSCRIPTION_UPDATE_FRAME, COMMIT, until="closed"),), + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + assert session.types == ("session.created", "session.updated", TRANSCRIPT_COMPLETED, "error", "closed"), session + assert session.transcripts == (blocked,), session + _assert_ended_by_the_guardrail(session, violations=1) + assert session.error_messages[0] == ENDER_MESSAGE, session + assert _sent_types(observed) == ("session.update", "input_audio_buffer.commit"), _sent(observed) + + +def test_end_session_after_one_fail_closes_the_transcription_session_on_the_first_violation( + guardrail_proxy: OwnedProxy, +) -> None: + blocked: Final = _blocked(ONE_STRIKE_WORD) + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(blocked)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + session: Final = _drive( + _ws_base(_owned_url(guardrail_proxy)), + f"model={model}&{TRANSCRIPTION_QUERY}&{_optin(ONE_STRIKE_GUARDRAIL)}", + key, + (_step(TRANSCRIPTION_UPDATE_FRAME, COMMIT, until="closed"),), + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + assert session.types == ("session.created", "session.updated", TRANSCRIPT_COMPLETED, "error", "closed"), session + _assert_ended_by_the_guardrail(session, violations=1) + assert _sent_types(observed) == ("session.update", "input_audio_buffer.commit"), _sent(observed) + + +def _twice_blocked_and_open(guardrail_proxy: OwnedProxy, scenario: Scenario, blocked: str, query: str) -> None: + handle: Final = _scripted(scenario, _transcription_scenario(blocked, blocked)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + steps: Final = ( + _step(TRANSCRIPTION_UPDATE_FRAME, COMMIT, until="error"), + _step(COMMIT, until="error"), + _probe(), + ) + session: Final = _drive( + _ws_base(_owned_url(guardrail_proxy)), f"model={model}&{TRANSCRIPTION_QUERY}{query}", key, steps + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + assert session.types == ( + "session.created", + "session.updated", + TRANSCRIPT_COMPLETED, + "error", + TRANSCRIPT_COMPLETED, + "error", + "session.updated", + ), session + assert session.transcripts == (blocked, blocked), session + assert session.errors == (GUARDRAIL_VIOLATION, GUARDRAIL_VIOLATION), session + assert _sent_types(observed) == ( + "session.update", + "input_audio_buffer.commit", + "input_audio_buffer.commit", + "session.update", + ), _sent(observed) + rows: Final = _spend_rows(key, 1) + assert rows[0]["call_type"] == "_arealtime", rows + + +def test_same_blocked_transcript_twice_reports_two_violations_and_keeps_the_session_open( + guardrail_proxy: OwnedProxy, +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + _twice_blocked_and_open(guardrail_proxy, scenario, BLOCKED_TRANSCRIPT, "") + + +def test_on_violation_warn_with_a_null_end_rule_reports_each_violation_and_keeps_the_session_open( + guardrail_proxy: OwnedProxy, +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + _twice_blocked_and_open(guardrail_proxy, scenario, _blocked(WARN_WORD), f"&{_optin(WARN_GUARDRAIL)}") + + +def test_first_configured_guardrail_wins_when_two_match_one_transcript(guardrail_proxy: OwnedProxy) -> None: + transcript: Final = f"please add {BLOCKED_WORD} and {ENDER_WORD} to the order" + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(transcript)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + session: Final = _transcribe_blocked( + _ws_base(_owned_url(guardrail_proxy)), f"model={model}&{TRANSCRIPTION_QUERY}&{_optin(ENDER_GUARDRAIL)}", key + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + _assert_transcription_blocked(session, observed, transcript, update=GA_TRANSCRIPTION_UPDATE) + assert ENDER_MESSAGE not in session.error_messages, session + + +def test_guardrail_raising_value_error_reports_the_exception_text_and_keeps_the_transcription_session_open( + guardrail_proxy: OwnedProxy, +) -> None: + blocked: Final = _blocked(VALUE_ERROR_WORD) + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(blocked)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + session: Final = _transcribe_blocked( + _ws_base(_owned_url(guardrail_proxy)), + f"model={model}&{TRANSCRIPTION_QUERY}&{_optin(VALUE_ERROR_GUARDRAIL)}", + key, + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + _assert_transcription_blocked(session, observed, blocked, update=GA_TRANSCRIPTION_UPDATE) + assert session.error_messages == (f"{VALUE_ERROR_WORD} is not allowed",), session + + +def test_guardrail_raising_runtime_error_closes_the_transcription_session_with_a_proxy_failure( + guardrail_proxy: OwnedProxy, +) -> None: + blocked: Final = _blocked(RUNTIME_ERROR_WORD) + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(blocked)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + session: Final = _drive( + _ws_base(_owned_url(guardrail_proxy)), + f"model={model}&{TRANSCRIPTION_QUERY}&{_optin(RUNTIME_ERROR_GUARDRAIL)}", + key, + (_step(TRANSCRIPTION_UPDATE_FRAME, COMMIT, until="closed"),), + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + assert session.types == ("session.created", "session.updated", TRANSCRIPT_COMPLETED, "error", "closed"), session + assert session.transcripts == (blocked,), session + _assert_closed_by_a_proxy_failure(session) + assert _sent_types(observed) == ("session.update", "input_audio_buffer.commit"), _sent(observed) + rows: Final = _spend_rows(key, 1) + assert rows[0]["call_type"] == "_arealtime", rows + + +@pytest.mark.parametrize("guardrail", [VALUE_ERROR_GUARDRAIL, RUNTIME_ERROR_GUARDRAIL]) +def test_raising_guardrails_leave_a_clean_transcription_session_alone( + guardrail_proxy: OwnedProxy, guardrail: str +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(CLEAN_TRANSCRIPT)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + session: Final = _transcribe( + _ws_base(_owned_url(guardrail_proxy)), f"model={model}&{TRANSCRIPTION_QUERY}&{_optin(guardrail)}", key + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + _assert_transcription_left_alone(session, observed, CLEAN_TRANSCRIPT, update=GA_TRANSCRIPTION_UPDATE) + + +def _voice_session( + guardrail_proxy: OwnedProxy, scenario: Scenario, response: RealtimeResponse, query: str, steps: tuple[Step, ...] +) -> tuple[Session, tuple[dict[str, JsonValue], ...]]: + handle: Final = _scripted(scenario, response) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id, model=VOICE_MODEL) + session: Final = _drive(_ws_base(_owned_url(guardrail_proxy)), f"model={model}{query}", key, steps) + return session, _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + + +def test_voice_session_guardrail_raising_runtime_error_closes_with_the_same_proxy_failure( + guardrail_proxy: OwnedProxy, +) -> None: + blocked: Final = _blocked(RUNTIME_ERROR_WORD) + with guardrail_proxy.gateway.scenario() as scenario: + session, observed = _voice_session( + guardrail_proxy, + scenario, + _voice_scenario(blocked), + f"&{_optin(RUNTIME_ERROR_GUARDRAIL)}", + (_step(VOICE_UPDATE_FRAME, COMMIT, until="closed"),), + ) + assert session.types == ( + "session.created", + "session.updated", + "error", + TRANSCRIPT_COMPLETED, + "error", + "closed", + ), session + assert session.errors[0] == MISSING_TURN_DETECTION_TYPE, session + _assert_closed_by_a_proxy_failure(session) + assert _sent_types(observed) == ("session.update", "session.update", "input_audio_buffer.commit"), _sent( + observed + ) + + +def test_voice_session_guardrail_raising_value_error_is_voiced_through_the_backend(guardrail_proxy: OwnedProxy) -> None: + blocked: Final = _blocked(VALUE_ERROR_WORD) + with guardrail_proxy.gateway.scenario() as scenario: + session, observed = _voice_session( + guardrail_proxy, + scenario, + _voice_scenario(blocked), + f"&{_optin(VALUE_ERROR_GUARDRAIL)}", + (_step(VOICE_UPDATE_FRAME, COMMIT, until=RESPONSE_DONE),), + ) + assert session.transcripts == (blocked,), session + assert session.errors == (MISSING_TURN_DETECTION_TYPE, GUARDRAIL_VIOLATION), session + assert session.error_messages[1] == f"{VALUE_ERROR_WORD} is not allowed", session + assert session.types[-1] == RESPONSE_DONE, session + assert _sent_types(observed) == ( + "session.update", + "session.update", + "input_audio_buffer.commit", + *VOICE_BLOCK_FRAMES, + ), _sent(observed) + + +def test_voice_session_second_violation_under_end_session_after_n_fails_closes_after_voicing_both( + guardrail_proxy: OwnedProxy, +) -> None: + blocked: Final = _blocked(TWO_STRIKES_WORD) + with guardrail_proxy.gateway.scenario() as scenario: + session, observed = _voice_session( + guardrail_proxy, + scenario, + _voice_scenario(blocked, blocked), + f"&{_optin(TWO_STRIKES_GUARDRAIL)}", + (_step(VOICE_UPDATE_FRAME, COMMIT, until=RESPONSE_DONE), _step(COMMIT, until="closed")), + ) + assert session.transcripts == (blocked, blocked), session + assert session.errors == (MISSING_TURN_DETECTION_TYPE, GUARDRAIL_VIOLATION, GUARDRAIL_VIOLATION), session + assert session.types[-1] == "closed", session + assert session.close_code == GUARDRAIL_END_CLOSE, session + assert _sent_types(observed) == ( + "session.update", + "session.update", + "input_audio_buffer.commit", + *VOICE_BLOCK_FRAMES, + "input_audio_buffer.commit", + *VOICE_BLOCK_FRAMES, + ), _sent(observed) + + +def _user_text_item(text: str) -> dict[str, JsonValue]: + return { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]}, + } + + +def _tool_output_item(output: str) -> dict[str, JsonValue]: + return { + "type": "conversation.item.create", + "item": {"type": "function_call_output", "call_id": "call_realtime_guard", "output": output}, + } + + +def test_blocked_user_text_on_a_transcription_session_is_dropped_without_a_voiced_block( + guardrail_proxy: OwnedProxy, +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(CLEAN_TRANSCRIPT)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + steps: Final = ( + _probe(), + _step(_user_text_item(BLOCKED_TRANSCRIPT), RESPONSE_CREATE, until="error"), + _probe(), + ) + session: Final = _drive( + _ws_base(_owned_url(guardrail_proxy)), f"model={model}&{TRANSCRIPTION_QUERY}", key, steps + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + assert session.types == ("session.created", "session.updated", "error", "session.updated"), session + assert session.errors == (GUARDRAIL_VIOLATION,), session + assert _sent_types(observed) == ("session.update", "session.update"), _sent(observed) + + +def test_blocked_tool_output_on_a_transcription_session_is_sanitized_without_a_voiced_block( + guardrail_proxy: OwnedProxy, +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(CLEAN_TRANSCRIPT)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + steps: Final = ( + _probe(), + _step(_tool_output_item(BLOCKED_TRANSCRIPT), until="error"), + _probe(), + ) + session: Final = _drive( + _ws_base(_owned_url(guardrail_proxy)), f"model={model}&{TRANSCRIPTION_QUERY}", key, steps + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + assert session.types == ("session.created", "session.updated", "error", "session.updated"), session + assert session.errors == (GUARDRAIL_VIOLATION,), session + assert _sent_types(observed) == ("session.update", "conversation.item.create", "session.update"), _sent( + observed + ) + assert object_value(_sent(observed)[1]["item"])["output"] == TOOL_OUTPUT_BLOCKED, _sent(observed) + + +def test_clean_user_text_on_a_transcription_session_is_forwarded_verbatim(guardrail_proxy: OwnedProxy) -> None: + item: Final = _user_text_item(CLEAN_TRANSCRIPT) + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(CLEAN_TRANSCRIPT)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + steps: Final = (_step(TRANSCRIPTION_UPDATE_FRAME, item, RESPONSE_CREATE, until=TRANSCRIPT_COMPLETED),) + session: Final = _drive( + _ws_base(_owned_url(guardrail_proxy)), f"model={model}&{TRANSCRIPTION_QUERY}", key, steps + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + assert session.types == ("session.created", "session.updated", TRANSCRIPT_COMPLETED), session + assert session.errors == (), session + assert _sent(observed) == (TRANSCRIPTION_UPDATE_FRAME, item, RESPONSE_CREATE), _sent(observed) + + +CLEAN_TRANSCRIPT_SHAPES: Final = (pytest.param("", id="empty"), pytest.param(FIVE_KB, id="five_kilobytes")) +NON_STRING_TRANSCRIPTS: Final = ( + pytest.param(None, id="null"), + pytest.param(123, id="integer"), + pytest.param(["a"], id="list"), +) + + +@pytest.mark.parametrize("transcript", CLEAN_TRANSCRIPT_SHAPES) +def test_clean_transcript_field_shapes_are_relayed_and_the_session_stays_open( + guardrail_proxy: OwnedProxy, transcript: str +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(transcript)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + steps: Final = (_step(TRANSCRIPTION_UPDATE_FRAME, COMMIT, until=TRANSCRIPT_COMPLETED), _probe()) + session: Final = _drive( + _ws_base(_owned_url(guardrail_proxy)), f"model={model}&{TRANSCRIPTION_QUERY}", key, steps + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + assert session.types == ("session.created", "session.updated", TRANSCRIPT_COMPLETED, "session.updated"), session + assert session.transcripts == (transcript,), session + assert session.errors == (), session + assert _sent_types(observed) == ("session.update", "input_audio_buffer.commit", "session.update"), _sent( + observed + ) + + +def test_transcript_event_without_the_field_is_relayed_and_the_session_stays_open(guardrail_proxy: OwnedProxy) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_events((_transcript_event_without_the_field(),))) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + steps: Final = (_step(TRANSCRIPTION_UPDATE_FRAME, COMMIT, until=TRANSCRIPT_COMPLETED), _probe()) + session: Final = _drive( + _ws_base(_owned_url(guardrail_proxy)), f"model={model}&{TRANSCRIPTION_QUERY}", key, steps + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + assert session.types == ("session.created", "session.updated", TRANSCRIPT_COMPLETED, "session.updated"), session + assert "transcript" not in session.events[2], session + assert session.errors == (), session + assert _sent_types(observed) == ("session.update", "input_audio_buffer.commit", "session.update"), _sent( + observed + ) + + +def test_five_kilobyte_transcript_ending_in_the_blocked_word_reports_a_violation(guardrail_proxy: OwnedProxy) -> None: + transcript: Final = f"{FIVE_KB} {BLOCKED_TRANSCRIPT}" + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(transcript)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + session: Final = _transcribe_blocked( + _ws_base(_owned_url(guardrail_proxy)), f"model={model}&{TRANSCRIPTION_QUERY}", key + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + _assert_transcription_blocked(session, observed, transcript, update=GA_TRANSCRIPTION_UPDATE) + + +@pytest.mark.parametrize("transcript", NON_STRING_TRANSCRIPTS) +def test_non_string_transcript_field_closes_the_transcription_session_with_a_proxy_failure( + guardrail_proxy: OwnedProxy, transcript: JsonValue +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + handle: Final = _scripted(scenario, _transcription_scenario(transcript)) + key: Final = scenario.key() + model: Final = _openai_deployment(scenario, handle.scenario_id) + session: Final = _drive( + _ws_base(_owned_url(guardrail_proxy)), + f"model={model}&{TRANSCRIPTION_QUERY}", + key, + (_step(TRANSCRIPTION_UPDATE_FRAME, COMMIT, until="closed"),), + ) + observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id) + assert session.types == ("session.created", "session.updated", TRANSCRIPT_COMPLETED, "error", "closed"), session + assert session.events[2]["transcript"] == transcript, session + _assert_closed_by_a_proxy_failure(session) + assert _sent_types(observed) == ("session.update", "input_audio_buffer.commit"), _sent(observed) + + +@pytest.mark.parametrize("transcript", NON_STRING_TRANSCRIPTS) +def test_non_string_transcript_field_closes_a_voice_session_with_the_same_proxy_failure( + guardrail_proxy: OwnedProxy, transcript: JsonValue +) -> None: + with guardrail_proxy.gateway.scenario() as scenario: + session, observed = _voice_session( + guardrail_proxy, + scenario, + _voice_scenario(transcript), + "", + (_step(VOICE_UPDATE_FRAME, COMMIT, until="closed"),), + ) + assert session.types == ( + "session.created", + "session.updated", + "error", + TRANSCRIPT_COMPLETED, + "error", + "closed", + ), session + assert session.events[3]["transcript"] == transcript, session + _assert_closed_by_a_proxy_failure(session) + assert _sent_types(observed) == ("session.update", "session.update", "input_audio_buffer.commit"), _sent( + observed + ) + + async def _hold_until_closed(ws_base: str, query: str, key: str, opened: asyncio.Queue[str]) -> Session: async with websockets.connect( f"{ws_base}/v1/realtime?{query}", additional_headers={"Authorization": f"Bearer {key}"} @@ -1108,7 +1967,7 @@ async def _frames_until_closed(socket: ClientConnection) -> AsyncIterator[dict[s def _relays_the_upstream_close(session: Session) -> bool: - return f"upstream websocket closed with code {session.close_code}" in session.error_messages[0] + return _relayed_close(session) in session.error_messages[-1] async def _drain(opened: asyncio.Queue[str], count: int) -> tuple[str, ...]: @@ -1249,6 +2108,8 @@ def test_worker_kill_then_proxy_restart_keep_transcription_sessions_uninjected( config: Final = _write_config(tmp_path, ca_bundle) with gateway.scenario() as scenario: handle: Final = _scripted(scenario, _transcription_scenario(CLEAN_TRANSCRIPT)) + blocked_after_kill: Final = _scripted(scenario, _transcription_scenario(BLOCKED_TRANSCRIPT)) + blocked_after_restart: Final = _scripted(scenario, _transcription_scenario(BLOCKED_TRANSCRIPT)) key: Final = scenario.key() with owned_proxy_process(gateway, tmp_path, _overrides(ca_bundle), config=config, workers=WORKERS) as owned: owned_url: Final = _owned_url(owned) @@ -1259,6 +2120,20 @@ def test_worker_kill_then_proxy_restart_keep_transcription_sessions_uninjected( f"openai/{TRANSCRIBE_MODEL}", handle.scenario_id, ) + blocked_model_after_kill: Final = _named_deployment( + owned.gateway, + scenario, + f"realtime-guard-owned-blocked-{uuid.uuid4().hex[:8]}", + f"openai/{TRANSCRIBE_MODEL}", + blocked_after_kill.scenario_id, + ) + blocked_model_after_restart: Final = _named_deployment( + owned.gateway, + scenario, + f"realtime-guard-owned-blocked-{uuid.uuid4().hex[:8]}", + f"openai/{TRANSCRIBE_MODEL}", + blocked_after_restart.scenario_id, + ) root: Final = psutil.Process(owned.process.pid) outcome: Final = asyncio.run(_sessions_through_worker_kill(_ws_base(owned_url), model, key, root)) record_property( @@ -1284,6 +2159,15 @@ def test_worker_kill_then_proxy_restart_keep_transcription_sessions_uninjected( CLEAN_TRANSCRIPT, update=GA_TRANSCRIPTION_UPDATE, ) + blocked_on_the_survivor: Final = _transcribe_blocked( + _ws_base(owned_url), f"model={blocked_model_after_kill}&{TRANSCRIPTION_QUERY}", key + ) + _assert_transcription_blocked( + blocked_on_the_survivor, + _observed(gateway.upstream_url, blocked_after_kill.scenario_id), + BLOCKED_TRANSCRIPT, + update=GA_TRANSCRIPTION_UPDATE, + ) codes: Final = asyncio.run( _sessions_through_proxy_shutdown( _ws_base(owned_url), model, key, lambda: stop_root_process(owned.process) @@ -1299,3 +2183,106 @@ def test_worker_kill_then_proxy_restart_keep_transcription_sessions_uninjected( CLEAN_TRANSCRIPT, update=GA_TRANSCRIPTION_UPDATE, ) + blocked_on_the_restart: Final = _transcribe_blocked( + _ws_base(_owned_url(restarted)), f"model={blocked_model_after_restart}&{TRANSCRIPTION_QUERY}", key + ) + _assert_transcription_blocked( + blocked_on_the_restart, + _observed(gateway.upstream_url, blocked_after_restart.scenario_id), + BLOCKED_TRANSCRIPT, + update=GA_TRANSCRIPTION_UPDATE, + ) + + +async def _frames_reporting_verdicts( + socket: ClientConnection, session_id: str, verdicts: asyncio.Queue[str] +) -> AsyncIterator[dict[str, JsonValue]]: + async for frame in _frames_until_closed(socket): + if frame.get("type") == "error" and object_value(frame["error"]).get("type") == GUARDRAIL_VIOLATION[0]: + await verdicts.put(session_id) + yield frame + + +async def _hold_blocked_until_closed( + ws_base: str, query: str, key: str, opened: asyncio.Queue[str], verdicts: asyncio.Queue[str] +) -> Session: + async with websockets.connect( + f"{ws_base}/v1/realtime?{query}", additional_headers={"Authorization": f"Bearer {key}"} + ) as socket: + created: Final = JSON_OBJECT.validate_json(await socket.recv()) + assert created.get("type") == "session.created", created + session_id: Final = string_value(object_value(created["session"])["id"]) + await opened.put(session_id) + await socket.send(json.dumps(TRANSCRIPTION_UPDATE_FRAME)) + await socket.send(json.dumps(COMMIT)) + return Session(tuple([frame async for frame in _frames_reporting_verdicts(socket, session_id, verdicts)]), None) + + +@dataclass(frozen=True, slots=True) +class BlockedBurst: + sessions: tuple[Session, ...] + before_the_outage: tuple[dict[str, JsonValue], ...] + + +async def _blocked_burst_through_outage( + ws_base: str, + proxy_url: str, + upstream_url: str, + scenario_id: str, + model: str, + key: str, + stop_upstream: Callable[[], None], +) -> BlockedBurst: + opened: Final[asyncio.Queue[str]] = asyncio.Queue() + verdicts: Final[asyncio.Queue[str]] = asyncio.Queue() + query: Final = f"model={model}&{TRANSCRIPTION_QUERY}" + holders: Final = tuple( + asyncio.ensure_future(_hold_blocked_until_closed(ws_base, query, key, opened, verdicts)) for _ in range(BURST) + ) + opened_sessions: Final = await asyncio.wait_for(_drain(opened, BURST), 60) + assert len(opened_sessions) == BURST, opened_sessions + judged_sessions: Final = await asyncio.wait_for(_drain(verdicts, BURST), 60) + assert sorted(judged_sessions) == sorted(opened_sessions), judged_sessions + before_the_outage: Final = await asyncio.to_thread(_observed, upstream_url, scenario_id) + await asyncio.to_thread(stop_upstream) + async with httpx.AsyncClient(base_url=proxy_url, timeout=15, trust_env=False) as client: + liveliness: Final = await client.get("/health/liveliness") + assert liveliness.status_code == 200, liveliness.text + return BlockedBurst(tuple(await asyncio.wait_for(asyncio.gather(*holders), 90)), before_the_outage) + + +@pytest.mark.timeout(240) +def test_upstream_outage_closes_every_blocked_transcription_session_and_the_verdict_survives_the_restart( + guardrail_proxy: OwnedProxy, tmp_path: Path, record_property: RecordProperty +) -> None: + with guardrail_proxy.gateway.scenario() as scenario, owned_upstream(tmp_path) as slot: + scenario_id: Final = f"realtime-guard-blocked-outage-{uuid.uuid4().hex[:12]}" + register_scenario(scenario_id, _transcription_scenario(BLOCKED_TRANSCRIPT), control_url=slot.url) + key: Final = scenario.key() + model: Final = scenario.model(model=f"openai/{TRANSCRIBE_MODEL}", api_key=scenario_id, api_base=slot.url) + proxy_url: Final = _owned_url(guardrail_proxy) + burst: Final = asyncio.run( + _blocked_burst_through_outage(_ws_base(proxy_url), proxy_url, slot.url, scenario_id, model, key, slot.stop) + ) + record_property( + "close_codes_during_upstream_outage", sorted(session.close_code or 0 for session in burst.sessions) + ) + assert [session.types for session in burst.sessions] == [ + ("session.updated", TRANSCRIPT_COMPLETED, "error", "error", "closed") + ] * BURST, burst.sessions + assert [session.errors for session in burst.sessions] == [(GUARDRAIL_VIOLATION, RELAYED_CLOSE)] * BURST, ( + burst.sessions + ) + assert all(_relays_the_upstream_close(session) for session in burst.sessions), burst.sessions + assert len({session.close_code for session in burst.sessions}) == 1, burst.sessions + assert sorted(map(str, _sent_types(burst.before_the_outage))) == sorted( + ("session.update", "input_audio_buffer.commit") * BURST + ), _sent(burst.before_the_outage) + slot.start() + register_scenario(scenario_id, _transcription_scenario(BLOCKED_TRANSCRIPT), control_url=slot.url) + recovered: Final = _transcribe_blocked(_ws_base(proxy_url), f"model={model}&{TRANSCRIPTION_QUERY}", key) + _assert_transcription_blocked( + recovered, _observed(slot.url, scenario_id), BLOCKED_TRANSCRIPT, update=GA_TRANSCRIPTION_UPDATE + ) + rows: Final = _spend_rows(key, BURST + 1) + assert {str(row["call_type"]) for row in rows} == {"_arealtime"}, rows diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 190a7adb6e5..9f9f0d340d4 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -1,11 +1,12 @@ import asyncio import json -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from dataclasses import dataclass -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import pytest +from typing_extensions import ReadOnly, TypedDict from websockets.exceptions import ConnectionClosed from websockets.frames import Close @@ -18,6 +19,7 @@ from litellm.litellm_core_utils.realtime_streaming import ( ) from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import GenericGuardrailAPIInputs def _make_transcript_event(text: str, item_id: str = "item_x") -> bytes: @@ -3708,3 +3710,112 @@ async def test_transcription_guardrail_still_disables_auto_response_on_realtime_ forwarded: Final = json.loads(backend_ws.send.await_args.args[0]) assert forwarded["session"]["audio"]["input"]["turn_detection"]["create_response"] is False, forwarded + + +class _ViolationSettings(TypedDict, total=False): + on_violation: ReadOnly[str] + end_session_after_n_fails: ReadOnly[int] + + +def _passthrough_transcription_config() -> MagicMock: + def transform_response( + message: str | bytes, + model: str, + logging_obj: object, + realtime_response_transform_input: object, + ) -> dict[str, object]: + return { + "response": json.loads(message), + "current_output_item_id": None, + "current_response_id": None, + "current_delta_chunks": None, + "current_conversation_id": None, + "current_item_chunks": None, + "current_delta_type": None, + "session_configuration_request": None, + } + + def transform_request(message: str, model: str, session_configuration_request: str | None = None) -> list[str]: + return [message] + + provider_config: Final = MagicMock() + provider_config.requires_session_configuration.return_value = False + provider_config.transform_realtime_response.side_effect = transform_response + provider_config.transform_realtime_request.side_effect = transform_request + return provider_config + + +@pytest.mark.asyncio +@pytest.mark.parametrize("uses_provider_config", [False, True]) +@pytest.mark.parametrize( + ("violation_settings", "expect_session_closed"), + [ + ({}, False), + ({"on_violation": "end_session"}, True), + ({"end_session_after_n_fails": 1}, True), + ], +) +async def test_transcription_session_guardrail_block_only_reports_violation( + monkeypatch: pytest.MonkeyPatch, + uses_provider_config: bool, + violation_settings: _ViolationSettings, + expect_session_closed: bool, +) -> None: + class BlockingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: Mapping[str, object], + input_type: Literal["request", "response"], + logging_obj: object | None = None, + ) -> GenericGuardrailAPIInputs: + if any("blocked" in text for text in inputs.get("texts", [])): + raise ValueError("blocked transcript") + return inputs + + monkeypatch.setattr( + litellm, + "callbacks", + [ + BlockingGuardrail( + guardrail_name="transcription-blocker", + event_hook=GuardrailEventHooks.realtime_input_transcription, + default_on=True, + **violation_settings, + ) + ], + ) + completed_type: Final = "conversation.item.input_audio_transcription.completed" + client_ws: Final = MagicMock() + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + blocked_event: Final = _make_transcript_event("a blocked transcript", item_id="item_1") + follow_up_events: Final = ( + () if expect_session_closed else (_make_transcript_event("a clean follow-up", item_id="item_2"),) + ) + backend_ws.recv = AsyncMock(side_effect=[blocked_event, *follow_up_events, ConnectionClosed(None, None)]) + backend_ws.send = AsyncMock() + backend_ws.close = AsyncMock() + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + MagicMock(), + provider_config=_passthrough_transcription_config() if uses_provider_config else None, + model="gpt-4o-transcribe", + force_transcription_model="gpt-4o-transcribe", + ) + + await streaming.backend_to_client_send_messages() + + sent_to_client: Final = [json.loads(call.args[0]) for call in client_ws.send_text.await_args_list] + expected_follow_up: Final = () if expect_session_closed else ((completed_type, "a clean follow-up"),) + assert [(event["type"], event.get("transcript")) for event in sent_to_client] == [ + (completed_type, "a blocked transcript"), + ("error", None), + *expected_follow_up, + ], sent_to_client + assert sent_to_client[1]["error"]["type"] == "guardrail_violation", sent_to_client + assert streaming._violation_count == 1 + sent_to_backend: Final = [call.args[0] for call in backend_ws.send.await_args_list] + assert sent_to_backend == [], sent_to_backend + assert backend_ws.close.await_count == (1 if expect_session_closed else 0) From 810a3106e1d518eaf9791591fc794330d85ea651 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:01:12 -0700 Subject: [PATCH 20/35] test(integration): hold the wire barrier until the test releases it or the wire closes (#45124) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/integration/_support/wire.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/integration/_support/wire.py b/tests/integration/_support/wire.py index 10bb7787cc9..d118da03daf 100644 --- a/tests/integration/_support/wire.py +++ b/tests/integration/_support/wire.py @@ -47,6 +47,13 @@ class Wire: return self.connected.qsize() +def _await_release(gate: threading.Event, closing: threading.Event) -> bool: + while not closing.is_set(): + if gate.wait(timeout=0.05): + return True + return gate.is_set() + + @contextmanager def wire_server( respond: Callable[[Request], Reply], @@ -60,6 +67,7 @@ def wire_server( errors: Final[SimpleQueue[Exception]] = SimpleQueue() disconnected: Final[SimpleQueue[str]] = SimpleQueue() connected: Final[SimpleQueue[str]] = SimpleQueue() + closing: Final = threading.Event() class Handler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" @@ -108,8 +116,12 @@ def wire_server( break self.wfile.write(b"%x\r\n%s\r\n" % (len(chunk), chunk)) self.wfile.flush() - if index == 0 and reply.gate_after_first is not None: - assert reply.gate_after_first.wait(timeout=5), "Stream barrier was never released" + if ( + index == 0 + and reply.gate_after_first is not None + and not _await_release(reply.gate_after_first, closing) + ): + break if reply.pause_between_chunks and index + 1 < len(reply.chunks): time.sleep(reply.pause_between_chunks) else: @@ -151,6 +163,7 @@ def wire_server( connected, ) finally: + closing.set() server.shutdown() thread.join(timeout=6) assert not thread.is_alive(), "Owned HTTP server survived cleanup" From f0031e9a7e4eb43475d8241f35cae1a65aafd223 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:05:10 +0000 Subject: [PATCH 21/35] fix(vertex-ai): correct claude-haiku-5-5 thinking and forced tool flags (#45116) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 12 ++++++------ model_prices_and_context_window.json | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 13503b28544..ef601bf522f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -81367,10 +81367,10 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "supports_forced_tool_use": false, - "thinking_always_on": true, + "supports_forced_tool_use": true, + "thinking_always_on": false, "prompt_cache_min_tokens": 512, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "vertex_ai/claude-haiku-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, @@ -81407,9 +81407,9 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "supports_forced_tool_use": false, - "thinking_always_on": true, + "supports_forced_tool_use": true, + "thinking_always_on": false, "prompt_cache_min_tokens": 512, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://platform.claude.com/docs/en/about-claude/pricing" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 13503b28544..ef601bf522f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -81367,10 +81367,10 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "supports_forced_tool_use": false, - "thinking_always_on": true, + "supports_forced_tool_use": true, + "thinking_always_on": false, "prompt_cache_min_tokens": 512, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "vertex_ai/claude-haiku-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, @@ -81407,9 +81407,9 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "supports_forced_tool_use": false, - "thinking_always_on": true, + "supports_forced_tool_use": true, + "thinking_always_on": false, "prompt_cache_min_tokens": 512, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://platform.claude.com/docs/en/about-claude/pricing" } } From aa3cd70c1833bdeccd044bd02b8e95a1265a7646 Mon Sep 17 00:00:00 2001 From: ANKITA SAHNI Date: Wed, 7 Oct 2026 21:10:12 +0200 Subject: [PATCH 22/35] feat(helm): allow custom labels, annotations, command and args on migrationJob (#42242) Co-authored-by: ryan-crabbe-berri --- .../templates/migrations-job.yaml | 18 +++++ .../tests/migrations-job_tests.yaml | 70 +++++++++++++++++++ helm/litellm-helm/values.yaml | 5 ++ 3 files changed, 93 insertions(+) diff --git a/helm/litellm-helm/templates/migrations-job.yaml b/helm/litellm-helm/templates/migrations-job.yaml index 5a873cbb965..b199ad2d8f3 100644 --- a/helm/litellm-helm/templates/migrations-job.yaml +++ b/helm/litellm-helm/templates/migrations-job.yaml @@ -6,6 +6,9 @@ metadata: name: {{ include "litellm.fullname" . }}-migrations labels: {{- include "litellm.labels" . | nindent 4 }} + {{- with .Values.migrationJob.jobLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} annotations: {{- if .Values.migrationJob.hooks.argocd.enabled }} argocd.argoproj.io/hook: PreSync @@ -17,6 +20,9 @@ metadata: helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "1" | quote }} {{- end }} checksum/config: {{ toYaml .Values | sha256sum }} + {{- with .Values.migrationJob.jobAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: template: metadata: @@ -25,6 +31,9 @@ spec: {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.migrationJob.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} annotations: {{- with .Values.migrationJob.annotations }} {{- toYaml . | nindent 8 }} @@ -47,7 +56,16 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.migrationJob.command }} + command: {{ toYaml .Values.migrationJob.command | nindent 12 }} + {{- else }} command: ["python", "litellm/proxy/prisma_migration.py"] + {{- end }} + {{- if .Values.migrationJob.args }} + args: {{ toYaml .Values.migrationJob.args | nindent 12 }} + {{- else if .Values.migrationJob.command }} + args: [] + {{- end }} workingDir: "/app" env: {{- if .Values.db.useExisting }} diff --git a/helm/litellm-helm/tests/migrations-job_tests.yaml b/helm/litellm-helm/tests/migrations-job_tests.yaml index dd4276ac60f..bd5dd457f95 100644 --- a/helm/litellm-helm/tests/migrations-job_tests.yaml +++ b/helm/litellm-helm/tests/migrations-job_tests.yaml @@ -360,3 +360,73 @@ tests: asserts: - notExists: path: spec.activeDeadlineSeconds + + - it: should set custom jobLabels and jobAnnotations on Job metadata + template: migrations-job.yaml + set: + migrationJob: + enabled: true + jobLabels: + environment: production + team: platform + jobAnnotations: + example.com/cost-center: "1234" + asserts: + - equal: + path: metadata.labels.environment + value: production + - equal: + path: metadata.labels.team + value: platform + - equal: + path: metadata.annotations['example.com/cost-center'] + value: "1234" + + - it: should set custom podLabels on Pod template + template: migrations-job.yaml + set: + migrationJob: + enabled: true + podLabels: + custom.io/pod-role: migration + asserts: + - equal: + path: spec.template.metadata.labels['custom.io/pod-role'] + value: migration + + - it: should override container command and args + template: migrations-job.yaml + set: + migrationJob: + enabled: true + command: + - sh + args: + - -c + - echo migrating + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: + - sh + - equal: + path: spec.template.spec.containers[0].args + value: + - -c + - echo migrating + + - it: should clear container args when only command is specified + template: migrations-job.yaml + set: + migrationJob: + enabled: true + command: + - sh + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: + - sh + - equal: + path: spec.template.spec.containers[0].args + value: [] diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 83dbb3c5aa0..42ed777e6bc 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -572,6 +572,11 @@ migrationJob: # In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default". serviceAccountName: "" annotations: {} + jobLabels: {} # Custom labels for the Job metadata + jobAnnotations: {} # Custom annotations for the Job metadata + podLabels: {} # Custom labels for the Job pod template + command: [] # Override container command (defaults to ["python", "litellm/proxy/prisma_migration.py"]) + args: [] # Override container args ttlSecondsAfterFinished: 120 resources: {} # Unset by default. This job runs the database migration and exits, so it does not From 40a9b959a6a12ba4355258b51366ec855a334489 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 7 Oct 2026 12:11:55 -0700 Subject: [PATCH 23/35] test(e2e): tag a2a, access_control, other, secret_manager and migrations tests with Subject metadata (#44965) * test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata * test(e2e): tag a2a, access_control, other, secret_manager and migrations tests with Subject metadata * docs(e2e): name every markerless harness test file that carries no Subject * test(e2e): keep the step discovery comprehensions to one for clause --- tests/e2e/a2a/a2a_client.py | 11 ++- tests/e2e/a2a/test_a2a_agent_e2e.py | 73 +++++++++++++++++- .../access_control/access_control_client.py | 10 +++ .../access_control/test_access_control_e2e.py | 43 +++++++++++ .../test_chat_auth_headers_e2e.py | 31 ++++++++ .../test_model_access_group_e2e.py | 30 ++++++++ tests/e2e/migrations/checks.py | 11 +++ tests/e2e/migrations/containers.py | 14 ++++ tests/e2e/migrations/database.py | 11 +++ tests/e2e/migrations/test_legacy.py | 16 ++++ tests/e2e/migrations/test_pooling.py | 6 ++ tests/e2e/migrations/test_recovery.py | 36 +++++++++ tests/e2e/migrations/test_rolling_upgrade.py | 11 +++ tests/e2e/migrations/test_shaped_database.py | 6 ++ tests/e2e/migrations/test_startup.py | 46 ++++++++++++ tests/e2e/migrations/test_upgrade.py | 16 ++++ tests/e2e/migrations/upgrade.py | 7 ++ tests/e2e/other/other_client.py | 12 +++ tests/e2e/other/owned_jwt_gateway.py | 4 + tests/e2e/other/test_health_lifecycle_e2e.py | 25 +++++++ tests/e2e/other/test_jwt_auth_e2e.py | 75 +++++++++++++++++++ tests/e2e/other/test_jwt_auto_register_e2e.py | 25 +++++++ tests/e2e/other/test_master_key_auth_e2e.py | 11 +++ tests/e2e/other/test_session_token_e2e.py | 16 ++++ .../secret_manager/secret_store_cyberark.py | 4 + .../secret_store_hashicorp_vault.py | 4 + .../secret_manager/test_secret_manager_e2e.py | 27 +++++++ 27 files changed, 578 insertions(+), 3 deletions(-) diff --git a/tests/e2e/a2a/a2a_client.py b/tests/e2e/a2a/a2a_client.py index 605dd8fb7e5..4017602c959 100644 --- a/tests/e2e/a2a/a2a_client.py +++ b/tests/e2e/a2a/a2a_client.py @@ -19,6 +19,7 @@ from pydantic import BaseModel, ConfigDict, Field from e2e_config import settle_propagation from e2e_http import NoBody, Result, Success, get_external, is_ok +from e2e_metadata import STEP_FRAMES, step from proxy_client import ProxyClient @@ -87,7 +88,7 @@ class A2ABridgeParams(BaseModel): custom_llm_provider: str model: str - api_key: str | None = None + api_key: str | None = Field(default=None, repr=False) class AgentRegisterBody(BaseModel): @@ -291,6 +292,7 @@ class A2AResponse(BaseModel): class A2AClient: proxy: ProxyClient + @step("Register the A2A agent {body.agent_name} through /v1/agents") def register_agent(self, body: AgentRegisterBody) -> Result[AgentResponse]: """Register an agent and, on success, wait until the data plane serves it. @@ -337,6 +339,7 @@ class A2AClient: ) time.sleep(self.proxy.poll_interval) + @step("Read the A2A agent back from /v1/agents/{{agent_id}}") def get_agent(self, agent_id: str) -> Result[AgentResponse]: return self.proxy.transport.get( f"/v1/agents/{agent_id}", @@ -345,6 +348,7 @@ class A2AClient: response_type=AgentResponse, ) + @step("Delete the A2A agent") def delete_agent(self, agent_id: str) -> None: result = self.proxy.transport.delete( f"/v1/agents/{agent_id}", @@ -353,8 +357,9 @@ class A2AClient: response_type=NoBody, ) if not is_ok(result): - warnings.warn(f"delete_agent({agent_id!r}) failed: {result}", stacklevel=2) + warnings.warn(f"delete_agent({agent_id!r}) failed: {result}", stacklevel=2 + STEP_FRAMES) + @step("Read the A2A agent's card from /a2a/{{agent_id}}/.well-known/agent-card.json with the given key") def agent_card(self, agent_id: str, key: str) -> Result[ServedAgentCard]: return self.proxy.transport.get( f"/a2a/{agent_id}/.well-known/agent-card.json", @@ -363,6 +368,7 @@ class A2AClient: response_type=ServedAgentCard, ) + @step("Send an A2A message to /a2a/{{agent_id}} with {body.params.message.parts}") def send_message(self, agent_id: str, key: str, body: A2AJsonRpcRequest) -> Result[A2AResponse]: return self.proxy.transport.post( f"/a2a/{agent_id}", @@ -376,6 +382,7 @@ def build_a2a_client(proxy: ProxyClient) -> A2AClient: return A2AClient(proxy=proxy) +@step("Fetch a published A2A agent card from its /.well-known endpoint") def fetch_agent_card(url: str, *, timeout: float = 20.0) -> Result[UpstreamAgentCard]: """Fetch a live A2A agent card from its /.well-known endpoint and parse it into the registration model, so a test can register a real published card verbatim rather diff --git a/tests/e2e/a2a/test_a2a_agent_e2e.py b/tests/e2e/a2a/test_a2a_agent_e2e.py index 8b89ce91806..c0427368a67 100644 --- a/tests/e2e/a2a/test_a2a_agent_e2e.py +++ b/tests/e2e/a2a/test_a2a_agent_e2e.py @@ -10,6 +10,8 @@ protocol version, and an unsupported version is refused at registration). from __future__ import annotations +from typing import Final + import pytest from a2a_client import ( @@ -31,6 +33,9 @@ from a2a_client import ( from e2e_config import unique_marker from e2e_http import Result, UnknownApiError, unwrap from lifecycle import ResourceManager +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta + +BRIDGE_MODEL: Final = "claude-haiku-4-5" # No api_key: litellm resolves ANTHROPIC_API_KEY from the proxy's own environment # for this provider, which is what the agent-owner flow relies on. Pinning @@ -42,7 +47,7 @@ from lifecycle import ResourceManager # omitted -> 200, "os.environ/..." -> 500 invalid x-api-key, literal key -> 200. BRIDGE = A2ABridgeParams( custom_llm_provider="anthropic", - model="claude-haiku-4-5", + model=BRIDGE_MODEL, ) MOVEHOME_AGENT_CARD_URL = "https://movehome.org/.well-known/agent.json" @@ -96,6 +101,12 @@ def _ask(text: str) -> A2AJsonRpcRequest: class TestA2AAgentLifecycle: @pytest.mark.covers("other.a2a.register.persists") + @meta( + Subject( + domain=Domain.AGENTS_API, + route=Route.A2A, + ) + ) def test_register_persists(self, client: A2AClient, resources: ResourceManager) -> None: agent = _register(client, resources, "0.3") fetched = unwrap(client.get_agent(agent.agent_id)) @@ -104,6 +115,15 @@ class TestA2AAgentLifecycle: assert fetched.agent_card_params.protocol_version == "0.3" @pytest.mark.covers("other.a2a.register.semver_version_accepted") + @meta( + Subject( + domain=Domain.AGENTS_API, + route=Route.A2A, + providers=(Provider.ANTHROPIC,), + models=(BRIDGE_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_semver_protocol_version_registers_and_serves(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: agent = _register(client, resources, "0.3.0") assert agent.agent_card_params.protocol_version == "0.3" @@ -116,6 +136,12 @@ class TestA2AAgentLifecycle: assert result.text != "" @pytest.mark.covers("other.a2a.message_send.real_world_agent_replies") + @meta( + Subject( + domain=Domain.AGENTS_API, + route=Route.A2A, + ) + ) def test_real_world_agent_replies_to_property_query(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: upstream = unwrap(fetch_agent_card(MOVEHOME_AGENT_CARD_URL)).model_copy(update={"url": MOVEHOME_ORIGIN}) assert upstream.protocol_version == "0.3.0" @@ -152,6 +178,12 @@ class TestA2AAgentLifecycle: assert all(listing.location.un_locode == location for listing in results.listings) @pytest.mark.covers("other.a2a.discovery.proxy_fronted_card") + @meta( + Subject( + domain=Domain.AGENTS_API, + route=Route.A2A, + ) + ) def test_discovery_card_is_proxy_fronted(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: agent = _register(client, resources, "0.3") card = unwrap(client.agent_card(agent.agent_id, scoped_key)) @@ -163,6 +195,15 @@ class TestA2AAgentLifecycle: assert card.supported_interfaces[0].url == card.url @pytest.mark.covers("other.a2a.message_send.bridge_invokes") + @meta( + Subject( + domain=Domain.AGENTS_API, + route=Route.A2A, + providers=(Provider.ANTHROPIC,), + models=(BRIDGE_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_message_send_runs_completion_bridge(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: agent = _register(client, resources, "0.3") request = _ask("Reply with exactly the word PONG and nothing else") @@ -177,6 +218,15 @@ class TestA2AAgentLifecycle: assert rows[0].model == f"a2a_agent/{agent.agent_card_params.name}" @pytest.mark.covers("other.a2a.version.serves_pinned_0_3") + @meta( + Subject( + domain=Domain.AGENTS_API, + route=Route.A2A, + providers=(Provider.ANTHROPIC,), + models=(BRIDGE_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_pinned_v0_3_serves_flat_message_shape(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: agent = _register(client, resources, "0.3") request = _ask("Say hi in one word") @@ -188,6 +238,15 @@ class TestA2AAgentLifecycle: assert result.text != "" @pytest.mark.covers("other.a2a.version.serves_pinned_1_0") + @meta( + Subject( + domain=Domain.AGENTS_API, + route=Route.A2A, + providers=(Provider.ANTHROPIC,), + models=(BRIDGE_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_pinned_v1_0_serves_nested_message_shape(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None: agent = _register(client, resources, "1.0") request = _ask("Say hi in one word") @@ -199,6 +258,12 @@ class TestA2AAgentLifecycle: assert result.text != "" @pytest.mark.covers("other.a2a.register.unsupported_version_rejected") + @meta( + Subject( + domain=Domain.AGENTS_API, + route=Route.A2A, + ) + ) def test_unsupported_protocol_version_rejected(self, client: A2AClient) -> None: result = _register_rejection(client, "9.9") match result: @@ -209,6 +274,12 @@ class TestA2AAgentLifecycle: pytest.fail(f"expected 400 for unsupported protocolVersion, got {result}") @pytest.mark.covers("other.a2a.register.malformed_version_rejected") + @meta( + Subject( + domain=Domain.AGENTS_API, + route=Route.A2A, + ) + ) def test_malformed_protocol_version_rejected(self, client: A2AClient) -> None: result = _register_rejection(client, "0.3.garbage") match result: diff --git a/tests/e2e/access_control/access_control_client.py b/tests/e2e/access_control/access_control_client.py index 5f459c09767..90ddf061bca 100644 --- a/tests/e2e/access_control/access_control_client.py +++ b/tests/e2e/access_control/access_control_client.py @@ -8,6 +8,7 @@ from dataclasses import dataclass from pydantic import BaseModel, ValidationError from proxy_client import ProxyClient +from e2e_metadata import step from e2e_http import NoBody, StreamingResponse, is_ok, unwrap from models import ( ChatBody, @@ -59,14 +60,17 @@ def error_envelope(body: str) -> ApiErrorEnvelope | None: class AccessControlClient: proxy: ProxyClient + @step("Generate a virtual key that can only call LLM API routes") def llm_only_key(self) -> str: return self.proxy.generate_key( KeyGenerateBody(models=[], allowed_routes=["llm_api_routes"]) ) + @step("Delete the virtual key") def delete_key(self, key: str) -> None: self.proxy.delete_key(key) + @step('Send a /chat/completions request to {model} with the prompt "{content}"') def chat_status( self, key: str, model: str, content: str, max_completion_tokens: int | None = None ) -> StreamingResponse: @@ -80,6 +84,7 @@ class AccessControlClient: ), ) + @step("Create the team {team_alias} with models: {models}") def create_team(self, team_alias: str, models: list[str]) -> str: team_id = unwrap( self.proxy.transport.post( @@ -92,6 +97,7 @@ class AccessControlClient: self._await_team(team_id) return team_id + @step("Set the team {team_alias}'s models to {models} through /team/update") def set_team_models(self, team_id: str, team_alias: str, models: list[str]) -> None: """Replace the team's allow-list. /model/new appends a team-scoped deployment's public name to it, so a test that means to grant only an access group has to @@ -105,6 +111,7 @@ class AccessControlClient: ) ) + @step("Delete the team") def delete_team(self, team_id: str) -> None: _ = self.proxy.transport.post( "/team/delete", @@ -113,6 +120,7 @@ class AccessControlClient: response_type=NoBody, ) + @step("List the deployments in the model access group {access_group}") def access_group_info(self, access_group: str) -> AccessGroupInfoResponse | None: result = self.proxy.transport.get( f"/access_group/{access_group}/info", @@ -122,6 +130,7 @@ class AccessControlClient: ) return unwrap(result) if is_ok(result) else None + @step("Read the team's models from /team/info") def team_models(self, team_id: str) -> list[str] | None: result = self.proxy.transport.get( "/team/info", @@ -139,6 +148,7 @@ class AccessControlClient: time.sleep(self.proxy.poll_interval) raise AssertionError(f"/team/info never resolved team {team_id!r} created by /team/new") + @step("Add a deployment named {model_name} that calls openai/gpt-4o-mini with the given key") def create_model_status(self, key: str, model_name: str) -> StreamingResponse: return self.proxy.transport.send( "/model/new", diff --git a/tests/e2e/access_control/test_access_control_e2e.py b/tests/e2e/access_control/test_access_control_e2e.py index 9d01f2915e7..419cae78df2 100644 --- a/tests/e2e/access_control/test_access_control_e2e.py +++ b/tests/e2e/access_control/test_access_control_e2e.py @@ -26,6 +26,7 @@ from e2e_http import Success, UnauthorizedError, UnknownApiError, unwrap from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatResponse, EmbedBody, LiteLLMParamsBody from proxy_client import ProxyClient +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta pytestmark = pytest.mark.e2e @@ -36,6 +37,14 @@ EMBEDDING_MODEL = "openai-text-embedding-3-small" class TestAccessControl: + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.GEMINI,), + models=(ALLOWED_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_allowed_model_is_permitted( self, client: AccessControlClient, resources: ResourceManager ) -> None: @@ -57,6 +66,13 @@ class TestAccessControl: f"200 must carry a real completion, not an error envelope: {result.body[:300]}" ) + @meta( + Subject( + domain=Domain.PROXY_AUTH, + models=(DISALLOWED_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_disallowed_model_is_denied_403( self, client: AccessControlClient, resources: ResourceManager ) -> None: @@ -73,6 +89,13 @@ class TestAccessControl: ) @pytest.mark.covers("other.auth.virtual_key.route_group_allowed") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.GEMINI, Provider.OPENAI,), + models=(ALLOWED_MODEL, EMBEDDING_MODEL,), + ) + ) def test_llm_api_routes_group_grants_every_llm_endpoint( self, client: AccessControlClient, resources: ResourceManager ) -> None: @@ -97,6 +120,12 @@ class TestAccessControl: f"the same key must still be shut out of /model/new, got {denied.status_code}: {denied.body[:300]}" ) + @meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.MODEL_MANAGEMENT, + ) + ) def test_llm_only_key_forbidden_from_management_route_403( self, client: AccessControlClient, resources: ResourceManager ) -> None: @@ -111,6 +140,12 @@ class TestAccessControl: f"403 body must be a route-permission denial, got: {result.body[:300]}" ) + @meta( + Subject( + domain=Domain.PROXY_AUTH, + mode=Mode.NONSTREAM, + ) + ) def test_unknown_model_returns_400( self, client: AccessControlClient, resources: ResourceManager ) -> None: @@ -140,6 +175,14 @@ class TestVirtualKeyAuth: "mgmt.virtual_key.invalid_denied", exercised_on=[], ) + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.ANTHROPIC,), + models=(VIRTUAL_KEY_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) def test_valid_key_allows_and_invalid_key_denied( self, proxy: ProxyClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/access_control/test_chat_auth_headers_e2e.py b/tests/e2e/access_control/test_chat_auth_headers_e2e.py index 197a54cc3a3..c0de14a0961 100644 --- a/tests/e2e/access_control/test_chat_auth_headers_e2e.py +++ b/tests/e2e/access_control/test_chat_auth_headers_e2e.py @@ -11,6 +11,7 @@ import pytest from e2e_http import AuthHeaders, NoBody, StreamingResponse, assert_auth_denied from models import ChatBody, ChatMessage from proxy_client import ProxyClient +from e2e_metadata import Domain, Route, Subject, meta pytestmark = pytest.mark.e2e @@ -32,26 +33,56 @@ def _chat_with_headers(proxy: ProxyClient, headers: AuthHeaders | NoBody) -> Str class TestChatAuthHeaders: @pytest.mark.covers("other.auth.llm_chat.missing_header_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.CHAT_COMPLETIONS, + ) + ) def test_missing_authorization_header_is_denied(self, proxy: ProxyClient) -> None: result = _chat_with_headers(proxy, NoBody()) assert_auth_denied(result, "missing Authorization") @pytest.mark.covers("other.auth.llm_chat.invalid_bearer_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.CHAT_COMPLETIONS, + ) + ) def test_bearer_invalid_token_is_denied(self, proxy: ProxyClient) -> None: result = _chat_with_headers(proxy, AuthHeaders(authorization="Bearer invalid_token")) assert_auth_denied(result, "Bearer invalid_token") @pytest.mark.covers("other.auth.llm_chat.no_bearer_prefix_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.CHAT_COMPLETIONS, + ) + ) def test_token_without_bearer_prefix_is_denied(self, proxy: ProxyClient) -> None: result = _chat_with_headers(proxy, AuthHeaders(authorization="invalid_token")) assert_auth_denied(result, "token without Bearer prefix") @pytest.mark.covers("other.auth.llm_chat.empty_bearer_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.CHAT_COMPLETIONS, + ) + ) def test_empty_bearer_token_is_denied(self, proxy: ProxyClient) -> None: result = _chat_with_headers(proxy, AuthHeaders(authorization="Bearer ")) assert_auth_denied(result, "empty Bearer token") @pytest.mark.covers("other.auth.llm_chat.not_bearer_scheme_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.CHAT_COMPLETIONS, + ) + ) def test_not_bearer_scheme_is_denied(self, proxy: ProxyClient) -> None: result = _chat_with_headers(proxy, AuthHeaders(authorization="NotBearer validtoken123")) assert_auth_denied(result, "NotBearer scheme") diff --git a/tests/e2e/access_control/test_model_access_group_e2e.py b/tests/e2e/access_control/test_model_access_group_e2e.py index 6dab51805fb..cfe2f061693 100644 --- a/tests/e2e/access_control/test_model_access_group_e2e.py +++ b/tests/e2e/access_control/test_model_access_group_e2e.py @@ -33,6 +33,7 @@ from models import ( ModelNewBody, TeamInfoResponse, ) +from e2e_metadata import Domain, Mode, Provider, Subject, meta pytestmark = pytest.mark.e2e @@ -177,6 +178,14 @@ class TestKeyScopedToAccessGroup: "other.auth.model_access_group.member_allowed", ) @pytest.mark.parametrize(("case", "select_model"), ALLOWED) + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.OPENAI,), + models=(GROUP_BACKEND, WILDCARD_BARE_MODEL, WILDCARD_PREFIXED_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_group_grants_every_deployment_in_it( self, case: str, @@ -202,6 +211,13 @@ class TestKeyScopedToAccessGroup: @pytest.mark.covers("other.auth.model_access_group.non_member_denied") @pytest.mark.parametrize(("case", "select_model"), DENIED) + @meta( + Subject( + domain=Domain.PROXY_AUTH, + models=(GROUP_BACKEND, UNCOVERED_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_group_grants_nothing_outside_it( self, case: str, @@ -228,6 +244,14 @@ class TestKeyScopedToAccessGroup: class TestTeamScopedToAccessGroup: @pytest.mark.covers("other.auth.model_access_group.team_wildcard_bare_name_allowed") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.OPENAI,), + models=(TEAM_WILDCARD_BARE_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_group_grants_the_teams_own_wildcard( self, client: AccessControlClient, team_grant: TeamGrant ) -> None: @@ -248,6 +272,12 @@ class TestTeamScopedToAccessGroup: ) @pytest.mark.covers("other.auth.model_access_group.team_non_member_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + mode=Mode.NONSTREAM, + ) + ) def test_group_grants_the_team_nothing_outside_it( self, client: AccessControlClient, team_grant: TeamGrant ) -> None: diff --git a/tests/e2e/migrations/checks.py b/tests/e2e/migrations/checks.py index 619ad3b0e6c..4d67dbaa20d 100644 --- a/tests/e2e/migrations/checks.py +++ b/tests/e2e/migrations/checks.py @@ -3,6 +3,7 @@ from contextlib import ExitStack from typing import Final from uuid import uuid4 +from e2e_metadata import step from psycopg import sql from .containers import Containers, Replica, failed, until @@ -21,12 +22,14 @@ GATED: Final = Migration( ) +@step("Start {count} proxy containers on the test database") def start_replicas( stack: ExitStack, containers: Containers, database: Database, migrations: tuple[Migration, ...] = (), count: int = 3 ) -> tuple[Replica, ...]: return tuple(stack.enter_context(containers.start(database, migrations)) for _ in range(count)) +@step("Check that the migration {migration.name} ran exactly once") def assert_completed(database: Database, migration: Migration = COMPLETE) -> None: assert database.query( 'SELECT finished_at IS NOT NULL, rolled_back_at IS NULL, applied_steps_count FROM ' @@ -36,6 +39,7 @@ def assert_completed(database: Database, migration: Migration = COMPLETE) -> Non assert database.query("SELECT id FROM migration_effect") == ((1,),) +@step("Apply the test migration by hand and record it in _prisma_migrations") def confirmed_history(database: Database) -> str: database.execute(COMPLETE_SQL) row_id: Final = str(uuid4()) @@ -46,6 +50,7 @@ def confirmed_history(database: Database) -> str: return row_id +@step("Check that the original _prisma_migrations row and its effect survived, with the row marked finished: {finished}") def assert_original_proof(database: Database, row_id: str, finished: bool) -> None: assert database.query( 'SELECT id, applied_steps_count, finished_at IS NOT NULL, rolled_back_at IS NULL FROM ' @@ -55,6 +60,7 @@ def assert_original_proof(database: Database, row_id: str, finished: bool) -> No assert database.query("SELECT id FROM migration_effect") == ((1,),) +@step("Install a trigger that pauses the migration before it is marked finished") def pause_completion(database: Database) -> None: database.execute( sql.SQL( @@ -67,6 +73,10 @@ def pause_completion(database: Database) -> None: ) +@step( + "Start a proxy container on the migration and kill it at its crash point, " + "with the migration SQL committed: {after_commit}" +) def interrupt_owner( containers: Containers, database: Database, after_commit: bool, *, stop_database_session: bool = True ) -> None: @@ -104,6 +114,7 @@ def interrupt_owner( ) +@step("Wait for every proxy container to refuse an unconfirmed migration and log recovery guidance") def unconfirmed(replicas: tuple[Replica, ...], database: Database) -> None: failed(replicas, "Migration completion could not be verified") started: Final = str( diff --git a/tests/e2e/migrations/containers.py b/tests/e2e/migrations/containers.py index 30374dedcf3..a05941373be 100644 --- a/tests/e2e/migrations/containers.py +++ b/tests/e2e/migrations/containers.py @@ -11,6 +11,7 @@ from typing import Final from uuid import uuid4 from e2e_http import NoBody, Success, unwrap +from e2e_metadata import step from models import KeyGenerateBody, KeyGenerateResponse, KeyInfoParams, KeyInfoResponse from transport import HttpTransport @@ -20,12 +21,14 @@ from .startup_models import ContainerState, Migration, Observation, Readiness MASTER_KEY: Final = "sk-migration-ci-fixture" +@step("Run a docker command") def docker(*args: str) -> str: result: Final = subprocess.run(("docker", *args), capture_output=True, text=True, timeout=90) assert result.returncode == 0, f"Docker operation failed: {result.stderr}" return result.stdout.strip() +@step("Wait for {description}") def until(description: str, condition: Callable[[], bool], seconds: float = 150) -> None: deadline: Final = time.monotonic() + seconds while time.monotonic() < deadline: @@ -41,9 +44,11 @@ class Replica: transport: HttpTransport output: Path + @step("Read the proxy container's state from docker inspect") def state(self) -> ContainerState: return ContainerState.model_validate_json(docker("inspect", "--format", "{{json .State}}", self.name)) + @step("Check whether the proxy container is running and ready on /health/readiness") def observe(self) -> Observation: state: Final = self.state() result: Final = self.transport.get( @@ -52,15 +57,18 @@ class Replica: ready: Final = isinstance(result, Success) and result.data.status == "healthy" and result.data.db == "connected" return Observation(None if state.Running else state.ExitCode, ready) + @step("Read the proxy container's logs") def logs(self) -> str: result: Final = subprocess.run(("docker", "logs", self.name), capture_output=True, text=True, timeout=30) assert result.returncode == 0, result.stderr return result.stdout + result.stderr + @step("Kill the proxy container") def kill(self) -> None: if self.state().Running: docker("kill", self.name) + @step("Generate a virtual key on the proxy container and read it back from /key/info and the database") def usable(self, database: Database) -> None: alias: Final = f"migration-{uuid4().hex}" key: Final = unwrap( @@ -86,6 +94,7 @@ class Replica: ) == ((alias,),) +@step("Wait for every proxy container to be ready, then generate and read back a virtual key on each") def ready(replicas: tuple[Replica, ...], database: Database) -> None: def all_ready() -> bool: observations: Final = tuple(replica.observe() for replica in replicas) @@ -97,6 +106,7 @@ def ready(replicas: tuple[Replica, ...], database: Database) -> None: replica.usable(database) +@step("Wait for the seed proxy container to be ready and finish building its request-log indexes") def seeded(seed: Replica, database: Database) -> None: ready((seed,), database) until("the seed replica to finish its request-log indexes", lambda: request_log_indexes_built(database)) @@ -110,6 +120,7 @@ def request_log_indexes_built(database: Database) -> bool: ) == ((2,),) +@step('Wait for every proxy container to refuse to start, logging "{marker}"') def failed(replicas: tuple[Replica, ...], marker: str) -> None: def all_stopped() -> bool: observations: Final = tuple(replica.observe() for replica in replicas) @@ -122,6 +133,7 @@ def failed(replicas: tuple[Replica, ...], marker: str) -> None: assert marker in replica.logs(), f"Startup failed outside the expected migration: {marker}" +@step("Check that every proxy container keeps waiting without serving for {seconds}s") def waiting(replicas: tuple[Replica, ...], seconds: float) -> None: deadline: Final = time.monotonic() + seconds while time.monotonic() < deadline: @@ -139,6 +151,7 @@ class Containers: def using(self, image: str) -> "Containers": return replace(self, image=image) + @step("Start a proxy container on the test database") @contextmanager def start( self, @@ -213,6 +226,7 @@ class Containers: subprocess.run(("docker", "rm", "-f", name), capture_output=True, text=True, timeout=30, check=True) +@step("Write the migration {migration.name} into the proxy container's migration directory") def write_migration(directory: Path, migration: Migration) -> None: path: Final = directory / "prisma" / "migrations" / migration.name path.mkdir(parents=True) diff --git a/tests/e2e/migrations/database.py b/tests/e2e/migrations/database.py index a370c21ba0b..929a6aee817 100644 --- a/tests/e2e/migrations/database.py +++ b/tests/e2e/migrations/database.py @@ -11,6 +11,8 @@ import psycopg from psycopg import sql from pydantic import TypeAdapter +from e2e_metadata import step + Scalar = str | int | bool | None ROWS: Final = TypeAdapter(tuple[tuple[Scalar, ...], ...]) GATE_KEY: Final = 39178002 @@ -35,6 +37,7 @@ class Database: container_url: str schema: str = "public" + @step("Open a connection to the test database") @contextmanager def connection(self) -> Generator[psycopg.Connection[tuple[object, ...]]]: with psycopg.connect(self.url, autocommit=True, connect_timeout=5) as connection: @@ -42,19 +45,23 @@ class Database: connection.execute("SET statement_timeout = '15s'") yield connection + @step("Run a SQL statement on the test database") def execute(self, statement: LiteralString | sql.Composed, params: tuple[Scalar, ...] = ()) -> None: with self.connection() as connection: connection.execute(statement, params or None) + @step("Query the test database") def query( self, statement: LiteralString | sql.Composed, params: tuple[Scalar, ...] = () ) -> tuple[tuple[Scalar, ...], ...]: with self.connection() as connection: return ROWS.validate_python(connection.execute(statement, params or None).fetchall()) + @step("Check whether {name} exists in the test database") def exists(self, name: str) -> bool: return self.query("SELECT to_regclass(%s) IS NOT NULL", (name,)) == ((True,),) + @step("Read the migration history from _prisma_migrations") def history(self) -> tuple[tuple[Scalar, ...], ...]: if not self.exists("_prisma_migrations"): return () @@ -63,6 +70,7 @@ class Database: "applied_steps_count, logs FROM _prisma_migrations ORDER BY id" ) + @step("List the database sessions waiting on an advisory lock") def blocked(self, key: int = GATE_KEY) -> tuple[tuple[Scalar, ...], ...]: return self.query( "SELECT pid FROM pg_locks WHERE locktype = 'advisory' AND NOT granted " @@ -71,6 +79,7 @@ class Database: (key >> 32, key & 0xFFFFFFFF), ) + @step("Hold an advisory lock on the test database") @contextmanager def lock(self, key: int = GATE_KEY) -> Generator[None]: with self.connection() as connection: @@ -86,6 +95,7 @@ class Databases: admin_url: str container_admin_url: str + @step("Create a test database") @contextmanager def create(self, template: Database | None = None, schema: str = "public") -> Generator[Database]: name: Final = f"litellm_migration_test_{uuid4().hex[:20]}" @@ -105,6 +115,7 @@ class Databases: connection.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(name))) +@step("Create a read-only database role on the test database") @contextmanager def restricted_user(database: Database) -> Generator[Database]: role: Final = f"migration_reader_{uuid4().hex[:16]}" diff --git a/tests/e2e/migrations/test_legacy.py b/tests/e2e/migrations/test_legacy.py index 4cad808d3ff..8d43fcd2deb 100644 --- a/tests/e2e/migrations/test_legacy.py +++ b/tests/e2e/migrations/test_legacy.py @@ -7,6 +7,7 @@ import pytest from .checks import COMPLETE, assert_completed, confirmed_history, assert_original_proof, start_replicas from .containers import Containers, failed, ready, seeded from .database import Database, Databases +from e2e_metadata import Domain, Subject, meta pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup] @@ -42,10 +43,20 @@ def adopt_legacy(containers: Containers, database: Database) -> None: class TestLegacyMigrations: + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_matching_schema_warns_and_starts(self, containers: Containers, database: Database) -> None: adopt_legacy(containers, database) @pytest.mark.parametrize("fault", ("schema_drift", "custom_migrations", "empty_ledger")) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_unrecognized_legacy_state_is_not_baselined( self, containers: Containers, database: Database, fault: str ) -> None: @@ -64,6 +75,11 @@ class TestLegacyMigrations: ) == ((0,),) @pytest.mark.parametrize("scenario", ("upgrade", "recovery", "legacy")) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_non_default_schema( self, containers: Containers, databases: Databases, scenario: Literal["upgrade", "recovery", "legacy"] ) -> None: diff --git a/tests/e2e/migrations/test_pooling.py b/tests/e2e/migrations/test_pooling.py index c4015549de3..6f6b8911f94 100644 --- a/tests/e2e/migrations/test_pooling.py +++ b/tests/e2e/migrations/test_pooling.py @@ -13,6 +13,7 @@ from psycopg import sql from .checks import COMPLETE, assert_completed from .containers import Containers, docker, ready, until from .database import Database, Databases, prisma_url, restricted_user +from e2e_metadata import Domain, Subject, meta POOL_IMAGE: Final = ( "ghcr.io/cloudnative-pg/pgbouncer@sha256:e6ddfe22d845e603825e235dd8334b21ecd125abea2a2172478f556b8dee2bb8" @@ -94,6 +95,11 @@ def pool(database: Database, output: Path) -> Generator[str]: class TestMigrationPooling: @pytest.mark.parametrize("scenario,replica_count", (("fresh", 3), ("upgrade", 3), ("legacy", 3), ("upgrade", 6))) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_direct_migrations_with_one_application_backend( self, containers: Containers, diff --git a/tests/e2e/migrations/test_recovery.py b/tests/e2e/migrations/test_recovery.py index 80e5747eaac..b1f019acbde 100644 --- a/tests/e2e/migrations/test_recovery.py +++ b/tests/e2e/migrations/test_recovery.py @@ -20,12 +20,18 @@ from .checks import ( from .containers import Containers, failed, ready, until, waiting from .database import COORDINATOR_LOCK, GATE_KEY, Database from .startup_models import Migration +from e2e_metadata import Domain, Subject, meta pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup] class TestMigrationRecovery: @pytest.mark.parametrize("after_commit", (False, True)) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_container_owner_crash(self, containers: Containers, database: Database, after_commit: bool) -> None: interrupt_owner(containers, database, after_commit, stop_database_session=False) history: Final = database.history() @@ -42,6 +48,11 @@ class TestMigrationRecovery: assert database.history() == history @pytest.mark.parametrize("after_commit", (False, True)) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_owner_and_database_session_crash( self, containers: Containers, database: Database, after_commit: bool ) -> None: @@ -60,6 +71,11 @@ class TestMigrationRecovery: assert database.history() == history @pytest.mark.parametrize("later_failure", (False, True)) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_remaining_migrations_after_recovery( self, containers: Containers, database: Database, later_failure: bool ) -> None: @@ -98,6 +114,11 @@ class TestMigrationRecovery: assert database.query("SELECT id FROM migration_next") == ((2,),) assert_original_proof(database, original, True) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_second_crash_during_recovery_is_atomic(self, containers: Containers, database: Database) -> None: original: Final = confirmed_history(database) pause_completion(database) @@ -114,6 +135,11 @@ class TestMigrationRecovery: ready(start_replicas(stack, containers, database, (COMPLETE,)), database) assert_original_proof(database, original, True) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_competing_recovery_rechecks_stale_failures(self, containers: Containers, database: Database) -> None: original: Final = confirmed_history(database) with ExitStack() as stack: @@ -132,6 +158,11 @@ class TestMigrationRecovery: @pytest.mark.parametrize( "fault", ("no_steps", "extra_steps", "failure_logs", "checksum", "duplicate_history", "missing_script") ) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_unproven_history_is_never_repaired( self, containers: Containers, @@ -172,6 +203,11 @@ class TestMigrationRecovery: assert database.history() == history assert database.query("SELECT id FROM migration_effect") == ((1,),) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_coordinator_timeout_preserves_proof(self, containers: Containers, database: Database) -> None: original: Final = confirmed_history(database) with database.lock(COORDINATOR_LOCK): diff --git a/tests/e2e/migrations/test_rolling_upgrade.py b/tests/e2e/migrations/test_rolling_upgrade.py index 5ad74e0ba8c..03f12cada3a 100644 --- a/tests/e2e/migrations/test_rolling_upgrade.py +++ b/tests/e2e/migrations/test_rolling_upgrade.py @@ -14,11 +14,17 @@ from .upgrade import ( migration_names, provision, ) +from e2e_metadata import Domain, Subject, meta pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup] class TestRollingUpgrade: + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_baseline_replica_keeps_serving_while_the_candidate_migrates( self, containers: Containers, baseline_image: str, baseline_database: Database ) -> None: @@ -38,6 +44,11 @@ class TestRollingUpgrade: assert CACHED_PLAN not in old.logs(), "The baseline replica hit a stale prepared statement" assert old.state().Running, "The baseline replica died during the upgrade" + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_both_releases_serve_and_share_keys_during_the_overlap( self, containers: Containers, baseline_image: str, baseline_database: Database ) -> None: diff --git a/tests/e2e/migrations/test_shaped_database.py b/tests/e2e/migrations/test_shaped_database.py index 20c4368ae33..07cc4b69597 100644 --- a/tests/e2e/migrations/test_shaped_database.py +++ b/tests/e2e/migrations/test_shaped_database.py @@ -5,6 +5,7 @@ import pytest from .containers import Containers, ready from .database import Database from .upgrade import assert_history_clean, assert_upgraded, confirm, migration_names, provision +from e2e_metadata import Domain, Subject, meta SPEND_ROWS: Final = 20_000 @@ -22,6 +23,11 @@ def seed_spend_logs(database: Database, rows: int) -> None: class TestPopulatedDatabaseUpgrade: + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_upgrade_completes_and_preserves_a_populated_spend_log( self, containers: Containers, baseline_image: str, baseline_database: Database ) -> None: diff --git a/tests/e2e/migrations/test_startup.py b/tests/e2e/migrations/test_startup.py index dc628a8ed7a..ff7112c3881 100644 --- a/tests/e2e/migrations/test_startup.py +++ b/tests/e2e/migrations/test_startup.py @@ -7,12 +7,18 @@ from .checks import COMPLETE, FATAL, GATED, assert_completed, start_replicas from .containers import Containers, failed, ready, until, waiting from .database import PRISMA_LOCK, Database, Databases, restricted_user from .startup_models import Migration +from e2e_metadata import Domain, Subject, meta pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup] class TestMigrationStartup: @pytest.mark.parametrize("replicas,v2", ((1, True), (3, True), (1, False))) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_fresh_database(self, containers: Containers, databases: Databases, replicas: int, v2: bool) -> None: with databases.create() as database, ExitStack() as stack: ready(tuple(stack.enter_context(containers.start(database, v2=v2)) for _ in range(replicas)), database) @@ -21,11 +27,21 @@ class TestMigrationStartup: ) == ((0,),) assert database.query("SELECT count(*) > 0 FROM _prisma_migrations") == ((True,),) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_concurrent_upgrade(self, containers: Containers, database: Database) -> None: with ExitStack() as stack: ready(start_replicas(stack, containers, database, (COMPLETE,)), database) assert_completed(database) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_waiters_survive_prolonged_contention(self, containers: Containers, database: Database) -> None: with ExitStack() as stack: with database.lock(): @@ -37,6 +53,11 @@ class TestMigrationStartup: ready((owner, *followers), database) assert_completed(database, GATED) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_lock_deadline_then_restart(self, containers: Containers, database: Database) -> None: history: Final = database.history() with database.lock(PRISMA_LOCK): @@ -51,6 +72,11 @@ class TestMigrationStartup: ready((restarted,), database) assert_completed(database) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_fatal_sql(self, containers: Containers, database: Database) -> None: with ExitStack() as stack: replicas: Final = start_replicas(stack, containers, database, (FATAL,)) @@ -61,6 +87,11 @@ class TestMigrationStartup: (COMPLETE.name, "%MIGRATION_TEST_FATAL%"), ) == ((1,),) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_duplicate_object_does_not_hide_incomplete_sql(self, containers: Containers, database: Database) -> None: database.execute( "CREATE TABLE migration_existing (id int PRIMARY KEY); INSERT INTO migration_existing VALUES (42)" @@ -77,6 +108,11 @@ class TestMigrationStartup: ) == ((True,),) @pytest.mark.parametrize("v2", (True, False)) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_restart_preserves_history_and_data(self, containers: Containers, database: Database, v2: bool) -> None: history: Final = database.history() before: Final = database.query('SELECT token FROM "LiteLLM_VerificationToken" ORDER BY token') @@ -86,12 +122,22 @@ class TestMigrationStartup: assert database.history() == history assert set(before).issubset(database.query('SELECT token FROM "LiteLLM_VerificationToken" ORDER BY token')) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_disabled_migrations(self, containers: Containers, database: Database) -> None: history: Final = database.history() with containers.start(database, (FATAL,), disabled=True) as replica: ready((replica,), database) assert database.history() == history + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_insufficient_privileges(self, containers: Containers, database: Database) -> None: history: Final = database.history() with restricted_user(database) as limited: diff --git a/tests/e2e/migrations/test_upgrade.py b/tests/e2e/migrations/test_upgrade.py index 23f0bbe9124..e9a29c970e9 100644 --- a/tests/e2e/migrations/test_upgrade.py +++ b/tests/e2e/migrations/test_upgrade.py @@ -7,11 +7,17 @@ from .checks import start_replicas from .containers import Containers, ready from .database import Database from .upgrade import assert_history_clean, assert_upgraded, confirm, migration_names, provision +from e2e_metadata import Domain, Subject, meta pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup] class TestReleaseUpgrade: + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_candidate_applies_the_pending_release_migrations( self, containers: Containers, baseline_database: Database ) -> None: @@ -21,6 +27,11 @@ class TestReleaseUpgrade: assert_upgraded(before, migration_names(baseline_database)) assert_history_clean(baseline_database) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_upgrade_preserves_keys_minted_by_the_baseline_release( self, containers: Containers, baseline_image: str, baseline_database: Database ) -> None: @@ -34,6 +45,11 @@ class TestReleaseUpgrade: assert_upgraded(before, migration_names(baseline_database)) confirm(new, key, alias) + @meta( + Subject( + domain=Domain.DB, + ) + ) def test_concurrent_replicas_upgrade_a_baseline_database_once( self, containers: Containers, baseline_database: Database ) -> None: diff --git a/tests/e2e/migrations/upgrade.py b/tests/e2e/migrations/upgrade.py index 2123f86450a..3dc6c348587 100644 --- a/tests/e2e/migrations/upgrade.py +++ b/tests/e2e/migrations/upgrade.py @@ -8,6 +8,7 @@ from typing import Final from uuid import uuid4 from e2e_http import Result, Success, unwrap +from e2e_metadata import step from models import ( KeyGenerateBody, KeyGenerateResponse, @@ -24,6 +25,7 @@ from .database import Database CACHED_PLAN: Final = "cached plan must not change result type" +@step("Generate a virtual key on the proxy container") def provision(replica: Replica) -> tuple[str, str]: alias: Final = f"upgrade-{uuid4().hex}" key: Final = unwrap( @@ -37,6 +39,7 @@ def provision(replica: Replica) -> tuple[str, str]: return key, alias +@step("Check that the key {alias} resolves on the proxy container through /key/info") def confirm(replica: Replica, key: str, alias: str) -> None: info: Final = unwrap( replica.transport.get( @@ -62,6 +65,7 @@ class Outcomes: self.failures.append(result.model_dump_json()) +@step("Send /v1/models requests with the virtual key to proxy container {replica.name} in the background") @contextmanager def auth_traffic(replica: Replica, key: str, interval: float = 0.05) -> Generator[Outcomes]: outcomes: Final = Outcomes() @@ -93,6 +97,7 @@ def auth_traffic(replica: Replica, key: str, interval: float = 0.05) -> Generato ) +@step("Wait for {calls} more successful /v1/models calls from {description}") def keep_serving(outcomes: Outcomes, description: str, calls: int = 20) -> int: target: Final = outcomes.served + calls until(description, lambda: outcomes.served >= target or bool(outcomes.failures)) @@ -100,10 +105,12 @@ def keep_serving(outcomes: Outcomes, description: str, calls: int = 20) -> int: return outcomes.served +@step("Read the applied migration names from _prisma_migrations") def migration_names(database: Database) -> frozenset[str]: return frozenset(str(row[0]) for row in database.query("SELECT migration_name FROM _prisma_migrations")) +@step("Check that _prisma_migrations holds no unfinished, rolled-back or duplicated migration") def assert_history_clean(database: Database) -> None: assert database.query( "SELECT count(*) FROM _prisma_migrations WHERE finished_at IS NULL OR rolled_back_at IS NOT NULL" diff --git a/tests/e2e/other/other_client.py b/tests/e2e/other/other_client.py index d7bad4f1ed1..17c5bcfd069 100644 --- a/tests/e2e/other/other_client.py +++ b/tests/e2e/other/other_client.py @@ -17,6 +17,7 @@ from dataclasses import dataclass from typing import Final from e2e_http import AnthropicHeaders, AuthHeaders, NoBody, ProbeResult, Result +from e2e_metadata import step from idp import Keycloak, keycloak_from_env from models import ( ChatBody, @@ -56,11 +57,13 @@ class OtherClient: """Resolved per use, so the suite's non-JWT tests never need the IdP env.""" return keycloak_from_env() + @step("Call /health/liveliness without credentials") def liveness(self) -> ProbeResult: """GET /health/liveliness. Unauthenticated; the probe returns status + raw body so the test can assert the worker reports itself alive.""" return self.proxy.transport.probe("/health/liveliness", params=NoBody()) + @step("Call /health/readiness without credentials") def readiness_public(self) -> Result[ReadinessResponse]: """GET /health/readiness with no credential at all, proving the probe is safe to expose to an unauthenticated load balancer.""" @@ -71,6 +74,7 @@ class OtherClient: response_type=ReadinessResponse, ) + @step("Call /health/readiness/details with the given key") def readiness_details(self, key: str) -> Result[ReadinessDetailsResponse]: return self.proxy.transport.get( "/health/readiness/details", @@ -79,6 +83,7 @@ class OtherClient: response_type=ReadinessDetailsResponse, ) + @step("Call /health/readiness/details without credentials") def readiness_details_unauthenticated(self) -> Result[ReadinessDetailsResponse]: return self.proxy.transport.get( "/health/readiness/details", @@ -87,6 +92,7 @@ class OtherClient: response_type=ReadinessDetailsResponse, ) + @step("Create the {body.user_role} user {body.user_email} through /user/new") def user_new(self, body: UserNewBody) -> Result[UserNewResponse]: """POST /user/new under the master key: seed the litellm user a JWT `sub` claim resolves to, before that token ever reaches the proxy.""" @@ -97,6 +103,7 @@ class OtherClient: response_type=UserNewResponse, ) + @step("Read the user's keys from /user/info") def user_info(self, user_id: str) -> Result[UserInfoWithKeysResponse]: """GET /user/info under the master key. Only the user's key rows are modelled: `token` is the stored key hash, never the plaintext key.""" @@ -107,6 +114,7 @@ class OtherClient: response_type=UserInfoWithKeysResponse, ) + @step("List the JWT-to-key mappings from /jwt/key/mapping/list") def jwt_mapping_list(self) -> Result[JwtKeyMappingListResponse]: """GET /jwt/key/mapping/list under the master key.""" return self.proxy.transport.get( @@ -116,6 +124,7 @@ class OtherClient: response_type=JwtKeyMappingListResponse, ) + @step("Delete the JWT-to-key mapping") def jwt_mapping_delete(self, mapping_id: str) -> Result[JwtKeyMappingDeleteResponse]: """POST /jwt/key/mapping/delete under the master key.""" return self.proxy.transport.post( @@ -125,6 +134,7 @@ class OtherClient: response_type=JwtKeyMappingDeleteResponse, ) + @step("Send a /chat/completions request to {body.model} as team {team} with the given token") def chat_as_team(self, token: str, team: str, body: ChatBody) -> Result[ChatResponse]: """POST /chat/completions under `token` with `x-litellm-team-id: team`.""" return self.proxy.transport.post( @@ -137,6 +147,7 @@ class OtherClient: response_type=ChatResponse, ) + @step("List the models from /v1/models with the given token, in the Anthropic shape: {anthropic}") def list_models_as(self, token: str, *, anthropic: bool = False) -> Result[ModelsListResponse]: """GET /v1/models under `token`, in the OpenAI shape or, with `anthropic`, the Anthropic Models API shape Claude Code reads. Both carry `data[].id`.""" @@ -148,6 +159,7 @@ class OtherClient: response_type=ModelsListResponse, ) + @step("List users from /user/list with the given key") def list_users_as(self, key: str) -> Result[UserListResponse]: """GET /user/list under `key`. Admin-only, so it doubles as the master key's authorization proof: the master key (proxy admin) reads it, a diff --git a/tests/e2e/other/owned_jwt_gateway.py b/tests/e2e/other/owned_jwt_gateway.py index b6c31479bd2..17d49f591d7 100644 --- a/tests/e2e/other/owned_jwt_gateway.py +++ b/tests/e2e/other/owned_jwt_gateway.py @@ -21,6 +21,7 @@ from typing import Final from e2e_config import INHERITED_ENV_PREFIXES, available_port from e2e_http import NoBody +from e2e_metadata import step from idp import Keycloak, stop_process_group from proxy_client import ProxyClient, build_proxy_client @@ -36,6 +37,7 @@ class OwnedJwtGateway: _log_path: Path _child: subprocess.Popen[bytes] | None = field(default=None, init=False, repr=False) + @step("Start the dedicated JWT proxy and wait for /health/liveliness") def start(self) -> None: with self._log_path.open("ab") as log: self._child = subprocess.Popen( @@ -54,12 +56,14 @@ class OwnedJwtGateway: time.sleep(0.5) raise AssertionError("owned JWT gateway did not become ready") + @step("Stop the dedicated JWT proxy") def stop(self) -> None: if self._child is not None: stop_process_group(self._child) assert self._child.poll() is not None, "old gateway process is still alive" +@step("Boot a dedicated proxy {name} with its own litellm_jwtauth config") def owned_jwt_gateway( idp: Keycloak, directory: Path, cleanup: ExitStack, *, litellm_jwtauth: str, name: str ) -> OwnedJwtGateway: diff --git a/tests/e2e/other/test_health_lifecycle_e2e.py b/tests/e2e/other/test_health_lifecycle_e2e.py index 2551352e8fa..3a63851a504 100644 --- a/tests/e2e/other/test_health_lifecycle_e2e.py +++ b/tests/e2e/other/test_health_lifecycle_e2e.py @@ -17,12 +17,19 @@ import pytest from e2e_config import MASTER_KEY from e2e_http import UnauthorizedError, unwrap from other_client import OtherClient +from e2e_metadata import Domain, Route, Subject, meta pytestmark = pytest.mark.e2e class TestHealthLifecycle: @pytest.mark.covers("other.lifecycle.liveness.ping") + @meta( + Subject( + domain=Domain.DEPLOY_OPS, + route=Route.HEALTH, + ) + ) def test_liveness_reports_alive_without_auth(self, client: OtherClient) -> None: probe = client.liveness() assert probe.status_code == 200, ( @@ -34,6 +41,12 @@ class TestHealthLifecycle: ) @pytest.mark.covers("other.lifecycle.readiness.public_probe") + @meta( + Subject( + domain=Domain.DEPLOY_OPS, + route=Route.HEALTH, + ) + ) def test_readiness_is_reachable_without_credentials(self, client: OtherClient) -> None: readiness = unwrap(client.readiness_public()) assert readiness.status == "healthy", ( @@ -41,6 +54,12 @@ class TestHealthLifecycle: ) @pytest.mark.covers("other.lifecycle.readiness.reports_db_status") + @meta( + Subject( + domain=Domain.DEPLOY_OPS, + route=Route.HEALTH, + ) + ) def test_readiness_reports_connected_db(self, client: OtherClient) -> None: readiness = unwrap(client.readiness_public()) assert readiness.db == "connected", ( @@ -49,6 +68,12 @@ class TestHealthLifecycle: ) @pytest.mark.covers("other.lifecycle.readiness_details.authenticated_diagnostics") + @meta( + Subject( + domain=Domain.DEPLOY_OPS, + route=Route.HEALTH, + ) + ) def test_readiness_details_require_auth_and_expose_diagnostics(self, client: OtherClient) -> None: anonymous = client.readiness_details_unauthenticated() assert isinstance(anonymous, UnauthorizedError), ( diff --git a/tests/e2e/other/test_jwt_auth_e2e.py b/tests/e2e/other/test_jwt_auth_e2e.py index a8aa474f04f..6da3924c284 100644 --- a/tests/e2e/other/test_jwt_auth_e2e.py +++ b/tests/e2e/other/test_jwt_auth_e2e.py @@ -15,6 +15,7 @@ from lifecycle import ResourceManager from models import ChatBody, ChatMessage, TeamNewBody from other_client import OtherClient from pydantic import BaseModel +from e2e_metadata import Domain, Mode, Provider, Subject, meta pytestmark = pytest.mark.e2e @@ -120,6 +121,14 @@ def _corrupt_signature(token: str) -> str: class TestJwtAuth: @pytest.mark.covers("other.auth.jwt.valid_token_allows", "other.auth.jwt.spend_attributed_to_claims") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_valid_token_for_an_existing_team_is_accepted_and_attributed( self, client: OtherClient, identity: Identity ) -> None: @@ -142,6 +151,13 @@ class TestJwtAuth: ) @pytest.mark.covers("other.auth.jwt.invalid_signature_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_tampered_signature_is_rejected(self, client: OtherClient, identity: Identity) -> None: tampered: Final = _corrupt_signature(client.idp.access_token(identity)) @@ -154,6 +170,13 @@ class TestJwtAuth: ) @pytest.mark.covers("other.auth.jwt.expired_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_expired_token_is_rejected(self, client: OtherClient, identity: Identity) -> None: expiring: Final = client.idp.access_token(identity, client_id=SHORT_LIVED_CLIENT_ID) delay: Final = _claims(expiring).exp - time.time() + 1 @@ -167,6 +190,13 @@ class TestJwtAuth: assert "expired" in result.body.lower(), f"the 401 must say the token expired, got {result.body[:300]}" @pytest.mark.covers("other.auth.jwt.wrong_issuer_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_signed_token_from_the_wrong_issuer_is_rejected(self, client: OtherClient, identity: Identity) -> None: token: Final = client.idp.access_token(identity, issuer_host="unexpected-issuer.invalid") claims: Final = _claims(token) @@ -177,6 +207,13 @@ class TestJwtAuth: assert "issuer" in result.body.lower(), f"expected issuer validation to reject the token: {result}" @pytest.mark.covers("other.auth.jwt.wrong_audience_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_signed_token_for_another_application_is_rejected(self, client: OtherClient, identity: Identity) -> None: token: Final = client.idp.access_token(identity, client_id=WRONG_AUDIENCE_CLIENT_ID) claims: Final = _claims(token) @@ -189,6 +226,13 @@ class TestJwtAuth: assert "audience" in result.body.lower(), f"expected audience validation to reject the token: {result}" @pytest.mark.covers("other.auth.jwt.unknown_team_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_token_naming_a_team_that_does_not_exist_is_rejected( self, client: OtherClient, resources: ResourceManager ) -> None: @@ -204,6 +248,14 @@ class TestJwtAuth: ) @pytest.mark.covers("other.auth.jwt.virtual_key_unaffected") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_plain_virtual_key_still_works_with_jwt_auth_enabled(self, client: OtherClient, scoped_key: str) -> None: response: Final = unwrap(client.proxy.chat(scoped_key, _ping())) assert response.choices, f"an sk- key must keep working on a proxy with enable_jwt_auth, got {response}" @@ -230,6 +282,14 @@ def _denial(client: OtherClient, token: str, team: str) -> str: class TestJwtTeamHeader: @pytest.mark.covers("other.auth.jwt.team_header_alias_binds_team") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_team_header_with_the_team_alias_binds_the_same_team_as_the_team_id( self, client: OtherClient, bound_team: BoundTeam ) -> None: @@ -249,6 +309,14 @@ class TestJwtTeamHeader: @pytest.mark.covers("other.auth.jwt.team_model_alias_listed_and_routes") @pytest.mark.parametrize("anthropic", [False, True], ids=["openai_shape", "anthropic_shape"]) + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.OPENAI,), + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_team_model_alias_is_listed_by_v1_models_under_the_same_token_that_routes_it( self, client: OtherClient, aliased_team: AliasedTeam, anthropic: bool ) -> None: @@ -265,6 +333,13 @@ class TestJwtTeamHeader: assert aliased_team.target in listed, f"the alias target {aliased_team.target!r} must stay listed, got {listed}" @pytest.mark.covers("other.auth.jwt.team_header_non_member_alias_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + models=(CHEAP_OPENAI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_team_header_with_the_alias_of_a_team_the_caller_is_not_in_is_rejected_like_an_unknown_value( self, client: OtherClient, resources: ResourceManager, bound_team: BoundTeam ) -> None: diff --git a/tests/e2e/other/test_jwt_auto_register_e2e.py b/tests/e2e/other/test_jwt_auto_register_e2e.py index 8f7c2c6a693..45a7ce2c182 100644 --- a/tests/e2e/other/test_jwt_auto_register_e2e.py +++ b/tests/e2e/other/test_jwt_auto_register_e2e.py @@ -22,6 +22,7 @@ from lifecycle import ResourceManager from models import ChatBody, ChatMessage, JwtKeyMappingRow, KeyGenerateBody, TeamNewBody, UserNewBody from other_client import OtherClient from owned_jwt_gateway import MODEL_NAME, OwnedJwtGateway, owned_jwt_gateway +from e2e_metadata import Domain, Mode, Provider, Subject, meta pytestmark = pytest.mark.e2e @@ -115,6 +116,14 @@ def minting_gateway(idp: Keycloak, tmp_path_factory: pytest.TempPathFactory) -> @pytest.mark.owned_gateway class TestJwtAutoRegisterMapExistingKey: @pytest.mark.covers("other.auth.jwt.auto_register_maps_existing_key") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.GEMINI,), + models=(MODEL_NAME,), + mode=Mode.NONSTREAM, + ) + ) def test_first_jwt_call_maps_to_the_users_existing_key_and_mints_none( self, client: OtherClient, idp: Keycloak, resources: ResourceManager, mapping_gateway: OwnedJwtGateway ) -> None: @@ -144,6 +153,14 @@ class TestJwtAutoRegisterMapExistingKey: ) @pytest.mark.covers("other.auth.jwt.auto_register_mints_when_keyless") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.GEMINI,), + models=(MODEL_NAME,), + mode=Mode.NONSTREAM, + ) + ) def test_first_jwt_call_mints_a_key_when_the_user_has_none( self, client: OtherClient, idp: Keycloak, resources: ResourceManager, mapping_gateway: OwnedJwtGateway ) -> None: @@ -160,6 +177,14 @@ class TestJwtAutoRegisterMapExistingKey: ) @pytest.mark.covers("other.auth.jwt.auto_register_default_mints") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + providers=(Provider.GEMINI,), + models=(MODEL_NAME,), + mode=Mode.NONSTREAM, + ) + ) def test_default_behavior_still_mints_when_the_user_already_has_a_key( self, client: OtherClient, idp: Keycloak, resources: ResourceManager, minting_gateway: OwnedJwtGateway ) -> None: diff --git a/tests/e2e/other/test_master_key_auth_e2e.py b/tests/e2e/other/test_master_key_auth_e2e.py index 6ab33c9b62a..cd506f746ba 100644 --- a/tests/e2e/other/test_master_key_auth_e2e.py +++ b/tests/e2e/other/test_master_key_auth_e2e.py @@ -15,12 +15,18 @@ import pytest from e2e_config import MASTER_KEY, unique_marker from e2e_http import UnauthorizedError, unwrap from other_client import OtherClient +from e2e_metadata import Domain, Subject, meta pytestmark = pytest.mark.e2e class TestMasterKeyAuth: @pytest.mark.covers("other.auth.master_key.valid_allows") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + ) + ) def test_master_key_authenticates_and_grants_admin_route(self, client: OtherClient) -> None: listing = unwrap(client.list_users_as(MASTER_KEY)) assert listing.total >= 0, ( @@ -29,6 +35,11 @@ class TestMasterKeyAuth: ) @pytest.mark.covers("other.auth.master_key.invalid_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + ) + ) def test_non_matching_master_key_is_denied(self, client: OtherClient) -> None: bogus = f"sk-{unique_marker()}" result = client.list_users_as(bogus) diff --git a/tests/e2e/other/test_session_token_e2e.py b/tests/e2e/other/test_session_token_e2e.py index 51791278026..16740390fd6 100644 --- a/tests/e2e/other/test_session_token_e2e.py +++ b/tests/e2e/other/test_session_token_e2e.py @@ -20,6 +20,7 @@ from e2e_http import UnauthorizedError, unwrap from lifecycle import ResourceManager from models import KeyGenerateBody, KeyLoggingCallback, KeyLoggingCallbackVars, KeyMetadata from other_client import OtherClient +from e2e_metadata import Domain, Subject, meta pytestmark = pytest.mark.e2e @@ -47,12 +48,22 @@ def _admin_session_token(expires_at: datetime) -> str: class TestSessionToken: @pytest.mark.covers("other.auth.session_token.valid_allows") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + ) + ) def test_unexpired_session_token_reaches_admin_route(self, client: OtherClient) -> None: token: Final = _admin_session_token(datetime.now(timezone.utc) + timedelta(minutes=10)) listing: Final = unwrap(client.list_users_as(token)) assert listing.total >= 0, f"an unexpired admin session token did not reach /user/list: {listing}" @pytest.mark.covers("other.auth.session_token.expired_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + ) + ) def test_expired_session_token_is_denied(self, client: OtherClient) -> None: token: Final = _admin_session_token(datetime.now(timezone.utc) - timedelta(minutes=1)) result: Final = client.list_users_as(token) @@ -60,6 +71,11 @@ class TestSessionToken: assert "expired" in result.body.lower(), f"expected the expired-key error, got {result.body[:300]}" @pytest.mark.covers("other.auth.session_token.encrypted_value_denied") + @meta( + Subject( + domain=Domain.PROXY_AUTH, + ) + ) def test_encrypted_stored_value_is_not_a_bearer_token( self, client: OtherClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/secret_manager/secret_store_cyberark.py b/tests/e2e/secret_manager/secret_store_cyberark.py index 375b997e02c..22c7c458d86 100644 --- a/tests/e2e/secret_manager/secret_store_cyberark.py +++ b/tests/e2e/secret_manager/secret_store_cyberark.py @@ -10,6 +10,7 @@ from urllib.parse import quote import pytest import yaml from e2e_http import ExternalWrite, Headers, send_text_external +from e2e_metadata import step from pydantic import Field from secret_store import SecretBackend @@ -94,6 +95,7 @@ class Conjur: if not result.ok: pytest.fail(f"Conjur refused to {action}: HTTP {result.status_code} {result.body[:300]}") + @step("Write the secret {name} to CyberArk Conjur") def write(self, name: str, value: str) -> None: self._update_root_policy("POST", f"- !variable {_policy_scalar(name)}\n", f"declare {name}") result: Final = send_text_external("POST", self._secret_url(name), headers=self._headers(), content=value) @@ -101,6 +103,7 @@ class Conjur: if not result.ok: pytest.fail(f"Conjur refused to write {name}: HTTP {result.status_code} {result.body[:300]}") + @step("Read the secret {name} from CyberArk Conjur") def read(self, name: str) -> str | None: result: Final = send_text_external("GET", self._secret_url(name), headers=self._headers()) self._fail_unless_reached(result, f"read {name}") @@ -110,6 +113,7 @@ class Conjur: pytest.fail(f"Conjur refused to read {name}: HTTP {result.status_code} {result.body[:300]}") return result.body + @step("Delete the secret {name} from CyberArk Conjur") def destroy(self, name: str) -> None: self._update_root_policy("PATCH", f"- !delete\n record: !variable {_policy_scalar(name)}\n", f"destroy {name}") diff --git a/tests/e2e/secret_manager/secret_store_hashicorp_vault.py b/tests/e2e/secret_manager/secret_store_hashicorp_vault.py index ccf8cefe716..5954719ccbb 100644 --- a/tests/e2e/secret_manager/secret_store_hashicorp_vault.py +++ b/tests/e2e/secret_manager/secret_store_hashicorp_vault.py @@ -14,6 +14,7 @@ from e2e_http import ( get_external, post_json_external, ) +from e2e_metadata import step from pydantic import BaseModel, Field from secret_store import SecretBackend @@ -68,6 +69,7 @@ class Vault: def _metadata_url(self, name: str) -> str: return f"{self.base_url}/v1/{self.mount}/metadata/{name}" + @step("Write the secret {name} to HashiCorp Vault") def write(self, name: str, value: str) -> None: write: Final = post_json_external( self._data_url(name), headers=self._headers(), json=KvWriteBody(data=KvData(key=value)) @@ -77,6 +79,7 @@ class Vault: if not write.ok: pytest.fail(f"Vault refused to write {name}: HTTP {write.status_code} {write.body[:300]}") + @step("Read the secret {name} from HashiCorp Vault") def read(self, name: str) -> str | None: result: Final = get_external(self._data_url(name), headers=self._headers(), response_type=KvReadResponse) match result: @@ -89,6 +92,7 @@ class Vault: case _: return pytest.fail(f"Vault refused to read {name}: {result}") + @step("Delete the secret {name} from HashiCorp Vault") def destroy(self, name: str) -> None: write: Final = delete_external(self._metadata_url(name), headers=self._headers()) if not write.ok and write.status_code != 404: diff --git a/tests/e2e/secret_manager/test_secret_manager_e2e.py b/tests/e2e/secret_manager/test_secret_manager_e2e.py index a9c9024718d..8eaabcaae6f 100644 --- a/tests/e2e/secret_manager/test_secret_manager_e2e.py +++ b/tests/e2e/secret_manager/test_secret_manager_e2e.py @@ -13,6 +13,7 @@ from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody, LiteLLMParamsBody from proxy_client import ProxyClient from secret_store import SecretStore +from e2e_metadata import Domain, Mode, Provider, Subject, meta pytestmark = [pytest.mark.e2e, pytest.mark.secret_manager] @@ -73,6 +74,14 @@ def _eventually(proxy: ProxyClient, read: Callable[[], str | None], expected: st class TestSecretManager: @pytest.mark.covers("other.config.secret_resolution.kms_integration") + @meta( + Subject( + domain=Domain.DEPLOY_OPS, + providers=(Provider.OPENAI,), + models=(BACKEND_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_deployment_key_resolves_from_the_manager( self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore, scoped_key: str ) -> None: @@ -83,6 +92,14 @@ class TestSecretManager: assert response.choices, f"the manager-backed deployment answered with no choices: {response}" @pytest.mark.covers("other.config.secret_resolution.manager_value_used") + @meta( + Subject( + domain=Domain.DEPLOY_OPS, + providers=(Provider.OPENAI,), + models=(BACKEND_MODEL,), + mode=Mode.NONSTREAM, + ) + ) def test_deployment_uses_the_value_the_manager_holds( self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore, scoped_key: str ) -> None: @@ -100,6 +117,11 @@ class TestSecretManager: pytest.fail(f"expected the provider to reject the manager-held key with 401, got {result}") @pytest.mark.covers("other.config.secret_manager.virtual_key_stored") + @meta( + Subject( + domain=Domain.DEPLOY_OPS, + ) + ) def test_generated_key_is_written_to_the_manager( self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore ) -> None: @@ -113,6 +135,11 @@ class TestSecretManager: @pytest.mark.requires_capability("deletes_stored_keys") @pytest.mark.covers("other.config.secret_manager.virtual_key_deleted") + @meta( + Subject( + domain=Domain.DEPLOY_OPS, + ) + ) def test_deleted_key_is_removed_from_the_manager( self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore ) -> None: From d174c43518a479fe584703eb7829d4a022f0f955 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:13:32 +0000 Subject: [PATCH 24/35] fix(azure): use integer prompt_cache_min_tokens for azure_ai/claude-haiku-5-5 (#45114) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ef601bf522f..cecc7d0856f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -81046,10 +81046,10 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "supports_forced_tool_use": false, - "thinking_always_on": true, + "supports_forced_tool_use": true, + "thinking_always_on": false, "prompt_cache_min_tokens": 512, - "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" + "source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide" }, "bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": { "bedrock_converse_supports_strict_tools": false, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ef601bf522f..cecc7d0856f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -81046,10 +81046,10 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "supports_forced_tool_use": false, - "thinking_always_on": true, + "supports_forced_tool_use": true, + "thinking_always_on": false, "prompt_cache_min_tokens": 512, - "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" + "source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide" }, "bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": { "bedrock_converse_supports_strict_tools": false, From 65dcf4325711a4c6b2a7c9dd52c576e75caced9e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:24:01 -0700 Subject: [PATCH 25/35] perf(lens): claim worker jobs from an indexed due queue instead of scanning every lens (#45095) * perf(lens): claim worker jobs from an indexed due queue instead of scanning every lens Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(lens): apply the due_at index concurrently on its own and default legacy rows to due Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * test(lens): pass repository to claim lifecycle tests Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(lens): page past unsupported due lenses and declare the full due_at index Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(lens): claim due lenses in a loop instead of recursion Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- .../20261008000000_lens_due_at/migration.sql | 1 + .../migration.sql | 2 + .../litellm_proxy_extras/schema.prisma | 3 + litellm/proxy/lens/endpoints.py | 60 +++- litellm/proxy/lens/repository.py | 78 ++++- litellm/proxy/lens/state.py | 9 + litellm/proxy/schema.prisma | 3 + schema.prisma | 3 + .../database/test_lens_repository.py | 284 +++++++++++++++++- .../database/test_lens_scheduler_load.py | 199 ++++++++++++ tests/proxy_behavior/lens/test_lifecycle.py | 23 +- tests/unit/proxy/lens/test_endpoints.py | 55 +++- tests/unit/proxy/lens/test_state.py | 40 +++ 13 files changed, 738 insertions(+), 22 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000000_lens_due_at/migration.sql create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000100_lens_due_at_index/migration.sql create mode 100644 tests/integration/database/test_lens_scheduler_load.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000000_lens_due_at/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000000_lens_due_at/migration.sql new file mode 100644 index 00000000000..676d9b124c9 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000000_lens_due_at/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "LiteLLM_Lens" ADD COLUMN IF NOT EXISTS "due_at" TIMESTAMP(3) DEFAULT '1970-01-01 00:00:00'; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000100_lens_due_at_index/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000100_lens_due_at_index/migration.sql new file mode 100644 index 00000000000..36311c4e749 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261008000100_lens_due_at_index/migration.sql @@ -0,0 +1,2 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_Lens_due_at_idx" +ON "LiteLLM_Lens" ("due_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 514df905866..ceb127c31bd 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1939,6 +1939,9 @@ model LiteLLM_Lens { id String @id version Int @default(0) data Json + due_at DateTime? @default(dbgenerated("'1970-01-01 00:00:00'::timestamp without time zone")) + + @@index([due_at], map: "LiteLLM_Lens_due_at_idx") } model LiteLLM_LensRun { diff --git a/litellm/proxy/lens/endpoints.py b/litellm/proxy/lens/endpoints.py index 21fde94edfe..d0b91aeea2f 100644 --- a/litellm/proxy/lens/endpoints.py +++ b/litellm/proxy/lens/endpoints.py @@ -1,10 +1,11 @@ import hashlib import secrets +from collections.abc import Awaitable, Callable from datetime import datetime, timedelta, timezone from functools import reduce from itertools import chain from types import MappingProxyType -from typing import Annotated, Final, TypeAlias +from typing import Annotated, Final, Protocol, TypeAlias from uuid import uuid4 from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response @@ -49,7 +50,7 @@ from litellm.proxy.lens.models import ( WorkerCreated, ) from litellm.proxy.lens.release import PROTOCOL_VERSION, release_tag, worker_image -from litellm.proxy.lens.repository import LensRepository, WriterDatabase +from litellm.proxy.lens.repository import DueLens, LensRepository, WriterDatabase from litellm.proxy.lens.reviews import criteria_key from litellm.proxy.lens.sources import ActivityAvailability, SourceReader, Storage, parse_execution from litellm.proxy.lens.state import ( @@ -71,11 +72,24 @@ from litellm.proxy.tracing_runtime import provide_storage from litellm.types.llms.base import LiteLLMBaseModel router: Final = APIRouter(prefix="/lens", tags=["Lens"]) +CLAIM_CANDIDATES: Final = 20 _bearer: Final = HTTPBearer() Auth: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] StorageDep: TypeAlias = Annotated[Storage | None, Depends(provide_storage)] +class _ClaimRepository(Protocol): + async def due( + self, scope: Scope, now: datetime, limit: int, after: DueLens | None = None + ) -> tuple[DueLens, ...]: ... + + async def sync_due(self, lens: Lens) -> None: ... + + async def update( + self, lens_id: str, transform: Callable[[Lens], Lens], attempts: int, *, changed_only: bool + ) -> Lens | None: ... + + def repository() -> LensRepository: from litellm.proxy.proxy_server import prisma_client @@ -501,13 +515,29 @@ async def claim(worker: WorkerAuth, protocol_version: int = 1, worker_release: s if worker.analysis_key_id is None: raise HTTPException(409, "Assign an analysis key to this worker in Lens setup") now: Final = datetime.now(timezone.utc) - await repository().heartbeat(worker.id, now.isoformat()) - for candidate in await repository().lenses(): - if not can_access(worker.scope, candidate.scope): - continue - if claimed := await claim_candidate(candidate, worker, now): - return claimed - return None + lens_repository: Final = repository() + await lens_repository.heartbeat(worker.id, now.isoformat()) + return await claim_due(worker, now, lens_repository) + + +async def claim_due( + worker: Worker, + now: datetime, + lens_repository: _ClaimRepository, + supports_model: Callable[[Worker, LensSettings], Awaitable[bool]] = worker_supports_model, +) -> Claim | None: + after: DueLens | None = None # rebind-ok: keyset cursor advances one page at a time + while True: + page = await lens_repository.due(worker.scope, now, CLAIM_CANDIDATES, after) + for candidate in page: + if not can_access(worker.scope, candidate.lens.scope): + continue + if claimed := await claim_candidate(candidate.lens, worker, now, lens_repository, supports_model): + return claimed + await lens_repository.sync_due(candidate.lens) + if len(page) < CLAIM_CANDIDATES: + return None + after = page[-1] @router.post("/worker/{lens_id}/{job_id}/progress", response_model=bool) @@ -752,9 +782,15 @@ async def heartbeat(lens_id: str, job_id: str, worker: WorkerAuth) -> bool: return await progress(lens_id, job_id, Progress(), worker) -async def claim_candidate(candidate: Lens, worker: Worker, now: datetime) -> Claim | None: +async def claim_candidate( + candidate: Lens, + worker: Worker, + now: datetime, + lens_repository: _ClaimRepository, + supports_model: Callable[[Worker, LensSettings], Awaitable[bool]] = worker_supports_model, +) -> Claim | None: active: Final = current_job(candidate) - if not await worker_supports_model(worker, active.settings if active else candidate.settings): + if not await supports_model(worker, active.settings if active else candidate.settings): return None job_id: Final = str(uuid4()) @@ -765,7 +801,7 @@ async def claim_candidate(candidate: Lens, worker: Worker, now: datetime) -> Cla return e return claim_job(scheduled, worker, now) - updated: Final = await repository().update(candidate.id, schedule, changed_only=True) + updated: Final = await lens_repository.update(candidate.id, schedule, attempts=1, changed_only=True) if updated is None: return None job: Final = current_job(updated) diff --git a/litellm/proxy/lens/repository.py b/litellm/proxy/lens/repository.py index ddbc1aad44a..cbb66f338fa 100644 --- a/litellm/proxy/lens/repository.py +++ b/litellm/proxy/lens/repository.py @@ -3,6 +3,7 @@ import json import random from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol @@ -24,7 +25,7 @@ from litellm.proxy.lens.models import ( Worker, ) from litellm.proxy.lens.reviews import criteria_key -from litellm.proxy.lens.state import apply_progress, current_job, replace_job +from litellm.proxy.lens.state import apply_progress, current_job, due_at, replace_job from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: @@ -39,6 +40,18 @@ class Database(Protocol): class Row(LiteLLMBaseModel): data: JsonValue + due_at: datetime | None = None + + +class DueRow(LiteLLMBaseModel): + data: JsonValue + due_at: datetime + + +@dataclass(frozen=True, slots=True) +class DueLens: + lens: Lens + due_at: datetime class FindingRun(LiteLLMBaseModel): @@ -47,6 +60,26 @@ class FindingRun(LiteLLMBaseModel): _ROWS: Final = TypeAdapter(tuple[Row, ...]) +_DUE_ROWS: Final = TypeAdapter(tuple[DueRow, ...]) +_DUE_QUERY: Final[LiteralString] = """SELECT data, due_at FROM "LiteLLM_Lens" +WHERE due_at IS NOT NULL AND due_at <= ($4::timestamptz AT TIME ZONE 'UTC') +AND ($1::boolean OR ( + COALESCE((data->'scope'->>'all_teams')::boolean, false) IS NOT TRUE + AND COALESCE(data->'scope'->>'team_id', '')=$2 + AND ($2 <> '' OR COALESCE(data->'scope'->>'api_key_hash', '')=$3) +)) +ORDER BY due_at, id +LIMIT $5""" +_DUE_AFTER_QUERY: Final[LiteralString] = """SELECT data, due_at FROM "LiteLLM_Lens" +WHERE due_at IS NOT NULL AND due_at <= ($4::timestamptz AT TIME ZONE 'UTC') +AND (due_at, id) > ($6::timestamp, $7) +AND ($1::boolean OR ( + COALESCE((data->'scope'->>'all_teams')::boolean, false) IS NOT TRUE + AND COALESCE(data->'scope'->>'team_id', '')=$2 + AND ($2 <> '' OR COALESCE(data->'scope'->>'api_key_hash', '')=$3) +)) +ORDER BY due_at, id +LIMIT $5""" UPDATE_ATTEMPTS: Final = 40 UPDATE_BACKOFF_SECONDS: Final = 0.02 @@ -146,6 +179,30 @@ class LensRepository: rows: Final = _ROWS.validate_python(await self.db.query_raw('SELECT data FROM "LiteLLM_Lens" ORDER BY id')) return tuple(Lens.model_validate(row.data) for row in rows) + async def due(self, scope: Scope, now: datetime, limit: int, after: DueLens | None = None) -> tuple[DueLens, ...]: + query: Final[LiteralString] = _DUE_QUERY if after is None else _DUE_AFTER_QUERY + parameters: Final[tuple[object, ...]] = ( + ( + scope.all_teams, + scope.team_id, + scope.api_key_hash, + now.isoformat(), + limit, + ) + if after is None + else ( + scope.all_teams, + scope.team_id, + scope.api_key_hash, + now.isoformat(), + limit, + after.due_at, + after.lens.id, + ) + ) + rows: Final = _DUE_ROWS.validate_python(await self.db.query_raw(query, *parameters), from_attributes=True) + return tuple(DueLens(lens=Lens.model_validate(row.data), due_at=row.due_at) for row in rows) + async def get(self, lens_id: str) -> Lens | None: rows: Final = _ROWS.validate_python( await self.db.query_raw( @@ -157,12 +214,25 @@ class LensRepository: async def create(self, lens: Lens) -> Lens: await self.db.execute_raw( - 'INSERT INTO "LiteLLM_Lens" (id, version, data) VALUES ($1,0,$2::jsonb)', + """INSERT INTO "LiteLLM_Lens" (id, version, data, due_at) + VALUES ($1,0,$2::jsonb,($3::timestamptz AT TIME ZONE 'UTC'))""", lens.id, lens.model_dump_json(), + scheduled_at.isoformat() if (scheduled_at := due_at(lens)) else None, ) return lens + async def sync_due(self, lens: Lens) -> None: + await self.db.execute_raw( + """UPDATE "LiteLLM_Lens" + SET due_at=($3::timestamptz AT TIME ZONE 'UTC') + WHERE id=$1 AND version=$2 + AND due_at IS DISTINCT FROM ($3::timestamptz AT TIME ZONE 'UTC')""", + lens.id, + lens.version, + scheduled_at.isoformat() if (scheduled_at := due_at(lens)) else None, + ) + async def update( self, lens_id: str, @@ -193,7 +263,8 @@ class LensRepository: """WITH previous AS MATERIALIZED ( SELECT data FROM "LiteLLM_Lens" WHERE id=$2 AND version=$3 FOR UPDATE ), updated AS ( - UPDATE "LiteLLM_Lens" SET data=$1::jsonb, version=version+1 + UPDATE "LiteLLM_Lens" SET data=$1::jsonb, version=version+1, + due_at=($4::timestamptz AT TIME ZONE 'UTC') WHERE id=$2 AND version=$3 AND EXISTS (SELECT 1 FROM previous) RETURNING id ) , archived AS (INSERT INTO "LiteLLM_LensRun" (id, lens_id, created_at, data) @@ -207,6 +278,7 @@ class LensRepository: updated.model_dump_json(), lens_id, previous.version, + scheduled_at.isoformat() if (scheduled_at := due_at(updated)) else None, ) ) return bool(rows and rows[0].data == 1), updated diff --git a/litellm/proxy/lens/state.py b/litellm/proxy/lens/state.py index 38efc7f23bb..599dca8f1c6 100644 --- a/litellm/proxy/lens/state.py +++ b/litellm/proxy/lens/state.py @@ -37,6 +37,15 @@ def current_job(lens: Lens) -> Job | None: return next((job for job in lens.jobs if job.status in ("queued", "running")), None) +def due_at(lens: Lens) -> datetime | None: + job: Final = current_job(lens) + if job is None: + return lens.next_run_at if lens.settings.enabled else None + if job.status == "queued": + return job.created_at + return job.lease_until or job.created_at + + def replace_job(lens: Lens, job: Job) -> Lens: return lens.model_copy( update=MappingProxyType({"jobs": tuple(job if old.id == job.id else old for old in lens.jobs)}) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 514df905866..ceb127c31bd 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1939,6 +1939,9 @@ model LiteLLM_Lens { id String @id version Int @default(0) data Json + due_at DateTime? @default(dbgenerated("'1970-01-01 00:00:00'::timestamp without time zone")) + + @@index([due_at], map: "LiteLLM_Lens_due_at_idx") } model LiteLLM_LensRun { diff --git a/schema.prisma b/schema.prisma index 514df905866..ceb127c31bd 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1939,6 +1939,9 @@ model LiteLLM_Lens { id String @id version Int @default(0) data Json + due_at DateTime? @default(dbgenerated("'1970-01-01 00:00:00'::timestamp without time zone")) + + @@index([due_at], map: "LiteLLM_Lens_due_at_idx") } model LiteLLM_LensRun { diff --git a/tests/integration/database/test_lens_repository.py b/tests/integration/database/test_lens_repository.py index e29e92a2505..cd2f4996657 100644 --- a/tests/integration/database/test_lens_repository.py +++ b/tests/integration/database/test_lens_repository.py @@ -14,6 +14,7 @@ import pytest_asyncio from fastapi import HTTPException from prisma import Prisma from psycopg import sql +from pydantic import TypeAdapter from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.db.prisma_client import PrismaWrapper @@ -35,7 +36,7 @@ from litellm.proxy.lens.models import ( Worker, ) from litellm.proxy.lens.repository import LensRepository, WriterDatabase -from litellm.proxy.lens.state import claim_job, queue_job +from litellm.proxy.lens.state import cancel_job, claim_job, current_job, due_at, end_job, queue_job, replace_job @pytest_asyncio.fixture(loop_scope="function") @@ -44,6 +45,274 @@ async def lens_db() -> AsyncIterator[Prisma]: yield db +def _scheduled_lens( + lens_id: str, + scope: Scope, + now: datetime, + next_run_at: datetime, + *, + enabled: bool = True, + jobs: tuple[Job, ...] = (), +) -> Lens: + return Lens( + id=lens_id, + scope=scope, + settings=LensSettings( + name="Scheduling test", + model="analysis", + context="Find unexpected behavior", + enabled=enabled, + ), + created_at=now, + next_run_at=next_run_at, + jobs=jobs, + budget_month=now.strftime("%Y-%m"), + ) + + +def _stored_due_at(lens_id: str) -> datetime | None: + with psycopg.connect(os.environ["DATABASE_URL"]) as connection: + row: Final = connection.execute('SELECT due_at FROM "LiteLLM_Lens" WHERE id=%s', (lens_id,)).fetchone() + return TypeAdapter(datetime | None).validate_python(row[0]) if row else None + + +async def _assert_due_column(repo: LensRepository, lens_id: str) -> None: + stored: Final = await repo.get(lens_id) + assert stored is not None + expected: Final = due_at(stored) + actual: Final = _stored_due_at(lens_id) + if expected is None: + assert actual is None + return + assert actual is not None + difference: Final = actual.replace(tzinfo=timezone.utc) - expected.astimezone(timezone.utc) + assert abs(difference.total_seconds()) <= 0.001 + + +@pytest.mark.asyncio +async def test_due_filters_by_schedule_and_scope(lens_db: Prisma) -> None: + utc_now: Final = datetime.now(timezone.utc).replace(microsecond=0) + worker_now: Final = utc_now.astimezone(timezone(timedelta(hours=3))) + team_id: Final = uuid4().hex + worker_scope: Final = Scope(team_id=team_id) + worker: Final = Worker(id=uuid4().hex, name="worker", scope=worker_scope, last_seen=worker_now) + repo: Final = LensRepository(WriterDatabase(PrismaWrapper(lens_db))) + due_lens: Final = _scheduled_lens(uuid4().hex, worker_scope, utc_now, utc_now - timedelta(minutes=20)) + future_lens: Final = _scheduled_lens(uuid4().hex, worker_scope, utc_now, utc_now + timedelta(minutes=20)) + disabled_lens: Final = _scheduled_lens( + uuid4().hex, worker_scope, utc_now, utc_now - timedelta(minutes=10), enabled=False + ) + live_queued: Final = queue_job( + _scheduled_lens(uuid4().hex, worker_scope, utc_now - timedelta(minutes=5), utc_now - timedelta(minutes=5)), + utc_now - timedelta(minutes=5), + uuid4().hex, + ) + live_lens: Final = claim_job(live_queued, worker, worker_now) + expired_queued: Final = queue_job( + _scheduled_lens(uuid4().hex, worker_scope, utc_now - timedelta(minutes=10), utc_now - timedelta(minutes=10)), + utc_now - timedelta(minutes=10), + uuid4().hex, + ) + expired_claimed: Final = claim_job(expired_queued, worker, utc_now - timedelta(minutes=10)) + expired_job: Final = expired_claimed.jobs[0].model_copy(update={"lease_until": utc_now - timedelta(minutes=5)}) + expired_lens: Final = expired_claimed.model_copy(update={"jobs": (expired_job,)}) + other_lens: Final = _scheduled_lens( + uuid4().hex, Scope(team_id=uuid4().hex), utc_now, utc_now - timedelta(minutes=3) + ) + worker_key: Final = uuid4().hex + key_lens: Final = _scheduled_lens( + uuid4().hex, Scope(api_key_hash=worker_key), utc_now, utc_now - timedelta(minutes=2) + ) + candidates: Final = (due_lens, future_lens, disabled_lens, live_lens, expired_lens, other_lens, key_lens) + await asyncio.gather(*(repo.create(candidate) for candidate in candidates)) + try: + await lens_db.execute_raw( + """UPDATE "LiteLLM_Lens" + SET data=jsonb_set(data, '{scope}', jsonb_build_object('team_id', $2)) + WHERE id=$1""", + due_lens.id, + team_id, + ) + await lens_db.execute_raw( + """UPDATE "LiteLLM_Lens" + SET data=jsonb_set(data, '{scope}', jsonb_build_object('api_key_hash', $2)) + WHERE id=$1""", + key_lens.id, + worker_key, + ) + team_due: Final = await repo.due(worker_scope, worker_now, 20) + assert tuple(candidate.lens.id for candidate in team_due) == tuple( + lens.id for lens in sorted((due_lens, expired_lens), key=lambda lens: (due_at(lens), lens.id)) + ) + assert team_due[0].lens.scope == worker_scope + key_due: Final = await repo.due(Scope(api_key_hash=worker_key), worker_now, 20) + assert tuple(candidate.lens.id for candidate in key_due) == (key_lens.id,) + all_due: Final = await repo.due(Scope(all_teams=True), worker_now, 20) + assert {candidate.lens.id for candidate in all_due} == { + due_lens.id, + expired_lens.id, + other_lens.id, + key_lens.id, + } + finally: + await lens_db.execute_raw( + 'DELETE FROM "LiteLLM_Lens" WHERE id=ANY($1::text[])', + tuple(lens.id for lens in candidates), + ) + + +@pytest.mark.asyncio +async def test_due_pages_lenses_with_equal_due_at_without_skipping_or_repeating(lens_db: Prisma) -> None: + now: Final = datetime.now(timezone.utc).replace(microsecond=0) + scope: Final = Scope(team_id=uuid4().hex) + repo: Final = LensRepository(WriterDatabase(PrismaWrapper(lens_db))) + lenses: Final = tuple(_scheduled_lens(uuid4().hex, scope, now, now - timedelta(minutes=1)) for _ in range(45)) + await asyncio.gather(*(repo.create(lens) for lens in lenses)) + try: + await lens_db.execute_raw( + """UPDATE "LiteLLM_Lens" + SET due_at=$2::timestamp + WHERE id=ANY($1::text[])""", + tuple(lens.id for lens in lenses), + "1970-01-01 00:00:00", + ) + first: Final = await repo.due(scope, now, 20) + second: Final = await repo.due(scope, now, 20, first[-1]) + third: Final = await repo.due(scope, now, 20, second[-1]) + assert tuple(len(page) for page in (first, second, third)) == (20, 20, 5) + ids: Final = tuple(candidate.lens.id for candidate in (*first, *second, *third)) + assert ids == tuple(sorted(lens.id for lens in lenses)) + finally: + await lens_db.execute_raw( + 'DELETE FROM "LiteLLM_Lens" WHERE id=ANY($1::text[])', + tuple(lens.id for lens in lenses), + ) + + +@pytest.mark.asyncio +async def test_due_at_stays_consistent_through_job_lifecycle(lens_db: Prisma) -> None: + now: Final = datetime.now(timezone.utc).replace(microsecond=0) + scope: Final = Scope(team_id=uuid4().hex) + lens: Final = _scheduled_lens(uuid4().hex, scope, now, now) + repo: Final = LensRepository(WriterDatabase(PrismaWrapper(lens_db))) + worker: Final = Worker(id=uuid4().hex, name="worker", scope=scope, last_seen=now) + await repo.create(lens) + try: + await _assert_due_column(repo, lens.id) + job_id: Final = uuid4().hex + claimed: Final = await repo.update( + lens.id, + lambda candidate: claim_job(queue_job(candidate, now, job_id), worker, now), + attempts=1, + ) + assert claimed is not None + await _assert_due_column(repo, lens.id) + active: Final = current_job(claimed) + assert active is not None + progressed: Final = await repo.progress(lens.id, active, Progress()) + assert progressed is not None + await _assert_due_column(repo, lens.id) + result_at: Final = datetime.now(timezone.utc) + + def finish(candidate: Lens) -> Lens: + active_job: Final = current_job(candidate) + if active_job is None: + return candidate + return replace_job(candidate, end_job(active_job, "completed", result_at)).model_copy( + update={"next_run_at": result_at + timedelta(minutes=candidate.settings.interval_minutes)} + ) + + completed: Final = await repo.update(lens.id, finish, attempts=1) + assert completed is not None + await _assert_due_column(repo, lens.id) + cancelled_at: Final = datetime.now(timezone.utc) + cancelled: Final = await repo.update( + lens.id, + lambda candidate: cancel_job( + queue_job(candidate, cancelled_at, uuid4().hex, trigger="manual"), + cancelled_at, + ), + attempts=1, + ) + assert cancelled is not None + await _assert_due_column(repo, lens.id) + finally: + await lens_db.execute_raw('DELETE FROM "LiteLLM_Lens" WHERE id=$1', lens.id) + + +@pytest.mark.asyncio +async def test_sync_due_repairs_legacy_rows_and_ignores_stale_versions(lens_db: Prisma) -> None: + now: Final = datetime.now(timezone.utc).replace(microsecond=0) + team_id: Final = uuid4().hex + scope: Final = Scope(team_id=team_id) + worker: Final = Worker(id=uuid4().hex, name="worker", scope=scope, last_seen=now) + repo: Final = LensRepository(WriterDatabase(PrismaWrapper(lens_db))) + due_idle: Final = _scheduled_lens(uuid4().hex, scope, now, now - timedelta(minutes=20)) + future_idle: Final = _scheduled_lens(uuid4().hex, scope, now, now + timedelta(minutes=20)) + disabled_idle: Final = _scheduled_lens(uuid4().hex, scope, now, now - timedelta(minutes=10), enabled=False) + queued_lens: Final = queue_job( + _scheduled_lens(uuid4().hex, scope, now, now + timedelta(minutes=20), enabled=False), + now - timedelta(minutes=3), + uuid4().hex, + trigger="manual", + ) + live_lens: Final = claim_job( + queue_job( + _scheduled_lens(uuid4().hex, scope, now, now + timedelta(minutes=20)), + now - timedelta(minutes=10), + uuid4().hex, + ), + worker, + now, + ) + expired_claimed: Final = claim_job( + queue_job( + _scheduled_lens(uuid4().hex, scope, now, now + timedelta(minutes=20)), + now - timedelta(minutes=10), + uuid4().hex, + ), + worker, + now - timedelta(minutes=10), + ) + expired_lens: Final = expired_claimed.model_copy( + update={"jobs": (expired_claimed.jobs[0].model_copy(update={"lease_until": now - timedelta(minutes=5)}),)} + ) + candidates: Final = (due_idle, future_idle, disabled_idle, queued_lens, live_lens, expired_lens) + await asyncio.gather(*(repo.create(candidate) for candidate in candidates)) + try: + past: Final = now - timedelta(hours=1) + await lens_db.execute_raw( + """UPDATE "LiteLLM_Lens" + SET due_at=($2::timestamptz AT TIME ZONE 'UTC') + WHERE id=ANY($1::text[])""", + tuple(lens.id for lens in candidates), + past.isoformat(), + ) + legacy_due: Final = await repo.due(scope, now, 20) + assert {candidate.lens.id for candidate in legacy_due} == {lens.id for lens in candidates} + for candidate in legacy_due: + await repo.sync_due(candidate.lens) + repaired_due: Final = await repo.due(scope, now, 20) + assert {candidate.lens.id for candidate in repaired_due} == {due_idle.id, queued_lens.id, expired_lens.id} + await asyncio.gather(*(_assert_due_column(repo, lens.id) for lens in candidates)) + stale: Final = await repo.get(future_idle.id) + assert stale is not None + await lens_db.execute_raw( + """UPDATE "LiteLLM_Lens" + SET version=version+1, due_at=($2::timestamptz AT TIME ZONE 'UTC') + WHERE id=$1""", + stale.id, + past.isoformat(), + ) + await repo.sync_due(stale) + assert _stored_due_at(stale.id) == past.replace(tzinfo=None) + finally: + await lens_db.execute_raw( + 'DELETE FROM "LiteLLM_Lens" WHERE id=ANY($1::text[])', + tuple(lens.id for lens in candidates), + ) + + @pytest.mark.asyncio async def test_concurrent_workers_cannot_both_acquire_the_same_job(lens_db: Prisma) -> None: now: Final = datetime.now(timezone.utc) @@ -291,6 +560,19 @@ def test_db_push_creates_fresh_lens_tables_and_preserves_them_on_restart(monkeyp assert connection.execute( sql.SQL("SELECT id, data FROM {}").format(sql.Identifier(schema, "LiteLLM_Lens")) ).fetchall() == [("saved", {"keep": True})] + assert ( + connection.execute( + sql.SQL("SELECT due_at FROM {} WHERE id='saved'").format(sql.Identifier(schema, "LiteLLM_Lens")) + ).fetchone()[0] + is not None + ) + due_index: Final = connection.execute( + """SELECT indexdef FROM pg_indexes + WHERE schemaname=%s AND tablename='LiteLLM_Lens' AND indexname='LiteLLM_Lens_due_at_idx'""", + (schema,), + ).fetchone() + assert due_index is not None + assert "WHERE" not in due_index[0] finally: connection.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) diff --git a/tests/integration/database/test_lens_scheduler_load.py b/tests/integration/database/test_lens_scheduler_load.py new file mode 100644 index 00000000000..0afa41bea74 --- /dev/null +++ b/tests/integration/database/test_lens_scheduler_load.py @@ -0,0 +1,199 @@ +import asyncio +import json +import os +import sys +from collections.abc import AsyncIterator +from contextlib import AbstractAsyncContextManager +from datetime import datetime, timedelta, timezone +from time import perf_counter +from typing import Final +from uuid import uuid4 + +import pytest +import pytest_asyncio +from prisma import Prisma +from pydantic import TypeAdapter +from typing_extensions import LiteralString + +from litellm.proxy.db.prisma_client import PrismaWrapper +from litellm.proxy.lens.endpoints import claim_due +from litellm.proxy.lens.models import Evidence, Finding, Lens, LensSettings, Scope, Worker +from litellm.proxy.lens.repository import Database, LensRepository, Row, WriterDatabase +from litellm.proxy.lens.state import current_job + + +@pytest_asyncio.fixture(loop_scope="function") +async def lens_db() -> AsyncIterator[Prisma]: + async with Prisma(datasource={"url": os.environ["DATABASE_URL"]}) as db: + yield db + + +class ReadMeter: + def __init__(self) -> None: + self.batches: tuple[tuple[int, ...], ...] = () + + def record(self, document_sizes: tuple[int, ...]) -> None: + self.batches = (*self.batches, document_sizes) + + @property + def document_count(self) -> int: + return sum(len(batch) for batch in self.batches) + + @property + def total_bytes(self) -> int: + return sum(sum(batch) for batch in self.batches) + + +class MeasuredDatabase: + def __init__(self, database: WriterDatabase, meter: ReadMeter) -> None: + self.database: Final = database + self.meter: Final = meter + + async def query_raw(self, query: LiteralString, *args: object) -> object: + rows: Final = await self.database.query_raw(query, *args) + if 'FROM "LiteLLM_Lens"' in query and "WHERE id" not in query: + documents: Final = TypeAdapter(tuple[Row, ...]).validate_python(rows) + self.meter.record( + tuple(len(json.dumps(row.data, separators=(",", ":")).encode("utf-8")) for row in documents) + ) + return rows + + async def execute_raw(self, query: LiteralString, *args: object) -> int: + return await self.database.execute_raw(query, *args) + + def transaction(self) -> AbstractAsyncContextManager[Database]: + return self.database.transaction() + + +def _large_lens(lens_id: str, scope: Scope, now: datetime, next_run_at: datetime) -> Lens: + findings: Final = tuple( + Finding( + id=f"f{index}", + title=f"Issue {index}", + description="Repeated operation returns an unexpected result.", + check_id="behavior", + evidence=( + Evidence( + execution_id=f"t{index}", + span_id=f"s{index}", + quote="Unexpected result", + ), + ), + first_seen=now, + last_seen=now, + revision=1, + ) + for index in range(100) + ) + return Lens( + id=lens_id, + scope=scope, + settings=LensSettings( + name="Claim scheduler load", + model="analysis", + context="Find unexpected behavior", + enabled=True, + ), + created_at=now, + next_run_at=next_run_at, + findings=findings, + budget_month=now.strftime("%Y-%m"), + ) + + +def _due_lens(lens_id: str, scope: Scope, now: datetime, model: str, next_run_at: datetime) -> Lens: + return Lens( + id=lens_id, + scope=scope, + settings=LensSettings( + name="Claim paging test", + model=model, + context="Find unexpected behavior", + enabled=True, + ), + created_at=now, + next_run_at=next_run_at, + budget_month=now.strftime("%Y-%m"), + ) + + +async def _supports_model(_worker: Worker, _settings: LensSettings) -> bool: + return True + + +async def _supports_supported_model(_worker: Worker, settings: LensSettings) -> bool: + return settings.model == "supported" + + +@pytest.mark.asyncio +async def test_claim_due_reaches_a_supported_lens_behind_a_full_page_of_unsupported_ones( + lens_db: Prisma, +) -> None: + now: Final = datetime.now(timezone.utc).replace(microsecond=0) + scope: Final = Scope(team_id=uuid4().hex) + worker: Final = Worker(id=uuid4().hex, name="paging-test-worker", scope=scope, last_seen=now) + unsupported_at: Final = now - timedelta(minutes=5) + supported_at: Final = now - timedelta(minutes=1) + unsupported: Final = tuple(_due_lens(uuid4().hex, scope, now, "unsupported", unsupported_at) for _ in range(25)) + supported: Final = _due_lens(uuid4().hex, scope, now, "supported", supported_at) + candidates: Final = (*unsupported, supported) + repository: Final = LensRepository(WriterDatabase(PrismaWrapper(lens_db))) + await asyncio.gather(*(repository.create(candidate) for candidate in candidates)) + try: + claim: Final = await claim_due(worker, now, repository, _supports_supported_model) + assert claim is not None + assert claim.lens_id == supported.id + assert claim.job.status == "running" + finally: + await lens_db.execute_raw( + 'DELETE FROM "LiteLLM_Lens" WHERE id=ANY($1::text[])', + tuple(candidate.id for candidate in candidates), + ) + + +@pytest.mark.asyncio +async def test_lens_claim_reads_scale_with_due_lenses_not_total_lenses(lens_db: Prisma) -> None: + now: Final = datetime.now(timezone.utc).replace(microsecond=0) + scope: Final = Scope(team_id=uuid4().hex) + worker: Final = Worker(id=uuid4().hex, name="load-test-worker", scope=scope, last_seen=now) + due_lens: Final = _large_lens(uuid4().hex, scope, now, now - timedelta(seconds=1)) + initial_future: Final = tuple(_large_lens(uuid4().hex, scope, now, now + timedelta(days=1)) for _ in range(20)) + additional_future: Final = tuple(_large_lens(uuid4().hex, scope, now, now + timedelta(days=1)) for _ in range(200)) + ids: Final = tuple(lens.id for lens in (due_lens, *initial_future, *additional_future)) + writer: Final = WriterDatabase(PrismaWrapper(lens_db)) + seed_repository: Final = LensRepository(writer) + await asyncio.gather(*(seed_repository.create(lens) for lens in (due_lens, *initial_future))) + try: + before_meter: Final = ReadMeter() + before_repository: Final = LensRepository(MeasuredDatabase(writer, before_meter)) + before_started: Final = perf_counter() + before_claim: Final = await claim_due(worker, now, before_repository, _supports_model) + before_seconds: Final = perf_counter() - before_started + assert before_claim is not None + assert before_claim.lens_id == due_lens.id + assert before_claim.job.status == "running" + claimed_lens: Final = await seed_repository.get(due_lens.id) + assert claimed_lens is not None + assert current_job(claimed_lens) == before_claim.job + await seed_repository.update( + due_lens.id, + lambda lens: lens.model_copy(update={"jobs": (), "next_run_at": now - timedelta(seconds=1)}), + attempts=1, + ) + await asyncio.gather(*(seed_repository.create(lens) for lens in additional_future)) + after_meter: Final = ReadMeter() + after_repository: Final = LensRepository(MeasuredDatabase(writer, after_meter)) + after_started: Final = perf_counter() + after_claim: Final = await claim_due(worker, now, after_repository, _supports_model) + after_seconds: Final = perf_counter() - after_started + assert after_claim is not None + assert after_claim.lens_id == due_lens.id + assert after_claim.job.status == "running" + sys.stdout.write( + f"claim read: before={before_meter.total_bytes} bytes, {before_seconds:.4f}s; " + f"after={after_meter.total_bytes} bytes, {after_seconds:.4f}s\n" + ) + assert before_meter.document_count == after_meter.document_count == 1 + assert before_meter.total_bytes == after_meter.total_bytes + finally: + await lens_db.execute_raw('DELETE FROM "LiteLLM_Lens" WHERE id=ANY($1::text[])', ids) diff --git a/tests/proxy_behavior/lens/test_lifecycle.py b/tests/proxy_behavior/lens/test_lifecycle.py index e3027474baa..5f30a11cf70 100644 --- a/tests/proxy_behavior/lens/test_lifecycle.py +++ b/tests/proxy_behavior/lens/test_lifecycle.py @@ -197,7 +197,10 @@ async def test_team_route_requires_a_worker_with_matching_model_access(lens_data ) try: wrong_team: Final = await endpoints.register_worker(endpoints.WorkerName(analysis_key_id=key_b), admin) - assert await endpoints.claim_candidate(lens, wrong_team.worker, datetime.now(timezone.utc)) is None + assert ( + await endpoints.claim_candidate(lens, wrong_team.worker, datetime.now(timezone.utc), endpoints.repository()) + is None + ) for operation in ( endpoints.create_lens(settings, admin), endpoints.run_lens(lens.id, RunRequest(), admin), @@ -212,7 +215,9 @@ async def test_team_route_requires_a_worker_with_matching_model_access(lens_data assert edited.settings.context == "Use sources" right_team: Final = await endpoints.register_worker(endpoints.WorkerName(analysis_key_id=key_a), admin) await endpoints.validate_workers(settings, lens.scope) - claim: Final = await endpoints.claim_candidate(lens, right_team.worker, datetime.now(timezone.utc)) + claim: Final = await endpoints.claim_candidate( + lens, right_team.worker, datetime.now(timezone.utc), endpoints.repository() + ) assert claim is not None and claim.job.worker_id == right_team.worker.id finally: await lens_database.db.execute_raw( @@ -257,7 +262,10 @@ async def test_scan_lifecycle_persists_results_and_revokes_worker(lens_database: assert lens.id in tuple(e.id for e in listing.lenses) assert worker.id in tuple(w.id for w in listing.workers) claims: Final = await asyncio.gather( - *(endpoints.claim_candidate(lens, worker, datetime.now(timezone.utc)) for _ in range(8)) + *( + endpoints.claim_candidate(lens, worker, datetime.now(timezone.utc), endpoints.repository()) + for _ in range(8) + ) ) winners: Final = tuple(claim for claim in claims if claim is not None) assert len(winners) == 1 @@ -265,7 +273,10 @@ async def test_scan_lifecycle_persists_results_and_revokes_worker(lens_database: assert claimed.job.worker_id == worker.id assert ( await endpoints.claim_candidate( - await endpoints.get_lens(lens.id, worker.scope), worker, datetime.now(timezone.utc) + await endpoints.get_lens(lens.id, worker.scope), + worker, + datetime.now(timezone.utc), + endpoints.repository(), ) is None ) @@ -420,7 +431,9 @@ async def test_failed_model_requests_release_lens_budget_reservations(lens_datab registration: Final = await endpoints.register_worker(endpoints.WorkerName(analysis_key_id=key_id), admin) worker: Final = registration.worker try: - claimed: Final = await endpoints.claim_candidate(lens, worker, datetime.now(timezone.utc)) + claimed: Final = await endpoints.claim_candidate( + lens, worker, datetime.now(timezone.utc), endpoints.repository() + ) assert claimed is not None for _ in range(3): with pytest.raises(HTTPException) as failed: diff --git a/tests/unit/proxy/lens/test_endpoints.py b/tests/unit/proxy/lens/test_endpoints.py index 34d51110761..5ac5b756dd2 100644 --- a/tests/unit/proxy/lens/test_endpoints.py +++ b/tests/unit/proxy/lens/test_endpoints.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Final @@ -10,6 +11,7 @@ import litellm from litellm import Router from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.lens.endpoints import ( + claim_due, list_agents, read_reviews, result, @@ -35,8 +37,9 @@ from litellm.proxy.lens.models import ( Scope, TraceFindingsRequest, TraceIdentity, + Worker, ) -from litellm.proxy.lens.repository import Row +from litellm.proxy.lens.repository import DueLens, Row from litellm.proxy.lens.state import claim_job, queue_job, replace_job from tests.unit.proxy.lens.test_agent_workspace import execution from tests.unit.proxy.lens.test_state import NOW, lens, worker @@ -624,3 +627,53 @@ async def test_unknown_gateway_release_refuses_registration_and_claims(monkeypat await claim(worker(), protocol_version=PROTOCOL_VERSION, worker_release="") assert claim_error.value.status_code == 503 assert claim_error.value.detail == registration_error.value.detail + + +@pytest.mark.asyncio +async def test_claim_due_pages_through_more_than_a_thousand_full_pages() -> None: + candidate_lens: Final = lens() + + def candidate_page(page_number: int, size: int) -> tuple[DueLens, ...]: + return tuple( + DueLens( + lens=candidate_lens.model_copy(update={"id": f"lens-{page_number * 20 + offset:05}"}), + due_at=NOW, + ) + for offset in range(size) + ) + + full_pages: Final = tuple(candidate_page(page_number, 20) for page_number in range(1_200)) + pages: Final = (*full_pages, candidate_page(1_200, 1)) + assigned_worker: Final = worker() + + class PagingRepository: + def __init__(self) -> None: + self.after_calls: tuple[DueLens | None, ...] = () + + async def due( + self, scope: Scope, now: datetime, limit: int, after: DueLens | None = None + ) -> tuple[DueLens, ...]: + assert scope == assigned_worker.scope + assert now == NOW + assert limit == 20 + self.after_calls = (*self.after_calls, after) + return pages[len(self.after_calls) - 1] + + async def sync_due(self, lens: Lens) -> None: + return None + + async def update( + self, lens_id: str, transform: Callable[[Lens], Lens], attempts: int, *, changed_only: bool + ) -> Lens | None: + raise AssertionError("Unsupported models must not update candidates") + + async def reject_model(_worker: Worker, _settings: LensSettings) -> bool: + return False + + repository: Final = PagingRepository() + claim: Final = await claim_due(assigned_worker, NOW, repository, reject_model) + expected_after: Final = (None, *(page[-1] for page in pages[:-1])) + + assert claim is None + assert len(repository.after_calls) == 1_201 + assert repository.after_calls == expected_after diff --git a/tests/unit/proxy/lens/test_state.py b/tests/unit/proxy/lens/test_state.py index 9db69557e28..80eb638a610 100644 --- a/tests/unit/proxy/lens/test_state.py +++ b/tests/unit/proxy/lens/test_state.py @@ -37,6 +37,7 @@ from litellm.proxy.lens.state import ( cancel_job, claim_job, current_job, + due_at, end_job, merge_finding, next_scan_start, @@ -89,6 +90,22 @@ def worker(team: str = "alpha", identity: str = "worker") -> Worker: return Worker(id=identity, name=identity, scope=Scope(team_id=team), last_seen=NOW) +def lens_with_job( + status: Literal["queued", "running", "completed"], + lease_until: datetime | None = None, + *, + enabled: bool = True, + trigger: Literal["schedule", "manual"] = "schedule", +) -> Lens: + original: Final = lens() + configured: Final = original.model_copy( + update={"settings": original.settings.model_copy(update={"enabled": enabled})} + ) + queued: Final = queue_job(configured, NOW, "job", trigger=trigger) + job: Final = queued.jobs[0].model_copy(update={"status": status, "lease_until": lease_until}) + return queued.model_copy(update={"jobs": (job,)}) + + def finding(execution: str) -> FindingDraft: return FindingDraft( title="Repeated failed searches", @@ -98,6 +115,29 @@ def finding(execution: str) -> FindingDraft: ) +@pytest.mark.parametrize( + ("candidate", "expected"), + ( + pytest.param(lens(), NOW, id="idle-enabled"), + pytest.param( + lens().model_copy(update={"settings": lens().settings.model_copy(update={"enabled": False})}), + None, + id="idle-disabled", + ), + pytest.param(lens_with_job("queued", enabled=False, trigger="manual"), NOW, id="queued-manual-while-disabled"), + pytest.param( + lens_with_job("running", NOW + timedelta(minutes=5)), + NOW + timedelta(minutes=5), + id="running-with-lease", + ), + pytest.param(lens_with_job("running"), NOW, id="running-without-lease"), + pytest.param(lens_with_job("completed"), NOW, id="completed-only"), + ), +) +def test_due_at_matches_the_current_scheduling_state(candidate: Lens, expected: datetime | None) -> None: + assert due_at(candidate) == expected + + @pytest.mark.parametrize( ("viewer", "target", "allowed"), ( From 8f85de740f84fe3fbd22e4712e7a04ea92ca8501 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:24:25 -0700 Subject: [PATCH 26/35] perf(lens): prune ClickHouse partitions when sampling and sample in one pass (#45087) * perf(lens): prune ClickHouse partitions when sampling and sample in one pass Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * test(lens): allow unused query fixture field in load tests Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(lens): shrink the sample page when a response exceeds the read limit Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(lens): qualify request sample window columns Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- .../crates/storage-clickhouse/src/read.rs | 38 ++++- .../traces-clickhouse/query/lens_sample.sql | 8 + .../traces-clickhouse/src/query/lens.rs | 9 +- .../crates/traces-clickhouse/tests/load.rs | 146 ++++++++++++++++++ .../crates/traces-clickhouse/tests/queries.rs | 109 ++++++++++++- litellm/proxy/lens/endpoints.py | 40 +++-- tests/unit/proxy/lens/test_endpoints.py | 72 +++++++++ 7 files changed, 400 insertions(+), 22 deletions(-) create mode 100644 litellm-rust/crates/traces-clickhouse/tests/load.rs diff --git a/litellm-rust/crates/storage-clickhouse/src/read.rs b/litellm-rust/crates/storage-clickhouse/src/read.rs index c4bfdef393a..03a5522e595 100644 --- a/litellm-rust/crates/storage-clickhouse/src/read.rs +++ b/litellm-rust/crates/storage-clickhouse/src/read.rs @@ -61,6 +61,16 @@ pub async fn execute_read( connection: &Connection, sql: &str, parameters: &BTreeMap, +) -> Result { + execute_read_with_limits(client, connection, sql, parameters, READ_LIMITS).await +} + +async fn execute_read_with_limits( + client: &Client, + connection: &Connection, + sql: &str, + parameters: &BTreeMap, + limits: ReadLimits, ) -> Result { if sql.trim().is_empty() { return Err(Error::EmptySql); @@ -89,12 +99,9 @@ pub async fn execute_read( .clear() .extend_pairs(existing_pairs) .append_pair("readonly", "1") - .append_pair("max_result_rows", &READ_LIMITS.result_rows.to_string()) + .append_pair("max_result_rows", &limits.result_rows.to_string()) .append_pair("result_overflow_mode", "throw") - .append_pair( - "max_execution_time", - &READ_LIMITS.execution_seconds.to_string(), - ) + .append_pair("max_execution_time", &limits.execution_seconds.to_string()) .append_pair("wait_end_of_query", "1") .append_pair("default_format", "JSON"); @@ -122,7 +129,7 @@ pub async fn execute_read( let mut body = Vec::new(); while let Some(chunk) = response.chunk().await.map_err(|_| Error::Transport)? { - if body.len() + chunk.len() > READ_LIMITS.response_bytes { + if body.len() + chunk.len() > limits.response_bytes { return Err(Error::ResponseTooLarge); } body.extend_from_slice(&chunk); @@ -141,6 +148,7 @@ pub trait Query { type Params: Serialize; type Row: DeserializeOwned; + const READ_LIMITS: ReadLimits = crate::read::READ_LIMITS; const SQL: &'static str; } @@ -159,7 +167,14 @@ pub async fn fetch( connection: &Connection, params: &Q::Params, ) -> Result, Error> { - let body = execute_read(client, connection, Q::SQL, ¶meters(params)?).await?; + let body = execute_read_with_limits( + client, + connection, + Q::SQL, + ¶meters(params)?, + Q::READ_LIMITS, + ) + .await?; decode_rows::(&body) } @@ -168,7 +183,14 @@ pub async fn fetch_json( connection: &Connection, params: &Q::Params, ) -> Result { - let body = execute_read(client, connection, Q::SQL, ¶meters(params)?).await?; + let body = execute_read_with_limits( + client, + connection, + Q::SQL, + ¶meters(params)?, + Q::READ_LIMITS, + ) + .await?; decode_rows::(&body)?; Ok(body) } diff --git a/litellm-rust/crates/traces-clickhouse/query/lens_sample.sql b/litellm-rust/crates/traces-clickhouse/query/lens_sample.sql index 92086c33c13..5df0c8a1145 100644 --- a/litellm-rust/crates/traces-clickhouse/query/lens_sample.sql +++ b/litellm-rust/crates/traces-clickhouse/query/lens_sample.sql @@ -18,10 +18,14 @@ SELECT *, selection_key FROM ( WHERE {source:String} IN ('traces','both') AND ({all_teams:UInt8}=1 OR TeamId={team:String}) AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) + -- The 7 day slack covers spans that started before the window and late ingestion + AND Timestamp >= fromUnixTimestamp64Milli(toInt64({start:UInt64})) - INTERVAL 7 DAY AND (TeamId,ApiKeyHash,TraceId) IN ( SELECT TeamId,ApiKeyHash,TraceId FROM otel_traces WHERE ({all_teams:UInt8}=1 OR TeamId={team:String}) AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) + AND Timestamp >= fromUnixTimestamp64Milli(toInt64({start:UInt64})) - INTERVAL 7 DAY + AND Timestamp < fromUnixTimestamp64Milli(toInt64({end:UInt64})) AND if(EngineReceivedMs>0,toInt64(EngineReceivedMs), toUnixTimestamp64Milli(Timestamp)+toInt64(intDiv(Duration,1000000))) >= {start:UInt64} ) @@ -42,6 +46,8 @@ SELECT *, selection_key FROM ( WHERE {source:String} IN ('requests','both') AND ({all_teams:UInt8}=1 OR team_id={team:String}) AND ({key_hash:String}='' OR api_key={key_hash:String}) + AND spend_logs.start_time >= fromUnixTimestamp64Milli(toInt64({start:UInt64})) - INTERVAL 7 DAY + AND spend_logs.start_time < fromUnixTimestamp64Milli(toInt64({end:UInt64})) AND if(EngineReceivedMs>0,toInt64(EngineReceivedMs),toUnixTimestamp64Milli(end_time)) >= {start:UInt64} AND EngineReceivedMs < {end:UInt64} AND toUnixTimestamp64Milli(end_time) < {end:UInt64} @@ -55,6 +61,8 @@ SELECT *, selection_key FROM ( SELECT TeamId,ApiKeyHash,LiteLLMRequestId FROM otel_traces WHERE ({all_teams:UInt8}=1 OR TeamId={team:String}) AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) AND LiteLLMRequestId!='' + AND Timestamp >= fromUnixTimestamp64Milli(toInt64({start:UInt64})) - INTERVAL 7 DAY + AND Timestamp < fromUnixTimestamp64Milli(toInt64({end:UInt64})) )) ) WHERE ({selected_team:String}='' OR team_id={selected_team:String}) diff --git a/litellm-rust/crates/traces-clickhouse/src/query/lens.rs b/litellm-rust/crates/traces-clickhouse/src/query/lens.rs index 622a014599e..6439e696dad 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/lens.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/lens.rs @@ -1,4 +1,10 @@ -use litellm_storage_clickhouse::Query; +use litellm_storage_clickhouse::{Query, ReadLimits}; + +const SAMPLE_READ_LIMITS: ReadLimits = ReadLimits { + result_rows: 10_000, + response_bytes: 16 * 1024 * 1024, + ..litellm_storage_clickhouse::READ_LIMITS +}; pub const LENS_QUERIES: [litellm_traces::ReadQuery; 5] = [ litellm_traces::ReadQuery::Availability, @@ -188,6 +194,7 @@ impl Query for LensSample { type Params = LensSampleParams; type Row = LensSampleRow; + const READ_LIMITS: ReadLimits = SAMPLE_READ_LIMITS; const SQL: &'static str = include_str!("../../query/lens_sample.sql"); } diff --git a/litellm-rust/crates/traces-clickhouse/tests/load.rs b/litellm-rust/crates/traces-clickhouse/tests/load.rs new file mode 100644 index 00000000000..07c1095dfc3 --- /dev/null +++ b/litellm-rust/crates/traces-clickhouse/tests/load.rs @@ -0,0 +1,146 @@ +use std::collections::BTreeMap; + +use litellm_storage_clickhouse::READ_LIMITS; +use litellm_traces_clickhouse::{Connection, Parameter, ReadQuery, execute_named_read}; +use rstest::rstest; +use serde_json::Value; + +#[path = "queries/support.rs"] +#[expect( + dead_code, + reason = "load tests share the query fixture but do not read through QueryReaders" +)] +mod fixtures; +mod support; + +use fixtures::{DATABASE, SeededDatabase, migrated_database}; +use support::TestResult; + +const SPANS_PER_DAY: u64 = 2_000; + +async fn seed_days(fixture: &SeededDatabase, first_day: u64, days: u64) -> TestResult { + let count = SPANS_PER_DAY * days; + let first_row = SPANS_PER_DAY * first_day; + let query = format!( + "INSERT INTO {DATABASE}.otel_traces \ + (Timestamp, TraceId, SpanId, ParentSpanId, SpanName, ServiceName, ObservationType, TeamId, ApiKeyHash, Duration, SpanAttributes) \ + SELECT now64(9) - toIntervalHour(intDiv(number, {SPANS_PER_DAY}) * 24 + 12 + {first_day} * 24), \ + concat('load-', toString(number + {first_row})), concat('span-', toString(number + {first_row})), \ + '', 'span', 'service', 'agent', 'load-team', '', 0, \ + if({first_day}=0 AND number < {SPANS_PER_DAY}, map('payload', repeat('x', 3000)), map()) \ + FROM numbers({count})" + ); + fixture + .database + .client + .post(&fixture.database.url) + .body(query) + .send() + .await? + .error_for_status()?; + Ok(()) +} + +fn sample_parameters(start: u64, end: u64) -> BTreeMap { + BTreeMap::from([ + ("source".into(), Parameter::Text("traces".into())), + ("all_teams".into(), Parameter::Integer(0)), + ("team".into(), Parameter::Text("load-team".into())), + ("key_hash".into(), Parameter::Text(String::new())), + ("start".into(), Parameter::Unsigned(start)), + ("end".into(), Parameter::Unsigned(end)), + ("agent_name".into(), Parameter::Text(String::new())), + ("service".into(), Parameter::Text(String::new())), + ("filter_keys".into(), Parameter::Strings(Vec::new())), + ("filter_values".into(), Parameter::Strings(Vec::new())), + ("selected_team".into(), Parameter::Text(String::new())), + ("execution_ids".into(), Parameter::Strings(Vec::new())), + ("sample_cap".into(), Parameter::Unsigned(0)), + ("sample_percent".into(), Parameter::Integer(100)), + ("preview".into(), Parameter::Integer(0)), + ("after".into(), Parameter::Text(String::new())), + ("limit".into(), Parameter::Unsigned(10_000)), + ("offset".into(), Parameter::Unsigned(0)), + ]) +} + +async fn sample( + fixture: &SeededDatabase, + start: u64, + end: u64, + query_id: &str, +) -> TestResult<(usize, usize)> { + let mut url = Connection::configured(&fixture.database.url, DATABASE, "default", "")? + .url() + .clone(); + url.query_pairs_mut().append_pair("query_id", query_id); + let connection = Connection::parse(url.as_str())?; + let response = execute_named_read( + &fixture.database.client, + &connection, + ReadQuery::Sample, + &sample_parameters(start, end), + ) + .await?; + let result: Value = serde_json::from_str(&response)?; + Ok(( + result["data"].as_array().ok_or("sample rows")?.len(), + response.len(), + )) +} + +async fn query_read_rows(fixture: &SeededDatabase, query_id: &str) -> TestResult { + fixture + .database + .client + .post(&fixture.database.url) + .body("SYSTEM FLUSH LOGS") + .send() + .await? + .error_for_status()?; + let response = fixture + .database + .client + .post(&fixture.database.url) + .body(format!( + "SELECT read_rows FROM system.query_log WHERE type = 'QueryFinish' \ + AND query_id = '{query_id}' ORDER BY event_time DESC LIMIT 1 FORMAT JSON" + )) + .send() + .await? + .error_for_status()? + .text() + .await?; + let result: Value = serde_json::from_str(&response)?; + result["data"][0]["read_rows"] + .as_u64() + .ok_or_else(|| "query log read_rows missing".into()) +} + +#[rstest] +#[tokio::test] +async fn lens_sample_reads_scale_with_window_not_retention( + #[future(awt)] migrated_database: TestResult, +) -> TestResult { + let fixture = migrated_database?; + seed_days(&fixture, 0, 8).await?; + let now_ms = time::OffsetDateTime::now_utc().unix_timestamp() as u64 * 1000; + let start = now_ms - 86_400_000; + let end = now_ms + 60_000; + let before_id = format!("lens_sample_before_{}", std::process::id()); + let (before_rows, response_bytes) = sample(&fixture, start, end, &before_id).await?; + assert_eq!(before_rows, SPANS_PER_DAY as usize); + assert!(response_bytes > READ_LIMITS.response_bytes); + let before = query_read_rows(&fixture, &before_id).await?; + + seed_days(&fixture, 8, 24).await?; + let after_id = format!("lens_sample_after_{}", std::process::id()); + let (after_rows, _) = sample(&fixture, start, end, &after_id).await?; + assert_eq!(after_rows, SPANS_PER_DAY as usize); + let after = query_read_rows(&fixture, &after_id).await?; + assert!( + after * 100 <= before * 105, + "read_rows grew from {before} to {after}" + ); + Ok(()) +} diff --git a/litellm-rust/crates/traces-clickhouse/tests/queries.rs b/litellm-rust/crates/traces-clickhouse/tests/queries.rs index 5cb0ee4bfcd..6e63adfc347 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/queries.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/queries.rs @@ -3,7 +3,8 @@ use std::collections::BTreeMap; use litellm_storage_clickhouse::fetch; use litellm_traces::query::named as contracts; use litellm_traces_clickhouse::{ - QueryScope, + Connection, InsertTable, Parameter, QueryScope, ReadQuery, execute_named_read, execute_read, + insert_rows, query::named::{ListTraces, ListTracesParams, TraceSpans, TraceSpansParams}, query_help, query_sql, }; @@ -18,6 +19,112 @@ mod support; use fixtures::{SeededDatabase, insert_export, migrated_database, seeded_database}; use support::TestResult; +#[rstest] +#[tokio::test] +async fn lens_sample_keeps_spans_before_window_start_and_excludes_old_only_traces( + #[future(awt)] migrated_database: TestResult, +) -> TestResult { + let fixture = migrated_database?; + let start_ms = time::OffsetDateTime::now_utc().unix_timestamp() * 1000 - 86_400_000; + let end_ms = start_ms + 86_460_000; + let rows = [ + ( + "late-root", + "trace-with-slack", + start_ms - 2 * 86_400_000, + "", + ), + ( + "in-window", + "trace-with-slack", + start_ms + 1_000, + "late-root", + ), + ("old-span", "trace-too-old", start_ms - 8 * 86_400_000, ""), + ] + .into_iter() + .map(|(span_id, trace_id, timestamp_ms, parent_span_id)| { + BTreeMap::from([ + ( + "Timestamp".into(), + serde_json::json!(timestamp_ms * 1_000_000), + ), + ("Duration".into(), serde_json::json!(1_000_000)), + ("TraceId".into(), serde_json::json!(trace_id)), + ("SpanId".into(), serde_json::json!(span_id)), + ("ParentSpanId".into(), serde_json::json!(parent_span_id)), + ("SpanName".into(), serde_json::json!(span_id)), + ("ObservationType".into(), serde_json::json!("agent")), + ("TeamId".into(), serde_json::json!("team-lens")), + ("ApiKeyHash".into(), serde_json::json!("")), + ]) + }) + .collect(); + let writer = Connection::writer(&fixture.database.url)?; + insert_rows( + &fixture.database.client, + &writer, + fixtures::DATABASE, + InsertTable::OtelTraces, + rows, + ) + .await?; + let connection = + Connection::configured(&fixture.database.url, fixtures::DATABASE, "default", "")?; + let parameters = BTreeMap::from([ + ("source".into(), Parameter::Text("traces".into())), + ("all_teams".into(), Parameter::Integer(0)), + ("team".into(), Parameter::Text("team-lens".into())), + ("key_hash".into(), Parameter::Text(String::new())), + ("start".into(), Parameter::Unsigned(start_ms as u64)), + ("end".into(), Parameter::Unsigned(end_ms as u64)), + ("agent_name".into(), Parameter::Text(String::new())), + ("service".into(), Parameter::Text(String::new())), + ("filter_keys".into(), Parameter::Strings(Vec::new())), + ("filter_values".into(), Parameter::Strings(Vec::new())), + ("selected_team".into(), Parameter::Text(String::new())), + ("execution_ids".into(), Parameter::Strings(Vec::new())), + ("sample_cap".into(), Parameter::Unsigned(0)), + ("sample_percent".into(), Parameter::Integer(100)), + ("preview".into(), Parameter::Integer(0)), + ("after".into(), Parameter::Text(String::new())), + ("limit".into(), Parameter::Unsigned(10_000)), + ("offset".into(), Parameter::Unsigned(0)), + ]); + let body = execute_named_read( + &fixture.database.client, + &connection, + ReadQuery::Sample, + ¶meters, + ) + .await?; + let result: serde_json::Value = serde_json::from_str(&body)?; + let executions = result["data"].as_array().ok_or("sample rows")?; + let trace = executions + .iter() + .find(|row| row["trace_id"] == "trace-with-slack") + .ok_or("sampled trace missing")?; + let original_start = execute_read( + &fixture.database.client, + &connection, + "SELECT toString(fromUnixTimestamp64Nano({timestamp:Int64})) AS start_time FORMAT JSON", + &BTreeMap::from([( + "timestamp".into(), + Parameter::Integer((start_ms - 2 * 86_400_000) * 1_000_000), + )]), + ) + .await?; + let original_start: serde_json::Value = serde_json::from_str(&original_start)?; + assert_eq!(trace["span_count"].as_u64(), Some(2)); + assert_eq!(trace["start_time"], original_start["data"][0]["start_time"]); + assert!( + !executions + .iter() + .any(|row| row["trace_id"] == "trace-too-old") + ); + Ok(()) +} + #[derive(Clone, Copy, strum::AsRefStr)] #[strum(serialize_all = "snake_case")] enum ScopeCase { diff --git a/litellm/proxy/lens/endpoints.py b/litellm/proxy/lens/endpoints.py index d0b91aeea2f..d8b89a4a90e 100644 --- a/litellm/proxy/lens/endpoints.py +++ b/litellm/proxy/lens/endpoints.py @@ -76,6 +76,9 @@ CLAIM_CANDIDATES: Final = 20 _bearer: Final = HTTPBearer() Auth: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] StorageDep: TypeAlias = Annotated[Storage | None, Depends(provide_storage)] +SAMPLE_PAGE_SIZE: Final = 10_000 +SAMPLE_PAGE_SIZES: Final = (SAMPLE_PAGE_SIZE, 5_000, 2_500, 1_250, 625, 312, 156, 100) +SAMPLE_RESPONSE_TOO_LARGE: Final = "ClickHouse query exceeded the response size limit" class _ClaimRepository(Protocol): @@ -570,24 +573,37 @@ async def sample(lens_id: str, job_id: str, worker: WorkerAuth, storage: Storage lens, job = await assigned(lens_id, job_id, worker) if job.sample is not None: return job.sample - pages: list[Sample] = [] # mutable-ok: freeze selection after stable cursor traversal + + async def read_page(cursor: str, sizes: tuple[int, ...]) -> tuple[Sample, tuple[int, ...]]: + page_size: Final = sizes[0] + try: + page: Final = await source_reader(storage).sample( + lens.scope, + job.settings, + int(job.start.timestamp() * 1000), + int(job.end.timestamp() * 1000), + page_size=page_size, + cursor=cursor, + ) + except RuntimeError as error: + if type(error) is not RuntimeError or str(error) != SAMPLE_RESPONSE_TOO_LARGE or len(sizes) == 1: + raise + return await read_page(cursor, sizes[1:]) + return page, sizes + + pages: list[tuple[Sample, tuple[int, ...]]] = [] # mutable-ok: freeze selection after stable cursor traversal cursor = "" # rebind-ok: advance by immutable identity, never by shifting row positions while True: - page = await source_reader(storage).sample( - lens.scope, - job.settings, - int(job.start.timestamp() * 1000), - int(job.end.timestamp() * 1000), - cursor=cursor, - ) - pages.append(page) - if not page.next_cursor or sum(len(p.executions) for p in pages) >= pages[0].selected: + sizes: Final = pages[-1][1] if pages else SAMPLE_PAGE_SIZES + page, usable_sizes = await read_page(cursor, sizes) + pages.append((page, usable_sizes)) + if not page.next_cursor or sum(len(p.executions) for p, _ in pages) >= pages[0][0].selected: break cursor = page.next_cursor executions: Final = tuple( - execution for p in pages for execution in p.executions + execution for p, _ in pages for execution in p.executions ) # comprehension-ok: flatten query pages - selected: Final = Sample(executions=executions, eligible=pages[0].eligible, selected=len(executions)) + selected: Final = Sample(executions=executions, eligible=pages[0][0].eligible, selected=len(executions)) def freeze(e: Lens) -> Lens: active: Final = current_job(e) diff --git a/tests/unit/proxy/lens/test_endpoints.py b/tests/unit/proxy/lens/test_endpoints.py index 5ac5b756dd2..6e393337524 100644 --- a/tests/unit/proxy/lens/test_endpoints.py +++ b/tests/unit/proxy/lens/test_endpoints.py @@ -24,6 +24,9 @@ from litellm.proxy.lens.endpoints import ( watching, worker_supports_model, ) +from litellm.proxy.lens.endpoints import ( + sample as worker_sample, +) from litellm.proxy.lens.models import ( ActivitySelection, Coverage, @@ -41,6 +44,7 @@ from litellm.proxy.lens.models import ( ) from litellm.proxy.lens.repository import DueLens, Row from litellm.proxy.lens.state import claim_job, queue_job, replace_job +from litellm.rust_bridge.trace.generated.models import ExecutionRow, LensSampleParams from tests.unit.proxy.lens.test_agent_workspace import execution from tests.unit.proxy.lens.test_state import NOW, lens, worker @@ -67,6 +71,74 @@ class ResultDatabase: return len(self.completed) +@pytest.mark.asyncio +async def test_worker_sample_retries_oversized_pages_and_keeps_all_executions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + claimed: Final = claim_job(queue_job(lens(), NOW, "job"), worker(), NOW) + active: Final = claimed.jobs[0].model_copy(update={"lease_until": datetime.max.replace(tzinfo=timezone.utc)}) + db: Final = ResultDatabase(replace_job(claimed, active)) + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=db)) + rows: Final = tuple( + ExecutionRow( + source="traces", + trace_id=trace_id, + team_id="team", + name=trace_id, + start_time="", + span_count=1, + root_seen=1, + eligible=3, + selected=3, + selection_key=trace_id, + ) + for trace_id in ("trace-1", "trace-2", "trace-3") + ) + + class SampleStorage: + def __init__(self) -> None: + self.limits: tuple[int, ...] = () + + async def lens_sample(self, parameters: LensSampleParams) -> tuple[ExecutionRow, ...]: + self.limits = (*self.limits, parameters.limit) + if parameters.limit > 2_500: + raise RuntimeError("ClickHouse query exceeded the response size limit") + return rows + + storage: Final = SampleStorage() + selected: Final = await worker_sample("lens", "job", worker(), storage) + assert storage.limits == (10_000, 5_000, 2_500) + assert tuple(execution.trace_id for execution in selected.executions) == ("trace-1", "trace-2", "trace-3") + assert selected.selected == 3 + + +@pytest.mark.asyncio +async def test_worker_sample_propagates_response_too_large_at_minimum_page_size( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + claimed: Final = claim_job(queue_job(lens(), NOW, "job"), worker(), NOW) + active: Final = claimed.jobs[0].model_copy(update={"lease_until": datetime.max.replace(tzinfo=timezone.utc)}) + db: Final = ResultDatabase(replace_job(claimed, active)) + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=db)) + + class SampleStorage: + def __init__(self) -> None: + self.limits: tuple[int, ...] = () + + async def lens_sample(self, parameters: LensSampleParams) -> tuple[ExecutionRow, ...]: + self.limits = (*self.limits, parameters.limit) + raise RuntimeError("ClickHouse query exceeded the response size limit") + + storage: Final = SampleStorage() + with pytest.raises(RuntimeError, match="response size limit"): + await worker_sample("lens", "job", worker(), storage) + assert storage.limits == (10_000, 5_000, 2_500, 1_250, 625, 312, 156, 100) + + @pytest.mark.asyncio @pytest.mark.parametrize( "selected,check_id,quoted", From c877e0f055e6647316ca8d21d2b5b51f2a2f9ccf Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:32:04 -0700 Subject: [PATCH 27/35] fix(logging): bound data URI regex so base64 truncation stays linear (#45132) * fix(logging): bound data URI regex so base64 truncation stays linear Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(logging): cover whitespace-free data: prefixes in data URI regex regression test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: nate Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/logging_utils.py | 2 +- tests/unit/litellm_core_utils/test_logging_utils.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/logging_utils.py b/litellm/litellm_core_utils/logging_utils.py index 5e247324cea..5ac6b5435dc 100644 --- a/litellm/litellm_core_utils/logging_utils.py +++ b/litellm/litellm_core_utils/logging_utils.py @@ -43,7 +43,7 @@ Helper utils used for logging callbacks # Regex matching data-URI base64 content: "data:;base64," # Captures: group(1)=mime_type, group(2)=base64_payload -_DATA_URI_RE: Final = re.compile(r"data:([^;]+);base64,([A-Za-z0-9+/=]+)") +_DATA_URI_RE: Final = re.compile(r"data:([^;,\s]{1,255});base64,([A-Za-z0-9+/=]+)") # Maximum nesting depth for _truncate_base64_in_value to guard against # pathological payloads. OpenAI message format is typically 3-4 levels deep. diff --git a/tests/unit/litellm_core_utils/test_logging_utils.py b/tests/unit/litellm_core_utils/test_logging_utils.py index 7b8db097d47..edf0dc7960b 100644 --- a/tests/unit/litellm_core_utils/test_logging_utils.py +++ b/tests/unit/litellm_core_utils/test_logging_utils.py @@ -95,6 +95,18 @@ class TestTruncateBase64InString: result = _truncate_base64_in_string(text) assert result.count("base64_data truncated") == 2 + @pytest.mark.timeout(10) + @pytest.mark.parametrize( + "text", + [ + 'data: {"choices": [{"delta": {"content": "hi"}}]}\n\n' * 50_000, + "data:" * 200_000, + ], + ids=["sse_lines", "whitespace_free_prefixes"], + ) + def test_repeated_data_prefixes_without_data_uris_are_scanned_in_linear_time(self, text: str): + assert _truncate_base64_in_string(text) == text + def test_no_data_uri(self): text = "hello world, no base64 here" assert _truncate_base64_in_string(text) == text From a9b97007900f241994eeaf8582e7fde6bc557d1f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:50:35 -0700 Subject: [PATCH 28/35] perf(lens): bound single trace reads by the sampled start time (#45088) * perf(lens): bound single trace reads by the sampled start time Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * test(lens): allow unused query fixture field in load tests Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * test(lens): pass start_time in every lens content and evidence test Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- .../traces-clickhouse/query/lens_content.sql | 2 + .../traces-clickhouse/query/lens_evidence.sql | 2 + .../traces-clickhouse/src/query/lens.rs | 2 + .../traces-clickhouse/src/query/number.rs | 2 +- .../crates/traces-clickhouse/tests/load.rs | 83 ++++++++++++++++++- .../traces-clickhouse/tests/migrations.rs | 30 ++++++- litellm/proxy/lens/sources.py | 2 + litellm/rust_bridge/trace/generated/models.py | 2 + .../traces-clickhouse/LensContentParams.json | 4 + .../traces-clickhouse/LensEvidenceParams.json | 4 + tests/unit/proxy/lens/test_sources.py | 17 +++- tests/unit/rust_bridge/trace/test_queries.py | 2 + 12 files changed, 148 insertions(+), 4 deletions(-) diff --git a/litellm-rust/crates/traces-clickhouse/query/lens_content.sql b/litellm-rust/crates/traces-clickhouse/query/lens_content.sql index 99fb56a5f48..52355a11061 100644 --- a/litellm-rust/crates/traces-clickhouse/query/lens_content.sql +++ b/litellm-rust/crates/traces-clickhouse/query/lens_content.sql @@ -17,6 +17,7 @@ SELECT * FROM ( FROM otel_traces WHERE {source:String}='traces' AND ({all_teams:UInt8}=1 OR TeamId={team:String}) AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) + AND Timestamp >= parseDateTime64BestEffortOrZero({start_time:String}, 9) - INTERVAL 7 DAY AND ({trace_ref:String}='' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))={trace_ref:String}) AND TraceId={id:String} AND TeamId={record_team:String} AND SpanId > {cursor:String} ORDER BY SpanId LIMIT 1 BY SpanId LIMIT 40 @@ -35,5 +36,6 @@ SELECT * FROM ( FROM spend_logs FINAL WHERE {source:String}='requests' AND ({all_teams:UInt8}=1 OR team_id={team:String}) AND ({key_hash:String}='' OR api_key={key_hash:String}) + AND spend_logs.start_time >= parseDateTime64BestEffortOrZero({start_time:String}, 3) - INTERVAL 7 DAY AND request_id={id:String} AND team_id={record_team:String} LIMIT 1 ) diff --git a/litellm-rust/crates/traces-clickhouse/query/lens_evidence.sql b/litellm-rust/crates/traces-clickhouse/query/lens_evidence.sql index a0d600cdfde..b53617364cf 100644 --- a/litellm-rust/crates/traces-clickhouse/query/lens_evidence.sql +++ b/litellm-rust/crates/traces-clickhouse/query/lens_evidence.sql @@ -2,6 +2,7 @@ SELECT sum(matches) AS count FROM ( SELECT count() AS matches FROM otel_traces WHERE {source:String}='traces' AND ({all_teams:UInt8}=1 OR TeamId={team:String}) AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) + AND Timestamp >= parseDateTime64BestEffortOrZero({start_time:String}, 9) - INTERVAL 7 DAY AND ({trace_ref:String}='' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))={trace_ref:String}) AND TraceId={id:String} AND TeamId={record_team:String} AND SpanId={span:String} AND position(concat('Input: ',Input,'\nOutput: ',Output,'\nStatus: ',StatusCode,' ',StatusMessage),{quote:String})>0 @@ -9,6 +10,7 @@ SELECT sum(matches) AS count FROM ( SELECT count() AS matches FROM spend_logs FINAL WHERE {source:String}='requests' AND ({all_teams:UInt8}=1 OR team_id={team:String}) AND ({key_hash:String}='' OR api_key={key_hash:String}) + AND spend_logs.start_time >= parseDateTime64BestEffortOrZero({start_time:String}, 3) - INTERVAL 7 DAY AND request_id={id:String} AND team_id={record_team:String} AND request_id={span:String} AND position(concat('Input: ',messages,'\nOutput: ',response,'\nError: ',error_str),{quote:String})>0 ) diff --git a/litellm-rust/crates/traces-clickhouse/src/query/lens.rs b/litellm-rust/crates/traces-clickhouse/src/query/lens.rs index 6439e696dad..77474c7143d 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/lens.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/lens.rs @@ -209,6 +209,7 @@ pub struct LensContentParams { pub source: ContentSource, pub id: String, pub record_team: String, + pub start_time: String, pub trace_ref: String, pub cursor: String, #[serde(deserialize_with = "super::number::deserialize")] @@ -252,6 +253,7 @@ pub struct LensEvidenceParams { pub source: ContentSource, pub id: String, pub record_team: String, + pub start_time: String, pub trace_ref: String, pub span: String, pub quote: String, diff --git a/litellm-rust/crates/traces-clickhouse/src/query/number.rs b/litellm-rust/crates/traces-clickhouse/src/query/number.rs index 9283903fee1..7a07845d0cd 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/number.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/number.rs @@ -99,7 +99,7 @@ mod tests { fn content_rejects_unsupported_sources(#[case] source: &str, #[case] valid: bool) { let parameters = serde_json::json!({ "all_teams": 0, "team": "team", "key_hash": "", "source": source, "id": "id", - "record_team": "team", "trace_ref": "", "cursor": "", "offset": 0 + "record_team": "team", "start_time": "", "trace_ref": "", "cursor": "", "offset": 0 }); assert_eq!( serde_json::from_value::(parameters).is_ok(), diff --git a/litellm-rust/crates/traces-clickhouse/tests/load.rs b/litellm-rust/crates/traces-clickhouse/tests/load.rs index 07c1095dfc3..aaec2c17e54 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/load.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/load.rs @@ -25,7 +25,8 @@ async fn seed_days(fixture: &SeededDatabase, first_day: u64, days: u64) -> TestR "INSERT INTO {DATABASE}.otel_traces \ (Timestamp, TraceId, SpanId, ParentSpanId, SpanName, ServiceName, ObservationType, TeamId, ApiKeyHash, Duration, SpanAttributes) \ SELECT now64(9) - toIntervalHour(intDiv(number, {SPANS_PER_DAY}) * 24 + 12 + {first_day} * 24), \ - concat('load-', toString(number + {first_row})), concat('span-', toString(number + {first_row})), \ + if({first_day} = 0, concat('load-', toString(number + {first_row})), 'load-0'), \ + concat('span-', toString(number + {first_row})), \ '', 'span', 'service', 'agent', 'load-team', '', 0, \ if({first_day}=0 AND number < {SPANS_PER_DAY}, map('payload', repeat('x', 3000)), map()) \ FROM numbers({count})" @@ -41,6 +42,62 @@ async fn seed_days(fixture: &SeededDatabase, first_day: u64, days: u64) -> TestR Ok(()) } +async fn trace_start_time(fixture: &SeededDatabase) -> TestResult { + let query = format!( + "SELECT toString(Timestamp, 'UTC') AS start_time FROM {DATABASE}.otel_traces \ + WHERE TraceId = 'load-0' LIMIT 1 FORMAT JSON" + ); + let response = fixture + .database + .client + .post(&fixture.database.url) + .body(query) + .send() + .await? + .error_for_status()? + .text() + .await?; + let result: Value = serde_json::from_str(&response)?; + result["data"][0]["start_time"] + .as_str() + .map(str::to_owned) + .ok_or_else(|| "trace start time missing".into()) +} + +fn content_parameters(start_time: &str) -> BTreeMap { + BTreeMap::from([ + ("source".into(), Parameter::Text("traces".into())), + ("all_teams".into(), Parameter::Integer(0)), + ("team".into(), Parameter::Text("load-team".into())), + ("key_hash".into(), Parameter::Text(String::new())), + ("id".into(), Parameter::Text("load-0".into())), + ("record_team".into(), Parameter::Text("load-team".into())), + ("start_time".into(), Parameter::Text(start_time.into())), + ("trace_ref".into(), Parameter::Text(String::new())), + ("cursor".into(), Parameter::Text(String::new())), + ("offset".into(), Parameter::Integer(1)), + ]) +} + +async fn content(fixture: &SeededDatabase, start_time: &str, query_id: &str) -> TestResult { + let connection = Connection::configured( + &format!("{}?query_id={query_id}", fixture.database.url), + DATABASE, + "default", + "", + )?; + let response = execute_named_read( + &fixture.database.client, + &connection, + ReadQuery::Content, + &content_parameters(start_time), + ) + .await?; + let result: Value = serde_json::from_str(&response)?; + assert!(!result["data"].as_array().ok_or("content rows")?.is_empty()); + Ok(()) +} + fn sample_parameters(start: u64, end: u64) -> BTreeMap { BTreeMap::from([ ("source".into(), Parameter::Text("traces".into())), @@ -144,3 +201,27 @@ async fn lens_sample_reads_scale_with_window_not_retention( ); Ok(()) } + +#[rstest] +#[tokio::test] +async fn lens_content_reads_scale_with_trace_not_retention( + #[future(awt)] migrated_database: TestResult, +) -> TestResult { + let fixture = migrated_database?; + seed_days(&fixture, 0, 8).await?; + let start_time = trace_start_time(&fixture).await?; + let before_id = format!("lens_content_before_{}", std::process::id()); + content(&fixture, &start_time, &before_id).await?; + let before = query_read_rows(&fixture, &before_id).await?; + + seed_days(&fixture, 8, 24).await?; + let after_id = format!("lens_content_after_{}", std::process::id()); + content(&fixture, &start_time, &after_id).await?; + let after = query_read_rows(&fixture, &after_id).await?; + println!("lens_content read_rows: before={before}, after={after}"); + assert!( + after * 100 <= before * 105, + "read_rows grew from {before} to {after}" + ); + Ok(()) +} diff --git a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs index e51d9083c59..7630b3033a7 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs @@ -1150,6 +1150,7 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate( ("source".into(), Parameter::Text("traces".into())), ("id".into(), Parameter::Text("shared".into())), ("record_team".into(), Parameter::Text("team".into())), + ("start_time".into(), Parameter::Text(String::new())), ("trace_ref".into(), Parameter::Text(first_ref.into())), ("cursor".into(), Parameter::Text(String::new())), ("offset".into(), Parameter::Integer(1)), @@ -1177,6 +1178,7 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate( ("source".into(), Parameter::Text("traces".into())), ("id".into(), Parameter::Text("shared".into())), ("record_team".into(), Parameter::Text("team".into())), + ("start_time".into(), Parameter::Text(String::new())), ("trace_ref".into(), Parameter::Text(first_ref.into())), ("span".into(), Parameter::Text("root".into())), ("quote".into(), Parameter::Text(opposite.into())), @@ -1339,7 +1341,7 @@ async fn lens_selection_pages_without_losing_or_repeating_runs( #[case::traces("traces", 9)] #[case::requests("requests", 3)] #[tokio::test] -async fn lens_content_keeps_original_span_and_request_timestamps( +async fn lens_content_keeps_original_timestamps_with_start_time_slack( #[future(awt)] database: TestResult, #[case] source: &str, #[case] precision: usize, @@ -1379,6 +1381,30 @@ async fn lens_content_keeps_original_span_and_request_timestamps( ) .await?; let connection = Connection::configured(&database.url, "trace_test", "default", "")?; + let start_time_body = execute_read( + &database.client, + &connection, + "SELECT toString(fromUnixTimestamp64Nano({timestamp:Int64})) AS start_time FORMAT JSON", + &BTreeMap::from([( + "timestamp".into(), + Parameter::Integer(root_start + 86_400_000_000_000), + )]), + ) + .await?; + let start_time: serde_json::Value = serde_json::from_str(&start_time_body)?; + let start_time = start_time["data"][0]["start_time"] + .as_str() + .ok_or("start time missing")? + .to_owned(); + let parsed_time_body = execute_read( + &database.client, + &connection, + "SELECT toString(parseDateTime64BestEffortOrZero({start_time:String}, 9)) AS start_time FORMAT JSON", + &BTreeMap::from([("start_time".into(), Parameter::Text(start_time.clone()))]), + ) + .await?; + let parsed_time: serde_json::Value = serde_json::from_str(&parsed_time_body)?; + assert_eq!(parsed_time["data"][0]["start_time"], start_time); let parameters = BTreeMap::from([ ("source".into(), Parameter::Text(source.into())), ("all_teams".into(), Parameter::Integer(0)), @@ -1386,6 +1412,7 @@ async fn lens_content_keeps_original_span_and_request_timestamps( ("record_team".into(), Parameter::Text("team".into())), ("key_hash".into(), Parameter::Text(String::new())), ("trace_ref".into(), Parameter::Text(String::new())), + ("start_time".into(), Parameter::Text(start_time)), ("id".into(), Parameter::Text("run".into())), ("cursor".into(), Parameter::Text(String::new())), ("offset".into(), Parameter::Integer(1)), @@ -1453,6 +1480,7 @@ async fn lens_content_keeps_output_visible_after_long_input( ("record_team".into(), Parameter::Text("team".into())), ("key_hash".into(), Parameter::Text(String::new())), ("trace_ref".into(), Parameter::Text(String::new())), + ("start_time".into(), Parameter::Text(String::new())), ("id".into(), Parameter::Text("request".into())), ("cursor".into(), Parameter::Text(String::new())), ("offset".into(), Parameter::Integer(1)), diff --git a/litellm/proxy/lens/sources.py b/litellm/proxy/lens/sources.py index 015be69ecca..b1d142a6748 100644 --- a/litellm/proxy/lens/sources.py +++ b/litellm/proxy/lens/sources.py @@ -142,6 +142,7 @@ class SourceReader: id=execution.trace_id, trace_ref=execution.trace_ref, record_team=execution.team_id, + start_time=execution.start_time, cursor=cursor, offset=offset + 1, ) @@ -175,6 +176,7 @@ class SourceReader: id=execution.trace_id, trace_ref=execution.trace_ref, record_team=execution.team_id, + start_time=execution.start_time, span=evidence.span_id, quote=evidence.quote, ) diff --git a/litellm/rust_bridge/trace/generated/models.py b/litellm/rust_bridge/trace/generated/models.py index ea2c8bda648..5d84003aba2 100644 --- a/litellm/rust_bridge/trace/generated/models.py +++ b/litellm/rust_bridge/trace/generated/models.py @@ -215,6 +215,7 @@ class LensContentParams(LiteLLMBaseModel): source: ContentSource id: str record_team: str + start_time: str trace_ref: str cursor: str offset: int = Field(..., ge=0, le=4294967295) @@ -232,6 +233,7 @@ class LensEvidenceParams(LiteLLMBaseModel): source: ContentSource id: str record_team: str + start_time: str trace_ref: str span: str quote: str diff --git a/scripts/trace_codegen/schemas/traces-clickhouse/LensContentParams.json b/scripts/trace_codegen/schemas/traces-clickhouse/LensContentParams.json index 5ee5ab558ce..6026ccd26e1 100644 --- a/scripts/trace_codegen/schemas/traces-clickhouse/LensContentParams.json +++ b/scripts/trace_codegen/schemas/traces-clickhouse/LensContentParams.json @@ -39,6 +39,9 @@ "source": { "$ref": "#/$defs/ContentSource" }, + "start_time": { + "type": "string" + }, "team": { "type": "string" }, @@ -53,6 +56,7 @@ "source", "id", "record_team", + "start_time", "trace_ref", "cursor", "offset" diff --git a/scripts/trace_codegen/schemas/traces-clickhouse/LensEvidenceParams.json b/scripts/trace_codegen/schemas/traces-clickhouse/LensEvidenceParams.json index 07b9c216083..dbe9b32fdd6 100644 --- a/scripts/trace_codegen/schemas/traces-clickhouse/LensEvidenceParams.json +++ b/scripts/trace_codegen/schemas/traces-clickhouse/LensEvidenceParams.json @@ -36,6 +36,9 @@ "span": { "type": "string" }, + "start_time": { + "type": "string" + }, "team": { "type": "string" }, @@ -50,6 +53,7 @@ "source", "id", "record_team", + "start_time", "trace_ref", "span", "quote" diff --git a/tests/unit/proxy/lens/test_sources.py b/tests/unit/proxy/lens/test_sources.py index 8512db04eff..ad1f90f97ce 100644 --- a/tests/unit/proxy/lens/test_sources.py +++ b/tests/unit/proxy/lens/test_sources.py @@ -10,8 +10,10 @@ from litellm.proxy.lens.sources import SourceReader, execution_id, parse_executi from litellm.rust_bridge.trace.generated.models import ( ActivityAvailability, AgentRow, + CountRow, ExecutionRow, LensContentParams, + LensEvidenceParams, PartRow, ) from tests.unit.proxy.lens.test_agent_workspace import python_data @@ -183,9 +185,17 @@ async def test_recorded_times_survive_source_catalog_reads_search_and_python( class ContentStorage: async def lens_content(self, parameters: LensContentParams) -> tuple[PartRow, ...]: - assert parameters.source == source and parameters.record_team == "team" + assert ( + parameters.source == source + and parameters.record_team == "team" + and parameters.start_time == run.start_time + ) return rows + async def lens_evidence(self, parameters: LensEvidenceParams) -> tuple[CountRow, ...]: + assert parameters.start_time == run.start_time + return (CountRow(count=1),) + reader: Final = SourceReader(ContentStorage()) async def read(identity: str, cursor: str, offset: int) -> ExecutionContent: @@ -217,6 +227,11 @@ async def test_recorded_times_survive_source_catalog_reads_search_and_python( assert computed.sessions[0].parts == expected assert min(computed.sessions[0].parts, key=lambda part: part.start_time).span_id == rows[-1].span_id assert await workspace.valid(Evidence(execution_id=run.id, span_id=rows[0].span_id, quote=rows[0].content)) + assert await reader.verify_evidence( + Scope(team_id="team"), + run, + Evidence(execution_id=run.id, span_id=rows[0].span_id, quote=rows[0].content), + ) @pytest.mark.asyncio diff --git a/tests/unit/rust_bridge/trace/test_queries.py b/tests/unit/rust_bridge/trace/test_queries.py index 3c0556b1d97..9fd2964af20 100644 --- a/tests/unit/rust_bridge/trace/test_queries.py +++ b/tests/unit/rust_bridge/trace/test_queries.py @@ -18,6 +18,7 @@ def test_named_query_rejects_offsets_outside_the_native_integer_range(offset: in "source": "traces", "id": "trace", "record_team": "team", + "start_time": "", "trace_ref": "ref", "cursor": "", "offset": offset, @@ -34,6 +35,7 @@ def test_named_query_rejects_parameters_for_a_different_query() -> None: source="traces", id="trace", record_team="team", + start_time="", trace_ref="ref", cursor="", offset=0, From 7921716f3928ebf2d33b71012576efe0d26c31fa Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:57:04 -0700 Subject: [PATCH 29/35] feat(lens-ui): show findings ranked by priority with frequency and highlighted evidence (#45143) * feat(lens-ui): compute how often a finding hits sampled traces per day Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): copy a finding for an agent as markdown Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): add affected, unaffected and quote highlight color tokens Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): add a frequency card with stacked affected traces per day Co-Authored-By: Claude Opus 5.5 * fix(lens-ui): keep the issue brief title out of the page heading outline Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): lay out a finding as summary, fix, frequency and highlighted examples Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): show findings as a dated list with percent affected beside the open finding Co-Authored-By: Claude Opus 5.5 * test(lens-ui): cover frequency and highlighted quotes on a finding Co-Authored-By: Claude Opus 5.5 * test(lens-ui): follow findings into the split list and example cards Co-Authored-By: Claude Opus 5.5 * style(lens-ui): take finding chart and quote colors from the dashboard theme Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): add a shared priority dot and pill for findings Co-Authored-By: Claude Opus 5.5 * style(lens-ui): soften the frequency card and show its date range Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): rank findings under high, medium and low priority headings Co-Authored-By: Claude Opus 5.5 * style(lens-ui): show finding priority, label quotes by content and collapse extra examples Co-Authored-By: Claude Opus 5.5 * test(lens-ui): prove findings are grouped and ordered by priority Co-Authored-By: Claude Opus 5.5 * test(lens-ui): cover finding priority, quote labels and example collapsing Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- ui/litellm-dashboard/src/app/globals.css | 12 + .../lens/LensWorkspace.integration.test.tsx | 7 +- .../FindingDetails.integration.test.tsx | 76 +++- .../lens/investigations/FindingDetails.tsx | 423 +++++++++++++----- .../FindingsView.integration.test.tsx | 47 +- .../lens/investigations/FindingsView.tsx | 215 ++++----- .../lens/investigations/FrequencyCard.tsx | 88 ++++ .../InvestigationsView.integration.test.tsx | 12 +- .../lens/investigations/IssueBrief.tsx | 4 +- .../lens/investigations/PriorityMark.tsx | 40 ++ .../components/lens/model/findings.test.ts | 31 +- .../src/components/lens/model/findings.ts | 13 + .../components/lens/model/frequency.test.ts | 64 +++ .../src/components/lens/model/frequency.ts | 58 +++ .../src/components/lens/model/inbox.ts | 4 + 15 files changed, 845 insertions(+), 249 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/FrequencyCard.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/PriorityMark.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/model/frequency.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/model/frequency.ts diff --git a/ui/litellm-dashboard/src/app/globals.css b/ui/litellm-dashboard/src/app/globals.css index 3bc500c4848..36c22097cdc 100644 --- a/ui/litellm-dashboard/src/app/globals.css +++ b/ui/litellm-dashboard/src/app/globals.css @@ -211,6 +211,10 @@ --trace-row-hover: oklch(0.975 0.008 215); --trace-row-selected: oklch(0.95 0.035 200); --trace-brand: oklch(0.6 0.13 195); + --finding-affected: var(--info); + --finding-unaffected: oklch(0.551 0.027 264.364 / 0.45); + --finding-quote: color-mix(in oklab, var(--warning) 16%, transparent); + --finding-ring: 0 0 0 1px oklch(0 0 0 / 0.06), 0 1px 2px -1px oklch(0 0 0 / 0.06), 0 2px 4px 0 oklch(0 0 0 / 0.04); --trace-border: oklch(0.92 0.01 230); --trace-line: oklch(0.88 0.03 205); --trace-card-border: oklch(0.93 0.01 230); @@ -288,6 +292,10 @@ --trace-row-hover: oklch(0.23 0.018 230); --trace-row-selected: oklch(0.29 0.05 210); --trace-brand: oklch(0.78 0.13 190); + --finding-affected: var(--info); + --finding-unaffected: oklch(0.707 0.022 261.325 / 0.35); + --finding-quote: color-mix(in oklab, var(--warning) 24%, transparent); + --finding-ring: 0 0 0 1px oklch(1 0 0 / 0.08); --trace-border: oklch(0.3 0.02 235); --trace-line: oklch(0.36 0.04 210); --trace-card-border: oklch(0.27 0.02 235); @@ -326,6 +334,10 @@ --color-trace-row-hover: var(--trace-row-hover); --color-trace-row-selected: var(--trace-row-selected); --color-trace-brand: var(--trace-brand); + --color-finding-affected: var(--finding-affected); + --color-finding-unaffected: var(--finding-unaffected); + --color-finding-quote: var(--finding-quote); + --shadow-finding-ring: var(--finding-ring); --color-trace-border: var(--trace-border); --color-trace-line: var(--trace-line); --color-trace-card-border: var(--trace-card-border); diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx index c251b36d190..262d7ff9a51 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx @@ -141,9 +141,7 @@ describe("Lens interactive demo", () => { await user.click(await screen.findByRole("row", { name: /Repeated lookups leave customers without an answer/ })); const finding = screen.getByRole("complementary", { name: "Finding details" }); expect(within(finding).getByText(/The support agent retries/)).toBeVisible(); - const summaries = within(finding).getAllByText("support_agent", { exact: true }); - await user.click(summaries[0]); - await user.click(within(finding).getAllByRole("button", { name: /Open original step/ })[0]); + await user.click(within(finding).getAllByRole("button", { name: "View span" })[0]); expect(await screen.findByRole("complementary", { name: "Span details" })).toHaveTextContent( "I will check that for you.", ); @@ -157,7 +155,8 @@ describe("Lens interactive demo", () => { await user.click(within(finding).getByRole("button", { name: "Back to finding" })); expect(within(finding).getByText(/The support agent retries/)).toBeVisible(); await user.click(within(finding).getByRole("button", { name: "Close finding (Esc)" })); - expect(await screen.findByRole("table", { name: "Findings" })).toBeVisible(); + expect(await screen.findByRole("grid", { name: "Findings" })).toBeVisible(); + expect(screen.queryByRole("complementary", { name: "Finding details" })).not.toBeInTheDocument(); expect(network).not.toHaveBeenCalled(); await expectUrl(onUrlUpdate, (url) => expect(url.get("demo")).toBe("true")); await expectUrl(onUrlUpdate, (url) => expect(url.has("span")).toBe(false)); diff --git a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx index e705ce17e81..a6c1e60cb3b 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx @@ -116,8 +116,7 @@ it("stacks a quote's original step over the finding and keeps the feedback draft const panel = screen.getByRole("complementary", { name: "Finding details" }); const reason = () => within(panel).getByRole("textbox", { name: "What should Lens remember?", hidden: true }); fireEvent.change(reason(), { target: { value: "Draft feedback" } }); - for (const summary of within(panel).getAllByText(/quote$/)) await user.click(summary); - await user.click(within(panel).getAllByRole("button", { name: "Open original step" })[0]); + await user.click(within(panel).getAllByRole("button", { name: "View span" })[0]); expect(await within(panel).findByTestId("run-view")).toHaveTextContent("trace-1 at step-a"); expect(reason()).not.toBeVisible(); expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); @@ -127,13 +126,51 @@ it("stacks a quote's original step over the finding and keeps the feedback draft expect(reason()).toBeVisible(); expect(reason()).toHaveValue("Draft feedback"); - await user.click(within(panel).getAllByRole("button", { name: "Open original step" })[1]); + await user.click(within(panel).getAllByRole("button", { name: "View span" })[1]); expect(await within(panel).findByTestId("run-view")).toHaveTextContent("trace-2 at step-b"); const url = new URLSearchParams(String(onUrlUpdate.mock.lastCall?.[0].queryString ?? "")); expect(url.get("evidence")).toBe(traceOf("trace-2")); expect(url.get("evidence_span")).toBe("step-b"); }); +it("reports how many sampled traces the finding affected and highlights each quoted line", () => { + const traceOf = (id: string) => btoa(JSON.stringify(["traces", "", id])); + const sampled = ["a", "b", "c", "d"].map((id) => ({ + id: traceOf(id), + name: `run ${id}`, + start_time: "2026-10-01T10:00:00Z", + metadata: [], + root_seen: true, + service: "support_agent", + source: "traces" as const, + span_count: 1, + team_id: "", + trace_id: id, + trace_ref: "", + })); + const current: Finding = { + ...finding, + occurrences: [traceOf("a")], + evidence: [{ execution_id: traceOf("a"), span_id: "s", quote: "files:read is missing", role: "support" }], + }; + renderWithLens( + + + , + ); + expect(screen.getByRole("region", { name: "Frequency" })).toHaveTextContent(/25%\s*1 of 4 traces affected/); + const example = screen.getByRole("article", { name: "run a" }); + expect(within(example).getByText("files:read is missing").tagName).toBe("MARK"); + expect(screen.queryByRole("article", { name: "run b" })).not.toBeInTheDocument(); +}); + it("shows contributing investigation runs and every affected trace, including older traces without retained quotes", async () => { const traceId = btoa(JSON.stringify(["traces", "", "older-trace", ""])); const current: Finding = { @@ -142,8 +179,33 @@ it("shows contributing investigation runs and every affected trace, including ol investigation_runs: ["first-investigation-run", "second-investigation-run"], }; renderWithLens(); - expect(screen.getByText("Found across 2 investigation runs")).toBeInTheDocument(); - expect(screen.getByText(/1 affected trace/)).toBeInTheDocument(); - fireEvent.click(screen.getByText("older-trace")); - expect(screen.getByRole("button", { name: "Open original trace" })).toBeInTheDocument(); + expect(screen.getByText("1 affected trace")).toBeVisible(); + expect(screen.getByText("Found across 2 investigation runs")).toBeVisible(); + const example = screen.getByRole("article", { name: "Trace older-tr" }); + expect(within(example).getByText("No quote was retained for this trace.")).toBeVisible(); + expect(within(example).getByRole("button", { name: "View trace" })).toBeVisible(); +}); + +it("shows the finding's priority and keeps the first three examples, revealing the rest on request", async () => { + const user = userEvent.setup(); + const traceOf = (id: string) => btoa(JSON.stringify(["traces", "", id])); + const ids = ["t1", "t2", "t3", "t4", "t5"]; + const current: Finding = { + ...finding, + occurrences: ids.map(traceOf), + evidence: ids.map((id) => ({ + execution_id: traceOf(id), + span_id: id, + quote: `Input: ${id}\nOutput: done`, + role: "support" as const, + })), + }; + renderWithLens(); + const panel = screen.getByRole("complementary", { name: "Finding details" }); + expect(within(panel).getByText("High priority")).toBeVisible(); + expect(within(panel).getAllByRole("article")).toHaveLength(3); + expect(within(panel).getAllByText("Call and result")).toHaveLength(3); + await user.click(within(panel).getByRole("button", { name: "Show 2 more examples" })); + expect(within(panel).getAllByRole("article")).toHaveLength(5); + expect(within(panel).queryByRole("button", { name: /Show \d+ more/ })).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx index 95157a878d1..59ad282bbe0 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx @@ -1,23 +1,32 @@ "use client"; import { useState } from "react"; -import { ArrowUpRight } from "lucide-react"; +import { ChevronRight, ClipboardCopy, X } from "lucide-react"; import { Inspector } from "@/components/shared/Inspector"; import { Button } from "@/components/ui/button"; import { Textarea } from "@/components/ui/textarea"; +import { useNow } from "@/hooks/useNow"; +import { copyToClipboard } from "@/utils/dataUtils"; import { AddToDatasetButton } from "../datasets/AddToDatasetDialog"; -import { evidenceTarget } from "../model/findings"; -import { runTime } from "../model/format"; +import { evidenceTarget, findingMarkdown } from "../model/findings"; +import { findingFrequency } from "../model/frequency"; +import { agoLabel, runTime } from "../model/format"; import { findingAgents, findingKey, type OwnedFinding, sampledExecutions } from "../model/inbox"; import type { Finding, Sample } from "../model/types"; import { EvidenceView } from "./Evidence"; +import { FrequencyCard } from "./FrequencyCard"; import { IssueBrief } from "./IssueBrief"; +import { PriorityPill } from "./PriorityMark"; import { type EvidenceRef, useEvidenceRoute } from "../route"; export const ownedFindingKey = (owned: OwnedFinding): string => findingKey(owned.lens, owned.finding); +type Quote = Finding["evidence"][number]; + +const SECTION_LABEL = "text-xs font-medium text-muted-foreground"; + export interface FindingDetailsProps { readonly finding: Finding; readonly lensId?: string; @@ -27,6 +36,240 @@ export interface FindingDetailsProps { readonly busy: boolean; readonly onOpenEvidence: (evidence: EvidenceRef) => void; readonly onReview: (status: Finding["status"], reason: string) => void; + readonly onClose?: () => void; +} + +function TopBar({ finding, onClose }: Pick) { + const now = useNow(30000); + return ( +
+

+ + {finding.id.slice(0, 8)} + + + + {agoLabel(Date.parse(finding.last_seen), now)} + +

+
+ + {onClose && ( + + )} +
+
+ ); +} + +function Disclosure({ title, children }: { title: string; children: React.ReactNode }) { + return ( +
+ + +
{children}
+
+ ); +} + +function ProseSection({ title, children }: { title: string; children: string }) { + return ( +
+

{title}

+

+ {children} +

+
+ ); +} + +const FIELD = /^(Input|Output|Status|Error)\s*:/gm; +const FIELD_LABEL: Readonly> = { + "Input,Output": "Call and result", + Input: "Call input", + Output: "Returned output", + Status: "Span status", + Error: "Error", +}; + +function quoteLabel(quote: Quote, isTrace: boolean): string { + if (quote.role === "counterexample") return "Counterexample"; + const fields = [...new Set(Array.from(quote.quote.matchAll(FIELD), (m) => m[1]))].join(","); + return FIELD_LABEL[fields] ?? (isTrace ? "Trace step" : "Logged request"); +} + +const MARK = { + support: "rounded-sm bg-finding-quote px-0.5 text-inherit", + counterexample: "rounded-sm bg-success/20 px-0.5 text-inherit", +} as const; + +function QuoteCard({ quote, onOpen }: { quote: Quote; onOpen: () => void }) { + const isTrace = evidenceTarget(quote.execution_id)?.source === "traces"; + return ( +
+
+ {quoteLabel(quote, isTrace)} + +
+ +
+ ); +} + +function EvidenceRail({ children }: { children: React.ReactNode }) { + return ( + <> +
+
+
+
+
+ +
Evidence
+
+
+ + + ); +} + +interface ExampleGroup { + readonly id: string; + readonly run: Sample["executions"][number] | undefined; + readonly quotes: readonly Quote[]; +} + +function Example({ group, onOpenEvidence }: { group: ExampleGroup; onOpenEvidence: (e: EvidenceRef) => void }) { + const traceId = evidenceTarget(group.id)?.id; + const name = group.run?.name ?? (traceId ? `Trace ${traceId.slice(0, 8)}` : "Recorded run"); + return ( +
+
+

+ {name} +

+ + {[group.run?.service, group.run && runTime(group.run.start_time)].filter(Boolean).join(" · ")} + +
+
+ {group.quotes.length === 0 ? ( +
+

No quote was retained for this trace.

+ +
+ ) : ( + + {group.quotes.map((quote, i) => ( + onOpenEvidence({ id: quote.execution_id, span: quote.span_id })} + /> + ))} + + )} +
+
+ ); +} + +const VISIBLE_EXAMPLES = 3; + +function Examples({ + groups, + onOpenEvidence, +}: { + groups: readonly ExampleGroup[]; + onOpenEvidence: (e: EvidenceRef) => void; +}) { + const [expanded, setExpanded] = useState(false); + if (groups.length === 0) return

No examples were recorded.

; + const shown = expanded ? groups : groups.slice(0, VISIBLE_EXAMPLES); + const hidden = groups.length - shown.length; + return ( + <> + {shown.map((group) => ( + + ))} + {hidden > 0 && ( + + )} + + ); +} + +function ReviewForm({ finding, busy, onReview }: Pick) { + const [reason, setReason] = useState(finding.reason ?? ""); + return ( +
+