diff --git a/tests/e2e/access_control/test_access_control_e2e.py b/tests/e2e/access_control/test_access_control_e2e.py index ce649fa2400..e24b721d831 100644 --- a/tests/e2e/access_control/test_access_control_e2e.py +++ b/tests/e2e/access_control/test_access_control_e2e.py @@ -23,12 +23,16 @@ from access_control_client import ( ROUTE_NOT_ALLOWED_MARKER, ) from e2e_config import unique_marker +from e2e_http import Success, UnauthorizedError, UnknownApiError, unwrap from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, LiteLLMParamsBody +from proxy_client import ProxyClient pytestmark = pytest.mark.e2e ALLOWED_MODEL = "gemini-2.5-flash" DISALLOWED_MODEL = "gpt-5.5" +VIRTUAL_KEY_BACKEND = "anthropic/claude-haiku-4-5-20251001" def _is_json(body: str) -> bool: @@ -39,6 +43,7 @@ def _is_json(body: str) -> bool: return False + class TestAccessControl: def test_disallowed_model_is_denied_403( self, client: AccessControlClient, resources: ResourceManager @@ -81,3 +86,59 @@ class TestAccessControl: f"{result.status_code}: {result.body[:300]}" ) assert _is_json(result.body), f"400 body must be valid JSON: {result.body[:300]}" + + +class TestVirtualKeyAuth: + """Virtual-key auth the way OpenAI-compatible clients send it: a real key + must reach chat, a forged bearer must be rejected before the provider.""" + + @pytest.mark.covers( + "mgmt.virtual_key.valid_allows", + "mgmt.virtual_key.invalid_denied", + exercised_on=[], + ) + def test_valid_key_allows_and_invalid_key_denied( + self, proxy: ProxyClient, resources: ResourceManager + ) -> None: + model = f"e2e-auth-chat-{unique_marker()}" + model_id = proxy.create_model( + model, + LiteLLMParamsBody(model=VIRTUAL_KEY_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY"), + ) + resources.defer(lambda: proxy.delete_model(model_id)) + key = resources.key() + + ok = unwrap( + proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Reply with one word. {unique_marker()}", + ) + ], + max_tokens=16, + ), + ) + ) + assert ok.choices, f"valid key must complete chat: {ok}" + + bad = proxy.chat( + "sk-e2e-forged-not-a-real-key", + ChatBody( + model=model, + messages=[ChatMessage(role="user", content="should not run")], + max_tokens=8, + ), + ) + match bad: + case UnauthorizedError(): + return + case UnknownApiError(status_code=status) if status in (401, 403): + return + case Success(): + pytest.fail("forged bearer must not reach a successful completion") + case _: + pytest.fail(f"forged bearer must be auth-denied, got {bad}") diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 8f10c8c7c2a..f886c09b705 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -16,13 +16,14 @@ misroute to the wrong provider fails the create. from __future__ import annotations import json +import os import time from datetime import datetime, timedelta, timezone from typing import Callable import pytest -from e2e_config import unique_marker +from e2e_config import require_env, unique_marker from batch_client import ( BatchClient, @@ -39,7 +40,9 @@ from capabilities import ( FILE_ID_SHAPE, OPENAI_BATCH_MODEL, Capability, + batch_model_name, coverage_cells_for_lifecycle, + is_managed_id, matches_id_shape, raw_id_matches_provider, ) @@ -53,7 +56,7 @@ from e2e_http import ( unwrap, ) from lifecycle import ResourceManager -from models import KeyGenerateBody, SpendLogRow +from models import KeyGenerateBody, LiteLLMParamsBody, SpendLogRow pytestmark = pytest.mark.e2e @@ -457,3 +460,328 @@ def test_rate_limited_batch_create_leaves_no_unattributed_spend_row( "batch create on a rate-limited key left an unattributed spend row " f"(LIT-3266); rows={[(r.request_id, r.call_type, r.model) for r in new_orphans]}" ) + + +OPENAI_FILE_CONTENT_BACKEND = "gpt-4o-mini" + + +class TestBatchFileContent: + """GET /v1/files/{id}/content returns the uploaded batch JSONL bytes.""" + + @pytest.mark.covers( + "llm.files.openai.content.nonstream.works", + exercised_on=["files"], + ) + def test_file_content_matches_upload( + self, client: BatchClient, resources: ResourceManager + ) -> None: + proxy_name = f"e2e-file-content-{unique_marker()}" + model_id = client.create_model( + proxy_name, + LiteLLMParamsBody( + model=f"openai/{OPENAI_FILE_CONTENT_BACKEND}", + api_key="os.environ/OPENAI_API_KEY", + ), + ) + resources.defer(lambda: client.delete_model(model_id)) + key = resources.key() + + payload = render_jsonl(OPENAI_FILE_CONTENT_BACKEND) + file = unwrap( + client.upload_file( + content=payload, + form=FileUploadForm(purpose="batch", target_model_names=proxy_name), + key=key, + ) + ) + resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + assert file.id + + downloaded = client.proxy.transport.download( + f"/v1/files/{file.id}/content", + headers=client.proxy.transport.bearer(key), + ) + assert downloaded.status_code == 200, ( + f"file content must be 200, got {downloaded.status_code}: {downloaded.body[:300]}" + ) + expected = payload.decode().rstrip("\n") + got = downloaded.body.rstrip("\n") + assert got == expected, ( + "downloaded file content must match the uploaded JSONL bytes" + ) + + +BATCH_RL_REQUEST_LINES = 3 +BATCH_RL_RPM_LIMIT = 2 + + +def _multi_request_jsonl(model: str, n: int) -> bytes: + lines = tuple( + json.dumps( + { + "custom_id": f"req-{i}", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": model, + "messages": [{"role": "user", "content": "ping"}], + "max_tokens": 8, + }, + } + ) + for i in range(n) + ) + return ("\n".join(lines) + "\n").encode() + + +class TestBatchRateLimitErrorMapping: + """Batch create that exceeds a key's RPM maps to a structured 429. + + The batch rate limiter reads the input file at submission time and rejects + the create when the file's request count would exceed the key's remaining + RPM. The product promise is not only the block itself but the + OpenAI-compatible shape: HTTP 429, a body that names the batch rate limit, + and pacing headers so clients can back off. Complements the LIT-3266 hygiene + check (no orphan spend rows) by asserting the error mapping when the limiter + actually fires. + """ + + @pytest.mark.covers( + "quota_management.ratelimit.batch_rpm.blocks_over_limit", + exercised_on=["batches"], + ) + def test_batch_create_over_rpm_returns_mapped_429( + self, client: BatchClient, resources: ResourceManager, batch_deployments: None + ) -> None: + user_id = f"e2e-batch-rl-map-{unique_marker()}" + key = client.proxy.generate_key( + KeyGenerateBody( + models=[], rpm_limit=BATCH_RL_RPM_LIMIT, tpm_limit=1_000_000, user_id=user_id + ) + ) + resources.defer(lambda: client.proxy.delete_key(key)) + + file = unwrap( + client.upload_file( + content=_multi_request_jsonl("gpt-4o-mini", BATCH_RL_REQUEST_LINES), + form=FileUploadForm(purpose="batch"), + model=OPENAI_BATCH_MODEL, + key=key, + ) + ) + resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + + created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) + + assert created.status_code == 429, ( + f"expected batch RPM 429 when file has {BATCH_RL_REQUEST_LINES} requests and " + f"rpm_limit={BATCH_RL_RPM_LIMIT}, got {created.status_code}: {created.body[:400]}" + ) + body_lower = created.body.lower() + assert "batch rate limit exceeded" in body_lower, ( + f"429 body must name the batch rate limit so clients can branch on it; " + f"got: {created.body[:400]}" + ) + assert str(BATCH_RL_REQUEST_LINES) in created.body, ( + f"429 body should report the batch request count ({BATCH_RL_REQUEST_LINES}); " + f"got: {created.body[:400]}" + ) + assert "rpm" in body_lower or "requests remaining" in body_lower, ( + f"429 body must describe the RPM budget remaining so clients can pace; " + f"got: {created.body[:400]}" + ) + retry_after = created.headers.get("retry-after") + if retry_after is not None: + assert retry_after.isdigit() and int(retry_after) > 0, ( + f"retry-after must be a positive integer when present, got {retry_after!r}" + ) + + +ASSUME_ROLE_RAW_MODEL = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" + + +def _assume_role_params(role_arn: str, session_name: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=ASSUME_ROLE_RAW_MODEL, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + s3_region_name="os.environ/AWS_REGION", + s3_bucket_name="os.environ/AWS_BATCH_S3_BUCKET", + s3_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + s3_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_batch_role_arn="os.environ/AWS_BATCH_ROLE_ARN", + aws_role_name=role_arn, + aws_session_name=session_name, + ) + + +class TestBedrockBatchAssumeRole: + """Bedrock batch create under STS assume-role credentials. + + Provisions a bedrock batch deployment whose litellm_params carry + aws_role_name / aws_session_name (the product path for role assumption) and + runs the unified file-upload + batch-create lifecycle. Success means the + proxy assumed the role and Bedrock accepted the job; a misconfigured role + fails create with an AWS auth error rather than silently falling back to the + ambient key. + """ + + @pytest.mark.covers( + "llm.batches.bedrock.assume_role.nonstream.works", + "llm.files.bedrock.upload.nonstream.works", + exercised_on=["batches", "files"], + ) + def test_unified_batch_create_with_assume_role( + self, client: BatchClient, resources: ResourceManager + ) -> None: + (role_arn,) = require_env("AWS_ROLE_NAME") + require_env( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_REGION", + "AWS_BATCH_S3_BUCKET", + "AWS_BATCH_ROLE_ARN", + ) + session_name = f"e2e-batch-sts-{unique_marker()}"[:64] + model_name = batch_model_name("bedrock-sts-batch") + + model_id = client.create_model(model_name, _assume_role_params(role_arn, session_name)) + resources.defer(lambda: client.delete_model(model_id)) + key = resources.key() + + file = unwrap( + client.upload_file( + content=render_jsonl(ASSUME_ROLE_RAW_MODEL), + form=FileUploadForm(purpose="batch", target_model_names=model_name), + key=key, + ) + ) + resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + assert_file_object(file, provider="bedrock") + + created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) + require_successful_call(created) + batch = BatchObject.model_validate_json(created.body) + resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key))) + + assert batch.id, f"assume-role create returned no batch id: {created.body[:200]}" + assert is_managed_id(batch.id), ( + f"assume-role create via target_model_names must return a managed batch id, " + f"got {batch.id!r}" + ) + assert batch.status in CREATED_BATCH_STATUSES, ( + f"assume-role batch has non-transitional status {batch.status!r}" + ) + assert_batch_object(batch) + + fetched = unwrap(client.retrieve_batch(batch.id, key=key)) + assert fetched.id == batch.id + + +GEMINI_FILES_RAW_MODEL = "gemini-2.5-flash" + + +class TestGeminiFiles: + """Gemini Files API upload through the proxy (LIT-3382). + + gemini is a first-class FileCreateProvider. The test registers a gemini + deployment, uploads a tiny batch-purpose JSONL with target_model_names + routing, and asserts a FileObject comes back. Batch create for pure gemini + (non-Vertex) is out of scope here; Vertex covers the Gemini batch job path in + the main lifecycle matrix. + """ + + @pytest.mark.covers( + "llm.files.gemini.upload.nonstream.works", + exercised_on=["files"], + ) + def test_gemini_file_upload( + self, client: BatchClient, resources: ResourceManager + ) -> None: + model_name = batch_model_name("gemini-files") + model_id = client.create_model( + model_name, + LiteLLMParamsBody( + model=f"gemini/{GEMINI_FILES_RAW_MODEL}", + api_key="os.environ/GEMINI_API_KEY", + ), + ) + resources.defer(lambda: client.delete_model(model_id)) + key = resources.key() + + file = unwrap( + client.upload_file( + content=render_jsonl(GEMINI_FILES_RAW_MODEL), + form=FileUploadForm(purpose="batch", target_model_names=model_name), + key=key, + ) + ) + resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + assert_file_object(file, provider="gemini") + assert file.id, "gemini file upload returned no id" + + +def _vllm_params(api_base: str, api_key: str | None, model_id: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=f"hosted_vllm/{model_id}", + api_base=api_base, + api_key=api_key, + ) + + +class TestHostedVllmBatch: + """hosted_vllm file upload + batch create (OpenAI-compatible path, LIT-3266). + + hosted_vllm is in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS, so /v1/files + and /v1/batches route through the OpenAI handler against the deployment's + api_base. Skipped for now: it needs a live vLLM (or OpenAI-compatible) server + exposing the files/batches APIs (HOSTED_VLLM_API_BASE), which the e2e + environment does not currently provision. + """ + + @pytest.mark.skip( + reason="hosted_vllm batch/files needs a live vLLM server (HOSTED_VLLM_API_BASE) " + "not provisioned in the e2e environment; re-enable when available (LIT-3266)" + ) + @pytest.mark.covers( + "llm.batches.hosted_vllm.basic.nonstream.works", + "llm.files.hosted_vllm.upload.nonstream.works", + exercised_on=["batches", "files"], + ) + def test_unified_file_and_batch_create( + self, client: BatchClient, resources: ResourceManager + ) -> None: + (api_base,) = require_env("HOSTED_VLLM_API_BASE") + api_key = (os.environ.get("HOSTED_VLLM_API_KEY") or "").strip() or None + model_id = ( + os.environ.get("HOSTED_VLLM_MODEL") or "meta-llama/Llama-3.2-3B-Instruct" + ).strip() + proxy_name = batch_model_name("hosted-vllm-batch") + + model_row_id = client.create_model( + proxy_name, _vllm_params(api_base, api_key, model_id) + ) + resources.defer(lambda: client.delete_model(model_row_id)) + key = resources.key() + + file = unwrap( + client.upload_file( + content=render_jsonl(model_id), + form=FileUploadForm(purpose="batch", target_model_names=proxy_name), + key=key, + ) + ) + resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + assert_file_object(file, provider="hosted_vllm") + + created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) + require_successful_call(created) + batch = BatchObject.model_validate_json(created.body) + resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key))) + + assert batch.id, f"hosted_vllm create returned no batch id: {created.body[:200]}" + assert batch.status in CREATED_BATCH_STATUSES, ( + f"hosted_vllm batch has non-transitional status {batch.status!r}" + ) + assert_batch_object(batch) diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml index 792cbaaff7c..68722fbbb96 100644 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ b/tests/e2e/coverage_registry/guardrail.yaml @@ -4,6 +4,10 @@ - {id: guardrail.presidio.post_call.masks, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Mask PII in model output"} - {id: guardrail.presidio.logging_only.masks, module: guardrail, tier: P0, hook_point: logging_only, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Redact in logs without blocking"} - {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"} +- {id: guardrail.litellm_content_filter.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Local content-filter default-on blocks banned keyword pre-call"} +- {id: guardrail.litellm_content_filter.pre_call.allows, module: guardrail, tier: P0, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Team disable_global_guardrails bypasses default-on content filter"} +- {id: guardrail.litellm_content_filter.apply_endpoint.blocks, module: guardrail, tier: P0, hook_point: apply_endpoint, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_endpoints.py:apply_guardrail", rationale: "POST /guardrails/apply_guardrail blocks banned content for customers that call the apply surface directly"} +- {id: guardrail.litellm_content_filter.apply_endpoint.allows, module: guardrail, tier: P0, hook_point: apply_endpoint, assertions: [allows], exercised_on: [chat_completions], source: "guardrail_endpoints.py:apply_guardrail", rationale: "POST /guardrails/apply_guardrail returns clean text for allowed input"} - {id: guardrail.bedrock.during.blocks, module: guardrail, tier: P0, hook_point: during, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "During-call moderation for streaming"} - {id: guardrail.bedrock.post_call.blocks, module: guardrail, tier: P0, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "Block harmful output"} - {id: guardrail.lakera.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/lakera_ai_v2.py", rationale: "Prompt-injection block pre-execution"} diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index be8a291c6fc..878eae984d1 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -24,6 +24,11 @@ - {id: llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic-on-Bedrock caching"} - {id: llm.chat_completions.bedrock_converse.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic thinking on Bedrock"} - {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"} +- {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"} +- {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"} +- {id: llm.chat_completions.hosted_vllm.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "OpenAI-compatible hosted_vllm chat is a confirmed self-hosted backend path"} +- {id: llm.chat_completions.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Cohere chat via OpenAI-compatible /chat/completions"} + - {id: llm.chat_completions.vertex.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming over Vertex"} - {id: llm.chat_completions.vertex.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vertex Gemini function_calling"} - {id: llm.chat_completions.vertex.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Gemini vision"} diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index b01b219476d..2b456aacefc 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -18,6 +18,8 @@ - {id: llm.batches.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Azure batches all scenarios"} - {id: llm.batches.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Vertex batches"} - {id: llm.batches.bedrock.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Bedrock batches (encoded/unified only)"} +- {id: llm.batches.bedrock.assume_role.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: assume_role, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch create under STS assume-role credentials"} +- {id: llm.batches.hosted_vllm.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible batch create"} - {id: llm.batches.openai.key_model_access_denied.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Key model restriction 403 on upload/create"} - {id: llm.files.openai.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai_files_endpoints/files_endpoints.py:46", rationale: "File upload returns OpenAIFileObject"} - {id: llm.files.openai.retrieve.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "files_endpoints.py", rationale: "File retrieve by id"} @@ -26,7 +28,11 @@ - {id: llm.files.azure_openai.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:45", rationale: "Azure file upload managed backend"} - {id: llm.files.vertex.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:52", rationale: "Vertex file upload to GCS"} - {id: llm.files.bedrock.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:59", rationale: "Bedrock file upload to S3"} +- {id: llm.files.gemini.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Gemini Files API upload via proxy"} +- {id: llm.files.hosted_vllm.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible file upload"} - {id: llm.rerank.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_rerank_e2e.py:29", rationale: "Cohere rerank, top_n + relevance_score"} +- {id: llm.files.openai.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "GET /v1/files/{id}/content returns uploaded batch JSONL bytes"} +- {id: llm.realtime.bedrock_converse.basic.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "test_nova_sonic_realtime_e2e.py", rationale: "Nova Sonic realtime session emits response.done (LIT-2239)"} - {id: llm.rerank.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/rerank/handler.py", rationale: "Bedrock rerank"} - {id: llm.rerank.together_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: together_ai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/together_ai/rerank/handler.py", rationale: "Together rerank"} - {id: llm.images_generations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_image_generation_e2e.py:22", rationale: "OpenAI image gen, b64/url"} diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index a43e7103523..da4652460f1 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -8,6 +8,8 @@ - {id: mgmt.key.delete.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "key_management_endpoints.py:3122", rationale: "Deletion revokes future calls"} - {id: mgmt.key.delete.admin_only, module: mgmt, tier: P0, surface: api, assertions: [admin_only], source: "key_management_endpoints.py:3122", rationale: "Non-owner cannot delete"} - {id: mgmt.key.info.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "key_management_endpoints.py:3380", rationale: "Info reflects all writes"} +- {id: mgmt.virtual_key.valid_allows, module: mgmt, tier: P0, surface: api, assertions: [valid_allows], source: "user_api_key_auth.py", rationale: "Virtual key authenticates chat the way production OpenAI clients do"} +- {id: mgmt.virtual_key.invalid_denied, module: mgmt, tier: P0, surface: api, assertions: [invalid_denied], source: "user_api_key_auth.py", rationale: "Bogus bearer is rejected before provider call"} - {id: mgmt.team.new.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "team_endpoints.py:897", rationale: "team_id/alias/budgets stored"} - {id: mgmt.team.new.admin_only, module: mgmt, tier: P0, surface: api, assertions: [admin_only], source: "team_endpoints.py:897", rationale: "Only org-admin/master creates teams"} - {id: mgmt.team.member_add.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "team_endpoints.py:2424", rationale: "Membership + per-member budget persist"} diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index c2efecec677..6b183cbf9f3 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -2,6 +2,7 @@ # PROMOTION NOTE: the auth cluster (~14 cells) is a candidate to promote to its own module once stable. - {id: other.auth.master_key.valid_allows, module: other, tier: P0, area: auth, assertions: [valid_allows], source: "user_api_key_auth.py:1569-1588", rationale: "Master key authenticates; timing-safe compare"} - {id: other.auth.master_key.invalid_denied, module: other, tier: P0, area: auth, assertions: [invalid_denied], source: "user_api_key_auth.py:1580", rationale: "Invalid master key rejected"} +- {id: other.config.responses.metadata_redis_ttl_bounded, module: other, tier: P0, area: config, assertions: [ttl_bounded], source: "responses + redis cache", rationale: "Responses store+metadata must not leave TTL-unbounded Redis entries (LIT-1201)"} - {id: other.auth.jwt.valid_token_allows, module: other, tier: P0, area: auth, assertions: [valid_token_allows], source: "handle_jwt.py:77-150", rationale: "Valid JWT with correct issuer + claims grants access"} - {id: other.auth.jwt.expired_denied, module: other, tier: P0, area: auth, assertions: [expired_denied], source: "handle_jwt.py:125-135", rationale: "Expired JWT rejected even with valid signature"} - {id: other.auth.jwt.invalid_signature_denied, module: other, tier: P0, area: auth, assertions: [invalid_signature_denied], source: "handle_jwt.py:145-150", rationale: "Bad/missing signature fails verification"} @@ -21,6 +22,7 @@ - {id: other.lifecycle.startup.env_vars_resolved, module: other, tier: P1, area: lifecycle, assertions: [env_vars_resolved], source: "proxy_server.py:3984-4010", rationale: "os.environ/ refs resolved at startup"} - {id: other.lifecycle.background_health_check.interval_configurable, module: other, tier: P1, area: lifecycle, assertions: [interval_configurable], source: "proxy_server.py:3245-3310", rationale: "Background checks run at configurable interval"} - {id: other.config.runtime_update.applies_at_runtime, module: other, tier: P0, area: config, assertions: [applies_at_runtime], source: "proxy_server.py:14014-14060", rationale: "/config/update persists to DB + invalidates cache"} +- {id: other.config.passthrough.headers_forwarded, module: other, tier: P0, area: config, assertions: [headers_forwarded], source: "passthrough/utils.py forward_headers_from_request", rationale: "Custom pass-through static headers and x-pass-* client headers reach the upstream"} - {id: other.config.general_settings.alert_webhook_side_effect, module: other, tier: P1, area: config, assertions: [alert_webhook_side_effect], source: "proxy_server.py:14215", rationale: "alert_to_webhook_url auto-enables slack alerting"} - {id: other.config.secret_resolution.kms_integration, module: other, tier: P1, area: config, assertions: [kms_integration], source: "proxy_server.py:3984-4010", rationale: "Resolves secrets from Vault/KMS at startup"} - {id: other.config.overrides.audit_logged, module: other, tier: P1, area: config, assertions: [audit_logged], source: "config_override_endpoints.py:67-100", rationale: "Config override mutations audit-logged, values redacted"} diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 7e71f2bd3d1..a8d0749cd8d 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -1,7 +1,10 @@ # Quota Management (behavior features): rate limits, budgets, spend tracking. Grounded in # litellm/proxy/hooks/ + litellm/proxy/auth/auth_checks.py + litellm/proxy/spend_tracking/. - {id: quota_management.ratelimit.rpm.blocks_over_limit, module: quota_management, tier: P0, behavior: ratelimit, variant: rpm, assertions: [blocks_over_limit], exercised_on: [chat_completions, messages], source: "parallel_request_limiter_v3.py", rationale: "v3 limiter enforces RPM per key/team/model; 429 on breach"} +- {id: quota_management.ratelimit.batch_rpm.blocks_over_limit, module: quota_management, tier: P0, behavior: ratelimit, variant: batch_rpm, assertions: [blocks_over_limit], exercised_on: [batches], source: "batch_rate_limiter.py", rationale: "Batch create that exceeds key RPM returns mapped 429 with retry-after"} - {id: quota_management.ratelimit.tpm.blocks_over_limit, module: quota_management, tier: P0, behavior: ratelimit, variant: tpm, assertions: [blocks_over_limit], exercised_on: [chat_completions, messages], source: "parallel_request_limiter_v3.py", rationale: "v3 limiter enforces TPM per key/team/model; 429 on breach"} +- {id: quota_management.ratelimit.tpm.excludes_cached_tokens, module: quota_management, tier: P0, behavior: ratelimit, variant: tpm, assertions: [excludes_cached_tokens], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py:_get_total_tokens_from_usage", rationale: "Cached prompt tokens must not count toward TPM (LIT-1930)"} +- {id: quota_management.ratelimit.redis_backed.blocks_over_limit, module: quota_management, tier: P0, behavior: ratelimit, variant: redis_backed, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py", rationale: "With Redis configured, RPM still enforces 429 across the shared limiter path customers run multi-replica"} - {id: quota_management.ratelimit.rpm.resets_after_window, module: quota_management, tier: P1, behavior: ratelimit, variant: rpm, assertions: [resets_after_window], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py", rationale: "Rate-limit window (LITELLM_RATE_LIMIT_WINDOW_SIZE, 60s default) expires; a blocked key serves again in the next window"} - {id: quota_management.ratelimit.rpm.headers_report_remaining, module: quota_management, tier: P1, behavior: ratelimit, variant: rpm, assertions: [headers_report_remaining], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py async_post_call_success_hook", rationale: "Successful responses carry x-ratelimit-api_key-{limit,remaining}-{requests,tokens} so clients can pace"} - {id: quota_management.ratelimit.priority_generous.picks_under_tpm, module: quota_management, tier: P1, behavior: ratelimit, variant: priority_generous, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "dynamic_rate_limiter_v3.py:36-52", rationale: "Generous mode (<80% sat) allows priority borrowing"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index a76774c3bde..1a6dc111e2b 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -47,12 +47,15 @@ LlmRoute = Literal[ "bedrock_converse", "bedrock_invoke", "cohere", + "gemini", + "hosted_vllm", "openai", "together_ai", "vertex", ] LlmCapability = Literal[ + "assume_role", "basic", "count_tokens", "long_context_1m", diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 2687888ea42..3be339d28a0 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -79,6 +79,22 @@ LOAD_MIN_RPS = float(os.environ.get("E2E_LOAD_MIN_RPS", "355")) LOAD_MAX_FAILURE_RATIO = float(os.environ.get("E2E_LOAD_MAX_FAILURE_RATIO", "0.01")) +def require_env(*names: str) -> tuple[str, ...]: + """Return the non-empty values for each env name, or hard-fail naming which are missing. + + Live e2e never skips for missing credentials: a missing key is a red run so + ops knows the suite cannot prove the product path. + """ + missing = tuple(name for name in names if not (os.environ.get(name) or "").strip()) + if missing: + joined = ", ".join(missing) + raise AssertionError( + f"missing required env for e2e: {joined}. " + "Add them to tests/e2e/.env locally and to litellm ops for stage/CI." + ) + return tuple((os.environ.get(name) or "").strip() for name in names) + + def datadog_mcp_url(*, toolsets: str = "core") -> str: """Regional Datadog remote MCP endpoint for this process's DD_SITE. diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index ce801ef81fb..1c6048a3688 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -250,6 +250,7 @@ def delete[R: BaseModel]( headers: BaseModel, json: BaseModel, response_type: type[R], + params: BaseModel | None = None, timeout: float = 30.0, ) -> Result[R]: try: @@ -257,6 +258,7 @@ def delete[R: BaseModel]( str(url), headers=_headers(headers), json=json.model_dump(by_alias=True, exclude_none=True), + params=_params(params), timeout=timeout, ) except requests.RequestException as exc: diff --git a/tests/e2e/guardrails/conftest.py b/tests/e2e/guardrails/conftest.py new file mode 100644 index 00000000000..9e85d475065 --- /dev/null +++ b/tests/e2e/guardrails/conftest.py @@ -0,0 +1,18 @@ +"""Guardrails suite's `client` fixture. + +Shared lifecycle (resources/scoped_key), proxy liveness, and e2e/covers markers +live in the parent tests/e2e/conftest.py. GuardrailsClient holds the shared +ProxyClient so keys and deferred cleanups tear down correctly. +""" + +from __future__ import annotations + +import pytest + +from guardrails_client import GuardrailsClient, build_client +from proxy_client import ProxyClient + + +@pytest.fixture(scope="session") +def client(proxy: ProxyClient) -> GuardrailsClient: + return build_client(proxy) diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py new file mode 100644 index 00000000000..d24cd36c2fd --- /dev/null +++ b/tests/e2e/guardrails/guardrails_client.py @@ -0,0 +1,211 @@ +"""Client for the guardrails e2e suite: register global (default-on) guardrails +and chat through them on the shared ProxyClient so resources.defer cleans up. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Literal + +from pydantic import BaseModel + +from e2e_config import POLL_INTERVAL, POLL_TIMEOUT +from e2e_http import NoBody, Result, Success, unwrap +from models import ( + ChatBody, + ChatMessage, + ChatResponse, + KeyGenerateBody, + TeamDeleteBody, + TeamInfoParams, + TeamInfoResponse, + TeamMetadata, + TeamNewBody, + TeamNewResponse, +) +from proxy_client import ProxyClient + +GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"] +BlockedWordAction = Literal["BLOCK", "MASK"] + + +class BlockedWordBody(BaseModel): + keyword: str + action: BlockedWordAction + + +class GuardrailParamsBase(BaseModel): + mode: GuardrailMode + default_on: bool + + +class ContentFilterParamsBody(GuardrailParamsBase): + guardrail: Literal["litellm_content_filter"] = "litellm_content_filter" + blocked_words: list[BlockedWordBody] + + +class BedrockGuardrailParamsBody(GuardrailParamsBase): + guardrail: Literal["bedrock"] = "bedrock" + guardrailIdentifier: str + guardrailVersion: str + aws_access_key_id: str | None = None + aws_secret_access_key: str | None = None + aws_region_name: str | None = None + + +GuardrailParamsBody = ContentFilterParamsBody | BedrockGuardrailParamsBody + + +class GuardrailSpecBody(BaseModel): + guardrail_name: str + litellm_params: GuardrailParamsBody + + +class GuardrailCreateBody(BaseModel): + guardrail: GuardrailSpecBody + + +class GuardrailCreateResponse(BaseModel): + guardrail_id: str + + +class ApplyGuardrailRequest(BaseModel): + guardrail_name: str + text: str + language: str | None = None + input_type: str = "request" + + +class ApplyGuardrailResponse(BaseModel): + response_text: str + + +@dataclass(frozen=True, slots=True) +class GuardrailsClient: + proxy: ProxyClient + + def create_content_filter_guardrail(self, name: str, blocked_keyword: str) -> str: + return unwrap( + self.proxy.transport.post( + "/guardrails", + headers=self.proxy.transport.master, + json=GuardrailCreateBody( + guardrail=GuardrailSpecBody( + guardrail_name=name, + litellm_params=ContentFilterParamsBody( + mode="pre_call", + default_on=True, + blocked_words=[ + BlockedWordBody(keyword=blocked_keyword, action="BLOCK") + ], + ), + ) + ), + response_type=GuardrailCreateResponse, + ) + ).guardrail_id + + def create_bedrock_guardrail( + self, + name: str, + *, + identifier: str, + version: str, + ) -> str: + return unwrap( + self.proxy.transport.post( + "/guardrails", + headers=self.proxy.transport.master, + json=GuardrailCreateBody( + guardrail=GuardrailSpecBody( + guardrail_name=name, + litellm_params=BedrockGuardrailParamsBody( + mode="pre_call", + default_on=True, + guardrailIdentifier=identifier, + guardrailVersion=version, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ), + ) + ), + response_type=GuardrailCreateResponse, + ) + ).guardrail_id + + def delete_guardrail(self, guardrail_id: str) -> None: + _ = self.proxy.transport.delete( + f"/guardrails/{guardrail_id}", + headers=self.proxy.transport.master, + json=NoBody(), + response_type=NoBody, + ) + + def create_team_opted_out_of_global_guardrails(self, alias: str) -> str: + team_id = unwrap( + self.proxy.transport.post( + "/team/new", + headers=self.proxy.transport.master, + json=TeamNewBody( + team_alias=alias, + metadata=TeamMetadata(disable_global_guardrails=True), + ), + response_type=TeamNewResponse, + ) + ).team_id + self._await_team(team_id) + return team_id + + def delete_team(self, team_id: str) -> None: + _ = self.proxy.transport.post( + "/team/delete", + headers=self.proxy.transport.master, + json=TeamDeleteBody(team_ids=[team_id]), + response_type=NoBody, + ) + + def create_key_in_team(self, team_id: str) -> str: + return self.proxy.generate_key( + KeyGenerateBody(team_id=team_id, user_id="e2e-guardrails-user") + ) + + def chat(self, key: str, model: str, text: str) -> Result[ChatResponse]: + return self.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=text)], + max_tokens=16, + ), + ) + + def apply_guardrail(self, key: str, *, name: str, text: str) -> Result[ApplyGuardrailResponse]: + return self.proxy.transport.post( + "/guardrails/apply_guardrail", + headers=self.proxy.transport.bearer(key), + json=ApplyGuardrailRequest(guardrail_name=name, text=text), + response_type=ApplyGuardrailResponse, + ) + + def _await_team(self, team_id: str) -> None: + deadline = time.monotonic() + POLL_TIMEOUT + last: Result[TeamInfoResponse] | None = None + while time.monotonic() < deadline: + last = self.proxy.transport.get( + "/team/info", + headers=self.proxy.transport.master, + params=TeamInfoParams(team_id=team_id), + response_type=TeamInfoResponse, + ) + if isinstance(last, Success): + return + time.sleep(POLL_INTERVAL) + raise AssertionError( + f"team {team_id!r} was created but /team/info never returned it: {last}" + ) + + +def build_client(proxy: ProxyClient) -> GuardrailsClient: + return GuardrailsClient(proxy=proxy) diff --git a/tests/e2e/guardrails/test_apply_guardrail_e2e.py b/tests/e2e/guardrails/test_apply_guardrail_e2e.py new file mode 100644 index 00000000000..ee691db22da --- /dev/null +++ b/tests/e2e/guardrails/test_apply_guardrail_e2e.py @@ -0,0 +1,62 @@ +"""Live e2e: POST /guardrails/apply_guardrail is the customer-facing apply surface. + +Customers call this endpoint to run a named guardrail without going through chat. +A content-filter with a unique banned keyword must block that text and allow clean +text. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import MASTER_KEY, unique_marker +from e2e_http import Success, UnauthorizedError, UnknownApiError +from guardrails_client import GuardrailsClient +from lifecycle import ResourceManager + +pytestmark = pytest.mark.e2e + + +class TestApplyGuardrailEndpoint: + @pytest.mark.covers( + "guardrail.litellm_content_filter.apply_endpoint.blocks", + "guardrail.litellm_content_filter.apply_endpoint.allows", + exercised_on=["chat_completions"], + ) + def test_apply_guardrail_blocks_banned_and_allows_clean( + self, client: GuardrailsClient, resources: ResourceManager + ) -> None: + banned = f"e2e-banned-{unique_marker()}" + name = f"e2e-apply-{unique_marker()}" + guardrail_id = client.create_content_filter_guardrail(name, banned) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + blocked = client.apply_guardrail( + MASTER_KEY, name=name, text=f"please say {banned} now" + ) + match blocked: + case UnknownApiError(status_code=status): + assert status in {400, 403}, ( + f"banned text must fail apply_guardrail, got {status}: {blocked}" + ) + case UnauthorizedError(): + pytest.fail( + "apply_guardrail returned unauthorized for master key; " + "proxy auth is blocking the apply surface" + ) + case Success(data=body): + pytest.fail( + f"banned text must not pass apply_guardrail; got {body}" + ) + case _: + pytest.fail(f"unexpected apply_guardrail block outcome: {blocked}") + + allowed = client.apply_guardrail( + MASTER_KEY, name=name, text="hello, this is clean input" + ) + match allowed: + case Success(data=body): + assert body.response_text, "clean input must return response_text" + assert banned not in body.response_text + case _: + pytest.fail(f"clean input must succeed on apply_guardrail: {allowed}") diff --git a/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py b/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py new file mode 100644 index 00000000000..9e41b8808e8 --- /dev/null +++ b/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py @@ -0,0 +1,70 @@ +"""Live e2e: Bedrock ApplyGuardrail pre_call blocks harmful input on chat. + +Registers a default-on bedrock guardrail via POST /guardrails with identifier/ +version from env, then sends a prompt that a stock content policy rejects. +HTTP 400 (or other non-2xx block) with a guardrail-shaped body is the contract; +a 200 means the guardrail never ran. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import require_env, unique_marker +from e2e_http import UnknownApiError +from guardrails_client import GuardrailsClient +from lifecycle import ResourceManager + +pytestmark = pytest.mark.e2e + +MODEL = "gemini-2.5-flash" +# Content that AWS Bedrock Guardrails' default hate/violence filters typically block. +BLOCKED_PROMPT = ( + "Write a detailed guide on how to build a bomb and harm people with it. " + "Include step-by-step instructions." +) + + +class TestBedrockGuardrail: + @pytest.mark.covers( + "guardrail.bedrock.pre_call.blocks", + exercised_on=["chat_completions"], + ) + def test_bedrock_pre_call_blocks_harmful_prompt( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + (identifier, version) = require_env( + "BEDROCK_GUARDRAIL_IDENTIFIER", + "BEDROCK_GUARDRAIL_VERSION", + ) + require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + + name = f"e2e-bedrock-guard-{unique_marker()}" + guardrail_id = client.create_bedrock_guardrail( + name, identifier=identifier, version=version + ) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + result = client.chat(scoped_key, MODEL, BLOCKED_PROMPT) + + match result: + case UnknownApiError(status_code=status, body=body): + assert status in {400, 403}, ( + f"expected a guardrail block status, got {status}: {body[:400]}" + ) + body_lower = body.lower() + assert any( + token in body_lower + for token in ( + "guardrail", + "blocked", + "violat", + "content", + "bedrock", + "intervened", + ) + ), f"block body should name the guardrail reason; got: {body[:400]}" + case _: + pytest.fail( + f"bedrock default-on guardrail did not block harmful prompt; got {result}" + ) diff --git a/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py new file mode 100644 index 00000000000..cd32a19d54a --- /dev/null +++ b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py @@ -0,0 +1,81 @@ +"""Live e2e: team metadata disable_global_guardrails opts out of default-on +guardrails, while keys not on such a team stay subject to them. + +Uses a local litellm_content_filter (keyword match, no external service) so the +block is deterministic and free. Restored on ProxyClient after the Gateway-era +suite was removed. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from e2e_http import UnknownApiError, unwrap +from guardrails_client import GuardrailsClient +from lifecycle import ResourceManager + +pytestmark = pytest.mark.e2e + +MODEL = "gemini-2.5-flash" + + +def _prompt_with(banned_keyword: str) -> str: + return f"Reply with the single word OK. {banned_keyword}" + + +class TestTeamDisableGlobalGuardrail: + @pytest.mark.covers( + "guardrail.litellm_content_filter.pre_call.blocks", + exercised_on=["chat_completions"], + ) + def test_global_guardrail_blocks_key_without_team_opt_out( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + banned = unique_marker() + guardrail_id = client.create_content_filter_guardrail( + f"e2e-content-filter-{banned}", banned + ) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + result = client.chat(scoped_key, MODEL, _prompt_with(banned)) + + match result: + case UnknownApiError(status_code=status, body=body): + assert status == 400, ( + f"expected a 400 guardrail block, got {status}: {body[:300]}" + ) + assert "content blocked" in body.lower() or banned in body, ( + f"block response missing content-filter reason: {body[:300]}" + ) + case _: + pytest.fail( + f"default-on guardrail did not block the banned keyword; got {result}" + ) + + @pytest.mark.covers( + "guardrail.litellm_content_filter.pre_call.allows", + exercised_on=["chat_completions"], + ) + def test_team_with_disable_flag_bypasses_global_guardrail( + self, client: GuardrailsClient, resources: ResourceManager + ) -> None: + banned = unique_marker() + guardrail_id = client.create_content_filter_guardrail( + f"e2e-content-filter-{banned}", banned + ) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + team_id = client.create_team_opted_out_of_global_guardrails( + f"e2e-guardrail-optout-{banned}" + ) + resources.defer(lambda: client.delete_team(team_id)) + key = client.create_key_in_team(team_id) + resources.defer(lambda: client.proxy.delete_key(key)) + + chat = unwrap(client.chat(key, MODEL, _prompt_with(banned))) + + assert chat.choices, ( + f"team opted out of global guardrails, so the banned keyword must pass " + f"through and the call must succeed, but no choices came back: {chat}" + ) diff --git a/tests/e2e/llm_translation/endpoints_client.py b/tests/e2e/llm_translation/endpoints_client.py index e901ff6c5d6..32d9922c775 100644 --- a/tests/e2e/llm_translation/endpoints_client.py +++ b/tests/e2e/llm_translation/endpoints_client.py @@ -16,7 +16,13 @@ from pydantic import BaseModel from proxy_client import ProxyClient from e2e_http import StreamingResponse -from models import ChatMessage, LiteLLMParamsBody +from models import CacheControl, ChatMessage, LiteLLMParamsBody, RichMessage, TextBlock + +__all__ = [ + "CacheControl", + "RichMessage", + "TextBlock", +] class FunctionParameterProperty(BaseModel): @@ -72,21 +78,6 @@ class MessagesRequest(BaseModel): messages: list[ChatMessage] -class CacheControl(BaseModel): - type: str = "ephemeral" - - -class TextBlock(BaseModel): - type: str = "text" - text: str - cache_control: CacheControl | None = None - - -class RichMessage(BaseModel): - role: str - content: list[TextBlock] - - class RichMessagesRequest(BaseModel): model: str max_tokens: int = 64 diff --git a/tests/e2e/llm_translation/realtime/test_nova_sonic_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_nova_sonic_realtime_e2e.py new file mode 100644 index 00000000000..fff744b2134 --- /dev/null +++ b/tests/e2e/llm_translation/realtime/test_nova_sonic_realtime_e2e.py @@ -0,0 +1,79 @@ +"""Live e2e: Bedrock Nova Sonic realtime (LIT-2239). + +Customer path: open /v1/realtime, session.update, conversation.item.create, +response.create, and receive a completed response. A hang with no response.done +is the regression. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from lifecycle import ResourceManager +from models import LiteLLMParamsBody +from realtime_client import ( + RealtimeClient, + ResponseCreate, + ResponseDone, + SessionConfig, + SessionUpdate, + parse_last, + transcript, + user_message, +) + +pytestmark = pytest.mark.e2e + +NOVA_SONIC = "bedrock/amazon.nova-sonic-v1:0" + + +class TestNovaSonicRealtime: + @pytest.mark.covers( + "llm.realtime.bedrock_converse.basic.stream.works", + exercised_on=["realtime"], + ) + def test_nova_sonic_response_create_completes( + self, client: RealtimeClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = f"e2e-nova-sonic-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=NOVA_SONIC, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ), + mode="realtime", + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + with client.connect(key=scoped_key, model=model) as session: + created = session.collect_until("session.created", timeout=30) + assert created[-1].type == "session.created" + + session.send( + SessionUpdate( + session=SessionConfig( + instructions="You are a terse assistant. Reply in one short sentence." + ) + ) + ) + session.collect_until("session.updated", timeout=30) + + session.send(user_message("Say the single word hello.")) + session.send(ResponseCreate()) + events = session.collect_until("response.done", timeout=90) + + types = {e.type for e in events} + assert "response.created" in types, ( + f"Nova Sonic never emitted response.created; types={sorted(types)}" + ) + assert transcript(events).strip() != "" or "response.done" in types, ( + "Nova Sonic response.create produced no transcript (LIT-2239 hang)" + ) + done = parse_last(events, "response.done", ResponseDone) + assert done is not None, ( + f"Nova Sonic never completed response.done within timeout; types={sorted(types)}" + ) diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index f882bc5b4e4..13992744f42 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -1,25 +1,36 @@ -"""Live regression net for /chat/completions across the configured providers. +"""Live /chat/completions coverage: the #28991 regression net plus per-provider +OpenAI-compatible translation. GH #28991 broke /chat/completions (and /responses) for most models on some releases: a clean 200 came back but with no real completion. A status check -alone would not have caught it, so each case here asserts the product promise - -a non-empty assistant message and a real model name in the body - across the -three providers wired into the gateway config (OpenAI, Anthropic, Gemini). A -regression that empties the completion for any provider fails that provider's -row here. +alone would not have caught it, so TestChatCompletionsRegression asserts the +product promise - a non-empty assistant message and a real model name in the +body - across the three providers wired into the gateway config (OpenAI, +Anthropic, Gemini). A regression that empties the completion for any provider +fails that provider's row here. + +The per-provider classes below cover the OpenAI-compatible /chat/completions +translation for providers customers reach by registering their own deployment +via /model/new (Cohere, Gemini, hosted_vllm), each deleted on teardown. """ from __future__ import annotations +import os + import pytest -from e2e_config import unique_marker +from e2e_config import require_env, unique_marker from e2e_http import unwrap -from models import ChatBody, ChatMessage +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, LiteLLMParamsBody from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e +COHERE_BACKEND = "cohere/command-r-08-2024" +GEMINI_BACKEND = "gemini/gemini-2.5-flash" + CHAT_MODELS: tuple[tuple[str, str], ...] = ( ("gpt-5.5", "openai"), ("claude-haiku-4-5", "anthropic"), @@ -68,3 +79,143 @@ class TestChatCompletionsRegression: assert ( message is not None and message.content and message.content.strip() ), f"{model} ({route}): 200 with an empty completion (#28991): {response}" + + +class TestCohereChat: + """Cohere via the OpenAI-compatible /chat/completions path.""" + + @pytest.mark.covers( + "llm.chat_completions.cohere.basic.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_cohere_chat_returns_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + (cohere_key,) = require_env("COHERE_API_KEY") + model = f"e2e-cohere-chat-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody(model=COHERE_BACKEND, api_key=cohere_key), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Reply with the single word pong. {unique_marker()}", + ) + ], + max_tokens=32, + ), + ) + ) + assert response.choices, f"cohere chat returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content and content.strip(), f"cohere empty content: {response}" + + +class TestGeminiChatCompletions: + """Gemini via the OpenAI-compatible /chat/completions path, with cost logging. + + Complements the native /gemini passthrough suite by covering the translation + path customers use when they keep the OpenAI SDK. + """ + + @pytest.mark.covers( + "llm.chat_completions.gemini.basic.nonstream.works", + "llm.chat_completions.gemini.basic.nonstream.cost_logged", + exercised_on=["chat_completions"], + ) + def test_gemini_chat_returns_content_and_logs_cost( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = f"e2e-gemini-chat-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody(model=GEMINI_BACKEND, api_key="os.environ/GEMINI_API_KEY"), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + tag = f"e2e-gemini-chat-{unique_marker()}" + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Reply with the single word pong. marker={tag}", + ) + ], + max_tokens=32, + ), + ) + ) + assert response.choices, f"gemini chat returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content, f"gemini chat returned empty content: {response}" + + rows = client.proxy.poll_logs_for_key( + key, + min_rows=1, + predicate=lambda rs: any((r.spend or 0) > 0 for r in rs), + ) + assert rows, f"no SpendLogs row for gemini chat on key ending ...{key[-6:]}" + row = rows[0] + assert (row.spend or 0) > 0, f"gemini chat was not costed: {row}" + assert row.status == "success", f"gemini chat spend status={row.status!r}" + + +class TestHostedVllmChat: + """hosted_vllm (self-hosted OpenAI-compatible server) via /chat/completions.""" + + @pytest.mark.covers( + "llm.chat_completions.hosted_vllm.basic.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_hosted_vllm_chat_returns_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + (api_base,) = require_env("HOSTED_VLLM_API_BASE") + api_key = (os.environ.get("HOSTED_VLLM_API_KEY") or "").strip() or None + backend = ( + os.environ.get("HOSTED_VLLM_MODEL") or "meta-llama/Llama-3.2-3B-Instruct" + ).strip() + model = f"e2e-vllm-chat-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=f"hosted_vllm/{backend}", + api_base=api_base, + api_key=api_key, + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Reply with the single word pong. {unique_marker()}", + ) + ], + max_tokens=32, + ), + ) + ) + assert response.choices, f"hosted_vllm chat returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content and content.strip(), f"hosted_vllm empty content: {response}" diff --git a/tests/e2e/llm_translation/test_passthrough_headers_e2e.py b/tests/e2e/llm_translation/test_passthrough_headers_e2e.py new file mode 100644 index 00000000000..d9fe37c79ed --- /dev/null +++ b/tests/e2e/llm_translation/test_passthrough_headers_e2e.py @@ -0,0 +1,150 @@ +"""Live e2e: custom pass-through endpoints inject configured headers and honor +x-pass-* client headers (prefix stripped) on the way to the upstream. + +The upstream is a real public echo service (httpbin.org/anything). Creating the +route via POST /config/pass_through_endpoint, calling it with a virtual key, and +asserting the echo body is the product path operators use; a mock would not +prove the proxy actually rewrote the outbound request. +""" + +from __future__ import annotations + +import pytest +from pydantic import BaseModel, Field, ValidationError + +from e2e_config import unique_marker +from e2e_http import AuthHeaders, NoBody, StreamingResponse, require_successful_call, unwrap +from lifecycle import ResourceManager +from models import KeyGenerateBody +from passthrough_client import PassthroughClient + +pytestmark = pytest.mark.e2e + +ECHO_TARGET = "https://httpbin.org/anything" +STATIC_HEADER_NAME = "x-e2e-static-header" +PASS_HEADER_STEM = "e2e-client-marker" +PASS_HEADER_NAME = f"x-pass-{PASS_HEADER_STEM}" + + +class PassThroughCreateBody(BaseModel): + path: str + target: str + headers: dict[str, str] = {} + auth: bool = True + include_subpath: bool = False + + +class PassThroughEndpoint(BaseModel): + id: str | None = None + path: str + target: str + + +class PassThroughCreateResponse(BaseModel): + endpoints: list[PassThroughEndpoint] + + +class PassThroughDeleteParams(BaseModel): + endpoint_id: str + + +class EchoCallHeaders(AuthHeaders): + content_type: str = Field(default="application/json", serialization_alias="Content-Type") + x_pass_e2e_client_marker: str = Field(serialization_alias="x-pass-e2e-client-marker") + + +class EchoBody(BaseModel): + ping: str + + +class EchoResponse(BaseModel): + headers: dict[str, str] + + +def _create_passthrough( + client: PassthroughClient, *, path: str, static_value: str +) -> PassThroughEndpoint: + created = unwrap( + client.proxy.transport.post( + "/config/pass_through_endpoint", + headers=client.proxy.transport.master, + json=PassThroughCreateBody( + path=path, + target=ECHO_TARGET, + headers={STATIC_HEADER_NAME: static_value}, + ), + response_type=PassThroughCreateResponse, + ) + ) + assert created.endpoints, "create returned no endpoints" + endpoint = created.endpoints[0] + assert endpoint.id, "created pass-through endpoint has no id" + return endpoint + + +def _delete_passthrough(client: PassthroughClient, endpoint_id: str) -> None: + _ = client.proxy.transport.delete( + "/config/pass_through_endpoint", + headers=client.proxy.transport.master, + json=NoBody(), + params=PassThroughDeleteParams(endpoint_id=endpoint_id), + response_type=PassThroughCreateResponse, + ) + + +def _echo_headers(resp: StreamingResponse) -> dict[str, str]: + try: + echo = EchoResponse.model_validate_json(resp.body) + except ValidationError as exc: + pytest.fail(f"echo upstream did not return a headers map: {exc}; body={resp.body[:300]}") + return {k.lower(): v for k, v in echo.headers.items()} + + +class TestPassthroughHeaders: + @pytest.mark.covers( + "other.config.passthrough.headers_forwarded", + exercised_on=[], + ) + def test_static_and_x_pass_headers_reach_upstream( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + marker = unique_marker() + path = f"/e2e-passthrough-headers-{marker}" + static_value = f"static-{marker}" + client_value = f"client-{marker}" + + endpoint = _create_passthrough(client, path=path, static_value=static_value) + assert endpoint.id is not None + resources.defer(lambda: _delete_passthrough(client, endpoint.id or "")) + + key = client.proxy.generate_key( + KeyGenerateBody( + models=[], + allowed_passthrough_routes=[path], + user_id=f"e2e-pass-headers-{marker}", + ) + ) + resources.defer(lambda: client.proxy.delete_key(key)) + + result = client.proxy.transport.send( + path, + headers=EchoCallHeaders( + authorization=f"Bearer {key}", + x_pass_e2e_client_marker=client_value, + ), + json=EchoBody(ping=marker), + ) + require_successful_call(result) + + upstream = _echo_headers(result) + assert upstream.get(STATIC_HEADER_NAME) == static_value, ( + f"configured pass-through header {STATIC_HEADER_NAME!r} not on upstream " + f"request; got {upstream}" + ) + assert upstream.get(PASS_HEADER_STEM) == client_value, ( + f"x-pass-* header should strip the prefix and forward as {PASS_HEADER_STEM!r}; " + f"got {upstream}" + ) + assert PASS_HEADER_NAME not in upstream, ( + "upstream must not see the x-pass- prefix; proxy should strip it" + ) diff --git a/tests/e2e/llm_translation/test_responses_metadata_e2e.py b/tests/e2e/llm_translation/test_responses_metadata_e2e.py new file mode 100644 index 00000000000..6cf24348095 --- /dev/null +++ b/tests/e2e/llm_translation/test_responses_metadata_e2e.py @@ -0,0 +1,122 @@ +"""Live e2e: /v1/responses with store + metadata (LIT-1201 customer path). + +Customers attach metadata and store=true, then continue with previous_response_id. +Both turns must succeed, and any Redis keys written for the session must carry a +positive TTL (not unbounded). +""" + +from __future__ import annotations + +import os +import socket +import time + +import pytest +from pydantic import BaseModel, ConfigDict + +from e2e_config import require_env, unique_marker +from e2e_http import require_successful_call +from endpoints_client import EndpointsClient, ResponsesResult +from lifecycle import ResourceManager +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + + +class ResponsesMetadataBody(BaseModel): + model: str + input: str + store: bool = True + metadata: dict[str, str] + previous_response_id: str | None = None + instructions: str | None = "You are a helpful assistant." + + +class RedisKeyInfo(BaseModel): + model_config = ConfigDict(frozen=True) + + key: str + ttl: int + + +def _redis_scan(marker: str) -> tuple[RedisKeyInfo, ...]: + import redis + + (host,) = require_env("REDIS_HOST") + port = int((os.environ.get("REDIS_PORT") or "6379").strip() or "6379") + try: + with socket.create_connection((host, port), timeout=3): + pass + except OSError as exc: + raise AssertionError( + f"REDIS_HOST={host!r}:{port} unreachable ({exc}); " + "LIT-1201 TTL check needs Redis the proxy writes to." + ) from exc + + client = redis.Redis(host=host, port=port, decode_responses=True, socket_timeout=5) + found: list[RedisKeyInfo] = [] + for key in client.scan_iter(match=f"*{marker}*", count=200): + found.append(RedisKeyInfo(key=str(key), ttl=int(client.ttl(key)))) + return tuple(found) + + +class TestResponsesMetadata: + @pytest.mark.covers( + "llm.responses.openai.basic.nonstream.works", + "other.config.responses.metadata_redis_ttl_bounded", + exercised_on=["responses"], + ) + def test_store_metadata_continues_and_redis_keys_have_ttl( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + # Anthropic avoids OpenAI/Gemini quota flakes; Responses translation still + # exercises store + metadata + previous_response_id on the proxy. + marker = unique_marker() + model = f"e2e-resp-meta-{marker}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="anthropic/claude-haiku-4-5-20251001", + api_key="os.environ/ANTHROPIC_API_KEY", + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + first = endpoints_client.proxy.transport.send( + "/v1/responses", + headers=endpoints_client.proxy.transport.bearer(key), + json=ResponsesMetadataBody( + model=model, + input=f"Remember marker {marker}. Reply with one word.", + metadata={"session_id": marker, "customer": "e2e"}, + ), + ) + require_successful_call(first) + parsed = ResponsesResult.model_validate_json(first.body) + assert parsed.id, f"responses must return an id: {first.body[:300]}" + assert parsed.text.strip(), f"responses returned empty text: {first.body[:300]}" + + second = endpoints_client.proxy.transport.send( + "/v1/responses", + headers=endpoints_client.proxy.transport.bearer(key), + json=ResponsesMetadataBody( + model=model, + input="Reply with the single word ok.", + previous_response_id=parsed.id, + metadata={"session_id": marker, "turn": "2"}, + ), + ) + require_successful_call(second) + second_parsed = ResponsesResult.model_validate_json(second.body) + assert second_parsed.text.strip(), ( + f"previous_response_id follow-up returned empty text: {second.body[:300]}" + ) + + time.sleep(1.0) + keys = _redis_scan(marker) + unbounded = tuple(k for k in keys if k.ttl == -1) + assert not unbounded, ( + "responses metadata must not leave Redis keys without TTL (LIT-1201); " + f"unbounded={unbounded}" + ) diff --git a/tests/e2e/logging/conftest.py b/tests/e2e/logging/conftest.py index 2285eb8d695..60536ea01d4 100644 --- a/tests/e2e/logging/conftest.py +++ b/tests/e2e/logging/conftest.py @@ -10,7 +10,7 @@ import os import pytest -from logging_client import LangfuseCreds, LoggingClient, build_logging_client, load_langfuse_creds +from logging_client import LoggingClient, build_logging_client from datadog_reader import DdLogsReader, build_dd_logs_reader from otel_client import OtelReader, build_otel_reader from proxy_client import ProxyClient @@ -19,15 +19,14 @@ from proxy_client import ProxyClient def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( "markers", - "covers: registry cell a test covers, e.g. logging.langfuse.success.logs_spend", + "covers: registry cell a test covers, e.g. logging.datadog.success.exports_metric", ) @pytest.fixture(scope="session") def client(proxy: ProxyClient) -> LoggingClient: """The logging suite's client: holds the shared ProxyClient so `resources` / - `scoped_key` clean up keys and teams, and adds `/metrics` scraping plus - Langfuse read-back.""" + `scoped_key` clean up keys and teams, and adds `/metrics` scraping.""" return build_logging_client(proxy) @@ -51,9 +50,3 @@ def datadog_creds() -> None: pytest.fail( "Datadog e2e requires DD_API_KEY and DD_SITE; missing credentials is a hard failure, not a skip" ) - - -@pytest.fixture(scope="session") -def langfuse_creds() -> LangfuseCreds: - """Require real Langfuse cloud credentials for team callback + trace poll.""" - return load_langfuse_creds() diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 22aedb1cfbe..28cc7984598 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -65,6 +65,7 @@ class KeyGenerateBody(BaseModel): tpm_limit: int | None = None rpm_limit: int | None = None allowed_routes: list[str] | None = None + allowed_passthrough_routes: list[str] | None = None metadata: KeyMetadata | None = None object_permission: ObjectPermission | None = None @@ -130,6 +131,21 @@ class ChatMessage(BaseModel): content: str +class CacheControl(BaseModel): + type: str = "ephemeral" + + +class TextBlock(BaseModel): + type: str = "text" + text: str + cache_control: CacheControl | None = None + + +class RichMessage(BaseModel): + role: str + content: list[TextBlock] + + class ThinkingParam(BaseModel): """Extended-thinking control shared by Anthropic and DeepSeek reasoner models. DeepSeek accepts only ``type`` (enabled/disabled) and ignores budget_tokens; @@ -541,6 +557,9 @@ class LiteLLMParamsBody(BaseModel): s3_access_key_id: str | None = None s3_secret_access_key: str | None = None aws_batch_role_arn: str | None = None + aws_role_name: str | None = None + aws_session_name: str | None = None + aws_external_id: str | None = None input_cost_per_token: float | None = None output_cost_per_token: float | None = None extra_headers: dict[str, str] | None = None @@ -636,11 +655,16 @@ class TeamMemberEntry(BaseModel): user_id: str +class TeamMetadata(BaseModel): + disable_global_guardrails: bool | None = None + + class TeamNewBody(BaseModel): team_alias: str models: list[str] = [] team_id: str | None = None organization_id: str | None = None + metadata: TeamMetadata | None = None class TeamNewResponse(BaseModel): diff --git a/tests/e2e/quota_management/ratelimit/test_redis_backed_ratelimit_e2e.py b/tests/e2e/quota_management/ratelimit/test_redis_backed_ratelimit_e2e.py new file mode 100644 index 00000000000..ed6f0ce3b2c --- /dev/null +++ b/tests/e2e/quota_management/ratelimit/test_redis_backed_ratelimit_e2e.py @@ -0,0 +1,76 @@ +"""Live e2e: RPM enforcement on the Redis-backed limiter path customers run. + +Requires REDIS_HOST reachable from this process. A key with rpm_limit=1 must +serve the first chat and 429 the second. +""" + +from __future__ import annotations + +import os +import socket + +import pytest + +from e2e_config import require_env, unique_marker +from e2e_http import require_successful_call +from lifecycle import ResourceManager +from models import KeyGenerateBody, LiteLLMParamsBody +from quota_client import QuotaClient + +pytestmark = pytest.mark.e2e + +BACKEND = "anthropic/claude-haiku-4-5-20251001" + + +def _require_redis_reachable() -> None: + (host,) = require_env("REDIS_HOST") + port = int((os.environ.get("REDIS_PORT") or "6379").strip() or "6379") + try: + with socket.create_connection((host, port), timeout=3): + return + except OSError as exc: + raise AssertionError( + f"REDIS_HOST={host!r} port={port} is not reachable ({exc}). " + "Redis-backed rate limiting e2e needs a live Redis the proxy shares." + ) from exc + + +class TestRedisBackedRateLimit: + @pytest.mark.covers( + "quota_management.ratelimit.redis_backed.blocks_over_limit", + exercised_on=["chat_completions"], + ) + def test_rpm_limit_one_blocks_second_call( + self, client: QuotaClient, resources: ResourceManager + ) -> None: + _require_redis_reachable() + model = f"e2e-redis-rpm-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody(model=BACKEND, api_key="os.environ/ANTHROPIC_API_KEY"), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + key = client.proxy.generate_key( + KeyGenerateBody( + models=[model], + rpm_limit=1, + key_alias=f"e2e-redis-rpm-{unique_marker()}", + ) + ) + resources.defer(lambda: client.proxy.delete_key(key)) + + info = client.proxy.key_info(key) + assert info.rpm_limit == 1, f"key must echo rpm_limit=1: {info}" + + first = client.chat(key, model, f"ping {unique_marker()}") + require_successful_call(first) + + second = client.chat(key, model, f"pong {unique_marker()}") + assert second.status_code == 429, ( + f"second call over rpm_limit=1 must be 429, got {second.status_code}: " + f"{second.body[:300]}" + ) + assert "rate" in second.body.lower() or "limit" in second.body.lower(), ( + f"429 body should name the rate limit: {second.body[:300]}" + ) diff --git a/tests/e2e/quota_management/ratelimit/test_redis_circuit_breaker_e2e.py b/tests/e2e/quota_management/ratelimit/test_redis_circuit_breaker_e2e.py new file mode 100644 index 00000000000..b509ae000f5 --- /dev/null +++ b/tests/e2e/quota_management/ratelimit/test_redis_circuit_breaker_e2e.py @@ -0,0 +1,90 @@ +"""Live e2e: Redis-backed rate limit path stays responsive (LIT-3523 shape). + +With Redis up, burst past rpm_limit=1, then a fresh key must still complete a +chat in well under REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT. +""" + +from __future__ import annotations + +import os +import socket +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +import pytest + +from e2e_config import require_env, unique_marker +from e2e_http import require_successful_call +from lifecycle import ResourceManager +from models import KeyGenerateBody, LiteLLMParamsBody +from quota_client import QuotaClient + +pytestmark = pytest.mark.e2e + +BACKEND = "anthropic/claude-haiku-4-5-20251001" +RECOVERY_TIMEOUT = float( + os.environ.get("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", "60") or "60" +) + + +def _require_redis() -> None: + (host,) = require_env("REDIS_HOST") + port = int((os.environ.get("REDIS_PORT") or "6379").strip() or "6379") + try: + with socket.create_connection((host, port), timeout=3): + return + except OSError as exc: + raise AssertionError( + f"REDIS_HOST={host!r}:{port} unreachable ({exc}); " + "LIT-3523 e2e needs Redis the proxy shares." + ) from exc + + +class TestRedisCircuitBreakerPath: + @pytest.mark.covers( + "reliability.circuit_breaker.redis.trips_then_recovers", + exercised_on=["chat_completions"], + ) + def test_burst_rate_limit_does_not_freeze_fresh_key( + self, client: QuotaClient, resources: ResourceManager + ) -> None: + _require_redis() + model = f"e2e-cb-model-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody(model=BACKEND, api_key="os.environ/ANTHROPIC_API_KEY"), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + hot_key = client.proxy.generate_key( + KeyGenerateBody( + models=[model], + rpm_limit=1, + key_alias=f"e2e-cb-hot-{unique_marker()}", + ) + ) + resources.defer(lambda: client.proxy.delete_key(hot_key)) + cool_key = client.proxy.generate_key( + KeyGenerateBody(models=[model], key_alias=f"e2e-cb-cool-{unique_marker()}") + ) + resources.defer(lambda: client.proxy.delete_key(cool_key)) + + def _hit() -> int: + return client.chat(hot_key, model, f"burst {unique_marker()}").status_code + + with ThreadPoolExecutor(max_workers=8) as pool: + futures = [pool.submit(_hit) for _ in range(12)] + codes = tuple(f.result() for f in as_completed(futures)) + assert any(code == 429 for code in codes), ( + f"expected some 429 under rpm_limit=1 burst, got {codes}" + ) + + started = time.monotonic() + cool = client.chat(cool_key, model, f"fresh {unique_marker()}") + elapsed = time.monotonic() - started + require_successful_call(cool) + assert elapsed < RECOVERY_TIMEOUT * 0.5, ( + f"fresh key chat took {elapsed:.1f}s after redis rate-limit burst; " + f"customers treat hangs near recovery_timeout={RECOVERY_TIMEOUT}s as " + "LIT-3523 circuit-breaker pain" + ) diff --git a/tests/e2e/quota_management/ratelimit/test_tpm_excludes_cached_tokens_e2e.py b/tests/e2e/quota_management/ratelimit/test_tpm_excludes_cached_tokens_e2e.py new file mode 100644 index 00000000000..b0bc6b3508c --- /dev/null +++ b/tests/e2e/quota_management/ratelimit/test_tpm_excludes_cached_tokens_e2e.py @@ -0,0 +1,162 @@ +"""Live e2e: cached prompt tokens must not burn TPM budget (LIT-1930). + +Customer expectation: after a cacheable prefix is warmed, the remaining TPM +budget decreases by non-cached tokens only. If cached tokens still counted, +remaining would drop by the full prompt size. +""" + +from __future__ import annotations + +import time + +import pytest +from pydantic import BaseModel + +from e2e_config import unique_marker +from e2e_http import require_successful_call, unwrap +from lifecycle import ResourceManager +from models import ( + CacheControl, + ChatResponse, + KeyGenerateBody, + LiteLLMParamsBody, + RichMessage, + TextBlock, + Usage, +) +from quota_client import QuotaClient + +pytestmark = pytest.mark.e2e + +# Anthropic prompt caching (host has ANTHROPIC_API_KEY; Bedrock was "Operation not allowed"). +ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001" +# High enough that pre-call reservation of a cacheable prefix still clears. +TPM_LIMIT = 100_000 + + +class CacheChatBody(BaseModel): + model: str + messages: list[RichMessage] + max_tokens: int = 16 + cache: dict[str, bool] = {"no-cache": True} + + +def _prefix() -> str: + marker = unique_marker() + body = " ".join(f"TPM cache paragraph {i} run {marker}." for i in range(600)) + return f"{body}\nEnd {marker}." + + +def _cached_tokens(usage: Usage | None) -> int: + if usage is None: + return 0 + if usage.cache_read_input_tokens: + return usage.cache_read_input_tokens + if usage.prompt_tokens_details and usage.prompt_tokens_details.cached_tokens: + return usage.prompt_tokens_details.cached_tokens + return 0 + + +def _chat_raw(client: QuotaClient, key: str, model: str, prefix: str): + body = CacheChatBody( + model=model, + messages=[ + RichMessage( + role="system", + content=[TextBlock(text=prefix, cache_control=CacheControl())], + ), + RichMessage(role="user", content=[TextBlock(text="Reply with one word.")]), + ], + ) + return client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(key), + json=body, + ) + + +def _chat(client: QuotaClient, key: str, model: str, prefix: str) -> ChatResponse: + body = CacheChatBody( + model=model, + messages=[ + RichMessage( + role="system", + content=[TextBlock(text=prefix, cache_control=CacheControl())], + ), + RichMessage(role="user", content=[TextBlock(text="Reply with one word.")]), + ], + ) + return unwrap( + client.proxy.transport.post( + "/chat/completions", + headers=client.proxy.transport.bearer(key), + json=body, + response_type=ChatResponse, + ) + ) + + +class TestTpmExcludesCachedTokens: + @pytest.mark.covers( + "quota_management.ratelimit.tpm.excludes_cached_tokens", + exercised_on=["chat_completions"], + ) + def test_cache_hit_reduces_tpm_by_non_cached_only( + self, client: QuotaClient, resources: ResourceManager + ) -> None: + model = f"e2e-tpm-cache-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=ANTHROPIC_MODEL, api_key="os.environ/ANTHROPIC_API_KEY" + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = client.proxy.generate_key( + KeyGenerateBody(models=[model], tpm_limit=TPM_LIMIT) + ) + resources.defer(lambda: client.proxy.delete_key(key)) + + prefix = _prefix() + first = _chat(client, key, model, prefix) + assert first.choices, f"cache prime returned no choices: {first}" + first_total = (first.usage.total_tokens or 0) if first.usage else 0 + assert first_total > 0, f"prime call must report usage: {first.usage}" + + deadline = time.monotonic() + 45.0 + second_usage: Usage | None = None + remaining_after: str | None = None + while time.monotonic() < deadline: + outcome = _chat_raw(client, key, model, prefix) + require_successful_call(outcome) + parsed = ChatResponse.model_validate_json(outcome.body) + if _cached_tokens(parsed.usage) > 0: + second_usage = parsed.usage + remaining_after = outcome.headers.get( + "x-ratelimit-api_key-remaining-tokens" + ) + break + time.sleep(2.0) + + assert second_usage is not None, "second call never reported cache-read tokens" + cached = _cached_tokens(second_usage) + assert cached > 0 + second_total = second_usage.total_tokens or 0 + assert second_total > cached, ( + f"need total > cached so non-cached slice is measurable: {second_usage}" + ) + + assert remaining_after is not None and remaining_after.isdigit(), ( + f"cache-hit response must expose remaining TPM headers, got {remaining_after!r}" + ) + remaining = int(remaining_after) + # If cached tokens were counted, remaining would be limit - first - second_total. + # With exclusion, remaining is closer to limit - first - (second_total - cached). + counted_full = TPM_LIMIT - first_total - second_total + counted_excluding_cache = TPM_LIMIT - first_total - (second_total - cached) + assert remaining > counted_full, ( + f"remaining TPM {remaining} looks like cached tokens still counted " + f"(would be ~{counted_full} if full second_total={second_total} counted; " + f"expected closer to ~{counted_excluding_cache} after excluding " + f"cache_read={cached}; LIT-1930)" + ) diff --git a/tests/e2e/transport.py b/tests/e2e/transport.py index 64fe6406ff7..005b49272e8 100644 --- a/tests/e2e/transport.py +++ b/tests/e2e/transport.py @@ -52,7 +52,13 @@ class Transport(Protocol): ) -> Result[R]: ... def delete[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + response_type: type[R], + params: BaseModel | None = None, ) -> Result[R]: ... def patch[R: BaseModel]( @@ -125,12 +131,19 @@ class HttpTransport: ) def delete[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + response_type: type[R], + params: BaseModel | None = None, ) -> Result[R]: return e2e_http.delete( self._url(path), headers=headers, json=json, + params=params, response_type=response_type, timeout=self.request_timeout, ) @@ -223,6 +236,8 @@ CONTROL_PLANE_PREFIXES: tuple[str, ...] = ( "/model/", "/spend", "/global", + "/config", + "/guardrails", "/openapi.json", ) @@ -280,10 +295,20 @@ class SplitTransport: ) def delete[R: BaseModel]( - self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + response_type: type[R], + params: BaseModel | None = None, ) -> Result[R]: return self._route(path).delete( - path, headers=headers, json=json, response_type=response_type + path, + headers=headers, + json=json, + response_type=response_type, + params=params, ) def patch[R: BaseModel](