From 3c7d66989ee75f7c0283b189d3dbc77a93969658 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Fri, 24 Jul 2026 16:16:53 -0700 Subject: [PATCH] test(e2e): rename suites, drop vendor_contract, fix greptile gaps Move shared status helpers into e2e_http, rename chat auth headers and chat security suites, remove vendor_contract and dev_config files_settings, and tighten transcription validation plus vector-store search assertions --- litellm/proxy/dev_config.yaml | 5 -- ...y_e2e.py => test_chat_auth_headers_e2e.py} | 8 +- tests/e2e/e2e_http.py | 64 +++++++++++++++ .../llm_translation/test_audio_speech_e2e.py | 3 +- .../test_audio_transcriptions_e2e.py | 16 ++-- .../test_bedrock_native_e2e.py | 11 ++- ...chat_completions_sec_vulnerability_e2e.py} | 10 +-- .../test_embeddings_endpoint_e2e.py | 10 +-- .../test_files_batches_contract_e2e.py | 3 +- .../test_image_generation_e2e.py | 10 +-- .../e2e/llm_translation/test_messages_e2e.py | 3 +- .../test_model_matrix_smoke_e2e.py | 3 +- .../llm_translation/test_moderations_e2e.py | 3 +- .../e2e/llm_translation/test_ocr_rust_e2e.py | 3 +- .../llm_translation/test_realtime_http_e2e.py | 3 +- .../e2e/llm_translation/test_responses_e2e.py | 16 ++-- tests/e2e/llm_translation/test_search_e2e.py | 3 +- .../llm_translation/test_vector_stores_e2e.py | 16 +++- tests/e2e/llm_translation/vendor_contract.py | 79 ------------------- 19 files changed, 126 insertions(+), 143 deletions(-) rename tests/e2e/access_control/{test_chat_auth_security_e2e.py => test_chat_auth_headers_e2e.py} (94%) rename tests/e2e/llm_translation/{test_chat_completions_vendor_contract_e2e.py => test_chat_completions_sec_vulnerability_e2e.py} (97%) delete mode 100644 tests/e2e/llm_translation/vendor_contract.py diff --git a/litellm/proxy/dev_config.yaml b/litellm/proxy/dev_config.yaml index 34724ca6d4b..f78431f694b 100644 --- a/litellm/proxy/dev_config.yaml +++ b/litellm/proxy/dev_config.yaml @@ -204,11 +204,6 @@ general_settings: # configured for the batched model. Defaults to false. # track_unmanaged_batch_cost: true -# for /v1/files (vector store upload, batches, etc.) -files_settings: - - custom_llm_provider: openai - api_key: os.environ/OPENAI_API_KEY - sandbox_tools: - sandbox_tool_name: e2b_sandbox litellm_params: diff --git a/tests/e2e/access_control/test_chat_auth_security_e2e.py b/tests/e2e/access_control/test_chat_auth_headers_e2e.py similarity index 94% rename from tests/e2e/access_control/test_chat_auth_security_e2e.py rename to tests/e2e/access_control/test_chat_auth_headers_e2e.py index 89ef85385a2..edad120a642 100644 --- a/tests/e2e/access_control/test_chat_auth_security_e2e.py +++ b/tests/e2e/access_control/test_chat_auth_headers_e2e.py @@ -1,8 +1,8 @@ -"""Vendor ยง11.1 auth security on LLM routes (LIT-4778). +"""Chat Authorization header matrix on LLM routes (LIT-4778). Virtual-key chat must reject missing and malformed Authorization headers before any provider call. These cases sit next to the existing valid/invalid key check -and pin the full bearer-token failure matrix from the vendor brief. +and pin the bearer-token failure matrix. """ from __future__ import annotations @@ -27,7 +27,7 @@ class RawAuthorizationHeaders(BaseModel): def _register_model(proxy: ProxyClient, resources: ResourceManager) -> str: - model = f"e2e-auth-sec-{unique_marker()}" + model = f"e2e-auth-headers-{unique_marker()}" model_id = proxy.create_model( model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY"), @@ -56,7 +56,7 @@ def _assert_auth_denied(result: StreamingResponse, context: str) -> None: ) -class TestChatAuthSecurity: +class TestChatAuthHeaders: @pytest.mark.covers("other.auth.llm_chat.missing_header_denied") def test_missing_authorization_header_is_denied( self, proxy: ProxyClient, resources: ResourceManager diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 1d4384e030e..52e2ac966ed 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -220,6 +220,70 @@ def require_successful_call(result: StreamingResponse) -> None: ) +def is_client_error(status: int) -> bool: + return 400 <= status < 500 + + +def is_auth_denied(status: int) -> bool: + return status in (401, 403) + + +def assert_not_server_error(result: StreamingResponse, context: str) -> None: + assert result.status_code not in (500, 502, 503), ( + f"{context}: proxy must not 5xx, got {result.status_code}: {result.body[:300]}" + ) + + +def assert_client_error(result: StreamingResponse, context: str) -> None: + assert is_client_error(result.status_code), ( + f"{context}: expected 4xx, got {result.status_code}: {result.body[:300]}" + ) + + +def assert_error_or_server_known(result: StreamingResponse, context: str) -> None: + """Missing required fields may be 4xx or 5xx per known acceptable proxy behavior.""" + assert result.status_code in range(400, 600), ( + f"{context}: expected error status, got {result.status_code}: {result.body[:300]}" + ) + assert result.status_code != 200 + + +def assert_auth_denied(result: StreamingResponse, context: str) -> None: + assert is_auth_denied(result.status_code), ( + f"{context}: expected 401/403, got {result.status_code}: {result.body[:300]}" + ) + + +def is_provider_account_denied(result: StreamingResponse) -> bool: + """True when the gateway reached the provider and the account/model is disabled.""" + if result.status_code not in (400, 403, 404): + return False + body = result.body.lower() + needles = ( + "operation not allowed", + "end of its life", + "accessdenied", + "not authorized", + "model use case details have not been submitted", + "you don't have access", + "do not have access", + ) + return any(n in body for n in needles) + + +def require_success_or_provider_denied(result: StreamingResponse, context: str) -> bool: + """Return True on success; return False when the provider denied the account. + + Raises on unexpected failures so real product regressions still fail hard. + """ + if result.ok and not result.stream_error: + return True + if is_provider_account_denied(result): + return False + require_successful_call(result) + return True + + def _headers(headers: BaseModel) -> dict[str, str]: dumped: dict[str, object] = headers.model_dump(by_alias=True, exclude_none=True) return {key: str(value) for key, value in dumped.items()} diff --git a/tests/e2e/llm_translation/test_audio_speech_e2e.py b/tests/e2e/llm_translation/test_audio_speech_e2e.py index 412668690ed..9243ce19a14 100644 --- a/tests/e2e/llm_translation/test_audio_speech_e2e.py +++ b/tests/e2e/llm_translation/test_audio_speech_e2e.py @@ -12,11 +12,10 @@ import pytest from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import require_successful_call +from e2e_http import require_successful_call, assert_error_or_server_known from endpoints_client import EndpointsClient from lifecycle import ResourceManager from models import LiteLLMParamsBody -from vendor_contract import assert_error_or_server_known pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py index e35c944f503..3a55bcb1073 100644 --- a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py +++ b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py @@ -79,10 +79,12 @@ class TestAudioTranscriptions: match result: case Success(): pytest.fail("empty audio file must not succeed as a transcript") - case UnknownApiError(status_code=status): - assert status in range(400, 600), f"unexpected {status}" - case _: + case UnknownApiError(status_code=status) if 400 <= status < 500: return + case UnknownApiError(status_code=status): + pytest.fail(f"empty audio expected 4xx, got {status}: {result}") + case _: + pytest.fail(f"empty audio unexpected result: {result}") @pytest.mark.covers("llm.audio_transcriptions.openai.input_validation.nonstream.works") def test_missing_model_returns_error( @@ -101,7 +103,9 @@ class TestAudioTranscriptions: match result: case Success(): pytest.fail("transcription without model must not succeed") - case UnknownApiError(status_code=status): - assert status in range(400, 600), f"unexpected {status}" - case _: + case UnknownApiError(status_code=status) if 400 <= status < 500: return + case UnknownApiError(status_code=status): + pytest.fail(f"missing model expected 4xx, got {status}: {result}") + case _: + pytest.fail(f"missing model unexpected result: {result}") diff --git a/tests/e2e/llm_translation/test_bedrock_native_e2e.py b/tests/e2e/llm_translation/test_bedrock_native_e2e.py index f6c01d99fdd..65e0ff8dab8 100644 --- a/tests/e2e/llm_translation/test_bedrock_native_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_native_e2e.py @@ -7,19 +7,18 @@ missing messages and invalid model handling without crashing the proxy. from __future__ import annotations import pytest +import requests from pydantic import BaseModel from e2e_config import unique_marker -import requests - -from lifecycle import ResourceManager -from models import LiteLLMParamsBody -from proxy_client import ProxyClient -from vendor_contract import ( +from e2e_http import ( assert_client_error, assert_error_or_server_known, require_success_or_provider_denied, ) +from lifecycle import ResourceManager +from models import LiteLLMParamsBody +from proxy_client import ProxyClient pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_chat_completions_vendor_contract_e2e.py b/tests/e2e/llm_translation/test_chat_completions_sec_vulnerability_e2e.py similarity index 97% rename from tests/e2e/llm_translation/test_chat_completions_vendor_contract_e2e.py rename to tests/e2e/llm_translation/test_chat_completions_sec_vulnerability_e2e.py index 068eb5f94bc..7eec437af42 100644 --- a/tests/e2e/llm_translation/test_chat_completions_vendor_contract_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_sec_vulnerability_e2e.py @@ -1,7 +1,7 @@ -"""Vendor API strategy coverage for /chat/completions (LIT-4778). +"""Chat completions security and input-sanitization e2e (LIT-4778). -Positive multi-turn history, input validation, boundary handling, contract shape, -and input sanitization against a live proxy and a real OpenAI-compatible model. +Multi-turn history, input validation, boundary handling, response shape, and +SQL/XSS payload sanitization against a live proxy and a real OpenAI-compatible model. """ from __future__ import annotations @@ -51,7 +51,7 @@ class ChatErrorEnvelope(BaseModel): def _register_chat_model(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]: - model = f"e2e-vendor-chat-{unique_marker()}" + model = f"e2e-chat-sec-{unique_marker()}" model_id = proxy.create_model( model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY"), @@ -80,7 +80,7 @@ def _assert_not_server_error(result: StreamingResponse, context: str) -> None: ) -class TestChatCompletionsVendorContract: +class TestChatCompletionsSecVulnerability: @pytest.mark.covers("llm.chat_completions.openai.multi_turn.nonstream.works") def test_multi_turn_history_is_honored( self, proxy: ProxyClient, resources: ResourceManager diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index 04d7bcc918a..cd642d51ca2 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -12,15 +12,15 @@ import pytest from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import require_successful_call -from endpoints_client import EmbeddingsResult, EndpointsClient -from lifecycle import ResourceManager -from models import LiteLLMParamsBody -from vendor_contract import ( +from e2e_http import ( assert_client_error, assert_error_or_server_known, require_success_or_provider_denied, + require_successful_call, ) +from endpoints_client import EmbeddingsResult, EndpointsClient +from lifecycle import ResourceManager +from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_files_batches_contract_e2e.py b/tests/e2e/llm_translation/test_files_batches_contract_e2e.py index a1496d938ae..8f19d84a425 100644 --- a/tests/e2e/llm_translation/test_files_batches_contract_e2e.py +++ b/tests/e2e/llm_translation/test_files_batches_contract_e2e.py @@ -10,11 +10,10 @@ import pytest from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import NoBody, Success, UnknownApiError +from e2e_http import NoBody, Success, UnknownApiError, assert_error_or_server_known from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient -from vendor_contract import assert_error_or_server_known pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_image_generation_e2e.py b/tests/e2e/llm_translation/test_image_generation_e2e.py index 4fe797ae6bf..cca1a23bbcd 100644 --- a/tests/e2e/llm_translation/test_image_generation_e2e.py +++ b/tests/e2e/llm_translation/test_image_generation_e2e.py @@ -11,15 +11,15 @@ import pytest from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import require_successful_call -from endpoints_client import EndpointsClient, ImagesResult -from lifecycle import ResourceManager -from models import LiteLLMParamsBody -from vendor_contract import ( +from e2e_http import ( assert_client_error, assert_error_or_server_known, require_success_or_provider_denied, + require_successful_call, ) +from endpoints_client import EndpointsClient, ImagesResult +from lifecycle import ResourceManager +from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index dadf1d83cc5..8142cf8b750 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -12,7 +12,7 @@ import pytest from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import require_successful_call, unwrap +from e2e_http import require_successful_call, unwrap, assert_error_or_server_known from endpoints_client import EndpointsClient, MessagesResult from lifecycle import ResourceManager from models import ( @@ -24,7 +24,6 @@ from models import ( SpendLogRow, ToolInputSchema, ) -from vendor_contract import assert_error_or_server_known pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_model_matrix_smoke_e2e.py b/tests/e2e/llm_translation/test_model_matrix_smoke_e2e.py index f8fccd4047e..6f72f94e8a8 100644 --- a/tests/e2e/llm_translation/test_model_matrix_smoke_e2e.py +++ b/tests/e2e/llm_translation/test_model_matrix_smoke_e2e.py @@ -11,11 +11,10 @@ from dataclasses import dataclass import pytest from e2e_config import unique_marker -from e2e_http import StreamingResponse, UnknownApiError, unwrap +from e2e_http import StreamingResponse, UnknownApiError, unwrap, is_provider_account_denied from lifecycle import ResourceManager from models import ChatBody, ChatMessage, LiteLLMParamsBody from proxy_client import ProxyClient -from vendor_contract import is_provider_account_denied pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_moderations_e2e.py b/tests/e2e/llm_translation/test_moderations_e2e.py index 190bf145a60..56a38c68b62 100644 --- a/tests/e2e/llm_translation/test_moderations_e2e.py +++ b/tests/e2e/llm_translation/test_moderations_e2e.py @@ -11,11 +11,10 @@ import pytest from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import unwrap +from e2e_http import unwrap, assert_error_or_server_known from endpoints_client import EndpointsClient from lifecycle import ResourceManager from models import LiteLLMParamsBody -from vendor_contract import assert_error_or_server_known pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_ocr_rust_e2e.py b/tests/e2e/llm_translation/test_ocr_rust_e2e.py index 31cc3355dac..472f2947c81 100644 --- a/tests/e2e/llm_translation/test_ocr_rust_e2e.py +++ b/tests/e2e/llm_translation/test_ocr_rust_e2e.py @@ -23,11 +23,10 @@ import pytest from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import unwrap +from e2e_http import unwrap, assert_error_or_server_known from endpoints_client import EndpointsClient from lifecycle import ResourceManager from models import LiteLLMParamsBody, OcrBody, OcrDocument, OcrResponse -from vendor_contract import assert_error_or_server_known pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_realtime_http_e2e.py b/tests/e2e/llm_translation/test_realtime_http_e2e.py index 36e8c8e01d2..182365bfc7f 100644 --- a/tests/e2e/llm_translation/test_realtime_http_e2e.py +++ b/tests/e2e/llm_translation/test_realtime_http_e2e.py @@ -10,11 +10,10 @@ import pytest from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import NoBody, unwrap +from e2e_http import NoBody, unwrap, assert_auth_denied from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient -from vendor_contract import assert_auth_denied pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index f099494a231..915c014f76d 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -14,7 +14,14 @@ import pytest from pydantic import BaseModel, ValidationError from e2e_config import unique_marker -from e2e_http import require_successful_call +from e2e_http import ( + assert_client_error, + assert_error_or_server_known, + assert_not_server_error, + is_client_error, + require_success_or_provider_denied, + require_successful_call, +) from endpoints_client import ( EndpointsClient, FunctionParameterProperty, @@ -26,13 +33,6 @@ from endpoints_client import ( ) from lifecycle import ResourceManager from models import LiteLLMParamsBody -from vendor_contract import ( - assert_client_error, - assert_error_or_server_known, - assert_not_server_error, - is_client_error, - require_success_or_provider_denied, -) pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_search_e2e.py b/tests/e2e/llm_translation/test_search_e2e.py index e7e92842aa0..f2ad746cd60 100644 --- a/tests/e2e/llm_translation/test_search_e2e.py +++ b/tests/e2e/llm_translation/test_search_e2e.py @@ -12,10 +12,9 @@ import pytest from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import NoBody, unwrap +from e2e_http import NoBody, unwrap, assert_client_error, assert_error_or_server_known from lifecycle import ResourceManager from proxy_client import ProxyClient -from vendor_contract import assert_client_error, assert_error_or_server_known pytestmark = pytest.mark.e2e diff --git a/tests/e2e/llm_translation/test_vector_stores_e2e.py b/tests/e2e/llm_translation/test_vector_stores_e2e.py index 59b1c151de7..84b18299715 100644 --- a/tests/e2e/llm_translation/test_vector_stores_e2e.py +++ b/tests/e2e/llm_translation/test_vector_stores_e2e.py @@ -13,11 +13,10 @@ import pytest from pydantic import BaseModel, ConfigDict from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker -from e2e_http import FileUploadForm, NoBody, unwrap +from e2e_http import FileUploadForm, NoBody, unwrap, assert_error_or_server_known from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient -from vendor_contract import assert_error_or_server_known pytestmark = pytest.mark.e2e @@ -221,7 +220,7 @@ class TestVectorStores: proxy.transport.upload( "/v1/files", headers=proxy.transport.bearer(key), - form=FileUploadForm(purpose="assistants"), + form=FileUploadForm(purpose="assistants", custom_llm_provider="openai"), filename="vs_doc.txt", content=content, file_content_type="text/plain", @@ -275,7 +274,16 @@ class TestVectorStores: response_type=VectorStoreSearchResponse, ) ) - assert search.data is not None, f"search returned no data field: {search}" + assert search.data, f"search returned no hits for marker {marker!r}: {search}" + hit_blob = " ".join( + " ".join(part.get("text", "") for part in (hit.content or [])) + + " " + + (hit.filename or "") + for hit in search.data + ) + assert marker in hit_blob or any( + (hit.file_id or "") == uploaded.id for hit in search.data + ), f"search hits must reference marker or uploaded file; marker={marker!r} hits={search.data}" deleted_file = unwrap( proxy.transport.delete( diff --git a/tests/e2e/llm_translation/vendor_contract.py b/tests/e2e/llm_translation/vendor_contract.py deleted file mode 100644 index afd41a854ac..00000000000 --- a/tests/e2e/llm_translation/vendor_contract.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Shared helpers for vendor API contract e2e tests (LIT-4778). - -Status-centric assertions used across endpoint negatives, sanitization, and -auth matrix cases so each test file stays thin. -""" - -from __future__ import annotations - -from e2e_http import StreamingResponse - - -def is_client_error(status: int) -> bool: - return 400 <= status < 500 - - -def is_auth_denied(status: int) -> bool: - return status in (401, 403) - - -def assert_not_server_error(result: StreamingResponse, context: str) -> None: - assert result.status_code not in (500, 502, 503), ( - f"{context}: proxy must not 5xx, got {result.status_code}: {result.body[:300]}" - ) - - -def assert_client_error(result: StreamingResponse, context: str) -> None: - assert is_client_error(result.status_code), ( - f"{context}: expected 4xx, got {result.status_code}: {result.body[:300]}" - ) - - -def assert_error_or_server_known(result: StreamingResponse, context: str) -> None: - """Missing required fields may be 4xx or 5xx per known acceptable proxy behavior.""" - assert result.status_code in range(400, 600), ( - f"{context}: expected error status, got {result.status_code}: {result.body[:300]}" - ) - assert result.status_code != 200 - - -def assert_auth_denied(result: StreamingResponse, context: str) -> None: - assert is_auth_denied(result.status_code), ( - f"{context}: expected 401/403, got {result.status_code}: {result.body[:300]}" - ) - - -def is_provider_account_denied(result: StreamingResponse) -> bool: - """True when the gateway reached the provider and the account/model is disabled. - - Common on local AWS accounts that list Bedrock models but cannot InvokeModel - ("Operation not allowed") or when a model version is EOL. - """ - if result.status_code not in (400, 403, 404): - return False - body = result.body.lower() - needles = ( - "operation not allowed", - "end of its life", - "accessdenied", - "not authorized", - "model use case details have not been submitted", - "you don't have access", - "do not have access", - ) - return any(n in body for n in needles) - - -def require_success_or_provider_denied(result: StreamingResponse, context: str) -> bool: - """Return True on success; return False when the provider denied the account. - - Raises on unexpected failures so real product regressions still fail hard. - """ - if result.ok and not result.stream_error: - return True - if is_provider_account_denied(result): - return False - from e2e_http import require_successful_call - - require_successful_call(result) - return True