From 83b1e9e0d7ac48e4806a91d1e1d7587725a82039 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 10 Sep 2026 12:46:50 -0700 Subject: [PATCH] test(router): keep classifier E2E verification local --- tests/e2e/coverage_registry/reliability.yaml | 1 - tests/e2e/router/test_classifier_audit_e2e.py | 143 ------------------ 2 files changed, 144 deletions(-) delete mode 100644 tests/e2e/router/test_classifier_audit_e2e.py diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 4d40f19e67b..6b69677d490 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -1,5 +1,4 @@ # Reliability & Performance (behavior features). Grounded in litellm/router.py + router_strategy/ + router_utils/. -- {id: reliability.routing.classifier_audit.separates_provider_input_and_source, module: reliability, tier: P1, behavior: routing, variant: classifier_audit, assertions: [separates_provider_input_and_source], exercised_on: [chat_completions, messages, responses], source: "litellm/litellm_core_utils/classifier_logging.py", rationale: "Classifier spend details distinguish provider input from the credential-masked originating request across all three request surfaces"} - {id: reliability.fallback.5xx.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: "5xx", assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:2024", rationale: "Reroute on provider 5xx to alternate deployment"} - {id: reliability.fallback.context_window.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: context_window, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6108", rationale: "Fallback when model exceeds context limit"} - {id: reliability.fallback.content_policy.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: content_policy, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6023", rationale: "Reroute on content-policy violation"} diff --git a/tests/e2e/router/test_classifier_audit_e2e.py b/tests/e2e/router/test_classifier_audit_e2e.py deleted file mode 100644 index 6b444441477..00000000000 --- a/tests/e2e/router/test_classifier_audit_e2e.py +++ /dev/null @@ -1,143 +0,0 @@ -import json -import os -from collections.abc import Iterator -from contextlib import ExitStack -from dataclasses import dataclass -from typing import Final, Literal - -import pytest -from pydantic import BaseModel, Field, JsonValue, TypeAdapter - -from e2e_config import unique_marker -from e2e_http import AnthropicHeaders, NoBody, unwrap -from models import ChatMessage, KeyGenerateBody, LiteLLMParamsBody -from proxy_client import ProxyClient - -pytestmark = pytest.mark.e2e - - -class AuditMetadata(BaseModel): - source_marker: str - authorization: str = "synthetic-audit-secret" - - -class AuditHeaders(AnthropicHeaders): - enable_redaction: str | None = Field(default=None, serialization_alias="x-litellm-enable-message-redaction") - - -class AuditRequest(BaseModel): - model: str - messages: list[ChatMessage] | None = None - system: str | None = None - instructions: str | None = None - input: str | None = None - max_tokens: int | None = None - max_output_tokens: int | None = None - metadata: AuditMetadata | None = None - litellm_metadata: AuditMetadata | None = None - - -class AuditResponse(BaseModel): - id: str - - -class AuditDetail(BaseModel): - proxy_server_request: dict[str, JsonValue] | str | None = None - response: dict[str, JsonValue] | str | None = None - - -@dataclass(frozen=True, slots=True) -class AuditDeployment: - alias: str - key: str - - -@pytest.fixture -def audit_deployment(proxy: ProxyClient, provider: str) -> Iterator[AuditDeployment]: - marker: Final = unique_marker() - classifier: Final = f"audit-classifier-{marker}" - target: Final = f"audit-target-{marker}" - alias: Final = f"audit-router-{marker}" - model: Final = os.environ.get( - f"E2E_CHEAP_{provider.upper()}_MODEL", "gpt-5.6" if provider == "openai" else "claude-haiku-4-5" - ) - params: Final = LiteLLMParamsBody( - model=f"{provider}/{model}", - api_key=os.environ.get(f"{provider.upper()}_API_KEY") or f"os.environ/{provider.upper()}_API_KEY", - api_base=os.environ.get(f"{provider.upper()}_API_BASE"), - ) - with ExitStack() as stack: - for name in (classifier, target): - stack.callback(proxy.delete_model, proxy.create_model(name, params)) - stack.callback(proxy.delete_model, proxy.create_model(alias, LiteLLMParamsBody( - model="auto_router/complexity_router", - complexity_router_config={ - "classifier_type": "llm", - "classifier_llm_config": {"model": classifier, "timeout_ms": 30000}, - "tiers": {tier: target for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")}, - }, - ))) - key: Final = proxy.generate_key(KeyGenerateBody(models=[alias, classifier, target])) - stack.callback(proxy.delete_key, key) - yield AuditDeployment(alias, key) - - -class TestClassifierAudit: - @pytest.mark.covers( - "reliability.routing.classifier_audit.separates_provider_input_and_source", - exercised_on=("chat_completions", "messages", "responses"), - ) - @pytest.mark.parametrize("surface", ["chat_completions", "messages", "responses"]) - @pytest.mark.parametrize("provider", ["openai", "anthropic"]) - @pytest.mark.parametrize("redact", [False, True]) - def test_classifier_audit_separates_input_and_masked_source( - self, proxy: ProxyClient, audit_deployment: AuditDeployment, surface: Literal["chat_completions", "messages", "responses"], - redact: bool, - ) -> None: - marker: Final = unique_marker() - source_marker: Final = f"source-only-{marker}" - prompt: Final = f"Reply with hello. Request label {marker}" - metadata: Final = AuditMetadata(source_marker=source_marker) - body: Final = AuditRequest( - model=audit_deployment.alias, - messages=[ChatMessage(role="user", content=prompt)] if surface != "responses" else None, - input=prompt if surface == "responses" else None, - system="Be concise" if surface == "messages" else None, - instructions="Be concise" if surface == "responses" else None, - max_tokens=128 if surface != "responses" else None, - max_output_tokens=128 if surface == "responses" else None, - metadata=metadata if surface != "responses" else None, - litellm_metadata=metadata if surface == "responses" else None, - ) - path: Final = {"chat_completions": "/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"}[surface] - response: Final = unwrap(proxy.transport.post( - path, headers=AuditHeaders(authorization=f"Bearer {audit_deployment.key}", enable_redaction="true" if redact else None), - json=body, response_type=AuditResponse, - )) - assert response.id - rows: Final = proxy.poll_logs_for_key(audit_deployment.key, min_rows=2) - assert len(rows) == 2, "Expected a classifier spend row and a routed response spend row" - details: Final = tuple( - unwrap(proxy.transport.get( - f"/spend/logs/ui/{row.request_id}", headers=proxy.transport.master, - params=NoBody(), response_type=AuditDetail, - )) for row in rows - ) - adapter: Final = TypeAdapter(dict[str, JsonValue]) - requests: Final = tuple( - adapter.validate_json(detail.proxy_server_request) if isinstance(detail.proxy_server_request, str) - else detail.proxy_server_request or {} for detail in details - ) - audits: Final = tuple(item for item in requests if "classifier_input" in item) - if redact: - assert audits == () - assert all("originating_request_masked" not in item for item in requests) - return - assert len(audits) == 1, "The audit belongs only to the classifier call" - audit: Final = audits[0] - assert marker in json.dumps(audit["classifier_input"]) - assert source_marker not in json.dumps(audit["classifier_input"]) - assert source_marker in json.dumps(audit["originating_request_masked"]) - assert "synthetic-audit-secret" not in json.dumps(audit) - assert audit_deployment.alias in json.dumps(audit["originating_request_masked"]) - assert any("tier" in str(detail.response) for detail in details)