From 24b7a38b5f8b202c125ea61cd5eadd25f29e1352 Mon Sep 17 00:00:00 2001 From: Moe Khalil Date: Sun, 20 Sep 2026 01:06:41 +0000 Subject: [PATCH] fix(auto-router): validate saved JEV probe payloads without credentials Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_auto_router_endpoints.py | 12 +++++++++--- .../add_model/JevConnectionTest.integration.test.tsx | 2 +- .../build_auto_router_routing_test_request.test.ts | 11 +++++++++-- .../build_auto_router_routing_test_request.ts | 9 +++++++-- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 9235a00bda6..03325d5296e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -2384,7 +2384,9 @@ async def test_jev_test_routing_authorizes_paid_evaluation_before_contacting_typ @pytest.mark.asyncio -@pytest.mark.parametrize("case", ["allowed", "missing", "blocked", "key", "budget", "team", "not-router"]) +@pytest.mark.parametrize( + "case", ["allowed", "credential-free", "missing", "blocked", "key", "budget", "team", "not-router"] +) async def test_saved_jev_probe_uses_authorized_server_configuration(monkeypatch: pytest.MonkeyPatch, case: str) -> None: router: Final = RecordingRouter("SIMPLE") stored_key: Final = "synthetic-server-jev-key" @@ -2429,7 +2431,11 @@ async def test_saved_jev_probe_uses_authorized_server_configuration(monkeypatch: "team_id": "member-preview-team" if case == "team" else None, }, classifier_type="jev", - jev_classifier_config={"api_key": "masked-key", "api_base": "https://browser-override.test"}, + jev_classifier_config=( + {"model": "jev-latest", "timeout_ms": 3000} + if case == "credential-free" + else {"api_key": "masked-key", "api_base": "https://browser-override.test"} + ), ) with respx.mock(assert_all_called=False) as http: handler: Final = http_handler.AsyncHTTPHandler() @@ -2466,7 +2472,7 @@ async def test_saved_jev_probe_uses_authorized_server_configuration(monkeypatch: assert result.routed_model == "cheap-model" assert evaluation.calls.last.request.headers["authorization"] == f"Bearer {stored_key}" assert stored_key not in result.model_dump_json() - assert evaluation.call_count == (1 if case == "allowed" else 0) + assert evaluation.call_count == (1 if case in ("allowed", "credential-free") else 0) assert router.recorded_calls == [] await handler.client.aclose() diff --git a/ui/litellm-dashboard/src/components/add_model/JevConnectionTest.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevConnectionTest.integration.test.tsx index 72acda7622a..c85c757e391 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevConnectionTest.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevConnectionTest.integration.test.tsx @@ -101,7 +101,7 @@ describe("JEV network probes", () => { const routingCall = fetchMock.mock.calls.find(([url]) => String(url).endsWith("/auto_router/test_routing")); const expectedRequest = { prompt: JEV_CONNECTION_TEST_PROMPT, - complexity_router_config: { ...config, jev_classifier_config: undefined }, + complexity_router_config: config, saved_model_id: "saved-id", }; expect(JSON.parse(String(routingCall?.[1]?.body))).toEqual(expectedRequest); diff --git a/ui/litellm-dashboard/src/components/add_model/build_auto_router_routing_test_request.test.ts b/ui/litellm-dashboard/src/components/add_model/build_auto_router_routing_test_request.test.ts index 174f93eae6c..de0fb6fe6e1 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_auto_router_routing_test_request.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_auto_router_routing_test_request.test.ts @@ -5,6 +5,7 @@ import { JEV_CONNECTION_TEST_PROMPT, } from "./build_auto_router_routing_test_request"; import { ComplexityRouterConfigPayload } from "./build_complexity_router_config"; +import { defaultJevClassifierConfig } from "./jev_classifier_config"; const CONFIG = { tiers: { SIMPLE: ["cheap"], MEDIUM: ["mid"], COMPLEX: ["strong"], REASONING: ["o3"] }, @@ -31,10 +32,16 @@ describe("buildAutoRouterRoutingTestRequest", () => { ); const expectedRequest = { prompt: JEV_CONNECTION_TEST_PROMPT, - complexity_router_config: { classifier_type: "jev", tiers: CONFIG.tiers }, + complexity_router_config: { + classifier_type: "jev", + tiers: CONFIG.tiers, + jev_classifier_config: defaultJevClassifierConfig(), + }, saved_model_id: "saved-id", }; expect(request).toEqual(expectedRequest); + expect(request?.complexity_router_config.jev_classifier_config).not.toHaveProperty("api_key"); + expect(request?.complexity_router_config.jev_classifier_config).not.toHaveProperty("api_base"); }); it.each(["object", "json"])("probes saved JEV %s configuration with custom tiers and team context", (format) => { const config = { @@ -47,7 +54,7 @@ describe("buildAutoRouterRoutingTestRequest", () => { }; const expectedRequest = { prompt: JEV_CONNECTION_TEST_PROMPT, - complexity_router_config: { ...config, jev_classifier_config: undefined }, + complexity_router_config: config, saved_model_id: "saved-id", team_id: "team-1", }; diff --git a/ui/litellm-dashboard/src/components/add_model/build_auto_router_routing_test_request.ts b/ui/litellm-dashboard/src/components/add_model/build_auto_router_routing_test_request.ts index 4679f3c50bf..6a9d1ce7d92 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_auto_router_routing_test_request.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_auto_router_routing_test_request.ts @@ -1,6 +1,7 @@ import { AutoRouterRoutingTestRequest } from "../networking"; import { ComplexityRouterConfigPayload } from "./build_complexity_router_config"; import { z } from "zod"; +import { jevClassifierConfigSchema } from "./jev_classifier_config"; export const JEV_CONNECTION_TEST_PROMPT = "What is 2 plus 2?"; @@ -21,13 +22,17 @@ export const buildSavedJevConnectionTestRequest = ( })() : rawConfig; const result = z - .object({ classifier_type: z.literal("jev"), tiers: z.record(z.unknown()) }) + .object({ + classifier_type: z.literal("jev"), + tiers: z.record(z.unknown()), + jev_classifier_config: jevClassifierConfigSchema.default({}), + }) .passthrough() .safeParse(parsed); if (!result.success) return undefined; return { prompt: JEV_CONNECTION_TEST_PROMPT, - complexity_router_config: { ...result.data, jev_classifier_config: undefined }, + complexity_router_config: result.data, saved_model_id: savedModelId, ...(teamId && { team_id: teamId }), };