fix(auto-router): preserve JEV accounting and context bounds

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/router_strategy/complexity_router/jev_classifier.py

@@ -100,8 +100,8 @@ class HttpJevClassifierClient:
             ),  # pyright: ignore[reportArgumentType]  # HTTP headers are not mutated by AsyncHTTPHandler
             timeout=timeout_s,
         )
-        self._log_response(request, response, request_kwargs, start_time)
         response.raise_for_status()
+        self._log_response(request, response, request_kwargs, start_time)
         return TypeAdapter(JevSystemOneResponse).validate_python(response.json())
 
     @staticmethod

tests/test_litellm/router_strategy/complexity_router/test_jev_classifier.py

@@ -3,6 +3,7 @@ import json
 from collections.abc import Mapping
 from datetime import datetime
 from typing import Final
+from unittest.mock import create_autospec
 
 import httpx
 import pytest
@@ -39,6 +40,37 @@ class _UsageRecorder(CustomLogger):
         self.calls = (*self.calls, kwargs)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("status_code", [400, 429, 500, 503])
+async def test_jev_http_errors_do_not_dispatch_successful_usage(
+    monkeypatch: pytest.MonkeyPatch, status_code: int
+) -> None:
+    recorder: Final = _UsageRecorder()
+    monkeypatch.setattr(litellm, "_async_success_callback", [recorder])
+    handler: Final = create_autospec(AsyncHTTPHandler, instance=True)
+    handler.post.return_value = httpx.Response(
+        status_code,
+        request=httpx.Request("POST", "https://typesafe.test/v1/systemone"),
+        json={
+            "model": "jev-accounting",
+            "usage": {"input_tokens": 3, "output_tokens": 2},
+            "answers": {"tier": _answer().model_dump()},
+        },
+    )
+    provider: Final = HttpJevClassifierClient("test", "https://typesafe.test", handler)
+    request: Final = build_jev_request(
+        "choose a tier", None, "jev-accounting", DEFAULT_JEV_INSTRUCTIONS, {"SIMPLE": "cheap"}
+    )
+
+    with pytest.raises(httpx.HTTPStatusError) as error:
+        await provider.evaluate(request, timeout_s=3)
+    await GLOBAL_LOGGING_WORKER.flush()
+
+    assert error.value.response.status_code == status_code
+    handler.post.assert_awaited_once()
+    assert recorder.calls == ()
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("answer", ["SIMPLE", "UNAVAILABLE", "malformed"])
 @pytest.mark.parametrize("private", [False, True])

ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx

@@ -8,7 +8,7 @@ import {
   chooseSelectOption,
 } from "../../../tests/test-utils";
 import userEvent from "@testing-library/user-event";
-import { vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import AddAutoRouterTab from "./add_auto_router_tab";
 import { toast } from "@/lib/toast";
 import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
@@ -1535,6 +1535,40 @@ describe("getSubmitBlockedReason", () => {
 describe("preset catalog fetch states", () => {
   afterEach(() => vi.mocked(useAutoRouterPresets).mockReturnValue(LOADED_PRESETS_QUERY));
 
+  it("preserves a JEV preset's per-turn bound in the create request", async () => {
+    vi.clearAllMocks();
+    testQueryClient.clear();
+    vi.mocked(handleAddAutoRouterSubmit).mockReset();
+    mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
+    vi.mocked(useAutoRouterPresets).mockReturnValue({
+      ...LOADED_PRESETS_QUERY,
+      data: [
+        {
+          ...ANTHROPIC_PRESET,
+          key: "bounded_jev",
+          label: "Bounded JEV",
+          complexity_router_config: {
+            ...ANTHROPIC_PRESET.complexity_router_config,
+            classifier_type: "jev",
+            jev_classifier_config: { model: "jev-test", timeout_ms: 3000 },
+            classifier_context_per_turn_chars: 450,
+          },
+        },
+      ],
+    });
+    renderWithProviders(<Harness />);
+    await waitForPresetEnabled("Bounded JEV");
+    await selectTemplate("Bounded JEV");
+    fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "bounded-router" } });
+    fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
+
+    await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
+    expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls[0][0].complexity_router_config).toMatchObject({
+      classifier_type: "jev",
+      classifier_context_per_turn_chars: 450,
+    });
+  });
+
   it("keeps showing cached presets without the error banner when only a refetch fails", () => {
     vi.mocked(useAutoRouterPresets).mockReturnValue({
       ...LOADED_PRESETS_QUERY,

ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx

@@ -415,6 +415,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
     classifierLlmConfig: complexityRouterConfig.classifier_llm_config,
     classifierContextWindowSize: complexityRouterConfig.classifier_context_window_size,
     classifierContextBudgetChars: complexityRouterConfig.classifier_context_budget_chars,
+    classifierContextPerTurnChars: complexityRouterConfig.classifier_context_per_turn_chars,
     classifierContextIncludeAssistantTurns: complexityRouterConfig.classifier_context_include_assistant_turns,
     classifierFallback: complexityRouterConfig.classifier_fallback,
     sessionAffinity: complexityRouterConfig.session_affinity ?? DEFAULT_SESSION_AFFINITY,

ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts

@@ -91,6 +91,7 @@ describe("buildComplexityRouterConfig", () => {
       classificationExamples: "stale examples",
       classifierContextWindowSize: 4,
       classifierContextBudgetChars: 2000,
+      classifierContextPerTurnChars: 450,
       classifierContextIncludeAssistantTurns: true,
       classifierFallback: "default_model",
       ...(custom && {
@@ -115,6 +116,7 @@ describe("buildComplexityRouterConfig", () => {
     expect(config.jev_classifier_config).toEqual(expectedJevConfig);
     expect(config.classifier_context_window_size).toBe(4);
     expect(config.classifier_context_budget_chars).toBe(2000);
+    expect(config.classifier_context_per_turn_chars).toBe(450);
     expect(config.classifier_context_include_assistant_turns).toBe(true);
     expect(config).not.toHaveProperty("classifier_llm_config");
     expect(config).not.toHaveProperty("classification_prompt");
@@ -876,13 +878,13 @@ describe("buildComplexityRouterConfig scorer knobs", () => {
     "%s with fallback %s only emits custom dimensions when its scorer decides",
     (classifierType, classifierFallback, emits) => {
       const dimension = { name: "d", weight: 0.4, keywords: ["orbitmesh"] };
-      const params = {
+      const uncheckedParams: unknown = {
         ...baseParams,
         classifierType,
         classifierFallback,
         customDimensions: [{ id: "row", ...dimension }],
       };
-      const payload = buildComplexityRouterConfig(params);
+      const payload = buildComplexityRouterConfig(uncheckedParams as BuildComplexityRouterConfigParams);
       if (emits) expect(payload.custom_dimensions).toEqual([dimension]);
       else expect(payload).not.toHaveProperty("custom_dimensions");
     },

ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts

@@ -156,6 +156,7 @@ export interface StoredComplexityRouterConfig {
   jev_classifier_config?: unknown;
   classifier_context_window_size?: unknown;
   classifier_context_budget_chars?: unknown;
+  classifier_context_per_turn_chars?: unknown;
   classifier_context_include_assistant_turns?: unknown;
   classifier_fallback?: unknown;
   classification_mode?: unknown;
@@ -195,6 +196,7 @@ export interface BuildComplexityRouterConfigParams {
   jevClassifierConfig?: JevClassifierConfig;
   classifierContextWindowSize: number | undefined;
   classifierContextBudgetChars: number | undefined;
+  classifierContextPerTurnChars?: number;
   classifierContextIncludeAssistantTurns: boolean | undefined;
   classifierFallback: ClassifierFallback | undefined;
   classificationPrompt: string | undefined;
@@ -533,6 +535,7 @@ const classifierWireFields = (
     hybridBoundaryMargin,
     classifierContextWindowSize,
     classifierContextBudgetChars,
+    classifierContextPerTurnChars,
     classifierContextIncludeAssistantTurns,
   }: Pick<
     BuildComplexityRouterConfigParams,
@@ -542,6 +545,7 @@ const classifierWireFields = (
     | "hybridBoundaryMargin"
     | "classifierContextWindowSize"
     | "classifierContextBudgetChars"
+    | "classifierContextPerTurnChars"
     | "classifierContextIncludeAssistantTurns"
   >,
 ): Partial<ComplexityRouterConfigPayload> => {
@@ -566,6 +570,10 @@ const classifierWireFields = (
       classifierContextBudgetChars !== undefined && {
         classifier_context_budget_chars: classifierContextBudgetChars,
       }),
+    ...(usesClassifierContext(effectiveType) &&
+      classifierContextPerTurnChars !== undefined && {
+        classifier_context_per_turn_chars: classifierContextPerTurnChars,
+      }),
     ...(usesClassifierContext(effectiveType) &&
       classifierContextIncludeAssistantTurns !== undefined && {
         classifier_context_include_assistant_turns: classifierContextIncludeAssistantTurns,
@@ -587,6 +595,7 @@ export const buildComplexityRouterConfig = ({
   jevClassifierConfig,
   classifierContextWindowSize,
   classifierContextBudgetChars,
+  classifierContextPerTurnChars,
   classifierContextIncludeAssistantTurns,
   classifierFallback,
   classificationPrompt,
@@ -648,6 +657,7 @@ export const buildComplexityRouterConfig = ({
     hybridBoundaryMargin,
     classifierContextWindowSize,
     classifierContextBudgetChars,
+    classifierContextPerTurnChars,
     classifierContextIncludeAssistantTurns,
   };
   const effectiveType = effectiveClassifierType({ custom_tier_set: customTierSet, classifier_type: classifierType });

ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts

@@ -80,6 +80,7 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
       },
       classifier_context_window_size: 7,
       classifier_context_budget_chars: 9000,
+      classifier_context_per_turn_chars: 450,
       classifier_context_include_assistant_turns: true,
       some_future_backend_key: { nested: true },
     };
@@ -87,12 +88,14 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
     expect(effectiveClassifierType(hydrated)).toBe("jev");
     expect(hydrated.classifier_llm_config).toBeUndefined();
     expect(hydrated.jev_classifier_config).toEqual(stored.jev_classifier_config);
+    expect(hydrated.classifier_context_per_turn_chars).toBe(450);
     const saved = buildUpdatedComplexityRouterConfig(stored, hydrated);
     const expectedSavedConfig = {
       classifier_type: "jev",
       jev_classifier_config: stored.jev_classifier_config,
       classifier_context_window_size: 7,
       classifier_context_budget_chars: 9000,
+      classifier_context_per_turn_chars: 450,
       classifier_context_include_assistant_turns: true,
       some_future_backend_key: { nested: true },
     };
@@ -100,6 +103,7 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
     expect(saved).not.toHaveProperty("classifier_llm_config");
     const reloaded = hydrateComplexityRouterConfig(saved, undefined);
     expect(reloaded.jev_classifier_config).toEqual(hydrated.jev_classifier_config);
+    expect(reloaded.classifier_context_per_turn_chars).toBe(450);
     expect(effectiveClassifierType(reloaded)).toBe("jev");
     const llm = buildUpdatedComplexityRouterConfig(saved, transitionClassifierType(reloaded, "llm"));
     expect(llm).not.toHaveProperty("jev_classifier_config");
@@ -287,6 +291,17 @@ describe("capability classifier configuration", () => {
 });
 
 describe("buildUpdatedComplexityRouterConfig classifier context window", () => {
+  it.each(["llm", "jev"] as const)("saves the form's per-turn bound over the stored %s bound", (classifier_type) => {
+    const formValue = {
+      ...hydrateComplexityRouterConfig({ ...STORED_LLM, classifier_type }, undefined),
+      classifier_context_per_turn_chars: 600,
+    };
+    const saved = buildUpdatedComplexityRouterConfig(STORED_LLM, formValue);
+
+    expect(saved.classifier_context_per_turn_chars).toBe(600);
+    expect(hydrateComplexityRouterConfig(saved, undefined).classifier_context_per_turn_chars).toBe(600);
+  });
+
   it("round-trips an untouched edit without changing the classifier context values", () => {
     const formValue = {
       tiers: STORED_LLM.tiers,

ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx

@@ -144,6 +144,10 @@ export const hydrateComplexityRouterConfig = (
       typeof parsedConfig.classifier_context_budget_chars === "number"
         ? parsedConfig.classifier_context_budget_chars
         : undefined,
+    classifier_context_per_turn_chars:
+      typeof parsedConfig.classifier_context_per_turn_chars === "number"
+        ? parsedConfig.classifier_context_per_turn_chars
+        : undefined,
     classifier_context_include_assistant_turns:
       typeof parsedConfig.classifier_context_include_assistant_turns === "boolean"
         ? parsedConfig.classifier_context_include_assistant_turns
@@ -342,6 +346,7 @@ export const buildUpdatedComplexityRouterConfig = (
     classifierLlmConfig: value.classifier_llm_config,
     classifierContextWindowSize: value.classifier_context_window_size,
     classifierContextBudgetChars: value.classifier_context_budget_chars,
+    classifierContextPerTurnChars: value.classifier_context_per_turn_chars,
     classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns,
     classifierFallback: value.classifier_fallback,
     sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY,