feat(ui): configure auto-router affinity idle TTL (#39679)

litellm/router_strategy/complexity_router/README.md

@@ -190,6 +190,9 @@ model_list:
 
         # Let that replacement also override a kept session pin, for image turns only (default: false)
         modality_pin_override: true
+
+        # Refreshes on every pin reuse, so this is idle time rather than total session length (default: 3600)
+        session_affinity_ttl_seconds: 300
 ```
 
 ## Usage
@@ -240,6 +243,10 @@ affinity write happens upstream of the gate and stores the session's own model,
 turn replays the original pin and the override is never pinned in its place. It does nothing
 unless `modality_routing` is also on.
 
+### Session pin retention
+
+`session_affinity_ttl_seconds` is the idle window for both the model pin selected by session affinity and the deployment pin. Every request that reuses a pin refreshes its TTL, so a session actively sending requests stays pinned. After the window passes with no pin reuse, the next request classifies again and creates a fresh pin. Omit the setting to track the default of 3600 seconds.
+
 ### Heuristic-first chaining
 
 `classifier_type: heuristic_first` runs the local scorer on every request and only calls the LLM

ui/litellm-dashboard/src/components/add_model/AffinityControls.tsx

@@ -1,26 +1,58 @@
 import React from "react";
 
+import { Input } from "@/components/ui/input";
 import { Switch } from "@/components/ui/switch";
 
 import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
-import { DEFAULT_DEPLOYMENT_AFFINITY } from "./ComplexityRouterConfig";
+import { DEFAULT_DEPLOYMENT_AFFINITY, DEFAULT_SESSION_AFFINITY_TTL_SECONDS } from "./ComplexityRouterConfig";
 
 export const AffinityControls: React.FC<{
   value: ComplexityRouterConfigValue;
   onChange: (value: ComplexityRouterConfigValue) => void;
-}> = ({ value, onChange }) => (
-  <>
-    <div className="flex items-center gap-2 mb-2">
-      <Switch
-        checked={value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY}
-        onCheckedChange={(deploymentAffinity) => onChange({ ...value, deployment_affinity: deploymentAffinity })}
-        aria-label="Pin a session to one deployment per model group"
-      />
-      <strong className="font-semibold">Pin a session to one deployment per model group</strong>
-    </div>
-    <span className="block text-xs text-muted-foreground">
-      Keeps a session on the same deployment within a group, so provider prompt caches stay warm. Turn off to
-      load-balance every turn.
-    </span>
-  </>
-);
+}> = ({ value, onChange }) => {
+  const [ttlDraft, setTtlDraft] = React.useState<string | null>(null);
+  const commitTtl = (raw: string) => {
+    setTtlDraft(null);
+    if (raw.trim() === "") {
+      onChange({ ...value, session_affinity_ttl_seconds: undefined });
+      return;
+    }
+    const parsed = Number(raw);
+    if (!Number.isFinite(parsed)) return;
+    onChange({ ...value, session_affinity_ttl_seconds: Math.max(1, Math.round(parsed)) });
+  };
+
+  return (
+    <>
+      <div className="flex items-center gap-2 mb-2">
+        <Switch
+          checked={value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY}
+          onCheckedChange={(deploymentAffinity) => onChange({ ...value, deployment_affinity: deploymentAffinity })}
+          aria-label="Pin a session to one deployment per model group"
+        />
+        <strong className="font-semibold">Pin a session to one deployment per model group</strong>
+      </div>
+      <span className="block text-xs mb-3 text-muted-foreground">
+        Keeps a session on the same deployment within a group, so provider prompt caches stay warm. Turn off to
+        load-balance every turn.
+      </span>
+      <div style={{ maxWidth: 320 }}>
+        <label className="block text-sm font-medium mb-1" htmlFor="session-affinity-ttl">
+          How long a pin survives idle (seconds)
+        </label>
+        <Input
+          id="session-affinity-ttl"
+          inputMode="numeric"
+          value={ttlDraft ?? value.session_affinity_ttl_seconds ?? ""}
+          placeholder={String(DEFAULT_SESSION_AFFINITY_TTL_SECONDS)}
+          onChange={(event) => setTtlDraft(event.target.value)}
+          onBlur={(event) => commitTtl(event.target.value)}
+        />
+        <span className="block text-xs mt-1 text-muted-foreground">
+          Refreshes after every request that reuses a pin. Empty tracks the backend default of{" "}
+          {DEFAULT_SESSION_AFFINITY_TTL_SECONDS} seconds.
+        </span>
+      </div>
+    </>
+  );
+};

ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx

@@ -947,6 +947,46 @@ describe("ComplexityRouterConfig affinity panel", () => {
 
     expect(screen.getByRole("switch", { name: "Pin a session to one deployment per model group" })).not.toBeChecked();
   });
+
+  it("writes an idle TTL on blur and keeps the partial input as a draft while typing", () => {
+    const onChange = vi.fn();
+    renderWithProviders(<ComplexityRouterConfig {...baseProps} onChange={onChange} />);
+    fireEvent.click(screen.getByText("Advanced: Affinity"));
+
+    const ttl = screen.getByLabelText("How long a pin survives idle (seconds)");
+    expect(ttl).toHaveAttribute("placeholder", "3600");
+    fireEvent.change(ttl, { target: { value: "300" } });
+    expect(onChange).not.toHaveBeenCalled();
+    fireEvent.blur(ttl);
+
+    expect(onChange).toHaveBeenCalledWith({ ...defaultValue, session_affinity_ttl_seconds: 300 });
+  });
+
+  it("clearing the idle TTL returns the router to its backend default", () => {
+    const onChange = vi.fn();
+    const value = { ...defaultValue, session_affinity_ttl_seconds: 300 };
+    renderWithProviders(<ComplexityRouterConfig {...baseProps} value={value} onChange={onChange} />);
+    fireEvent.click(screen.getByText("Advanced: Affinity"));
+
+    const ttl = screen.getByLabelText("How long a pin survives idle (seconds)");
+    expect(ttl).toHaveValue("300");
+    fireEvent.change(ttl, { target: { value: "" } });
+    fireEvent.blur(ttl);
+
+    expect(onChange).toHaveBeenCalledWith({ ...value, session_affinity_ttl_seconds: undefined });
+  });
+
+  it("clamps a non-positive idle TTL to the backend's minimum", () => {
+    const onChange = vi.fn();
+    renderWithProviders(<ComplexityRouterConfig {...baseProps} onChange={onChange} />);
+    fireEvent.click(screen.getByText("Advanced: Affinity"));
+
+    const ttl = screen.getByLabelText("How long a pin survives idle (seconds)");
+    fireEvent.change(ttl, { target: { value: "0" } });
+    fireEvent.blur(ttl);
+
+    expect(onChange).toHaveBeenCalledWith({ ...defaultValue, session_affinity_ttl_seconds: 1 });
+  });
 });
 
 describe("ComplexityRouterConfig default model", () => {

ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx

@@ -58,6 +58,7 @@ export const DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE = 3;
 export const DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS = 8000;
 export const MIN_QUOTED_CONTEXT_TURN_CHARS = 120;
 export const DEFAULT_SESSION_AFFINITY = false;
+export const DEFAULT_SESSION_AFFINITY_TTL_SECONDS = 3600;
 export const DEFAULT_DEPLOYMENT_AFFINITY = true;
 
 export type ClassificationMode = "every_request" | "user_turn";
@@ -411,6 +412,7 @@ export interface ComplexityRouterConfigValue {
   hybrid_boundary_margin?: number;
   classification_mode?: ClassificationMode;
   session_affinity?: boolean;
+  session_affinity_ttl_seconds?: number;
   modality_routing?: boolean;
   modality_pin_override?: boolean;
   deployment_affinity?: boolean;

ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx

@@ -493,7 +493,7 @@ describe("AddAutoRouterTab", () => {
     });
   });
 
-  it("carries session affinity turned on through to the create payload", async () => {
+  it("carries session affinity turned on and its idle window through to the create payload", async () => {
     const user = userEvent.setup();
     vi.mocked(getMissingTiersError).mockReturnValue(null);
 
@@ -503,12 +503,17 @@ describe("AddAutoRouterTab", () => {
     expandDetailedConfiguration();
     await user.click(screen.getByText("Advanced: Classification Method"));
     await user.click(await screen.findByRole("radio", { name: /Once per session/ }));
+    await user.click(screen.getByText("Advanced: Affinity"));
+    const ttl = await screen.findByLabelText("How long a pin survives idle (seconds)");
+    fireEvent.change(ttl, { target: { value: "300" } });
+    fireEvent.blur(ttl);
 
     await user.click(screen.getByRole("button", { name: /add auto router/i }));
 
     await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
     expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({
       session_affinity: true,
+      session_affinity_ttl_seconds: 300,
     });
   });
 

ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx

@@ -388,6 +388,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
     reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score,
     enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation,
     contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer,
+    sessionAffinityTtlSeconds: complexityRouterConfig.session_affinity_ttl_seconds,
   };
 
   const submitRecommendedRouter = async (name: string) => {

ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts

@@ -73,6 +73,15 @@ describe("buildComplexityRouterConfig", () => {
     expect(config.context_window_escalation_buffer).toBe(0.9);
   });
 
+  it("omits session_affinity_ttl_seconds when untouched, so the router tracks the backend default", () => {
+    expect(buildComplexityRouterConfig(baseParams)).not.toHaveProperty("session_affinity_ttl_seconds");
+  });
+
+  it("emits an explicit session affinity idle window", () => {
+    const config = buildComplexityRouterConfig({ ...baseParams, sessionAffinityTtlSeconds: 300 });
+    expect(config.session_affinity_ttl_seconds).toBe(300);
+  });
+
   it("trims escalation keywords and drops blank entries", () => {
     const config = buildComplexityRouterConfig({
       ...baseParams,

ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts

@@ -138,6 +138,7 @@ export interface BuildComplexityRouterConfigParams {
   tierModelParams?: TierModelParamsByTier;
   enableContextWindowEscalation?: boolean;
   contextWindowEscalationBuffer?: number;
+  sessionAffinityTtlSeconds?: number;
 }
 
 /**
@@ -175,6 +176,7 @@ export interface ComplexityRouterConfigPayload {
   hybrid_boundary_margin?: number;
   classification_mode: ClassificationMode;
   session_affinity: boolean;
+  session_affinity_ttl_seconds?: number;
   deployment_affinity: boolean;
   modality_routing: boolean;
   modality_pin_override: boolean;
@@ -456,6 +458,7 @@ export const buildComplexityRouterConfig = ({
   tierModelParams,
   enableContextWindowEscalation,
   contextWindowEscalationBuffer,
+  sessionAffinityTtlSeconds,
 }: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => {
   const serializedTierModelConfigs = customTierSet
     ? serializeTierModelConfigs(
@@ -522,6 +525,9 @@ export const buildComplexityRouterConfig = ({
     ...(contextWindowEscalationBuffer !== undefined && {
       context_window_escalation_buffer: contextWindowEscalationBuffer,
     }),
+    ...(sessionAffinityTtlSeconds !== undefined && {
+      session_affinity_ttl_seconds: sessionAffinityTtlSeconds,
+    }),
     ...scorerKnobs,
   };
   if (!customTierSet) return payload;

ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts

@@ -257,6 +257,43 @@ describe("buildUpdatedComplexityRouterConfig session affinity", () => {
   });
 });
 
+describe("buildUpdatedComplexityRouterConfig session affinity ttl", () => {
+  it("writes an edited idle window", () => {
+    const result = buildUpdatedComplexityRouterConfig(STORED, { ...FORM_VALUE, session_affinity_ttl_seconds: 300 });
+    expect(result.session_affinity_ttl_seconds).toBe(300);
+  });
+
+  it("carries a stored idle window through an untouched open-and-save", () => {
+    const stored = { ...STORED, session_affinity_ttl_seconds: 900 };
+    const result = buildUpdatedComplexityRouterConfig(stored, hydrateComplexityRouterConfig(stored, undefined));
+    expect(result.session_affinity_ttl_seconds).toBe(900);
+  });
+
+  it("drops the key when the field is cleared, so the router goes back to tracking the backend default", () => {
+    const result = buildUpdatedComplexityRouterConfig(
+      { ...STORED, session_affinity_ttl_seconds: 900 },
+      { ...FORM_VALUE, session_affinity_ttl_seconds: undefined },
+    );
+    expect(result).not.toHaveProperty("session_affinity_ttl_seconds");
+  });
+
+  it("keeps the idle window on a custom tier set, whose deployment pin still uses it", () => {
+    const result = buildUpdatedComplexityRouterConfig(STORED, {
+      ...FORM_VALUE,
+      session_affinity_ttl_seconds: 300,
+      custom_tier_set: {
+        tiers: [
+          { id: "a", name: "CASUAL", definition: "small talk", models: ["gpt-4o-mini"] },
+          { id: "b", name: "AUDIT", definition: "security review", models: ["o1"] },
+        ],
+        fallback_tier_id: "a",
+      },
+    });
+    expect(result.session_affinity).toBe(false);
+    expect(result.session_affinity_ttl_seconds).toBe(300);
+  });
+});
+
 describe("buildUpdatedComplexityRouterConfig modality pin override", () => {
   it("writes modality_pin_override explicitly both ways", () => {
     expect(
@@ -529,6 +566,7 @@ describe("managed keys survive an untouched open-and-save", () => {
     classifier_fallback: "default_model",
     classification_mode: "user_turn",
     session_affinity: true,
+    session_affinity_ttl_seconds: 300,
     modality_routing: true,
     modality_pin_override: true,
     deployment_affinity: false,

ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx

@@ -550,6 +550,49 @@ describe("EditAutoRouterModal deployment affinity", () => {
     expect(savedConfig().deployment_affinity).toBe(false);
   });
 
+  it("preserves an idle TTL through an untouched save", async () => {
+    const user = userEvent.setup();
+    renderWithStoredConfig({ ...STORED_CONFIG, session_affinity_ttl_seconds: 300 });
+
+    await user.click(await screen.findByText("Advanced: Affinity"));
+    expect(await screen.findByLabelText("How long a pin survives idle (seconds)")).toHaveValue("300");
+
+    await user.click(screen.getByRole("button", { name: /save changes/i }));
+
+    await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
+    expect(savedConfig().session_affinity_ttl_seconds).toBe(300);
+  });
+
+  it("persists an edited idle TTL", async () => {
+    const user = userEvent.setup();
+    renderWithStoredConfig(STORED_CONFIG);
+
+    await user.click(await screen.findByText("Advanced: Affinity"));
+    const ttl = await screen.findByLabelText("How long a pin survives idle (seconds)");
+    fireEvent.change(ttl, { target: { value: "300" } });
+    fireEvent.blur(ttl);
+
+    await user.click(screen.getByRole("button", { name: /save changes/i }));
+
+    await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
+    expect(savedConfig().session_affinity_ttl_seconds).toBe(300);
+  });
+
+  it("removes the idle TTL when cleared", async () => {
+    const user = userEvent.setup();
+    renderWithStoredConfig({ ...STORED_CONFIG, session_affinity_ttl_seconds: 300 });
+
+    await user.click(await screen.findByText("Advanced: Affinity"));
+    const ttl = await screen.findByLabelText("How long a pin survives idle (seconds)");
+    fireEvent.change(ttl, { target: { value: "" } });
+    fireEvent.blur(ttl);
+
+    await user.click(screen.getByRole("button", { name: /save changes/i }));
+
+    await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
+    expect(savedConfig()).not.toHaveProperty("session_affinity_ttl_seconds");
+  });
+
   // modality_pin_override is a managed key, so the modal rewrites it from form state on save. A
   // hydration gap would silently turn a stored override off on the next untouched save.
   it("shows a stored modality_pin_override=true as on and preserves it through an untouched save", async () => {

ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx

@@ -104,6 +104,7 @@ export interface StoredComplexityRouterConfig {
   dimension_weights?: unknown;
   reasoning_override_min_score?: unknown;
   session_affinity?: unknown;
+  session_affinity_ttl_seconds?: unknown;
   modality_routing?: unknown;
   modality_pin_override?: unknown;
   deployment_affinity?: unknown;
@@ -182,6 +183,11 @@ export const hydrateComplexityRouterConfig = (
     reasoning_override_min_score: hydrateReasoningOverrideMinScore(parsedConfig.reasoning_override_min_score),
     session_affinity:
       typeof parsedConfig.session_affinity === "boolean" ? parsedConfig.session_affinity : DEFAULT_SESSION_AFFINITY,
+    session_affinity_ttl_seconds:
+      typeof parsedConfig.session_affinity_ttl_seconds === "number" &&
+      Number.isFinite(parsedConfig.session_affinity_ttl_seconds)
+        ? parsedConfig.session_affinity_ttl_seconds
+        : undefined,
     modality_routing: typeof parsedConfig.modality_routing === "boolean" ? parsedConfig.modality_routing : false,
     modality_pin_override:
       typeof parsedConfig.modality_pin_override === "boolean" ? parsedConfig.modality_pin_override : false,
@@ -224,6 +230,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
   "hybrid_boundary_margin",
   "classification_mode",
   "session_affinity",
+  "session_affinity_ttl_seconds",
   "modality_routing",
   "modality_pin_override",
   "deployment_affinity",
@@ -322,6 +329,7 @@ export const buildUpdatedComplexityRouterConfig = (
     classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns,
     classifierFallback: value.classifier_fallback,
     sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY,
+    sessionAffinityTtlSeconds: value.session_affinity_ttl_seconds,
     modalityRouting: value.modality_routing ?? false,
     modalityPinOverride: value.modality_pin_override ?? false,
     deploymentAffinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY,

ui/litellm-dashboard/src/lib/autorouter_presets.test.ts

@@ -83,9 +83,19 @@ describe("autorouter_presets", () => {
       expect(config.tier_boundaries).toBeUndefined();
       expect(config.token_thresholds).toBeUndefined();
       expect(config.dimension_weights).toBeUndefined();
+      expect(config.session_affinity_ttl_seconds).toBeUndefined();
     }
   });
 
+  it("carries a preset's session affinity idle window into the prefilled form state", () => {
+    const config = getPresetByKey("anthropic_family")!.complexity_router_config;
+    const prefill = buildPresetPrefill({ ...config, session_affinity_ttl_seconds: 300 }, groupsOnly([]));
+    expect(prefill.complexityRouterConfig.session_affinity_ttl_seconds).toBe(300);
+    expect(
+      buildPresetPrefill(config, groupsOnly([])).complexityRouterConfig.session_affinity_ttl_seconds,
+    ).toBeUndefined();
+  });
+
   it("keeps the model-family presets on the heuristic classifier", () => {
     for (const key of ["anthropic_family", "gemini_family", "openai_family"]) {
       expect(getPresetByKey(key)!.complexity_router_config.classifier_type).toBe("heuristic");

ui/litellm-dashboard/src/lib/autorouter_presets.ts

@@ -284,6 +284,7 @@ export const buildPresetPrefill = (
       classifier_context_include_assistant_turns: config.classifier_context_include_assistant_turns,
       classification_mode: config.classification_mode ?? DEFAULT_CLASSIFICATION_MODE,
       session_affinity: config.session_affinity ?? DEFAULT_SESSION_AFFINITY,
+      session_affinity_ttl_seconds: config.session_affinity_ttl_seconds,
       deployment_affinity: config.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY,
       modality_routing: config.modality_routing ?? false,
       modality_pin_override: config.modality_pin_override ?? false,