mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
fix(ui): repair router settings controls
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Krrish Dholakia
parent
221183b2f4
commit
e1e233f88a
12 changed files with 236 additions and 282 deletions
|
|
@ -224,36 +224,9 @@ ROUTER_SETTINGS_FIELDS: List[RouterSettingsField] = [
|
|||
field_name="default_litellm_params",
|
||||
field_type="Dictionary",
|
||||
field_value=None,
|
||||
field_description=(
|
||||
"Default parameters for Router.chat.completion.create. E.g. set "
|
||||
"cache_control_injection_points here to enable Anthropic/Bedrock "
|
||||
"prompt caching for every model on this proxy."
|
||||
),
|
||||
field_description="Default parameters for Router.chat.completion.create",
|
||||
field_default=None,
|
||||
ui_field_name="Default LiteLLM Params",
|
||||
link="https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing",
|
||||
),
|
||||
RouterSettingsField(
|
||||
field_name="optional_pre_call_checks",
|
||||
field_type="List",
|
||||
field_value=None,
|
||||
field_description=(
|
||||
"Extra checks the router runs before picking a deployment. Add "
|
||||
"'prompt_caching' to route repeat requests back to the deployment "
|
||||
"that cached the prompt."
|
||||
),
|
||||
field_default=[],
|
||||
options=[
|
||||
"prompt_caching",
|
||||
"router_budget_limiting",
|
||||
"responses_api_deployment_check",
|
||||
"deployment_affinity",
|
||||
"session_affinity",
|
||||
"enforce_model_rate_limits",
|
||||
"encrypted_content_affinity",
|
||||
],
|
||||
ui_field_name="Optional Pre-call Checks",
|
||||
link="https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing",
|
||||
),
|
||||
RouterSettingsField(
|
||||
field_name="set_verbose",
|
||||
|
|
|
|||
|
|
@ -78,46 +78,6 @@ class TestRouterSettingsEndpoints:
|
|||
assert isinstance(routing_strategy_field["options"], list)
|
||||
assert len(routing_strategy_field["options"]) > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_router_fields_includes_optional_pre_call_checks(self):
|
||||
"""
|
||||
Regression test: `optional_pre_call_checks` (e.g. "prompt_caching", used for
|
||||
Claude Code prompt cache routing) must be exposed as a configurable field so
|
||||
the Admin UI can render and save it, not just `default_litellm_params`.
|
||||
"""
|
||||
response = client.get(
|
||||
"/router/fields", headers={"Authorization": "Bearer sk-1234"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
fields = response.json()["fields"]
|
||||
field = next(
|
||||
(f for f in fields if f["field_name"] == "optional_pre_call_checks"), None
|
||||
)
|
||||
assert field is not None
|
||||
assert "prompt_caching" in field["options"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_router_fields_excludes_unimplemented_forward_client_headers_check(self):
|
||||
"""
|
||||
Regression test: "forward_client_headers_by_model_group" is a literal in the
|
||||
OptionalPreCallChecks type union, but Router.add_optional_pre_call_checks has
|
||||
no handler for it - selecting it from the Admin UI's multi-select would save
|
||||
successfully and show as enabled while doing nothing. It must not be offered
|
||||
as an option until it's actually implemented.
|
||||
"""
|
||||
response = client.get(
|
||||
"/router/fields", headers={"Authorization": "Bearer sk-1234"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
fields = response.json()["fields"]
|
||||
field = next(
|
||||
(f for f in fields if f["field_name"] == "optional_pre_call_checks"), None
|
||||
)
|
||||
assert field is not None
|
||||
assert "forward_client_headers_by_model_group" not in field["options"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_router_settings_includes_routing_groups_from_live_router(
|
||||
self, monkeypatch
|
||||
|
|
|
|||
|
|
@ -1,24 +1,18 @@
|
|||
import { describe, it, expect, vi } from "vitest";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import { fireEvent, render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import DefaultLitellmParamsSection from "./DefaultLitellmParamsSection";
|
||||
|
||||
describe("DefaultLitellmParamsSection", () => {
|
||||
it("should render the non-cache-control keys as JSON in the textarea", () => {
|
||||
render(
|
||||
<DefaultLitellmParamsSection
|
||||
value={{ timeout: 30, max_retries: 0 }}
|
||||
routerFieldsMetadata={{}}
|
||||
onChange={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
render(<DefaultLitellmParamsSection value={{ timeout: 30, max_retries: 0 }} onChange={vi.fn()} />);
|
||||
const textarea = screen.getByRole("textbox") as HTMLTextAreaElement;
|
||||
expect(textarea.value).toContain('"timeout": 30');
|
||||
expect(textarea.value).toContain('"max_retries": 0');
|
||||
});
|
||||
|
||||
it("should not show the cache control editor when no injection points are set", () => {
|
||||
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} routerFieldsMetadata={{}} onChange={vi.fn()} />);
|
||||
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} onChange={vi.fn()} />);
|
||||
expect(screen.queryByTestId("cache-control-location-select-0")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
|
|
@ -26,17 +20,29 @@ describe("DefaultLitellmParamsSection", () => {
|
|||
render(
|
||||
<DefaultLitellmParamsSection
|
||||
value={{ cache_control_injection_points: [{ location: "message", role: "system" }] }}
|
||||
routerFieldsMetadata={{}}
|
||||
onChange={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByTestId("cache-control-location-select-0")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should preserve unsupported cache control points in the JSON editor", () => {
|
||||
render(
|
||||
<DefaultLitellmParamsSection
|
||||
value={{ cache_control_injection_points: [{ location: "tool_config" }] }}
|
||||
onChange={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
|
||||
const textarea = screen.getByRole("textbox") as HTMLTextAreaElement;
|
||||
expect(textarea.value).toContain('"location": "tool_config"');
|
||||
expect(screen.queryByTestId("cache-control-location-select-0")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should call onChange with cache_control_injection_points added when the toggle is switched on", async () => {
|
||||
const onChange = vi.fn();
|
||||
const user = userEvent.setup();
|
||||
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} routerFieldsMetadata={{}} onChange={onChange} />);
|
||||
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} onChange={onChange} />);
|
||||
|
||||
await user.click(screen.getByRole("switch"));
|
||||
|
||||
|
|
@ -49,7 +55,6 @@ describe("DefaultLitellmParamsSection", () => {
|
|||
render(
|
||||
<DefaultLitellmParamsSection
|
||||
value={{ timeout: 30, cache_control_injection_points: [{ location: "message", role: "system" }] }}
|
||||
routerFieldsMetadata={{}}
|
||||
onChange={onChange}
|
||||
/>,
|
||||
);
|
||||
|
|
@ -65,7 +70,6 @@ describe("DefaultLitellmParamsSection", () => {
|
|||
render(
|
||||
<DefaultLitellmParamsSection
|
||||
value={{ timeout: 30, cache_control_injection_points: [{ location: "message" }] }}
|
||||
routerFieldsMetadata={{}}
|
||||
onChange={onChange}
|
||||
/>,
|
||||
);
|
||||
|
|
@ -81,7 +85,7 @@ describe("DefaultLitellmParamsSection", () => {
|
|||
it("should not call onChange and should flag the field as invalid when the textarea has invalid JSON on blur", async () => {
|
||||
const onChange = vi.fn();
|
||||
const user = userEvent.setup();
|
||||
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} routerFieldsMetadata={{}} onChange={onChange} />);
|
||||
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} onChange={onChange} />);
|
||||
|
||||
const textarea = screen.getByRole("textbox") as HTMLTextAreaElement;
|
||||
await user.clear(textarea);
|
||||
|
|
@ -95,7 +99,7 @@ describe("DefaultLitellmParamsSection", () => {
|
|||
it("should clear the invalid state once the textarea is edited again", async () => {
|
||||
const onChange = vi.fn();
|
||||
const user = userEvent.setup();
|
||||
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} routerFieldsMetadata={{}} onChange={onChange} />);
|
||||
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} onChange={onChange} />);
|
||||
|
||||
const textarea = screen.getByRole("textbox") as HTMLTextAreaElement;
|
||||
await user.clear(textarea);
|
||||
|
|
@ -108,4 +112,16 @@ describe("DefaultLitellmParamsSection", () => {
|
|||
|
||||
expect(textarea).not.toHaveClass("ant-input-status-error");
|
||||
});
|
||||
|
||||
it("should reject valid JSON that is not an object", async () => {
|
||||
const onChange = vi.fn();
|
||||
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} onChange={onChange} />);
|
||||
|
||||
const textarea = screen.getByRole("textbox") as HTMLTextAreaElement;
|
||||
fireEvent.change(textarea, { target: { value: "[]" } });
|
||||
fireEvent.blur(textarea);
|
||||
|
||||
expect(onChange).not.toHaveBeenCalled();
|
||||
expect(textarea).toHaveClass("ant-input-status-error");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -6,59 +6,104 @@ import CacheControlInjectionPointsEditor, {
|
|||
import NotificationsManager from "../molecules/notifications_manager";
|
||||
|
||||
interface DefaultLitellmParamsSectionProps {
|
||||
value: { [key: string]: any };
|
||||
routerFieldsMetadata: { [key: string]: any };
|
||||
onChange: (value: { [key: string]: any }) => void;
|
||||
value: Record<string, unknown>;
|
||||
onChange: (value: Record<string, unknown>) => void;
|
||||
}
|
||||
|
||||
const DefaultLitellmParamsSection: React.FC<DefaultLitellmParamsSectionProps> = ({
|
||||
value,
|
||||
routerFieldsMetadata,
|
||||
onChange,
|
||||
}) => {
|
||||
const meta = routerFieldsMetadata["default_litellm_params"];
|
||||
const { cache_control_injection_points, ...otherParams } = value || {};
|
||||
type ParsedDefaultParams = { status: "valid"; value: Record<string, unknown> } | { status: "invalid"; message: string };
|
||||
|
||||
const CACHE_CONTROL_ROLES = ["user", "system", "assistant"] as const;
|
||||
|
||||
const parseDefaultParams = (text: string): ParsedDefaultParams => {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(text || "{}");
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
||||
return { status: "invalid", message: "Expected a JSON object" };
|
||||
}
|
||||
return { status: "valid", value: parsed as Record<string, unknown> };
|
||||
} catch (error) {
|
||||
return { status: "invalid", message: error instanceof Error ? error.message : "Invalid JSON" };
|
||||
}
|
||||
};
|
||||
|
||||
const isCacheControlInjectionPoint = (value: unknown): value is CacheControlInjectionPoint => {
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return false;
|
||||
}
|
||||
if (!("location" in value) || value.location !== "message") {
|
||||
return false;
|
||||
}
|
||||
const role = "role" in value ? value.role : undefined;
|
||||
const index = "index" in value ? value.index : undefined;
|
||||
const hasValidRole = role === undefined || CACHE_CONTROL_ROLES.some((validRole) => validRole === role);
|
||||
const hasValidIndex = index === undefined || (typeof index === "number" && Number.isInteger(index));
|
||||
return hasValidRole && hasValidIndex;
|
||||
};
|
||||
|
||||
const DefaultLitellmParamsSection: React.FC<DefaultLitellmParamsSectionProps> = ({ value, onChange }) => {
|
||||
const cacheControlInjectionPoints = React.useMemo(
|
||||
() =>
|
||||
Array.isArray(value.cache_control_injection_points) &&
|
||||
value.cache_control_injection_points.every(isCacheControlInjectionPoint)
|
||||
? value.cache_control_injection_points
|
||||
: undefined,
|
||||
[value.cache_control_injection_points],
|
||||
);
|
||||
const otherParams = React.useMemo(
|
||||
() =>
|
||||
Object.fromEntries(
|
||||
Object.entries(value).filter(
|
||||
([key]) => key !== "cache_control_injection_points" || cacheControlInjectionPoints === undefined,
|
||||
),
|
||||
),
|
||||
[value, cacheControlInjectionPoints],
|
||||
);
|
||||
|
||||
const [otherParamsText, setOtherParamsText] = React.useState(() => JSON.stringify(otherParams, null, 2));
|
||||
const [showCacheControl, setShowCacheControl] = React.useState((cache_control_injection_points?.length ?? 0) > 0);
|
||||
const [hasInvalidJson, setHasInvalidJson] = React.useState(false);
|
||||
|
||||
const parseOtherParams = (): { [key: string]: any } => {
|
||||
try {
|
||||
return JSON.parse(otherParamsText || "{}");
|
||||
} catch {
|
||||
return otherParams;
|
||||
}
|
||||
};
|
||||
const showCacheControl = (cacheControlInjectionPoints?.length ?? 0) > 0;
|
||||
|
||||
const handleOtherParamsBlur = () => {
|
||||
try {
|
||||
const parsed = JSON.parse(otherParamsText || "{}");
|
||||
setHasInvalidJson(false);
|
||||
onChange({ ...parsed, cache_control_injection_points });
|
||||
} catch (error) {
|
||||
const parsed = parseDefaultParams(otherParamsText);
|
||||
if (parsed.status === "invalid") {
|
||||
setHasInvalidJson(true);
|
||||
NotificationsManager.warning(`Default LiteLLM Params is not valid JSON, change not saved: ${error}`);
|
||||
NotificationsManager.warning(`Default LiteLLM Params is not valid JSON: ${parsed.message}`);
|
||||
return;
|
||||
}
|
||||
setHasInvalidJson(false);
|
||||
onChange(
|
||||
cacheControlInjectionPoints
|
||||
? { ...parsed.value, cache_control_injection_points: cacheControlInjectionPoints }
|
||||
: parsed.value,
|
||||
);
|
||||
};
|
||||
|
||||
const handleCacheControlPointsChange = (points: CacheControlInjectionPoint[]) => {
|
||||
const base = parseOtherParams();
|
||||
const parsed = parseDefaultParams(otherParamsText);
|
||||
const base = parsed.status === "valid" ? parsed.value : otherParams;
|
||||
onChange(points.length > 0 ? { ...base, cache_control_injection_points: points } : base);
|
||||
};
|
||||
|
||||
const handleCacheControlToggle = (checked: boolean) => {
|
||||
setShowCacheControl(checked);
|
||||
handleCacheControlPointsChange(checked ? [{ location: "message" }] : []);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div className="max-w-3xl space-y-2">
|
||||
<span className="text-xs font-medium text-gray-700 uppercase tracking-wide">
|
||||
{meta?.ui_field_name || "default_litellm_params"}
|
||||
</span>
|
||||
<p className="text-xs text-gray-500 mt-0.5 mb-2">{meta?.field_description || ""}</p>
|
||||
<span className="text-xs font-medium text-gray-700 uppercase tracking-wide">Default LiteLLM Params</span>
|
||||
<p className="text-xs text-gray-500 mt-0.5 mb-2">
|
||||
Default parameters for Router.chat.completion.create. Set cache control injection points to enable prompt
|
||||
caching for every model on this proxy.{" "}
|
||||
<a
|
||||
href="https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-blue-600 hover:text-blue-800 underline"
|
||||
>
|
||||
Learn more
|
||||
</a>
|
||||
</p>
|
||||
<Input.TextArea
|
||||
value={otherParamsText}
|
||||
onChange={(e) => {
|
||||
|
|
@ -87,7 +132,7 @@ const DefaultLitellmParamsSection: React.FC<DefaultLitellmParamsSectionProps> =
|
|||
{showCacheControl && (
|
||||
<div className="ml-6 pl-4 border-l-2 border-gray-200">
|
||||
<CacheControlInjectionPointsEditor
|
||||
value={cache_control_injection_points || [{ location: "message" }]}
|
||||
value={cacheControlInjectionPoints || [{ location: "message" }]}
|
||||
onChange={handleCacheControlPointsChange}
|
||||
/>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { describe, it, expect, vi } from "vitest";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import OptionalPreCallChecksSelector from "./OptionalPreCallChecksSelector";
|
||||
import OptionalPreCallChecksSelector, { OPTIONAL_PRE_CALL_CHECK_OPTIONS } from "./OptionalPreCallChecksSelector";
|
||||
|
||||
vi.mock("antd", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("antd")>();
|
||||
|
|
@ -24,57 +24,29 @@ vi.mock("antd", async (importOriginal) => {
|
|||
};
|
||||
});
|
||||
|
||||
const baseMetadata = {
|
||||
optional_pre_call_checks: {
|
||||
ui_field_name: "Optional Pre-call Checks",
|
||||
field_description: "Extra checks the router runs before picking a deployment",
|
||||
link: null,
|
||||
},
|
||||
};
|
||||
|
||||
const options = ["prompt_caching", "router_budget_limiting", "session_affinity"];
|
||||
|
||||
describe("OptionalPreCallChecksSelector", () => {
|
||||
it("should render one option per entry in options", () => {
|
||||
render(<OptionalPreCallChecksSelector value={[]} options={options} routerFieldsMetadata={{}} onChange={vi.fn()} />);
|
||||
it("should render the frontend-defined options", () => {
|
||||
render(<OptionalPreCallChecksSelector value={[]} onChange={vi.fn()} />);
|
||||
const select = screen.getByTestId("optional-pre-call-checks-select") as HTMLSelectElement;
|
||||
expect(Array.from(select.options).map((o) => o.value)).toEqual(options);
|
||||
expect(Array.from(select.options).map((o) => o.value)).toEqual(OPTIONAL_PRE_CALL_CHECK_OPTIONS);
|
||||
});
|
||||
|
||||
it("should display default label when no metadata is provided", () => {
|
||||
render(<OptionalPreCallChecksSelector value={[]} options={options} routerFieldsMetadata={{}} onChange={vi.fn()} />);
|
||||
it("should display the frontend-defined label and description", () => {
|
||||
render(<OptionalPreCallChecksSelector value={[]} onChange={vi.fn()} />);
|
||||
expect(screen.getByText("Optional Pre-call Checks")).toBeInTheDocument();
|
||||
expect(screen.getByText(/extra checks the router runs before picking a deployment/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should display the label and description from metadata when provided", () => {
|
||||
render(
|
||||
<OptionalPreCallChecksSelector
|
||||
value={[]}
|
||||
options={options}
|
||||
routerFieldsMetadata={baseMetadata}
|
||||
onChange={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByText("Extra checks the router runs before picking a deployment")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should render a Learn more link when metadata provides one", () => {
|
||||
const metadata = {
|
||||
optional_pre_call_checks: { ...baseMetadata.optional_pre_call_checks, link: "https://docs.example.com/checks" },
|
||||
};
|
||||
render(
|
||||
<OptionalPreCallChecksSelector value={[]} options={options} routerFieldsMetadata={metadata} onChange={vi.fn()} />,
|
||||
);
|
||||
it("should render the prompt caching documentation link", () => {
|
||||
render(<OptionalPreCallChecksSelector value={[]} onChange={vi.fn()} />);
|
||||
const link = screen.getByRole("link", { name: /learn more/i });
|
||||
expect(link).toHaveAttribute("href", "https://docs.example.com/checks");
|
||||
expect(link).toHaveAttribute("href", "https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing");
|
||||
});
|
||||
|
||||
it("should call onChange with the selected checks", async () => {
|
||||
const onChange = vi.fn();
|
||||
const user = userEvent.setup();
|
||||
render(
|
||||
<OptionalPreCallChecksSelector value={[]} options={options} routerFieldsMetadata={{}} onChange={onChange} />,
|
||||
);
|
||||
render(<OptionalPreCallChecksSelector value={[]} onChange={onChange} />);
|
||||
|
||||
await user.selectOptions(screen.getByTestId("optional-pre-call-checks-select"), "prompt_caching");
|
||||
|
||||
|
|
@ -82,14 +54,7 @@ describe("OptionalPreCallChecksSelector", () => {
|
|||
});
|
||||
|
||||
it("should reflect an already-selected value", () => {
|
||||
render(
|
||||
<OptionalPreCallChecksSelector
|
||||
value={["router_budget_limiting"]}
|
||||
options={options}
|
||||
routerFieldsMetadata={{}}
|
||||
onChange={vi.fn()}
|
||||
/>,
|
||||
);
|
||||
render(<OptionalPreCallChecksSelector value={["router_budget_limiting"]} onChange={vi.fn()} />);
|
||||
const select = screen.getByTestId("optional-pre-call-checks-select") as HTMLSelectElement;
|
||||
const selected = Array.from(select.selectedOptions).map((o) => o.value);
|
||||
expect(selected).toEqual(["router_budget_limiting"]);
|
||||
|
|
|
|||
|
|
@ -1,48 +1,43 @@
|
|||
import React from "react";
|
||||
import { Select } from "antd";
|
||||
|
||||
export const OPTIONAL_PRE_CALL_CHECK_OPTIONS = [
|
||||
"prompt_caching",
|
||||
"router_budget_limiting",
|
||||
"responses_api_deployment_check",
|
||||
"deployment_affinity",
|
||||
"session_affinity",
|
||||
"enforce_model_rate_limits",
|
||||
"encrypted_content_affinity",
|
||||
] as const;
|
||||
|
||||
interface OptionalPreCallChecksSelectorProps {
|
||||
value: string[];
|
||||
options: string[];
|
||||
routerFieldsMetadata: { [key: string]: any };
|
||||
onChange: (value: string[]) => void;
|
||||
}
|
||||
|
||||
const OptionalPreCallChecksSelector: React.FC<OptionalPreCallChecksSelectorProps> = ({
|
||||
value,
|
||||
options,
|
||||
routerFieldsMetadata,
|
||||
onChange,
|
||||
}) => {
|
||||
const meta = routerFieldsMetadata["optional_pre_call_checks"];
|
||||
|
||||
const OptionalPreCallChecksSelector: React.FC<OptionalPreCallChecksSelectorProps> = ({ value, onChange }) => {
|
||||
return (
|
||||
<div className="space-y-2 max-w-3xl">
|
||||
<label className="block">
|
||||
<span className="text-xs font-medium text-gray-700 uppercase tracking-wide">
|
||||
{meta?.ui_field_name || "Optional Pre-call Checks"}
|
||||
</span>
|
||||
<span className="text-xs font-medium text-gray-700 uppercase tracking-wide">Optional Pre-call Checks</span>
|
||||
<p className="text-xs text-gray-500 mt-0.5 mb-2">
|
||||
{meta?.field_description || ""}
|
||||
{meta?.link && (
|
||||
<>
|
||||
{" "}
|
||||
<a
|
||||
href={meta.link}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-blue-600 hover:text-blue-800 underline"
|
||||
>
|
||||
Learn more
|
||||
</a>
|
||||
</>
|
||||
)}
|
||||
Extra checks the router runs before picking a deployment. Add 'prompt_caching' to route repeat
|
||||
requests back to the deployment that cached the prompt.{" "}
|
||||
<a
|
||||
href="https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-blue-600 hover:text-blue-800 underline"
|
||||
>
|
||||
Learn more
|
||||
</a>
|
||||
</p>
|
||||
<Select
|
||||
mode="multiple"
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
options={options.map((option) => ({ value: option, label: option }))}
|
||||
options={OPTIONAL_PRE_CALL_CHECK_OPTIONS.map((option) => ({ value: option, label: option }))}
|
||||
placeholder="No pre-call checks enabled"
|
||||
className="w-full"
|
||||
data-testid="optional-pre-call-checks-select"
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ describe("RouterSettingsForm", () => {
|
|||
const user = userEvent.setup();
|
||||
render(<RouterSettingsForm {...baseProps} onChange={onChange} />);
|
||||
|
||||
await user.click(screen.getByRole("switch"));
|
||||
await user.click(screen.getAllByRole("switch")[0]);
|
||||
|
||||
expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ enableTagFiltering: true }));
|
||||
});
|
||||
|
|
@ -129,26 +129,13 @@ describe("RouterSettingsForm", () => {
|
|||
expect(screen.getByText("Reliability & Retries")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should not show the optional pre-call checks selector when no options are known", () => {
|
||||
it("should show frontend-defined optional pre-call checks without backend field metadata", () => {
|
||||
render(<RouterSettingsForm {...baseProps} />);
|
||||
expect(screen.queryByTestId("optional-pre-call-checks-select")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should show the optional pre-call checks selector populated from field metadata options", () => {
|
||||
const props = {
|
||||
...baseProps,
|
||||
routerFieldsMetadata: {
|
||||
optional_pre_call_checks: {
|
||||
ui_field_name: "Optional Pre-call Checks",
|
||||
options: ["prompt_caching", "router_budget_limiting"],
|
||||
},
|
||||
},
|
||||
};
|
||||
render(<RouterSettingsForm {...props} />);
|
||||
|
||||
const select = screen.getByTestId("optional-pre-call-checks-select") as HTMLSelectElement;
|
||||
const optionValues = Array.from(select.options).map((o) => o.value);
|
||||
expect(optionValues).toEqual(["prompt_caching", "router_budget_limiting"]);
|
||||
expect(optionValues).toContain("prompt_caching");
|
||||
expect(optionValues).toContain("router_budget_limiting");
|
||||
});
|
||||
|
||||
it("should call onChange with the updated optional_pre_call_checks when the selector changes", async () => {
|
||||
|
|
@ -157,9 +144,6 @@ describe("RouterSettingsForm", () => {
|
|||
const props = {
|
||||
...baseProps,
|
||||
onChange,
|
||||
routerFieldsMetadata: {
|
||||
optional_pre_call_checks: { ui_field_name: "Optional Pre-call Checks", options: ["prompt_caching"] },
|
||||
},
|
||||
};
|
||||
render(<RouterSettingsForm {...props} />);
|
||||
|
||||
|
|
@ -172,17 +156,8 @@ describe("RouterSettingsForm", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("should not show the Default LiteLLM Params section before router_settings has loaded", () => {
|
||||
it("should show frontend-defined Default LiteLLM Params without backend field metadata", () => {
|
||||
render(<RouterSettingsForm {...baseProps} />);
|
||||
expect(screen.queryByText("Cache Control Injection Points")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should show the Default LiteLLM Params section once router_settings has loaded", () => {
|
||||
const props = {
|
||||
...baseProps,
|
||||
value: { ...defaultValue, routerSettings: { default_litellm_params: { timeout: 30 } } },
|
||||
};
|
||||
render(<RouterSettingsForm {...props} />);
|
||||
expect(screen.getByText("Cache Control Injection Points")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -48,15 +48,13 @@ const RouterSettingsForm: React.FC<RouterSettingsFormProps> = ({
|
|||
});
|
||||
};
|
||||
|
||||
const handleDefaultLitellmParamsChange = (params: { [key: string]: any }) => {
|
||||
const handleDefaultLitellmParamsChange = (params: Record<string, unknown>) => {
|
||||
onChange({
|
||||
...value,
|
||||
routerSettings: { ...value.routerSettings, default_litellm_params: params },
|
||||
});
|
||||
};
|
||||
|
||||
const optionalPreCallCheckOptions: string[] = routerFieldsMetadata["optional_pre_call_checks"]?.options || [];
|
||||
|
||||
return (
|
||||
<div className="w-full space-y-8 py-2">
|
||||
{/* Routing Settings Section */}
|
||||
|
|
@ -85,14 +83,10 @@ const RouterSettingsForm: React.FC<RouterSettingsFormProps> = ({
|
|||
/>
|
||||
|
||||
{/* Optional Pre-call Checks */}
|
||||
{optionalPreCallCheckOptions.length > 0 && (
|
||||
<OptionalPreCallChecksSelector
|
||||
value={value.routerSettings.optional_pre_call_checks || []}
|
||||
options={optionalPreCallCheckOptions}
|
||||
routerFieldsMetadata={routerFieldsMetadata}
|
||||
onChange={handleOptionalPreCallChecksChange}
|
||||
/>
|
||||
)}
|
||||
<OptionalPreCallChecksSelector
|
||||
value={value.routerSettings.optional_pre_call_checks || []}
|
||||
onChange={handleOptionalPreCallChecksChange}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Divider */}
|
||||
|
|
@ -104,16 +98,12 @@ const RouterSettingsForm: React.FC<RouterSettingsFormProps> = ({
|
|||
)}
|
||||
|
||||
{/* Default LiteLLM Params */}
|
||||
{"default_litellm_params" in value.routerSettings && (
|
||||
<>
|
||||
<DefaultLitellmParamsSection
|
||||
value={value.routerSettings.default_litellm_params || {}}
|
||||
routerFieldsMetadata={routerFieldsMetadata}
|
||||
onChange={handleDefaultLitellmParamsChange}
|
||||
/>
|
||||
<div className="border-t border-gray-200" />
|
||||
</>
|
||||
)}
|
||||
<DefaultLitellmParamsSection
|
||||
key={JSON.stringify(value.routerSettings.default_litellm_params || {})}
|
||||
value={value.routerSettings.default_litellm_params || {}}
|
||||
onChange={handleDefaultLitellmParamsChange}
|
||||
/>
|
||||
<div className="border-t border-gray-200" />
|
||||
|
||||
{/* Other Settings */}
|
||||
<ReliabilityRetriesSection routerSettings={value.routerSettings} routerFieldsMetadata={routerFieldsMetadata} />
|
||||
|
|
|
|||
|
|
@ -210,7 +210,9 @@ describe("RouterSettings", () => {
|
|||
vi.mocked(getCallbacksCall).mockResolvedValue({
|
||||
router_settings: {
|
||||
...mockCallbacksResponse.router_settings,
|
||||
default_litellm_params: { cache_control_injection_points: [{ location: "message", role: "system" }] },
|
||||
default_litellm_params: {
|
||||
cache_control_injection_points: [{ location: "message", role: "system", index: 2 }],
|
||||
},
|
||||
optional_pre_call_checks: ["prompt_caching"],
|
||||
},
|
||||
});
|
||||
|
|
@ -228,7 +230,9 @@ describe("RouterSettings", () => {
|
|||
"test-token",
|
||||
expect.objectContaining({
|
||||
router_settings: expect.objectContaining({
|
||||
default_litellm_params: { cache_control_injection_points: [{ location: "message", role: "system" }] },
|
||||
default_litellm_params: {
|
||||
cache_control_injection_points: [{ location: "message", role: "system", index: 2 }],
|
||||
},
|
||||
optional_pre_call_checks: ["prompt_caching"],
|
||||
}),
|
||||
}),
|
||||
|
|
@ -243,19 +247,6 @@ describe("RouterSettings", () => {
|
|||
optional_pre_call_checks: [],
|
||||
},
|
||||
});
|
||||
vi.mocked(getRouterSettingsCall).mockResolvedValue({
|
||||
...mockRouterSettingsResponse,
|
||||
fields: [
|
||||
...mockRouterSettingsResponse.fields,
|
||||
{
|
||||
field_name: "optional_pre_call_checks",
|
||||
ui_field_name: "Optional Pre-call Checks",
|
||||
field_description: "Extra checks the router runs before picking a deployment",
|
||||
options: ["prompt_caching", "router_budget_limiting"],
|
||||
link: null,
|
||||
},
|
||||
],
|
||||
});
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<RouterSettings {...defaultProps} />);
|
||||
|
||||
|
|
|
|||
|
|
@ -50,7 +50,6 @@ const RouterSettings: React.FC<RouterSettingsProps> = ({ accessToken, userRole,
|
|||
fieldsMap[field.field_name] = {
|
||||
ui_field_name: field.ui_field_name,
|
||||
field_description: field.field_description,
|
||||
field_type: field.field_type,
|
||||
options: field.options,
|
||||
link: field.link,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import React from "react";
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
|
|
@ -52,10 +53,44 @@ describe("CacheControlInjectionPointsEditor", () => {
|
|||
/>,
|
||||
);
|
||||
|
||||
const removeIcons = document.querySelectorAll(".anticon-minus-circle");
|
||||
expect(removeIcons).toHaveLength(2);
|
||||
await user.click(removeIcons[0] as HTMLElement);
|
||||
const removeButtons = screen.getAllByRole("button", { name: /remove injection point/i });
|
||||
expect(removeButtons).toHaveLength(2);
|
||||
await user.click(removeButtons[0]);
|
||||
|
||||
expect(onChange).toHaveBeenCalledWith([{ location: "message", role: "user" }]);
|
||||
});
|
||||
|
||||
it("should accept an integer index", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onChange = vi.fn();
|
||||
|
||||
const ControlledEditor = () => {
|
||||
const [points, setPoints] = React.useState([{ location: "message" as const }]);
|
||||
const handleChange = (updatedPoints: typeof points) => {
|
||||
setPoints(updatedPoints);
|
||||
onChange(updatedPoints);
|
||||
};
|
||||
return <CacheControlInjectionPointsEditor value={points} onChange={handleChange} />;
|
||||
};
|
||||
|
||||
render(<ControlledEditor />);
|
||||
|
||||
const indexInput = screen.getByTestId("cache-control-index-input-0");
|
||||
await user.type(indexInput, "-1");
|
||||
|
||||
expect(indexInput).toHaveValue("-1");
|
||||
expect(onChange).toHaveBeenLastCalledWith([{ location: "message", index: -1 }]);
|
||||
});
|
||||
|
||||
it("should increment the index with the stepper", async () => {
|
||||
const onChange = vi.fn();
|
||||
const user = userEvent.setup();
|
||||
render(<CacheControlInjectionPointsEditor value={[{ location: "message", index: 0 }]} onChange={onChange} />);
|
||||
|
||||
const incrementButton = document.querySelector(".ant-input-number-handler-up");
|
||||
expect(incrementButton).not.toBeNull();
|
||||
await user.click(incrementButton as HTMLElement);
|
||||
|
||||
expect(onChange).toHaveBeenLastCalledWith([{ location: "message", index: 1 }]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import React from "react";
|
||||
import { Select } from "antd";
|
||||
import { Button, Flex, InputNumber, Select, Typography } from "antd";
|
||||
import { PlusOutlined, MinusCircleOutlined } from "@ant-design/icons";
|
||||
import NumericalInput from "./numerical_input";
|
||||
|
||||
const { Text } = Typography;
|
||||
|
||||
export interface CacheControlInjectionPoint {
|
||||
location: "message";
|
||||
|
|
@ -24,9 +25,11 @@ const CacheControlInjectionPointsEditor: React.FC<CacheControlInjectionPointsEdi
|
|||
return (
|
||||
<>
|
||||
{points.map((point, index) => (
|
||||
<div key={index} className="flex items-center mb-4 gap-4">
|
||||
<div style={{ width: "180px" }}>
|
||||
<span className="text-xs text-gray-500">Type</span>
|
||||
<Flex key={index} align="end" gap="middle" wrap className="mb-4">
|
||||
<div className="flex-1 min-w-40">
|
||||
<Text type="secondary" className="block text-xs mb-1">
|
||||
Type
|
||||
</Text>
|
||||
<Select
|
||||
disabled
|
||||
value="message"
|
||||
|
|
@ -36,8 +39,10 @@ const CacheControlInjectionPointsEditor: React.FC<CacheControlInjectionPointsEdi
|
|||
/>
|
||||
</div>
|
||||
|
||||
<div style={{ width: "180px" }}>
|
||||
<span className="text-xs text-gray-500">Role</span>
|
||||
<div className="flex-1 min-w-40">
|
||||
<Text type="secondary" className="block text-xs mb-1">
|
||||
Role
|
||||
</Text>
|
||||
<Select
|
||||
placeholder="Select a role"
|
||||
allowClear
|
||||
|
|
@ -53,36 +58,41 @@ const CacheControlInjectionPointsEditor: React.FC<CacheControlInjectionPointsEdi
|
|||
/>
|
||||
</div>
|
||||
|
||||
<div style={{ width: "180px" }}>
|
||||
<span className="text-xs text-gray-500">Index</span>
|
||||
<NumericalInput
|
||||
type="number"
|
||||
<div className="flex-1 min-w-40">
|
||||
<Text type="secondary" className="block text-xs mb-1">
|
||||
Index
|
||||
</Text>
|
||||
<InputNumber
|
||||
placeholder="Optional"
|
||||
step={1}
|
||||
precision={0}
|
||||
value={point.index}
|
||||
onChange={(newIndex: string) =>
|
||||
updatePoint(index, { index: newIndex === "" ? undefined : Number(newIndex) })
|
||||
}
|
||||
onChange={(newIndex) => updatePoint(index, { index: newIndex ?? undefined })}
|
||||
className="w-full"
|
||||
data-testid={`cache-control-index-input-${index}`}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{points.length > 1 && (
|
||||
<MinusCircleOutlined
|
||||
className="text-red-500 cursor-pointer text-lg ml-12"
|
||||
<Button
|
||||
type="text"
|
||||
danger
|
||||
aria-label={`Remove injection point ${index + 1}`}
|
||||
icon={<MinusCircleOutlined />}
|
||||
onClick={() => onChange(points.filter((_, i) => i !== index))}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</Flex>
|
||||
))}
|
||||
|
||||
<button
|
||||
type="button"
|
||||
className="flex items-center justify-center w-full border border-dashed border-gray-300 py-2 px-4 text-gray-600 hover:text-blue-600 hover:border-blue-300 transition-all rounded"
|
||||
<Button
|
||||
type="dashed"
|
||||
block
|
||||
icon={<PlusOutlined />}
|
||||
onClick={() => onChange([...points, { location: "message" as const }])}
|
||||
>
|
||||
<PlusOutlined className="mr-2" />
|
||||
Add Injection Point
|
||||
</button>
|
||||
</Button>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue