mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-12 03:48:56 +00:00
Merge 677aec8563 into 2a01fe6e5c
This commit is contained in:
ishaan-berri
GitHub
commit
b48c285da5
22 changed files with 582 additions and 59 deletions
|
|
@ -1,3 +1,4 @@
|
|||
import fnmatch
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
|
|
@ -5,7 +6,7 @@ from dataclasses import dataclass
|
|||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from typing import TYPE_CHECKING, Any, Final, cast # noqa: TID251 # narrows untyped fallback dict shapes
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_router_logger
|
||||
|
|
@ -548,32 +549,42 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li
|
|||
- exact match
|
||||
- stripped model group match
|
||||
- provider-prefixed model group match
|
||||
- fnmatch pattern key match (e.g. "openai/*")
|
||||
- generic fallback
|
||||
"""
|
||||
generic_fallback_idx: int | None = None
|
||||
stripped_model_fallback: list[str] | None = None
|
||||
pattern_model_fallback: list[str] | None = None # mutable-ok: set in loop # rebind-ok: set in loop
|
||||
fallback_model_group: list[str] | None = None
|
||||
fallback_keys: Final = tuple(next(iter(item)) for item in fallbacks if isinstance(item, dict) and item)
|
||||
prefixed_model_group: Final = _provider_prefixed_model_group(model_group, fallback_keys)
|
||||
## check for specific model group-specific fallbacks
|
||||
for idx, item in enumerate(fallbacks):
|
||||
if isinstance(item, dict):
|
||||
fallback_key = next(iter(item))
|
||||
fallback_key = cast(str, next(iter(item))) # cast-ok: fallback keys are str
|
||||
if fallback_key == model_group: # check exact match
|
||||
fallback_model_group = item[model_group]
|
||||
break
|
||||
elif fallback_key == prefixed_model_group or _check_stripped_model_group(
|
||||
model_group=model_group, fallback_key=fallback_key
|
||||
): # check generic fallback
|
||||
stripped_model_fallback = item[fallback_key]
|
||||
stripped_model_fallback = cast(list[str], item[fallback_key]) # cast-ok: chains are str lists
|
||||
elif fallback_key == "*": # check generic fallback
|
||||
generic_fallback_idx = idx
|
||||
elif (
|
||||
pattern_model_fallback is None
|
||||
and "*" in fallback_key
|
||||
and fnmatch.fnmatchcase(model_group, fallback_key)
|
||||
):
|
||||
pattern_model_fallback = cast(list[str], item[fallback_key]) # cast-ok: chains are str lists
|
||||
elif isinstance(item, str):
|
||||
fallback_model_group = [item]
|
||||
## if none, check for generic fallback
|
||||
if fallback_model_group is None:
|
||||
if stripped_model_fallback is not None:
|
||||
fallback_model_group = stripped_model_fallback
|
||||
elif pattern_model_fallback is not None:
|
||||
fallback_model_group = pattern_model_fallback # rebind-ok: one branch assignment like stripped
|
||||
elif generic_fallback_idx is not None:
|
||||
fallback_model_group = fallbacks[generic_fallback_idx]["*"]
|
||||
|
||||
|
|
|
|||
|
|
@ -805,6 +805,40 @@ def test_get_fallback_model_group_does_not_mutate_fallbacks():
|
|||
assert fallbacks == [{"gpt-3.5-turbo": ["claude-3-haiku"]}, "gpt-4o-mini"]
|
||||
|
||||
|
||||
class TestGetFallbackModelGroupPatternKeys:
|
||||
def test_provider_wildcard_key_matches_model_group(self):
|
||||
fallbacks = [{"openai/*": ["claude-3-haiku"]}]
|
||||
|
||||
assert get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-6-astra")[0] == [
|
||||
"claude-3-haiku"
|
||||
]
|
||||
|
||||
def test_provider_wildcard_key_does_not_match_other_providers(self):
|
||||
fallbacks = [{"openai/*": ["claude-3-haiku"]}]
|
||||
|
||||
assert get_fallback_model_group(fallbacks=fallbacks, model_group="anthropic/claude-x")[0] is None
|
||||
|
||||
def test_exact_key_beats_pattern_key(self):
|
||||
fallbacks = [{"openai/*": ["generic-backup"]}, {"openai/gpt-4": ["specific-backup"]}]
|
||||
|
||||
assert get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-4")[0] == ["specific-backup"]
|
||||
|
||||
def test_pattern_key_beats_generic_star(self):
|
||||
fallbacks = [{"*": ["star-backup"]}, {"openai/*": ["openai-backup"]}]
|
||||
|
||||
assert get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-4")[0] == ["openai-backup"]
|
||||
|
||||
def test_first_matching_pattern_key_in_list_order_wins(self):
|
||||
fallbacks = [{"openai/*": ["first-backup"]}, {"*gpt*": ["second-backup"]}]
|
||||
|
||||
assert get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-4")[0] == ["first-backup"]
|
||||
|
||||
def test_generic_star_still_applies_when_no_pattern_matches(self):
|
||||
fallbacks = [{"anthropic/*": ["anthropic-backup"]}, {"*": ["star-backup"]}]
|
||||
|
||||
assert get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-4") == (["star-backup"], 1)
|
||||
|
||||
|
||||
class TestTriggerCooldownForFailedDeployment:
|
||||
def test_calls_set_cooldown_deployments_with_stamped_deployment_id(self):
|
||||
mock_router = MagicMock()
|
||||
|
|
|
|||
|
|
@ -427,6 +427,42 @@ def test_router_fallbacks_with_wildcard_model_name():
|
|||
assert response["choices"][0]["message"]["content"] == "Hi this is claude!"
|
||||
|
||||
|
||||
def test_router_fallbacks_with_provider_wildcard_fallback_key():
|
||||
"""
|
||||
A fallback key of "openai/*" must catch a failing request routed through a
|
||||
wildcard "openai/*" deployment (model_group is the requested model name,
|
||||
e.g. "openai/gpt-6-astra").
|
||||
"""
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "claude-3-haiku",
|
||||
"litellm_params": {
|
||||
"model": "claude-haiku-4-5-20251001",
|
||||
"api_key": os.getenv("ANTHROPIC_API_KEY"),
|
||||
"mock_response": "Hi this is claude!",
|
||||
},
|
||||
},
|
||||
],
|
||||
fallbacks=[{"openai/*": ["claude-3-haiku"]}],
|
||||
)
|
||||
|
||||
response = router.completion(
|
||||
model="openai/gpt-6-astra",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
mock_testing_fallbacks=True,
|
||||
)
|
||||
|
||||
assert response["choices"][0]["message"]["content"] == "Hi this is claude!"
|
||||
|
||||
|
||||
def test_get_fallback_model_group():
|
||||
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
|
||||
|
||||
|
|
|
|||
17
ui/litellm-dashboard/src/app/(dashboard)/fallbacks/page.tsx
Normal file
17
ui/litellm-dashboard/src/app/(dashboard)/fallbacks/page.tsx
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
"use client";
|
||||
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks";
|
||||
|
||||
export default function FallbacksPage() {
|
||||
const { accessToken, userRole, userId } = useAuthorized();
|
||||
return (
|
||||
<div className="flex flex-col gap-4 px-8 py-6">
|
||||
<div className="flex flex-col gap-2">
|
||||
<h1 className="text-2xl font-bold">Fallbacks</h1>
|
||||
<p className="text-sm text-muted-foreground">Automatically retry a different model when the primary fails</p>
|
||||
</div>
|
||||
<Fallbacks accessToken={accessToken} userRole={userRole} userID={userId} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -40,6 +40,7 @@ const LEGACY_PAGE_ROUTES: ReadonlyMap<string, string> = new Map(
|
|||
"model-insights": "model-insights",
|
||||
agents: "agents",
|
||||
"router-settings": "router-settings",
|
||||
fallbacks: "fallbacks",
|
||||
users: "users",
|
||||
teams: "teams",
|
||||
organizations: "organizations",
|
||||
|
|
|
|||
|
|
@ -63,8 +63,6 @@ export default function AddFallbacks({ accessToken, value = [], onChange }: AddF
|
|||
}
|
||||
}, [accessToken, isModalVisible]);
|
||||
|
||||
const availableModels = Array.from(new Set(modelInfo.map((option) => option.model_group))).sort();
|
||||
|
||||
const handleCancel = () => {
|
||||
setIsModalVisible(false);
|
||||
// Reset to initial state
|
||||
|
|
@ -125,7 +123,7 @@ export default function AddFallbacks({ accessToken, value = [], onChange }: AddF
|
|||
key={modalKey}
|
||||
groups={groups}
|
||||
onGroupsChange={setGroups}
|
||||
availableModels={availableModels}
|
||||
modelGroups={modelInfo}
|
||||
maxFallbacks={10}
|
||||
maxGroups={5}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import React, { useMemo, useState } from "react";
|
|||
import { fetchAvailableModels } from "@/components/llm_calls/fetch_models";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { AddFallbacksModal } from "./AddFallbacksModal";
|
||||
import { fallbackChainOptions, primaryModelOptions } from "./fallbackOptions";
|
||||
import { FallbackGroup, FallbackGroupConfig } from "./FallbackGroupConfig";
|
||||
|
||||
export type FallbackEntry = { [modelName: string]: string[] };
|
||||
|
|
@ -51,10 +52,8 @@ export default function EditFallbacks({
|
|||
enabled: Boolean(accessToken),
|
||||
});
|
||||
|
||||
const availableModels = useMemo(
|
||||
() => Array.from(new Set(modelGroups.map((option) => option.model_group))).sort(),
|
||||
[modelGroups],
|
||||
);
|
||||
const primaryOptions = useMemo(() => primaryModelOptions(modelGroups), [modelGroups]);
|
||||
const chainOptions = useMemo(() => fallbackChainOptions(modelGroups), [modelGroups]);
|
||||
|
||||
const handleSave = async () => {
|
||||
const primaryModel = group.primaryModel;
|
||||
|
|
@ -83,7 +82,8 @@ export default function EditFallbacks({
|
|||
<FallbackGroupConfig
|
||||
group={group}
|
||||
onChange={setGroup}
|
||||
availableModels={availableModels}
|
||||
primaryOptions={primaryOptions}
|
||||
fallbackOptions={chainOptions}
|
||||
maxFallbacks={maxFallbacks}
|
||||
disablePrimaryModel
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -4,9 +4,11 @@
|
|||
*/
|
||||
|
||||
import { MultiSelect } from "@/components/shared/MultiSelect";
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { SearchSelect, SearchSelectOption } from "@/components/shared/SearchSelect";
|
||||
import { ProviderLogo } from "@/components/molecules/models/ProviderLogo";
|
||||
import { AlertCircle, ArrowDown, X } from "lucide-react";
|
||||
import React, { useId } from "react";
|
||||
import { FallbackOption } from "./fallbackOptions";
|
||||
|
||||
export interface FallbackGroup {
|
||||
id: string;
|
||||
|
|
@ -17,20 +19,27 @@ export interface FallbackGroup {
|
|||
interface FallbackGroupConfigProps {
|
||||
group: FallbackGroup;
|
||||
onChange: (updatedGroup: FallbackGroup) => void;
|
||||
availableModels: string[];
|
||||
primaryOptions: FallbackOption[];
|
||||
fallbackOptions: FallbackOption[];
|
||||
maxFallbacks: number;
|
||||
disablePrimaryModel?: boolean;
|
||||
}
|
||||
|
||||
const withWildcardIcons = (options: FallbackOption[]): SearchSelectOption[] =>
|
||||
options.map((option) =>
|
||||
option.value.endsWith("/*") ? { ...option, icon: <ProviderLogo provider={option.value.slice(0, -2)} /> } : option,
|
||||
);
|
||||
|
||||
export function FallbackGroupConfig({
|
||||
group,
|
||||
onChange,
|
||||
availableModels,
|
||||
primaryOptions,
|
||||
fallbackOptions,
|
||||
maxFallbacks,
|
||||
disablePrimaryModel = false,
|
||||
}: FallbackGroupConfigProps) {
|
||||
// Filter available options for fallbacks (exclude primary only, allow already selected to be shown for deselection)
|
||||
const availableFallbackOptions = availableModels.filter((m) => m !== group.primaryModel);
|
||||
// Exclude the primary only, allow already selected to be shown for deselection
|
||||
const availableFallbackOptions = fallbackOptions.filter((option) => option.value !== group.primaryModel);
|
||||
|
||||
const handlePrimaryChange = (value: string | null) => {
|
||||
const newFallbacks = group.fallbackModels.filter((model) => model !== value);
|
||||
|
|
@ -71,7 +80,7 @@ export function FallbackGroupConfig({
|
|||
</label>
|
||||
<SearchSelect
|
||||
inputId={primaryModelInputId}
|
||||
options={availableModels.map((m) => ({ label: m, value: m }))}
|
||||
options={withWildcardIcons(primaryOptions)}
|
||||
value={group.primaryModel}
|
||||
onValueChange={handlePrimaryChange}
|
||||
placeholder="Select primary model"
|
||||
|
|
@ -110,7 +119,7 @@ export function FallbackGroupConfig({
|
|||
{/* Add Fallback Input */}
|
||||
<div className="mb-4">
|
||||
<MultiSelect
|
||||
options={availableFallbackOptions.map((m) => ({ label: m, value: m }))}
|
||||
options={availableFallbackOptions}
|
||||
value={group.fallbackModels}
|
||||
onValueChange={handleFallbackSelect}
|
||||
placeholder={
|
||||
|
|
|
|||
|
|
@ -6,7 +6,11 @@ import type { FallbackGroup } from "./FallbackGroupConfig";
|
|||
import { toast } from "@/lib/toast";
|
||||
|
||||
const mockOnGroupsChange = vi.fn();
|
||||
const AVAILABLE_MODELS = ["gpt-4", "gpt-3.5-turbo", "claude-3-opus"];
|
||||
const AVAILABLE_MODEL_GROUPS = [
|
||||
{ model_group: "gpt-4", providers: ["openai"] },
|
||||
{ model_group: "gpt-3.5-turbo", providers: ["openai"] },
|
||||
{ model_group: "claude-3-opus", providers: ["anthropic"] },
|
||||
];
|
||||
|
||||
describe("FallbackSelectionForm", () => {
|
||||
beforeEach(() => {
|
||||
|
|
@ -16,14 +20,14 @@ describe("FallbackSelectionForm", () => {
|
|||
|
||||
it("should render the component", () => {
|
||||
render(
|
||||
<FallbackSelectionForm groups={[]} onGroupsChange={mockOnGroupsChange} availableModels={AVAILABLE_MODELS} />,
|
||||
<FallbackSelectionForm groups={[]} onGroupsChange={mockOnGroupsChange} modelGroups={AVAILABLE_MODEL_GROUPS} />,
|
||||
);
|
||||
expect(screen.getByText(/no fallback groups configured/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should show Create First Group button when no groups exist", () => {
|
||||
render(
|
||||
<FallbackSelectionForm groups={[]} onGroupsChange={mockOnGroupsChange} availableModels={AVAILABLE_MODELS} />,
|
||||
<FallbackSelectionForm groups={[]} onGroupsChange={mockOnGroupsChange} modelGroups={AVAILABLE_MODEL_GROUPS} />,
|
||||
);
|
||||
expect(screen.getByRole("button", { name: /create first group/i })).toBeInTheDocument();
|
||||
});
|
||||
|
|
@ -31,7 +35,7 @@ describe("FallbackSelectionForm", () => {
|
|||
it("should call onGroupsChange when Create First Group is clicked", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(
|
||||
<FallbackSelectionForm groups={[]} onGroupsChange={mockOnGroupsChange} availableModels={AVAILABLE_MODELS} />,
|
||||
<FallbackSelectionForm groups={[]} onGroupsChange={mockOnGroupsChange} modelGroups={AVAILABLE_MODEL_GROUPS} />,
|
||||
);
|
||||
|
||||
await user.click(screen.getByRole("button", { name: /create first group/i }));
|
||||
|
|
@ -49,7 +53,11 @@ describe("FallbackSelectionForm", () => {
|
|||
it("should display tabs when groups exist", () => {
|
||||
const groups: FallbackGroup[] = [{ id: "1", primaryModel: null, fallbackModels: [] }];
|
||||
render(
|
||||
<FallbackSelectionForm groups={groups} onGroupsChange={mockOnGroupsChange} availableModels={AVAILABLE_MODELS} />,
|
||||
<FallbackSelectionForm
|
||||
groups={groups}
|
||||
onGroupsChange={mockOnGroupsChange}
|
||||
modelGroups={AVAILABLE_MODEL_GROUPS}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByRole("tab", { name: /group 1/i })).toBeInTheDocument();
|
||||
});
|
||||
|
|
@ -57,16 +65,36 @@ describe("FallbackSelectionForm", () => {
|
|||
it("should display primary model as tab label when set", () => {
|
||||
const groups: FallbackGroup[] = [{ id: "1", primaryModel: "gpt-4", fallbackModels: [] }];
|
||||
render(
|
||||
<FallbackSelectionForm groups={groups} onGroupsChange={mockOnGroupsChange} availableModels={AVAILABLE_MODELS} />,
|
||||
<FallbackSelectionForm
|
||||
groups={groups}
|
||||
onGroupsChange={mockOnGroupsChange}
|
||||
modelGroups={AVAILABLE_MODEL_GROUPS}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByRole("tab", { name: "gpt-4" })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should display friendly label for provider wildcard primary", () => {
|
||||
const groups: FallbackGroup[] = [{ id: "1", primaryModel: "openai/*", fallbackModels: [] }];
|
||||
render(
|
||||
<FallbackSelectionForm
|
||||
groups={groups}
|
||||
onGroupsChange={mockOnGroupsChange}
|
||||
modelGroups={AVAILABLE_MODEL_GROUPS}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByRole("tab", { name: "All OpenAI Models" })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should call onGroupsChange when add tab button is clicked", async () => {
|
||||
const user = userEvent.setup();
|
||||
const groups: FallbackGroup[] = [{ id: "1", primaryModel: null, fallbackModels: [] }];
|
||||
render(
|
||||
<FallbackSelectionForm groups={groups} onGroupsChange={mockOnGroupsChange} availableModels={AVAILABLE_MODELS} />,
|
||||
<FallbackSelectionForm
|
||||
groups={groups}
|
||||
onGroupsChange={mockOnGroupsChange}
|
||||
modelGroups={AVAILABLE_MODEL_GROUPS}
|
||||
/>,
|
||||
);
|
||||
|
||||
const addTabButton = screen.getByRole("button", { name: /add fallback group/i });
|
||||
|
|
@ -94,7 +122,7 @@ describe("FallbackSelectionForm", () => {
|
|||
<FallbackSelectionForm
|
||||
groups={groups}
|
||||
onGroupsChange={mockOnGroupsChange}
|
||||
availableModels={AVAILABLE_MODELS}
|
||||
modelGroups={AVAILABLE_MODEL_GROUPS}
|
||||
maxGroups={5}
|
||||
/>,
|
||||
);
|
||||
|
|
@ -107,7 +135,7 @@ describe("FallbackSelectionForm", () => {
|
|||
<FallbackSelectionForm
|
||||
groups={groups}
|
||||
onGroupsChange={mockOnGroupsChange}
|
||||
availableModels={AVAILABLE_MODELS}
|
||||
modelGroups={AVAILABLE_MODEL_GROUPS}
|
||||
maxGroups={3}
|
||||
/>,
|
||||
);
|
||||
|
|
@ -121,7 +149,11 @@ describe("FallbackSelectionForm", () => {
|
|||
{ id: "2", primaryModel: "gpt-3.5-turbo", fallbackModels: [] },
|
||||
];
|
||||
render(
|
||||
<FallbackSelectionForm groups={groups} onGroupsChange={mockOnGroupsChange} availableModels={AVAILABLE_MODELS} />,
|
||||
<FallbackSelectionForm
|
||||
groups={groups}
|
||||
onGroupsChange={mockOnGroupsChange}
|
||||
modelGroups={AVAILABLE_MODEL_GROUPS}
|
||||
/>,
|
||||
);
|
||||
|
||||
const removeButtons = screen.getAllByRole("button", { name: /^remove /i });
|
||||
|
|
@ -138,7 +170,11 @@ describe("FallbackSelectionForm", () => {
|
|||
it("should render FallbackGroupConfig for each group", () => {
|
||||
const groups: FallbackGroup[] = [{ id: "1", primaryModel: null, fallbackModels: [] }];
|
||||
render(
|
||||
<FallbackSelectionForm groups={groups} onGroupsChange={mockOnGroupsChange} availableModels={AVAILABLE_MODELS} />,
|
||||
<FallbackSelectionForm
|
||||
groups={groups}
|
||||
onGroupsChange={mockOnGroupsChange}
|
||||
modelGroups={AVAILABLE_MODEL_GROUPS}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByRole("combobox", { name: /primary model/i })).toHaveValue("");
|
||||
expect(screen.getByText("Primary Model")).toBeInTheDocument();
|
||||
|
|
@ -147,7 +183,11 @@ describe("FallbackSelectionForm", () => {
|
|||
it("should display group with primary and fallback models in FallbackGroupConfig", () => {
|
||||
const groups: FallbackGroup[] = [{ id: "1", primaryModel: "gpt-4", fallbackModels: ["gpt-3.5-turbo"] }];
|
||||
render(
|
||||
<FallbackSelectionForm groups={groups} onGroupsChange={mockOnGroupsChange} availableModels={AVAILABLE_MODELS} />,
|
||||
<FallbackSelectionForm
|
||||
groups={groups}
|
||||
onGroupsChange={mockOnGroupsChange}
|
||||
modelGroups={AVAILABLE_MODEL_GROUPS}
|
||||
/>,
|
||||
);
|
||||
expect(screen.getByRole("tab", { name: "gpt-4" })).toBeInTheDocument();
|
||||
expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0);
|
||||
|
|
@ -165,7 +205,7 @@ describe("FallbackSelectionForm", () => {
|
|||
<FallbackSelectionForm
|
||||
groups={groups}
|
||||
onGroupsChange={mockOnGroupsChange}
|
||||
availableModels={AVAILABLE_MODELS}
|
||||
modelGroups={AVAILABLE_MODEL_GROUPS}
|
||||
maxGroups={5}
|
||||
/>,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -5,16 +5,18 @@
|
|||
*/
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { Plus, X } from "lucide-react";
|
||||
import React, { useEffect, useState } from "react";
|
||||
import React, { useEffect, useMemo, useState } from "react";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { fallbackChainOptions, fallbackPrimaryLabel, primaryModelOptions } from "./fallbackOptions";
|
||||
import { FallbackGroup, FallbackGroupConfig } from "./FallbackGroupConfig";
|
||||
|
||||
interface FallbackSelectionFormProps {
|
||||
groups: FallbackGroup[];
|
||||
onGroupsChange: (groups: FallbackGroup[]) => void;
|
||||
availableModels: string[];
|
||||
modelGroups: ModelGroup[];
|
||||
maxFallbacks?: number;
|
||||
maxGroups?: number;
|
||||
}
|
||||
|
|
@ -22,10 +24,12 @@ interface FallbackSelectionFormProps {
|
|||
export function FallbackSelectionForm({
|
||||
groups,
|
||||
onGroupsChange,
|
||||
availableModels,
|
||||
modelGroups,
|
||||
maxFallbacks = 10,
|
||||
maxGroups = 5,
|
||||
}: FallbackSelectionFormProps) {
|
||||
const primaryOptions = useMemo(() => primaryModelOptions(modelGroups), [modelGroups]);
|
||||
const chainOptions = useMemo(() => fallbackChainOptions(modelGroups), [modelGroups]);
|
||||
const [activeKey, setActiveKey] = useState(groups.length > 0 ? groups[0].id : "1");
|
||||
|
||||
// Reset activeKey when groups change (e.g., when modal reopens)
|
||||
|
|
@ -77,7 +81,7 @@ export function FallbackSelectionForm({
|
|||
};
|
||||
|
||||
const groupLabel = (group: FallbackGroup, index: number) =>
|
||||
group.primaryModel ? group.primaryModel : `Group ${index + 1}`;
|
||||
group.primaryModel ? fallbackPrimaryLabel(group.primaryModel) : `Group ${index + 1}`;
|
||||
|
||||
if (groups.length === 0) {
|
||||
return (
|
||||
|
|
@ -128,7 +132,8 @@ export function FallbackSelectionForm({
|
|||
<FallbackGroupConfig
|
||||
group={group}
|
||||
onChange={handleGroupUpdate}
|
||||
availableModels={availableModels}
|
||||
primaryOptions={primaryOptions}
|
||||
fallbackOptions={chainOptions}
|
||||
maxFallbacks={maxFallbacks}
|
||||
/>
|
||||
</TabsContent>
|
||||
|
|
|
|||
|
|
@ -115,6 +115,27 @@ describe("Fallbacks", () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it("should resolve a provider logo for wildcard and prefixed primary keys", async () => {
|
||||
vi.mocked(networkingModule.getCallbacksCall).mockResolvedValueOnce({
|
||||
router_settings: {
|
||||
fallbacks: [
|
||||
{ "anthropic/*": ["gpt-4"] },
|
||||
{ "openai/gpt-6-astra": ["gpt-4"] },
|
||||
{ "bedrock/us.anthropic.claude-opus-4-8": ["gpt-4"] },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const { container } = renderWithQueryClient(<Fallbacks {...defaultProps} />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("All Anthropic Models")).toBeInTheDocument();
|
||||
});
|
||||
expect(screen.getByText("All Anthropic Models").parentElement).toHaveAttribute("title", "anthropic/*");
|
||||
const logos = container.querySelectorAll("img");
|
||||
expect(logos.length).toBeGreaterThanOrEqual(3);
|
||||
});
|
||||
|
||||
it("should render the component", async () => {
|
||||
renderWithQueryClient(<Fallbacks {...defaultProps} />);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { ArrowRight, Pencil, Play, Trash2 } from "lucide-react";
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
|
||||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
|
|
@ -9,6 +10,8 @@ import { ProviderLogo } from "../../../molecules/models/ProviderLogo";
|
|||
import { toast } from "@/lib/toast";
|
||||
import { getCallbacksCall, setCallbacksCall } from "../../../networking";
|
||||
import { isProxyAdminRole } from "@/utils/roles";
|
||||
import { fetchAvailableModels } from "@/components/llm_calls/fetch_models";
|
||||
import { fallbackPrimaryLabel, resolveFallbackProvider } from "./fallbackOptions";
|
||||
import AddFallbacks from "./AddFallbacks";
|
||||
import EditFallbacks from "./EditFallbacks";
|
||||
|
||||
|
|
@ -20,12 +23,14 @@ const modelCardClass =
|
|||
|
||||
const iconWrapperClass = "inline-flex shrink-0 items-center justify-center px-1.5 py-1.5";
|
||||
|
||||
function renderModelNameCell(modelName: string, getProviderFromModel?: (modelName: string) => string): React.ReactNode {
|
||||
const provider = getProviderFromModel?.(modelName) ?? modelName;
|
||||
export type ModelDisplayResolver = (modelName: string) => { provider: string; label: string; title?: string };
|
||||
|
||||
function renderModelNameCell(modelName: string, resolveModel?: ModelDisplayResolver): React.ReactNode {
|
||||
const resolved = resolveModel?.(modelName) ?? { provider: modelName, label: modelName };
|
||||
return (
|
||||
<span className={modelCardClass}>
|
||||
<ProviderLogo provider={provider} className="w-4 h-4 shrink-0" />
|
||||
<span className="break-words">{modelName}</span>
|
||||
<span className={modelCardClass} title={resolved.title}>
|
||||
<ProviderLogo provider={resolved.provider} className="w-4 h-4 shrink-0" />
|
||||
<span className="break-words">{resolved.label}</span>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
|
@ -33,17 +38,17 @@ function renderModelNameCell(modelName: string, getProviderFromModel?: (modelNam
|
|||
function renderFallbacksChain(
|
||||
_primaryModel: string,
|
||||
fallbackModels: string[],
|
||||
getProviderFromModel?: (modelName: string) => string,
|
||||
resolveModel?: ModelDisplayResolver,
|
||||
): React.ReactNode {
|
||||
const list = Array.isArray(fallbackModels) ? fallbackModels : [];
|
||||
if (list.length === 0) return null;
|
||||
|
||||
const ChainCard = ({ modelName }: { modelName: string }) => {
|
||||
const provider = getProviderFromModel?.(modelName) ?? modelName;
|
||||
const resolved = resolveModel?.(modelName) ?? { provider: modelName, label: modelName };
|
||||
return (
|
||||
<span className={modelCardClass}>
|
||||
<ProviderLogo provider={provider} className="w-4 h-4 shrink-0" />
|
||||
<span className="break-words">{modelName}</span>
|
||||
<span className={modelCardClass} title={resolved.title}>
|
||||
<ProviderLogo provider={resolved.provider} className="w-4 h-4 shrink-0" />
|
||||
<span className="break-words">{resolved.label}</span>
|
||||
</span>
|
||||
);
|
||||
};
|
||||
|
|
@ -127,11 +132,18 @@ const Fallbacks: React.FC<FallbacksProps> = ({ accessToken, userRole, userID })
|
|||
const [fallbackToEdit, setFallbackToEdit] = useState<FallbackEntry | null>(null);
|
||||
|
||||
const { data: modelCostMapData } = useModelCostMap();
|
||||
const getProviderFromModel = (model: string): string => {
|
||||
if (modelCostMapData != null && typeof modelCostMapData === "object" && model in modelCostMapData) {
|
||||
return modelCostMapData[model]["litellm_provider"] ?? "";
|
||||
}
|
||||
return "";
|
||||
const { data: modelGroups = [] } = useQuery({
|
||||
queryKey: ["availableModels", "fallbacks"],
|
||||
queryFn: () => fetchAvailableModels(accessToken ?? ""),
|
||||
enabled: Boolean(accessToken),
|
||||
});
|
||||
const resolveModelDisplay = (model: string): { provider: string; label: string; title?: string } => {
|
||||
const label = fallbackPrimaryLabel(model);
|
||||
return {
|
||||
provider: resolveFallbackProvider(model, modelCostMapData, modelGroups),
|
||||
label,
|
||||
...(label !== model && { title: model }),
|
||||
};
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -281,10 +293,10 @@ const Fallbacks: React.FC<FallbacksProps> = ({ accessToken, userRole, userID })
|
|||
Object.entries(item).map(([key, value]) => (
|
||||
<TableRow key={index.toString() + key}>
|
||||
<TableCell className="align-top whitespace-normal">
|
||||
{renderModelNameCell(key, getProviderFromModel)}
|
||||
{renderModelNameCell(key, resolveModelDisplay)}
|
||||
</TableCell>
|
||||
<TableCell className="align-top whitespace-normal">
|
||||
{renderFallbacksChain(key, Array.isArray(value) ? value : [], getProviderFromModel)}
|
||||
{renderFallbacksChain(key, Array.isArray(value) ? value : [], resolveModelDisplay)}
|
||||
</TableCell>
|
||||
<TableCell className="align-top">
|
||||
{canModify && (
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import {
|
||||
fallbackChainOptions,
|
||||
fallbackPrimaryLabel,
|
||||
primaryModelOptions,
|
||||
resolveFallbackProvider,
|
||||
} from "./fallbackOptions";
|
||||
|
||||
const modelGroups: ModelGroup[] = [
|
||||
{ model_group: "gpt-4", providers: ["openai"] },
|
||||
{ model_group: "claude-3-opus", providers: ["anthropic"] },
|
||||
{ model_group: "bedrock/us.anthropic.claude-opus-4-8", providers: ["bedrock"] },
|
||||
];
|
||||
|
||||
describe("primaryModelOptions", () => {
|
||||
it("lists provider wildcards first with friendly labels, then model groups", () => {
|
||||
const options = primaryModelOptions(modelGroups);
|
||||
|
||||
expect(options.slice(0, 3)).toEqual([
|
||||
{ value: "anthropic/*", label: "All Anthropic Models" },
|
||||
{ value: "bedrock/*", label: "All Amazon Bedrock Models" },
|
||||
{ value: "openai/*", label: "All OpenAI Models" },
|
||||
]);
|
||||
expect(options.slice(3).map((option) => option.value)).toEqual([
|
||||
"bedrock/us.anthropic.claude-opus-4-8",
|
||||
"claude-3-opus",
|
||||
"gpt-4",
|
||||
]);
|
||||
});
|
||||
|
||||
it("derives a provider wildcard from the model group prefix when providers are absent", () => {
|
||||
const options = primaryModelOptions([{ model_group: "openai/gpt-6-astra" }]);
|
||||
|
||||
expect(options[0]).toEqual({ value: "openai/*", label: "All OpenAI Models" });
|
||||
expect(options).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("ignores prefixes that are not known provider slugs", () => {
|
||||
const options = primaryModelOptions([{ model_group: "my-router/custom-tier" }]);
|
||||
|
||||
expect(options).toEqual([{ value: "my-router/custom-tier", label: "my-router/custom-tier" }]);
|
||||
});
|
||||
|
||||
it("folds a model group literally named openai/* into the friendly option", () => {
|
||||
const options = primaryModelOptions([{ model_group: "openai/*", providers: ["openai"] }]);
|
||||
|
||||
expect(options).toEqual([{ value: "openai/*", label: "All OpenAI Models" }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fallbackChainOptions", () => {
|
||||
it("excludes wildcard entries, which can never be fallback targets", () => {
|
||||
const options = fallbackChainOptions([...modelGroups, { model_group: "openai/*", providers: ["openai"] }]);
|
||||
|
||||
expect(options.map((option) => option.value)).toEqual([
|
||||
"bedrock/us.anthropic.claude-opus-4-8",
|
||||
"claude-3-opus",
|
||||
"gpt-4",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveFallbackProvider", () => {
|
||||
it("prefers the cost map litellm_provider", () => {
|
||||
expect(resolveFallbackProvider("gpt-4", { "gpt-4": { litellm_provider: "azure" } }, modelGroups)).toBe("azure");
|
||||
});
|
||||
|
||||
it("falls back to the model group providers when the cost map misses", () => {
|
||||
expect(resolveFallbackProvider("claude-3-opus", null, modelGroups)).toBe("anthropic");
|
||||
});
|
||||
|
||||
it("resolves provider wildcards and prefixed names from the prefix", () => {
|
||||
expect(resolveFallbackProvider("anthropic/*", null, [])).toBe("anthropic");
|
||||
expect(resolveFallbackProvider("openai/gpt-6-astra", null, [])).toBe("openai");
|
||||
expect(resolveFallbackProvider("bedrock/us.anthropic.claude-opus-4-8", null, [])).toBe("bedrock");
|
||||
});
|
||||
|
||||
it("returns empty when nothing resolves", () => {
|
||||
expect(resolveFallbackProvider("custom-group", null, [])).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("fallbackPrimaryLabel", () => {
|
||||
it("renders provider wildcards as All <Provider> Models", () => {
|
||||
expect(fallbackPrimaryLabel("openai/*")).toBe("All OpenAI Models");
|
||||
expect(fallbackPrimaryLabel("anthropic/*")).toBe("All Anthropic Models");
|
||||
});
|
||||
|
||||
it("keeps concrete names and unknown wildcards as-is", () => {
|
||||
expect(fallbackPrimaryLabel("gpt-4")).toBe("gpt-4");
|
||||
expect(fallbackPrimaryLabel("custom/*")).toBe("custom/*");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
import { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { getProviderLogoAndName, provider_map, resolveLitellmProviderSlug } from "@/components/provider_info_helpers";
|
||||
|
||||
export interface FallbackOption {
|
||||
label: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
const KNOWN_PROVIDER_SLUGS: ReadonlySet<string> = new Set(
|
||||
Object.values(provider_map).map((slug) => slug.toLowerCase()),
|
||||
);
|
||||
|
||||
const modelGroupProviderPrefix = (modelGroup: string): string | null => {
|
||||
const slashIndex = modelGroup.indexOf("/");
|
||||
if (slashIndex <= 0) {
|
||||
return null;
|
||||
}
|
||||
const prefix = modelGroup.slice(0, slashIndex).toLowerCase();
|
||||
return KNOWN_PROVIDER_SLUGS.has(prefix) ? prefix : null;
|
||||
};
|
||||
|
||||
export const providerSlugsFor = (modelGroups: ModelGroup[]): string[] => {
|
||||
const slugs = new Set<string>();
|
||||
for (const modelGroup of modelGroups) {
|
||||
for (const provider of modelGroup.providers ?? []) {
|
||||
slugs.add(resolveLitellmProviderSlug(provider));
|
||||
}
|
||||
const prefix = modelGroupProviderPrefix(modelGroup.model_group);
|
||||
if (prefix) {
|
||||
slugs.add(prefix);
|
||||
}
|
||||
}
|
||||
return Array.from(slugs).sort((a, b) => a.localeCompare(b));
|
||||
};
|
||||
|
||||
export const providerWildcardOption = (slug: string): FallbackOption => {
|
||||
const { displayName } = getProviderLogoAndName(slug);
|
||||
return { value: `${slug}/*`, label: `All ${displayName} Models` };
|
||||
};
|
||||
|
||||
export const providerWildcardOptions = (modelGroups: ModelGroup[]): FallbackOption[] =>
|
||||
providerSlugsFor(modelGroups).map(providerWildcardOption);
|
||||
|
||||
export const primaryModelOptions = (modelGroups: ModelGroup[]): FallbackOption[] => {
|
||||
const wildcardValues = new Set(providerWildcardOptions(modelGroups).map((option) => option.value));
|
||||
const groupOptions = Array.from(new Set(modelGroups.map((modelGroup) => modelGroup.model_group)))
|
||||
.filter((name) => !wildcardValues.has(name))
|
||||
.sort((a, b) => a.localeCompare(b))
|
||||
.map((name) => ({ label: name, value: name }));
|
||||
return [...providerWildcardOptions(modelGroups), ...groupOptions];
|
||||
};
|
||||
|
||||
export const fallbackChainOptions = (modelGroups: ModelGroup[]): FallbackOption[] =>
|
||||
Array.from(new Set(modelGroups.map((modelGroup) => modelGroup.model_group)))
|
||||
.filter((name) => !name.includes("*"))
|
||||
.sort((a, b) => a.localeCompare(b))
|
||||
.map((name) => ({ label: name, value: name }));
|
||||
|
||||
export const resolveFallbackProvider = (
|
||||
model: string,
|
||||
modelCostMap: Record<string, { litellm_provider?: string }> | null | undefined,
|
||||
modelGroups: ModelGroup[],
|
||||
): string => {
|
||||
const fromCostMap = modelCostMap?.[model]?.litellm_provider;
|
||||
if (typeof fromCostMap === "string" && fromCostMap !== "") {
|
||||
return fromCostMap;
|
||||
}
|
||||
const provider = modelGroups.find((modelGroup) => modelGroup.model_group === model)?.providers?.[0];
|
||||
if (provider) {
|
||||
return resolveLitellmProviderSlug(provider);
|
||||
}
|
||||
return modelGroupProviderPrefix(model) ?? "";
|
||||
};
|
||||
|
||||
export const fallbackPrimaryLabel = (model: string): string => {
|
||||
if (model.endsWith("/*")) {
|
||||
const slug = model.slice(0, -2);
|
||||
if (KNOWN_PROVIDER_SLUGS.has(slug.toLowerCase())) {
|
||||
const { displayName } = getProviderLogoAndName(slug);
|
||||
return `All ${displayName} Models`;
|
||||
}
|
||||
}
|
||||
return model;
|
||||
};
|
||||
|
|
@ -16,8 +16,13 @@ vi.mock("@/components/llm_calls/fetch_models", () => ({
|
|||
}));
|
||||
|
||||
vi.mock("../Settings/RouterSettings/Fallbacks/FallbackSelectionForm", () => ({
|
||||
FallbackSelectionForm: ({ availableModels }: { availableModels: string[] }) => (
|
||||
<div data-testid="available-models">{availableModels.join(",")}</div>
|
||||
FallbackSelectionForm: ({ modelGroups }: { modelGroups: { model_group: string }[] }) => (
|
||||
<div data-testid="available-models">
|
||||
{modelGroups
|
||||
.map((modelGroup) => modelGroup.model_group)
|
||||
.sort()
|
||||
.join(",")}
|
||||
</div>
|
||||
),
|
||||
}));
|
||||
|
||||
|
|
|
|||
|
|
@ -327,8 +327,6 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
|
|||
setFallbacks(newFallbacks);
|
||||
};
|
||||
|
||||
const availableModels = Array.from(new Set(modelInfo.map((option) => option.model_group))).sort();
|
||||
|
||||
// Expose getValue method via ref
|
||||
useImperativeHandle(ref, () => ({
|
||||
getValue: () => {
|
||||
|
|
@ -363,7 +361,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
|
|||
<FallbackSelectionForm
|
||||
groups={fallbackGroups}
|
||||
onGroupsChange={handleFallbackGroupsChange}
|
||||
availableModels={availableModels}
|
||||
modelGroups={modelInfo}
|
||||
maxGroups={5}
|
||||
/>
|
||||
</TabsContent>
|
||||
|
|
|
|||
|
|
@ -251,6 +251,7 @@ describe("Sidebar (leftnav)", () => {
|
|||
"Virtual Keys",
|
||||
"Playground",
|
||||
"Models + Endpoints",
|
||||
"Fallbacks",
|
||||
"Agentic",
|
||||
"MCP Servers",
|
||||
"Guardrails",
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ import {
|
|||
ScrollText,
|
||||
Search,
|
||||
Server,
|
||||
Shuffle,
|
||||
Settings as SettingsIcon,
|
||||
Shield,
|
||||
ShieldCheck,
|
||||
|
|
@ -226,6 +227,13 @@ const menuGroups: MenuGroup[] = [
|
|||
icon: <Network {...ICON} />,
|
||||
roles: rolesAllowedToViewWriteScopedPages,
|
||||
},
|
||||
{
|
||||
key: "fallbacks",
|
||||
page: "fallbacks",
|
||||
label: "Fallbacks",
|
||||
icon: <Shuffle {...ICON} />,
|
||||
roles: all_admin_roles,
|
||||
},
|
||||
{
|
||||
key: "agentic",
|
||||
page: "agentic",
|
||||
|
|
|
|||
|
|
@ -666,3 +666,92 @@ describe("preferred LiteAdmin model", () => {
|
|||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("router fallback operations", () => {
|
||||
const routerSettings = () => ({
|
||||
routing_strategy: "least-busy",
|
||||
fallbacks: [{ "gpt-4": ["claude-3-haiku"] }, { "openai/*": ["claude-3-haiku", "gpt-4"] }],
|
||||
model_group_retry_policy: { "gpt-4": { AuthenticationErrorRetries: 0 } },
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
managementFetch.mockImplementation(async (input) => {
|
||||
const path = new URL(String(input)).pathname;
|
||||
if (path.endsWith("/get/config/callbacks")) return json({ router_settings: routerSettings() });
|
||||
if (path.endsWith("/config/update")) return json({ status: "success" });
|
||||
throw new Error(`Unexpected request: ${path}`);
|
||||
});
|
||||
});
|
||||
|
||||
const configUpdateBody = () => {
|
||||
const call = managementFetch.mock.calls.find(([url]) => String(url).includes("/config/update"));
|
||||
expect(call).toBeDefined();
|
||||
return JSON.parse(String(call?.[1]?.body));
|
||||
};
|
||||
|
||||
it("fallbacks_list reads router_settings.fallbacks without confirmation", async () => {
|
||||
const input = options();
|
||||
const model = transport([completion([call("fallbacks_list", {})]), completion()]);
|
||||
|
||||
await runLiteAdmin(input, model.client);
|
||||
|
||||
expect(managementFetch).toHaveBeenCalledTimes(1);
|
||||
expect(new URL(String(managementFetch.mock.calls[0][0])).pathname).toBe("/root/get/config/callbacks");
|
||||
expect(input.confirm).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fallback_set appends a new entry and saves full router_settings minus the retry policy", async () => {
|
||||
const input = options();
|
||||
const model = transport([
|
||||
completion([call("fallback_set", { model: "anthropic/*", fallbacks: ["gpt-4"] })]),
|
||||
completion(),
|
||||
]);
|
||||
|
||||
await runLiteAdmin(input, model.client);
|
||||
|
||||
expect(configUpdateBody()).toEqual({
|
||||
router_settings: {
|
||||
routing_strategy: "least-busy",
|
||||
fallbacks: [
|
||||
{ "gpt-4": ["claude-3-haiku"] },
|
||||
{ "openai/*": ["claude-3-haiku", "gpt-4"] },
|
||||
{ "anthropic/*": ["gpt-4"] },
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(input.confirm).toHaveBeenCalledExactlyOnceWith(
|
||||
expect.objectContaining({ name: "fallback_set", destructive: false }),
|
||||
);
|
||||
});
|
||||
|
||||
it("fallback_set replaces the entry for an existing model in place", async () => {
|
||||
const model = transport([
|
||||
completion([call("fallback_set", { model: "gpt-4", fallbacks: ["gpt-4o-mini"] })]),
|
||||
completion(),
|
||||
]);
|
||||
|
||||
await runLiteAdmin(options(), model.client);
|
||||
|
||||
expect(configUpdateBody().router_settings.fallbacks).toEqual([
|
||||
{ "gpt-4": ["gpt-4o-mini"] },
|
||||
{ "openai/*": ["claude-3-haiku", "gpt-4"] },
|
||||
]);
|
||||
});
|
||||
|
||||
it("fallback_delete removes only that entry and keeps the rest of router_settings", async () => {
|
||||
const input = options();
|
||||
const model = transport([completion([call("fallback_delete", { model: "openai/*" })]), completion()]);
|
||||
|
||||
await runLiteAdmin(input, model.client);
|
||||
|
||||
expect(configUpdateBody()).toEqual({
|
||||
router_settings: {
|
||||
routing_strategy: "least-busy",
|
||||
fallbacks: [{ "gpt-4": ["claude-3-haiku"] }],
|
||||
},
|
||||
});
|
||||
expect(input.confirm).toHaveBeenCalledExactlyOnceWith(
|
||||
expect.objectContaining({ name: "fallback_delete", destructive: true }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -104,6 +104,22 @@ const requestLogsFields = {
|
|||
model: optionalText,
|
||||
status_filter: optional(z.enum(["success", "failure"])),
|
||||
};
|
||||
const fallbackTarget = text.regex(/^[^*]+$/, "Wildcard entries cannot be fallback targets.");
|
||||
const fallbackSetFields = {
|
||||
model: text.describe(
|
||||
"The primary model group, or a provider wildcard like openai/* meaning all models from that provider.",
|
||||
),
|
||||
fallbacks: z
|
||||
.array(fallbackTarget)
|
||||
.min(1)
|
||||
.max(10)
|
||||
.describe("Concrete model groups to try in order. Never a wildcard."),
|
||||
};
|
||||
const fallbackDeleteFields = { model: text };
|
||||
|
||||
type RouterSettings = Record<string, unknown>;
|
||||
const fallbackEntries = (settings: RouterSettings): Record<string, string[]>[] =>
|
||||
Array.isArray(settings.fallbacks) ? (settings.fallbacks as Record<string, string[]>[]) : [];
|
||||
|
||||
export function createLiteAdminOperations(context: OperationContext) {
|
||||
const active = () => {
|
||||
|
|
@ -111,6 +127,14 @@ export function createLiteAdminOperations(context: OperationContext) {
|
|||
context.assertCurrent();
|
||||
};
|
||||
const auth = { accessToken: context.accessToken, signal: context.signal };
|
||||
const readRouterSettings = async (): Promise<RouterSettings> => {
|
||||
const data = await apiClient.get<{ router_settings?: RouterSettings }>("/get/config/callbacks", auth);
|
||||
const settings = { ...(data?.router_settings ?? {}) };
|
||||
delete settings.model_group_retry_policy;
|
||||
return settings;
|
||||
};
|
||||
const saveRouterSettings = (routerSettings: RouterSettings) =>
|
||||
apiClient.post<unknown>("/config/update", { ...auth, body: { router_settings: routerSettings } });
|
||||
const operation =
|
||||
(mode: "read" | "write" | "delete", kind: ResultKind, extraArguments?: Record<string, unknown>) =>
|
||||
<Schema extends z.ZodType<Record<string, unknown>>>(
|
||||
|
|
@ -339,5 +363,39 @@ export function createLiteAdminOperations(context: OperationContext) {
|
|||
query: { ...a, start_date: `${a.start_date} 00:00:00`, end_date: `${a.end_date} 23:59:59` },
|
||||
}),
|
||||
),
|
||||
operation("read", "fallback")("fallbacks_list", "List configured router fallbacks", object({}), async () => ({
|
||||
fallbacks: fallbackEntries(await readRouterSettings()),
|
||||
})),
|
||||
operation("write", "fallback")(
|
||||
"fallback_set",
|
||||
"Set the fallback chain for a model",
|
||||
object(fallbackSetFields),
|
||||
async (a) => {
|
||||
const settings = await readRouterSettings();
|
||||
const current = fallbackEntries(settings);
|
||||
const existingIndex = current.findIndex(
|
||||
(entry) => entry !== null && typeof entry === "object" && a.model in entry,
|
||||
);
|
||||
const updated =
|
||||
existingIndex >= 0
|
||||
? current.map((entry, index) => (index === existingIndex ? { [a.model]: a.fallbacks } : entry))
|
||||
: [...current, { [a.model]: a.fallbacks }];
|
||||
await saveRouterSettings({ ...settings, fallbacks: updated });
|
||||
return { fallbacks: updated };
|
||||
},
|
||||
),
|
||||
operation("delete", "fallback")(
|
||||
"fallback_delete",
|
||||
"Remove the fallback chain for a primary model",
|
||||
object(fallbackDeleteFields),
|
||||
async (a) => {
|
||||
const settings = await readRouterSettings();
|
||||
const updated = fallbackEntries(settings)
|
||||
.map((entry) => Object.fromEntries(Object.entries(entry).filter(([key]) => key !== a.model)))
|
||||
.filter((entry) => Object.keys(entry).length > 0);
|
||||
await saveRouterSettings({ ...settings, fallbacks: updated });
|
||||
return { fallbacks: updated };
|
||||
},
|
||||
),
|
||||
];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { z } from "zod";
|
||||
|
||||
export type ResultKind = "key" | "team" | "user" | "budget" | "spend" | "log";
|
||||
export type ResultKind = "key" | "team" | "user" | "budget" | "spend" | "log" | "fallback";
|
||||
|
||||
const limitedList = <Schema extends z.ZodType<unknown>>(schema: Schema, limit = 50) =>
|
||||
z.preprocess((value) => (Array.isArray(value) ? value.slice(0, limit) : value), z.array(schema));
|
||||
|
|
@ -150,6 +150,7 @@ export function projectToolResult(kind: ResultKind, value: unknown, secrets: rea
|
|||
]),
|
||||
spend: limitedList(spend),
|
||||
log: z.object({ ...pagination, data: limitedList(log) }),
|
||||
fallback: z.object({ fallbacks: limitedList(z.record(z.string(), limitedList(text))).nullish() }),
|
||||
};
|
||||
const parsed = schemas[kind].safeParse(value);
|
||||
if (!parsed.success)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ export const pageDescriptions: Record<string, string> = {
|
|||
"api-keys": "Manage virtual keys for API access and authentication",
|
||||
"llm-playground": "Interactive playground for testing LLM requests",
|
||||
models: "Configure and manage LLM models and endpoints",
|
||||
fallbacks: "Configure which models to retry when the primary model fails",
|
||||
agents: "Create and manage AI agents",
|
||||
agentic: "Manage agentic resources: agents, workflow runs, and memory",
|
||||
workflows: "Track and inspect durable workflow run history",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue