mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
Merge pull request #41617 from BerriAI/litellm_fuse_model_profile_presets
feat(router): add maintained Fuse model and harness presets
This commit is contained in:
tin-berri
GitHub
commit
252a0f1eac
16 changed files with 1066 additions and 45 deletions
|
|
@ -23,6 +23,7 @@ from litellm.proxy._types import (
|
|||
)
|
||||
from litellm.proxy.utils import get_custom_url
|
||||
from litellm.repositories.table_repositories import ClaudeCodePluginRepository
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import FusePresetCatalog, get_fuse_presets
|
||||
from litellm.types.agents import AgentCard
|
||||
from litellm.types.mcp import MCPPublicServer
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
|
|
@ -424,6 +425,14 @@ async def get_complexity_scorer_defaults() -> ComplexityScorerDefaults:
|
|||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/public/complexity_router/fuse_presets",
|
||||
response_model=FusePresetCatalog,
|
||||
)
|
||||
async def get_public_fuse_presets() -> FusePresetCatalog:
|
||||
return get_fuse_presets()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/public/litellm_model_cost_map",
|
||||
tags=["public", "model management"],
|
||||
|
|
|
|||
100
litellm/router_strategy/complexity_router/fuse_presets.json
Normal file
100
litellm/router_strategy/complexity_router/fuse_presets.json
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
{
|
||||
"version": "2026-09-17-v1",
|
||||
"models": [
|
||||
{
|
||||
"id": "gpt-6-astra-v1",
|
||||
"label": "GPT-6 Astra",
|
||||
"model": "gpt-6-astra",
|
||||
"text": "OpenAI model for demanding end-to-end work, including reasoning, coding, research, and document tasks",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-astra"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-sol-v1",
|
||||
"label": "GPT-5.6 Sol",
|
||||
"model": "gpt-5.6-sol",
|
||||
"text": "OpenAI model for complex professional work, supporting reasoning and tool calling",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-sol"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-luna-v1",
|
||||
"label": "GPT-5.6 Luna",
|
||||
"model": "gpt-5.6-luna",
|
||||
"text": "OpenAI model for high-volume workloads, supporting reasoning and tool calling",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-luna"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-terra-v1",
|
||||
"label": "GPT-5.6 Terra",
|
||||
"model": "gpt-5.6-terra",
|
||||
"text": "OpenAI general-purpose model supporting reasoning, text and image input, and tool calling",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-terra"]
|
||||
},
|
||||
{
|
||||
"id": "claude-haiku-4-5-v1",
|
||||
"label": "Claude Haiku 4.5",
|
||||
"model": "claude-haiku-4-5",
|
||||
"text": "Anthropic latency-focused model supporting text and image input, tool use, and extended thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/haiku-4-5/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-sonnet-5-v1",
|
||||
"label": "Claude Sonnet 5",
|
||||
"model": "claude-sonnet-5",
|
||||
"text": "Anthropic model balancing speed and capability, with adaptive thinking and tool use",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/sonnet-5/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-opus-5-v1",
|
||||
"label": "Claude Opus 5",
|
||||
"model": "claude-opus-5",
|
||||
"text": "Anthropic model for complex agentic coding and enterprise work, with adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/opus-5/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-fable-5-v1",
|
||||
"label": "Claude Fable 5",
|
||||
"model": "claude-fable-5",
|
||||
"text": "Anthropic model for demanding reasoning and long-running agent tasks, with always-on adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/fable-5/introducing-claude-fable-5-and-claude-mythos-5"]
|
||||
},
|
||||
{
|
||||
"id": "claude-fable-5-1-v1",
|
||||
"label": "Claude Fable 5.1",
|
||||
"model": "claude-fable-5-1",
|
||||
"text": "Anthropic model for demanding reasoning, long-running agentic coding, and multistep research, with always-on adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/fable-5-1/overview"]
|
||||
}
|
||||
],
|
||||
"harnesses": [
|
||||
{
|
||||
"id": "unspecified-v1",
|
||||
"label": "Unspecified runtime",
|
||||
"text": "Agent runtime is unspecified. Assess the task using the supplied context without assuming repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://code.claude.com/docs/en/how-claude-code-works", "https://mini-swe-agent.com/latest/faq/"]
|
||||
},
|
||||
{
|
||||
"id": "claude-code-v1",
|
||||
"label": "Claude Code",
|
||||
"text": "Claude Code supplies an agent loop with context management and configured tools. Available actions depend on the session's tools, permissions, and execution environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://code.claude.com/docs/en/how-claude-code-works"]
|
||||
},
|
||||
{
|
||||
"id": "codex-cli-v1",
|
||||
"label": "Codex CLI",
|
||||
"text": "Codex CLI supplies a terminal-based coding agent. File operations, command execution, and integrations depend on the session's tools, permissions, and sandbox. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://learn.chatgpt.com/docs/codex/cli", "https://learn.chatgpt.com/codex/permissions"]
|
||||
},
|
||||
{
|
||||
"id": "opencode-v1",
|
||||
"label": "OpenCode",
|
||||
"text": "OpenCode supplies a configurable agent runtime. Available actions depend on the selected agent, tools, permissions, and execution environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://opencode.ai/docs/agents/"]
|
||||
},
|
||||
{
|
||||
"id": "mini-swe-agent-v1",
|
||||
"label": "mini-SWE-agent",
|
||||
"text": "The standard mini-SWE-agent setup uses a bash-only action interface and separate command executions. Available commands and resources depend on its configured environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://mini-swe-agent.com/latest/faq/"]
|
||||
}
|
||||
]
|
||||
}
|
||||
52
litellm/router_strategy/complexity_router/fuse_presets.py
Normal file
52
litellm/router_strategy/complexity_router/fuse_presets.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
from functools import lru_cache
|
||||
from importlib.resources import files
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StringConstraints
|
||||
|
||||
ProfileText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=4000)]
|
||||
|
||||
|
||||
class FuseModelPreset(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
id: str
|
||||
label: str
|
||||
text: ProfileText
|
||||
sources: tuple[str, ...] = Field(min_length=1)
|
||||
model: str
|
||||
|
||||
|
||||
class FuseHarnessPreset(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
id: str
|
||||
label: str
|
||||
text: ProfileText
|
||||
sources: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class FusePresetCatalog(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
version: str
|
||||
models: tuple[FuseModelPreset, ...]
|
||||
harnesses: tuple[FuseHarnessPreset, ...]
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_fuse_presets() -> FusePresetCatalog:
|
||||
return FusePresetCatalog.model_validate_json(
|
||||
files(__package__).joinpath("fuse_presets.json").read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
|
||||
def resolve_fuse_profile(text: str | None, preset_id: str | None, kind: Literal["model", "harness"]) -> str | None:
|
||||
if preset_id is None:
|
||||
return text
|
||||
catalog: Final = get_fuse_presets()
|
||||
presets: Final = catalog.models if kind == "model" else catalog.harnesses
|
||||
preset: Final = next((entry for entry in presets if entry.id == preset_id), None)
|
||||
if preset is None:
|
||||
return None
|
||||
return text if text is not None else preset.text
|
||||
|
|
@ -7,15 +7,15 @@ from dataclasses import dataclass
|
|||
from sys import float_info
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StringConstraints, TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StringConstraints, TypeAdapter, model_validator
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.llms.base_llm.base_utils import (
|
||||
type_to_response_format_param, # pyright: ignore[reportUnknownVariableType] # legacy output validated below
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import ProfileText, resolve_fuse_profile
|
||||
|
||||
ShortText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=512)]
|
||||
ProfileText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=4000)]
|
||||
|
||||
|
||||
class _SolverProfile(TypedDict):
|
||||
|
|
@ -139,20 +139,41 @@ class LLMV2Config(BaseModel):
|
|||
|
||||
efficient_tier: str = "SIMPLE"
|
||||
capable_tier: str = "REASONING"
|
||||
efficient_profile: ProfileText
|
||||
capable_profile: ProfileText
|
||||
harness: ProfileText
|
||||
efficient_profile: ProfileText | None = None
|
||||
capable_profile: ProfileText | None = None
|
||||
harness: ProfileText | None = None
|
||||
efficient_profile_preset: str | None = None
|
||||
capable_profile_preset: str | None = None
|
||||
harness_preset: str | None = None
|
||||
max_quality_gap: float = Field(ge=0.0, le=1.0, description="Maximum estimated success loss allowed for efficient.")
|
||||
max_output_tokens: int = Field(default=1024, ge=1)
|
||||
response_format: Literal["json_schema", "json_object"] = "json_schema"
|
||||
calibration: LLMV2Calibration | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_profiles(self) -> LLMV2Config:
|
||||
self._profile_texts()
|
||||
return self
|
||||
|
||||
def _profile_texts(self) -> tuple[str, str, str]:
|
||||
efficient: Final = resolve_fuse_profile(self.efficient_profile, self.efficient_profile_preset, "model")
|
||||
capable: Final = resolve_fuse_profile(self.capable_profile, self.capable_profile_preset, "model")
|
||||
harness: Final = resolve_fuse_profile(self.harness, self.harness_preset, "harness")
|
||||
if efficient is None:
|
||||
raise ValueError("efficient_profile requires text or a known efficient_profile_preset")
|
||||
if capable is None:
|
||||
raise ValueError("capable_profile requires text or a known capable_profile_preset")
|
||||
if harness is None:
|
||||
raise ValueError("harness requires text or a known harness_preset")
|
||||
return efficient, capable, harness
|
||||
|
||||
def system_prompt(self, efficient_model: str, capable_model: str) -> str:
|
||||
efficient, capable, harness = self._profile_texts()
|
||||
profiles: Final[_SolverProfiles] = {
|
||||
"prompt_version": LLM_V2_PROMPT_VERSION,
|
||||
"harness": self.harness,
|
||||
"efficient": {"model": efficient_model, "profile": self.efficient_profile},
|
||||
"capable": {"model": capable_model, "profile": self.capable_profile},
|
||||
"harness": harness,
|
||||
"efficient": {"model": efficient_model, "profile": efficient},
|
||||
"capable": {"model": capable_model, "profile": capable},
|
||||
}
|
||||
schema: Final = (
|
||||
"\n\nResponse JSON schema:\n" + json.dumps(LLMV2Verdict.model_json_schema())
|
||||
|
|
|
|||
|
|
@ -302,6 +302,7 @@ editable-profile = "dev"
|
|||
include = [
|
||||
"litellm/proxy/_experimental/out/**",
|
||||
"litellm/router_strategy/complexity_router/artifacts/*.json",
|
||||
"litellm/router_strategy/complexity_router/fuse_presets.json",
|
||||
"litellm/proxy/client/cli/commands/codex_base_instructions.md",
|
||||
]
|
||||
exclude = [
|
||||
|
|
|
|||
|
|
@ -11,12 +11,23 @@ from fastapi.testclient import TestClient
|
|||
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.public_endpoints import router
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelGroupInfoProxy,
|
||||
)
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
||||
def test_fuse_presets_route_serves_the_shared_catalog_without_authentication() -> None:
|
||||
app: Final = FastAPI()
|
||||
app.include_router(router)
|
||||
client: Final = TestClient(app)
|
||||
response: Final = client.get("/public/complexity_router/fuse_presets")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == get_fuse_presets().model_dump(mode="json")
|
||||
assert client.get("/public/complexity_router/fuse_presets").json() == response.json()
|
||||
|
||||
|
||||
def test_get_supported_providers_returns_enum_values():
|
||||
app_instance = FastAPI()
|
||||
app_instance.include_router(router)
|
||||
|
|
|
|||
69
tests/test_litellm/router_strategy/test_fuse_presets.py
Normal file
69
tests/test_litellm/router_strategy/test_fuse_presets.py
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
import json
|
||||
from hashlib import sha256
|
||||
from importlib.resources import files
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets, resolve_fuse_profile
|
||||
|
||||
|
||||
def test_catalog_is_loaded_once_and_preserves_bundled_content() -> None:
|
||||
get_fuse_presets.cache_clear()
|
||||
first: Final = get_fuse_presets()
|
||||
second: Final = get_fuse_presets()
|
||||
assert first is second
|
||||
bundled: Final = json.loads(
|
||||
files("litellm.router_strategy.complexity_router").joinpath("fuse_presets.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert first.model_dump(mode="json") == bundled
|
||||
entries: Final = (*first.models, *first.harnesses)
|
||||
assert len({entry.id for entry in entries}) == len(entries)
|
||||
assert all(entry.sources and all(source.startswith("https://") for source in entry.sources) for entry in entries)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kind", "preset_id", "expected_digest"),
|
||||
(
|
||||
("model", "gpt-6-astra-v1", "a9403b0c00ea64081b7b08b5b968850670f3a047d219a7e0668f2169146ae96e"),
|
||||
("model", "gpt-5.6-sol-v1", "2b91a6c43e0e93183aaaf9c355e1bbb8ed2e9817aab6b0c2f50148f53a23247b"),
|
||||
("model", "gpt-5.6-luna-v1", "fff94a9e01bf4519798d5be4e76a3f9d57b75a2d9966a59dc92cbfeb5cd08d07"),
|
||||
("model", "gpt-5.6-terra-v1", "75de040f3bea841fa4764885738303893ee7ac0804aed1e932cd3959185ff893"),
|
||||
("model", "claude-haiku-4-5-v1", "91c1920953073462b6b70ef810596a5325f08286b5e62630cff47938fc4157db"),
|
||||
("model", "claude-sonnet-5-v1", "133f4414c644a707cd8cf565a486153856f4836ca4e4f75ee0553f2b7a1e3663"),
|
||||
("model", "claude-opus-5-v1", "9cbfcae45d2e3a2575e44ce5adf618f56614abff4b3221d35900c647200b99ef"),
|
||||
("model", "claude-fable-5-v1", "25c275d7403f1572ffb4fe899d5feecd9a434ebdc37b4dd9ef601a8ecf4850fc"),
|
||||
("model", "claude-fable-5-1-v1", "37693107c878ab6266530395bbdc2d2813d676d179bf281d05e5a5ec1b9d4c60"),
|
||||
("harness", "unspecified-v1", "d9eb30b61509456f0c71ca805b33d821cab6605578d567a29ab421d8f602ce7b"),
|
||||
("harness", "claude-code-v1", "7ee8e9d50f1cf44a8a58461efff66d6182f245d25499702c144d1c642c101ed9"),
|
||||
("harness", "codex-cli-v1", "0678047e34562ef05b5e2fba099c1f9e5876304f7eaf3b0d8c3809e707eb3311"),
|
||||
("harness", "opencode-v1", "8b6cc240d90091ac2ef9b374b535f981a55abb91e25d4c04fdb9fc206eeb907e"),
|
||||
("harness", "mini-swe-agent-v1", "21e2dc4a8a2320a5a554a498b30516326dc3592ebf20b0f4db20ddb33e879a39"),
|
||||
),
|
||||
)
|
||||
def test_existing_preset_text_is_unchanged(
|
||||
kind: Literal["model", "harness"], preset_id: str, expected_digest: str
|
||||
) -> None:
|
||||
text: Final = resolve_fuse_profile(None, preset_id, kind)
|
||||
assert text is not None
|
||||
assert sha256(text.encode("utf-8")).hexdigest() == expected_digest
|
||||
|
||||
|
||||
def test_every_catalog_entry_resolves_without_changing_custom_ownership() -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
for entry in catalog.models:
|
||||
assert resolve_fuse_profile(None, entry.id, "model") == entry.text
|
||||
assert resolve_fuse_profile("Custom text", entry.id, "model") == "Custom text"
|
||||
for entry in catalog.harnesses:
|
||||
assert resolve_fuse_profile(None, entry.id, "harness") == entry.text
|
||||
assert resolve_fuse_profile("Custom text", entry.id, "harness") == "Custom text"
|
||||
assert resolve_fuse_profile("Custom text", None, "model") == "Custom text"
|
||||
assert resolve_fuse_profile("Custom text", None, "harness") == "Custom text"
|
||||
|
||||
|
||||
def test_cached_catalog_and_records_cannot_be_modified() -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
for record, field in ((catalog, "version"), (catalog.models[0], "text"), (catalog.harnesses[0], "text")):
|
||||
with pytest.raises(ValidationError, match="frozen"):
|
||||
setattr(record, field, "Changed")
|
||||
|
|
@ -11,8 +11,10 @@ from litellm import ModelResponse, Router
|
|||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, ComplexityTier
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
from litellm.router_strategy.complexity_router.llm_v2 import (
|
||||
LLM_V2_PROMPT_VERSION,
|
||||
LLM_V2_SYSTEM_PROMPT,
|
||||
LLMV2Calibration,
|
||||
LLMV2Config,
|
||||
LLMV2ProbabilityCalibration,
|
||||
|
|
@ -174,6 +176,114 @@ def test_invalid_forecast_settings_are_rejected(overrides: dict[str, object]) ->
|
|||
LLMV2Config.model_validate({**base.model_dump(), **overrides})
|
||||
|
||||
|
||||
def _preset_config(**overrides: object) -> LLMV2Config:
|
||||
catalog: Final = get_fuse_presets()
|
||||
return LLMV2Config.model_validate(
|
||||
{
|
||||
"efficient_profile_preset": catalog.models[0].id,
|
||||
"capable_profile_preset": catalog.models[-1].id,
|
||||
"harness_preset": catalog.harnesses[-1].id,
|
||||
"max_quality_gap": 0.05,
|
||||
**overrides,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_preset_roundtrip_keeps_references_without_materializing_text() -> None:
|
||||
config: Final = _preset_config()
|
||||
serialized: Final = config.model_dump(exclude_none=True)
|
||||
assert serialized["efficient_profile_preset"] == config.efficient_profile_preset
|
||||
assert serialized["capable_profile_preset"] == config.capable_profile_preset
|
||||
assert serialized["harness_preset"] == config.harness_preset
|
||||
assert not {"efficient_profile", "capable_profile", "harness"}.intersection(serialized)
|
||||
assert LLMV2Config.model_validate(config.model_dump()) == config
|
||||
assert LLMV2Config.model_validate_json(config.model_dump_json()) == config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
def test_preset_explicit_override_wins_and_survives_roundtrip(field: str) -> None:
|
||||
config: Final = _preset_config(**{field: " Operator description "})
|
||||
roundtrip: Final = LLMV2Config.model_validate_json(config.model_dump_json())
|
||||
assert roundtrip.model_dump()[field] == "Operator description"
|
||||
assert roundtrip.efficient_profile_preset == config.efficient_profile_preset
|
||||
assert roundtrip.capable_profile_preset == config.capable_profile_preset
|
||||
assert roundtrip.harness_preset == config.harness_preset
|
||||
payload: Final = json.loads(
|
||||
roundtrip.system_prompt("opaque-efficient", "opaque-capable").split("Configured solver profiles:\n")[1]
|
||||
)
|
||||
if field == "harness":
|
||||
assert payload["harness"] == "Operator description"
|
||||
else:
|
||||
assert payload[field.removesuffix("_profile")]["profile"] == "Operator description"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
@pytest.mark.parametrize("invalid", ("", " \n\t", "x" * 4001))
|
||||
def test_preset_does_not_bypass_supplied_text_bounds(field: str, invalid: str) -> None:
|
||||
with pytest.raises(ValidationError, match=field):
|
||||
_preset_config(**{field: invalid})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
@pytest.mark.parametrize("override", (None, "Custom override"))
|
||||
@pytest.mark.parametrize("invalid_id", ("missing-v1", ""))
|
||||
def test_preset_unknown_reference_rejects_even_when_overridden(
|
||||
field: str, override: str | None, invalid_id: str
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match=f"{field}.*preset"):
|
||||
_preset_config(**{field: override, f"{field}_preset": invalid_id})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
def test_preset_missing_text_and_reference_rejects(field: str) -> None:
|
||||
with pytest.raises(ValidationError, match=field):
|
||||
_preset_config(**{f"{field}_preset": None})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
def test_preset_reference_rejects_the_wrong_catalog_kind(field: str) -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
wrong_id: Final = catalog.models[0].id if field == "harness" else catalog.harnesses[0].id
|
||||
with pytest.raises(ValidationError, match=field):
|
||||
_preset_config(**{f"{field}_preset": wrong_id})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ("json_schema", "json_object"))
|
||||
def test_custom_profile_prompt_bytes_are_unchanged(mode: str) -> None:
|
||||
base: Final = _config().llm_v2_config
|
||||
assert base is not None
|
||||
config: Final = LLMV2Config.model_validate({**base.model_dump(), "response_format": mode})
|
||||
old_payload: Final = {
|
||||
"prompt_version": LLM_V2_PROMPT_VERSION,
|
||||
"harness": config.harness,
|
||||
"efficient": {"model": "opaque-efficient", "profile": config.efficient_profile},
|
||||
"capable": {"model": "opaque-capable", "profile": config.capable_profile},
|
||||
}
|
||||
schema: Final = (
|
||||
"\n\nResponse JSON schema:\n" + json.dumps(LLMV2Verdict.model_json_schema()) if mode == "json_object" else ""
|
||||
)
|
||||
assert config.system_prompt("opaque-efficient", "opaque-capable") == (
|
||||
LLM_V2_SYSTEM_PROMPT + "\n\nConfigured solver profiles:\n" + json.dumps(old_payload) + schema
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preset_router_passes_catalog_text_and_opaque_group_names_to_judge() -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
config: Final = _config(llm_v2_config=_preset_config().model_dump())
|
||||
router, client = _router(_verdict().model_dump_json(), config)
|
||||
outcome: Final = await router.aclassify("Complete the supplied task")
|
||||
assert outcome.tier == ComplexityTier.SIMPLE
|
||||
prompt: Final = client.acompletion.call_args.kwargs["messages"][0]["content"]
|
||||
payload: Final = json.loads(prompt.split("Configured solver profiles:\n")[1])
|
||||
assert payload == {
|
||||
"prompt_version": LLM_V2_PROMPT_VERSION,
|
||||
"harness": catalog.harnesses[-1].text,
|
||||
"efficient": {"model": "efficient", "profile": catalog.models[0].text},
|
||||
"capable": {"model": "capable", "profile": catalog.models[-1].text},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_judge_fuses_whole_task_and_keeps_caller_text_out_of_system_prompt() -> None:
|
||||
router, client = _router(_verdict().model_dump_json())
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
carries_complexity_router_settings,
|
||||
classify_strategy_router_model,
|
||||
|
|
@ -171,6 +174,52 @@ def test_validate_accepts_loadable_complexity_config(complexity_router_config):
|
|||
assert validate_complexity_router_config_write(complexity_router_config=complexity_router_config) is None
|
||||
|
||||
|
||||
def _fuse_write_config(profiles: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {
|
||||
"classifier_type": "llm_v2",
|
||||
"classifier_llm_config": {"model": "judge"},
|
||||
"tiers": {"SIMPLE": ["opaque-efficient"], "REASONING": ["opaque-capable"]},
|
||||
"llm_v2_config": {"max_quality_gap": 0.05, **profiles},
|
||||
}
|
||||
|
||||
|
||||
def test_fuse_write_accepts_presets_and_custom_text_with_the_same_entitlement() -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
presets: Final = _fuse_write_config(
|
||||
{
|
||||
"efficient_profile_preset": catalog.models[0].id,
|
||||
"capable_profile_preset": catalog.models[-1].id,
|
||||
"harness_preset": catalog.harnesses[0].id,
|
||||
}
|
||||
)
|
||||
custom: Final = _fuse_write_config(
|
||||
{
|
||||
"efficient_profile": catalog.models[0].text,
|
||||
"capable_profile": catalog.models[-1].text,
|
||||
"harness": catalog.harnesses[0].text,
|
||||
}
|
||||
)
|
||||
assert validate_complexity_router_config_write(presets) is None
|
||||
assert validate_complexity_router_config_write(custom) is None
|
||||
assert claimed_capability(presets) is claimed_capability(custom)
|
||||
assert claimed_capability(presets) is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
def test_fuse_write_rejects_unknown_preset_even_with_custom_text(field: str) -> None:
|
||||
config: Final = _fuse_write_config(
|
||||
{
|
||||
"efficient_profile": "Custom efficient solver",
|
||||
"capable_profile": "Custom capable solver",
|
||||
"harness": "Custom runtime",
|
||||
f"{field}_preset": "unknown-v1",
|
||||
}
|
||||
)
|
||||
violation: Final = validate_complexity_router_config_write(config)
|
||||
assert violation is not None
|
||||
assert f"{field}_preset" in violation
|
||||
|
||||
|
||||
def test_naming_check_ignores_the_config_entirely():
|
||||
"""The naming contract and the config's contents are separate questions with separate owners;
|
||||
a write may carry a config without naming a model, so neither can stand in for the other."""
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import React, { useState } from "react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
|
||||
import { act, fireEvent, renderWithProviders, screen, testQueryClient, waitFor } from "../../../tests/test-utils";
|
||||
import ClassificationMethodConfig from "./ClassificationMethodConfig";
|
||||
import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs";
|
||||
import ForecastClassifierConfig from "./ForecastClassifierConfig";
|
||||
|
|
@ -37,6 +37,53 @@ const fuseInitial: ComplexityRouterConfigValue = {
|
|||
},
|
||||
};
|
||||
const options = ["judge", "efficient", "capable"].map((model) => ({ value: model, label: model }));
|
||||
const catalog = {
|
||||
version: "catalog-v1",
|
||||
models: [
|
||||
{
|
||||
id: "efficient-v1",
|
||||
label: "Efficient preset",
|
||||
text: "Maintained efficient profile",
|
||||
sources: ["https://example.com/efficient"],
|
||||
model: "efficient-model",
|
||||
},
|
||||
{
|
||||
id: "capable-v1",
|
||||
label: "Capable preset",
|
||||
text: "Maintained capable profile",
|
||||
sources: ["https://example.com/capable"],
|
||||
model: "capable-model",
|
||||
},
|
||||
],
|
||||
harnesses: [
|
||||
{
|
||||
id: "runtime-v1",
|
||||
label: "Runtime preset",
|
||||
text: "Maintained runtime profile",
|
||||
sources: ["https://example.com/runtime"],
|
||||
},
|
||||
],
|
||||
};
|
||||
const presetConfig = {
|
||||
efficient_profile_preset: catalog.models[0].id,
|
||||
capable_profile_preset: catalog.models[1].id,
|
||||
harness_preset: catalog.harnesses[0].id,
|
||||
max_quality_gap: 0.05,
|
||||
};
|
||||
const presetInitial = { ...fuseInitial, llm_v2_config: presetConfig };
|
||||
|
||||
beforeEach(() => {
|
||||
testQueryClient.clear();
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockImplementation(async () => Response.json(catalog)),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
testQueryClient.clear();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
function Form({ initialValue = initial }: { initialValue?: ComplexityRouterConfigValue }) {
|
||||
const [value, setValue] = useState(initialValue);
|
||||
|
|
@ -72,6 +119,193 @@ function Form({ initialValue = initial }: { initialValue?: ComplexityRouterConfi
|
|||
}
|
||||
|
||||
describe("forecast classifier form", () => {
|
||||
it("selects all three maintained presets, previews provenance, and saves only references", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Form initialValue={fuseInitial} />);
|
||||
await user.click(screen.getByRole("combobox", { name: "Efficient solver profile preset" }));
|
||||
await user.click(await screen.findByRole("option", { name: /^Efficient preset/ }));
|
||||
await user.click(screen.getByRole("combobox", { name: "Capable solver profile preset" }));
|
||||
await user.click(screen.getByRole("option", { name: /^Capable preset/ }));
|
||||
await user.click(screen.getByRole("combobox", { name: "Harness and budget preset" }));
|
||||
await user.click(screen.getByRole("option", { name: /^Runtime preset/ }));
|
||||
expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(catalog.models[0].text);
|
||||
expect(screen.getByLabelText("Efficient solver profile")).toHaveAttribute("readonly");
|
||||
expect(screen.getByLabelText("Capable solver profile")).toHaveValue(catalog.models[1].text);
|
||||
expect(screen.getByLabelText("Harness and budget")).toHaveValue(catalog.harnesses[0].text);
|
||||
expect(screen.getAllByText(`Catalog version: ${catalog.version}`)).toHaveLength(3);
|
||||
expect(screen.getByText(`Model: ${catalog.models[0].model}`)).toBeInTheDocument();
|
||||
expect(screen.getAllByRole("link", { name: "Source 1" }).map((link) => link.getAttribute("href"))).toEqual([
|
||||
catalog.models[0].sources[0],
|
||||
catalog.models[1].sources[0],
|
||||
catalog.harnesses[0].sources[0],
|
||||
]);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
expect(JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config).toEqual(
|
||||
presetConfig,
|
||||
);
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
expect(fetch).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ url: expect.stringMatching(/\/public\/complexity_router\/fuse_presets$/) }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([undefined, null, "Explicit override"])(
|
||||
"copies effective text to Custom and clears only that reference, override=%s",
|
||||
async (override) => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(
|
||||
<Form initialValue={{ ...presetInitial, llm_v2_config: { ...presetConfig, efficient_profile: override } }} />,
|
||||
);
|
||||
const effectiveText = override ?? catalog.models[0].text;
|
||||
await waitFor(() => expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(effectiveText));
|
||||
await user.click(screen.getByRole("combobox", { name: "Efficient solver profile preset" }));
|
||||
await user.click(screen.getByRole("option", { name: "Custom" }));
|
||||
expect(screen.getByLabelText("Efficient solver profile")).not.toHaveAttribute("readonly");
|
||||
expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(effectiveText);
|
||||
fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Custom budget" } });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
const { efficient_profile_preset: _preset, ...rest } = presetConfig;
|
||||
expect(
|
||||
JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config,
|
||||
).toEqual({
|
||||
...rest,
|
||||
efficient_profile: "Custom budget",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ ...fuseInitial.llm_v2_config!, efficient_profile: catalog.models[0].text },
|
||||
{
|
||||
...presetConfig,
|
||||
efficient_profile: "Explicit override",
|
||||
capable_profile: "Capable override",
|
||||
harness: "Harness override",
|
||||
},
|
||||
])("keeps existing custom ownership and references on an unchanged save: %j", async (settings) => {
|
||||
renderWithProviders(<Form initialValue={{ ...fuseInitial, llm_v2_config: settings }} />);
|
||||
await waitFor(() => expect(screen.queryByText(/Loading profile presets/)).not.toBeInTheDocument());
|
||||
expect(screen.getByRole("combobox", { name: "Efficient solver profile preset" })).toHaveValue("Custom");
|
||||
expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(settings.efficient_profile);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
expect(JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config).toEqual(
|
||||
settings,
|
||||
);
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
"keeps edits and stored IDs while the pending catalog settles, success=%s",
|
||||
async (success) => {
|
||||
let resolveCatalog: (response: Response) => void = () => {};
|
||||
vi.mocked(fetch).mockReturnValue(
|
||||
new Promise<Response>((resolve) => {
|
||||
resolveCatalog = resolve;
|
||||
}),
|
||||
);
|
||||
const settings = { ...presetConfig, efficient_profile: "Original override" };
|
||||
renderWithProviders(<Form initialValue={{ ...fuseInitial, llm_v2_config: settings }} />);
|
||||
expect(screen.getByText(/Loading profile presets/)).toBeInTheDocument();
|
||||
fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Typed while loading" } });
|
||||
await act(async () =>
|
||||
resolveCatalog(success ? Response.json(catalog) : Response.json({ error: "unavailable" }, { status: 503 })),
|
||||
);
|
||||
if (success) await screen.findAllByText(`Catalog version: ${catalog.version}`);
|
||||
else expect(await screen.findByText(/Profile presets could not be loaded/)).toBeInTheDocument();
|
||||
expect(screen.getByLabelText("Efficient solver profile")).toHaveValue("Typed while loading");
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
expect(
|
||||
JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config,
|
||||
).toEqual({ ...settings, efficient_profile: "Typed while loading" });
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["efficient_profile", "Efficient solver profile"],
|
||||
["capable_profile", "Capable solver profile"],
|
||||
["harness", "Harness and budget"],
|
||||
] as const)(
|
||||
"preserves the saved %s reference during a catalog outage until Custom text replaces it",
|
||||
async (field, label) => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(fetch).mockImplementation(async () => Response.json({ error: "unavailable" }, { status: 503 }));
|
||||
renderWithProviders(<Form initialValue={presetInitial} />);
|
||||
expect(await screen.findByText(/Profile presets could not be loaded/)).toBeInTheDocument();
|
||||
const save = screen.getByRole("button", { name: "Save configuration" });
|
||||
const output = screen.getByRole("status", { name: "Saved configuration" });
|
||||
expect(save).toBeEnabled();
|
||||
await user.click(save);
|
||||
expect(JSON.parse(output.textContent!).llm_v2_config).toEqual(presetConfig);
|
||||
|
||||
await user.click(screen.getByRole("combobox", { name: `${label} preset` }));
|
||||
await user.click(screen.getByRole("option", { name: "Custom" }));
|
||||
expect(screen.getByLabelText(label)).toHaveValue("");
|
||||
expect(screen.getByLabelText(label)).not.toHaveAttribute("readonly");
|
||||
expect(screen.getByRole("combobox", { name: `${label} preset` })).toHaveValue("Custom");
|
||||
expect(save).toBeEnabled();
|
||||
await user.click(save);
|
||||
expect(JSON.parse(output.textContent!).llm_v2_config).toEqual(presetConfig);
|
||||
|
||||
fireEvent.change(screen.getByLabelText(label), { target: { value: " " } });
|
||||
expect(save).toBeDisabled();
|
||||
await user.click(screen.getByRole("button", { name: `Keep saved ${label.toLowerCase()} preset` }));
|
||||
expect(screen.getByLabelText(label)).toHaveAttribute("readonly");
|
||||
expect(save).toBeEnabled();
|
||||
await user.click(save);
|
||||
expect(JSON.parse(output.textContent!).llm_v2_config).toEqual(presetConfig);
|
||||
|
||||
await user.click(screen.getByRole("combobox", { name: `${label} preset` }));
|
||||
await user.click(screen.getByRole("option", { name: "Custom" }));
|
||||
const replacement = "Manually authored replacement";
|
||||
fireEvent.change(screen.getByLabelText(label), { target: { value: replacement } });
|
||||
expect(save).toBeEnabled();
|
||||
await user.click(save);
|
||||
const referenceKey = `${field}_preset` as const;
|
||||
const { [referenceKey]: _reference, ...remaining } = presetConfig;
|
||||
expect(JSON.parse(output.textContent!).llm_v2_config).toEqual({ ...remaining, [field]: replacement });
|
||||
},
|
||||
);
|
||||
|
||||
it.each([true, false])(
|
||||
"keeps a reference selected as Custom while the catalog settles, success=%s",
|
||||
async (success) => {
|
||||
const user = userEvent.setup();
|
||||
const response = Promise.withResolvers<Response>();
|
||||
vi.mocked(fetch).mockReturnValue(response.promise);
|
||||
renderWithProviders(<Form initialValue={presetInitial} />);
|
||||
await user.click(screen.getByRole("combobox", { name: "Efficient solver profile preset" }));
|
||||
await user.click(screen.getByRole("option", { name: "Custom" }));
|
||||
await act(async () => response.resolve(success ? Response.json(catalog) : Response.json({}, { status: 503 })));
|
||||
if (success) await screen.findAllByText(`Catalog version: ${catalog.version}`);
|
||||
else await screen.findByText(/Profile presets could not be loaded/);
|
||||
expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(success ? catalog.models[0].text : "");
|
||||
await user.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
expect(
|
||||
JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config,
|
||||
).toEqual(presetConfig);
|
||||
fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Replacement" } });
|
||||
await user.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
const { efficient_profile_preset: _reference, ...remaining } = presetConfig;
|
||||
expect(
|
||||
JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config,
|
||||
).toEqual({
|
||||
...remaining,
|
||||
efficient_profile: "Replacement",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps unknown saved IDs visible with unavailable previews rather than replacing them", async () => {
|
||||
const settings = { ...presetConfig, efficient_profile_preset: "unavailable-v8" };
|
||||
renderWithProviders(<Form initialValue={{ ...fuseInitial, llm_v2_config: settings }} />);
|
||||
await screen.findAllByText(`Catalog version: ${catalog.version}`);
|
||||
expect(screen.getByRole("combobox", { name: "Efficient solver profile preset" })).toHaveValue("unavailable-v8");
|
||||
expect(screen.getByText("Preset preview unavailable. The saved reference is preserved")).toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
expect(JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config).toEqual(
|
||||
settings,
|
||||
);
|
||||
});
|
||||
|
||||
it("switches a populated standard router to Capability without saving hidden pools or their overrides", () => {
|
||||
renderWithProviders(
|
||||
<Form
|
||||
|
|
@ -171,7 +405,7 @@ describe("forecast classifier form", () => {
|
|||
fireEvent.click(screen.getByRole("tab", { name: "Complexity" }));
|
||||
fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${target}`) }));
|
||||
await user.click(screen.getByRole("combobox", { name: "Classifier Model" }));
|
||||
await user.click(screen.getByRole("option", { name: "judge", exact: true }));
|
||||
await user.click(screen.getByRole("option", { name: "judge" }));
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
const output = screen.getByRole("status", { name: "Saved configuration" });
|
||||
expect(output).toHaveTextContent('"classification_rubric":"agentic"');
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/component
|
|||
import { ChevronRight } from "lucide-react";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import FuseProfilePresets from "./FuseProfilePresets";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { MultiSelect } from "@/components/shared/MultiSelect";
|
||||
|
|
@ -239,35 +239,13 @@ const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptions
|
|||
</>
|
||||
) : (
|
||||
<>
|
||||
{(["efficient_profile", "capable_profile", "harness"] as const).map((field) => {
|
||||
const label = {
|
||||
efficient_profile: "Efficient solver profile",
|
||||
capable_profile: "Capable solver profile",
|
||||
harness: "Harness and budget",
|
||||
}[field];
|
||||
return (
|
||||
<div key={field} className="space-y-1">
|
||||
<Label htmlFor={`${id}-${field}`}>{label}</Label>
|
||||
<Textarea
|
||||
id={`${id}-${field}`}
|
||||
value={fuse[field]}
|
||||
maxLength={4000}
|
||||
placeholder={
|
||||
field === "harness"
|
||||
? "Tools, execution environment, verification, and budget available to each solver"
|
||||
: "Describe this solver's strengths, limitations, and settings"
|
||||
}
|
||||
onChange={(event) => updateFuse({ ...fuse, [field]: event.target.value })}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<FuseProfilePresets value={fuse} onChange={updateFuse} />
|
||||
<NumberField
|
||||
label="Maximum quality gap"
|
||||
value={fuse.max_quality_gap}
|
||||
min={0}
|
||||
max={1}
|
||||
help="Allowed difference between capable and efficient success probabilities, from 0 to 1. This is an estimate, not a measured quality guarantee"
|
||||
help="Allowed difference between capable and efficient success probabilities, from 0 to 1. Tune on held-out tasks from your workload; this estimate is not a measured quality guarantee. A gap of 0 still selects efficient on tied or higher forecasts. Route directly to one model to avoid judging when you do not want model selection"
|
||||
onChange={(max_quality_gap) => updateFuse({ ...fuse, max_quality_gap })}
|
||||
/>
|
||||
</>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,208 @@
|
|||
import React from "react";
|
||||
import { $api } from "@/lib/http/api";
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import {
|
||||
fuseProfileFields,
|
||||
selectFuseProfile,
|
||||
type FuseProfileField,
|
||||
type FuseSettings,
|
||||
} from "./forecast_classifier_config";
|
||||
|
||||
const catalogQueryOptions = {
|
||||
staleTime: Infinity,
|
||||
gcTime: Infinity,
|
||||
retry: false,
|
||||
refetchOnMount: false,
|
||||
refetchOnWindowFocus: false,
|
||||
refetchOnReconnect: false,
|
||||
};
|
||||
|
||||
const profileLabels: Readonly<Record<FuseProfileField, string>> = {
|
||||
efficient_profile: "Efficient solver profile",
|
||||
capable_profile: "Capable solver profile",
|
||||
harness: "Harness and budget",
|
||||
};
|
||||
|
||||
type FusePresetEntry = {
|
||||
id: string;
|
||||
label: string;
|
||||
text: string;
|
||||
sources: readonly string[];
|
||||
model?: string;
|
||||
};
|
||||
|
||||
type FieldProps = {
|
||||
id: string;
|
||||
field: FuseProfileField;
|
||||
value: FuseSettings;
|
||||
onChange: (value: FuseSettings) => void;
|
||||
presets: readonly FusePresetEntry[] | undefined;
|
||||
catalogVersion: string | undefined;
|
||||
customWithoutPreview: ReadonlySet<FuseProfileField>;
|
||||
setCustomWithoutPreview: React.Dispatch<React.SetStateAction<ReadonlySet<FuseProfileField>>>;
|
||||
};
|
||||
|
||||
const profileSelectionLabel = (awaitingCustomText: boolean, custom: boolean): string => {
|
||||
if (awaitingCustomText) return "Saved preset remains active until replacement text is entered";
|
||||
if (custom) return "Custom text overrides preset";
|
||||
return "Preset";
|
||||
};
|
||||
|
||||
function FuseProfilePresetField({
|
||||
id,
|
||||
field,
|
||||
value,
|
||||
onChange,
|
||||
presets,
|
||||
catalogVersion,
|
||||
customWithoutPreview,
|
||||
setCustomWithoutPreview,
|
||||
}: FieldProps) {
|
||||
const label = profileLabels[field];
|
||||
const presetId = value[`${field}_preset`];
|
||||
const preset = presets?.find((entry) => entry.id === presetId);
|
||||
const awaitingCustomText = customWithoutPreview.has(field) && value[field] == null && presetId != null;
|
||||
const custom = value[field] != null || presetId == null || awaitingCustomText;
|
||||
const effectiveText = value[field] ?? preset?.text ?? "";
|
||||
const selectionLabel = profileSelectionLabel(awaitingCustomText, custom);
|
||||
const chooseProfile = (selected: string | null) => {
|
||||
if (!selected) return;
|
||||
const missingPreview = presetId != null && preset == null && value[field] == null;
|
||||
if (selected === "custom" && missingPreview) {
|
||||
setCustomWithoutPreview((fields) => new Set([...fields, field]));
|
||||
return;
|
||||
}
|
||||
setCustomWithoutPreview((fields) => new Set([...fields].filter((entry) => entry !== field)));
|
||||
onChange(selectFuseProfile(value, field, selected === "custom" ? undefined : selected, effectiveText));
|
||||
};
|
||||
const editProfile = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
|
||||
if (customWithoutPreview.has(field) && event.target.value.trim().length > 0) {
|
||||
setCustomWithoutPreview((fields) => new Set([...fields].filter((entry) => entry !== field)));
|
||||
onChange(selectFuseProfile(value, field, undefined, event.target.value));
|
||||
return;
|
||||
}
|
||||
onChange({ ...value, [field]: event.target.value });
|
||||
};
|
||||
const keepSavedPreset = () => {
|
||||
if (presetId == null) return;
|
||||
setCustomWithoutPreview((fields) => new Set([...fields].filter((entry) => entry !== field)));
|
||||
onChange(selectFuseProfile(value, field, presetId, ""));
|
||||
};
|
||||
const placeholder =
|
||||
field === "harness"
|
||||
? "Tools, execution environment, verification, and budget available to each solver"
|
||||
: "Describe this solver's strengths, limitations, and settings";
|
||||
|
||||
return (
|
||||
<div className="space-y-2 min-w-0">
|
||||
<Label htmlFor={`${id}-${field}-preset`}>{label} preset</Label>
|
||||
<SearchSelect
|
||||
inputId={`${id}-${field}-preset`}
|
||||
aria-label={`${label} preset`}
|
||||
value={custom ? "custom" : presetId}
|
||||
allowClear={false}
|
||||
options={[
|
||||
{ value: "custom", label: "Custom" },
|
||||
...(presets ?? []).map((entry) => ({ value: entry.id, label: entry.label, sublabel: entry.id })),
|
||||
]}
|
||||
onValueChange={chooseProfile}
|
||||
/>
|
||||
<Textarea
|
||||
aria-label={label}
|
||||
value={effectiveText}
|
||||
readOnly={!custom}
|
||||
maxLength={4000}
|
||||
rows={4}
|
||||
placeholder={placeholder}
|
||||
onChange={editProfile}
|
||||
/>
|
||||
{customWithoutPreview.has(field) && presetId != null && (
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
aria-label={`Keep saved ${label.toLowerCase()} preset`}
|
||||
onClick={keepSavedPreset}
|
||||
>
|
||||
Keep saved preset
|
||||
</Button>
|
||||
)}
|
||||
{presetId && (
|
||||
<div className="space-y-1 text-xs text-muted-foreground break-words">
|
||||
<p>
|
||||
{selectionLabel}: {presetId}
|
||||
</p>
|
||||
{preset ? (
|
||||
<>
|
||||
<p>Catalog version: {catalogVersion}</p>
|
||||
{preset.model && <p>Model: {preset.model}</p>}
|
||||
<div className="flex flex-wrap gap-x-3 gap-y-1">
|
||||
{preset.sources.map((source, index) => (
|
||||
<a key={source} href={source} target="_blank" rel="noopener noreferrer" className="underline">
|
||||
Source {index + 1}
|
||||
</a>
|
||||
))}
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<p>Preset preview unavailable. The saved reference is preserved</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function FuseProfilePresets({
|
||||
value,
|
||||
onChange,
|
||||
}: {
|
||||
value: FuseSettings;
|
||||
onChange: (value: FuseSettings) => void;
|
||||
}) {
|
||||
const id = React.useId();
|
||||
const [customWithoutPreview, setCustomWithoutPreview] = React.useState<ReadonlySet<FuseProfileField>>(
|
||||
() => new Set(),
|
||||
);
|
||||
const { data, isPending, isError } = $api.useQuery(
|
||||
"get",
|
||||
"/public/complexity_router/fuse_presets",
|
||||
{},
|
||||
catalogQueryOptions,
|
||||
);
|
||||
return (
|
||||
<div className="space-y-4 min-w-0">
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Choose profiles that match every deployment in each solver group and its actual settings. Profile selection is
|
||||
independent of routing model names. Presets describe the solvers and runtime; they do not set a quality gap or
|
||||
calibration. Validate those separately for your workload, judge, and exact profile versions.
|
||||
</p>
|
||||
{isPending && (
|
||||
<p role="status" className="text-xs text-muted-foreground">
|
||||
Loading profile presets. Custom editing is available
|
||||
</p>
|
||||
)}
|
||||
{isError && (
|
||||
<p role="status" className="text-xs text-muted-foreground">
|
||||
Profile presets could not be loaded. Saved references are preserved and Custom editing is available
|
||||
</p>
|
||||
)}
|
||||
{fuseProfileFields.map((field) => (
|
||||
<FuseProfilePresetField
|
||||
key={field}
|
||||
id={id}
|
||||
field={field}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
presets={field === "harness" ? data?.harnesses : data?.models}
|
||||
catalogVersion={data?.version}
|
||||
customWithoutPreview={customWithoutPreview}
|
||||
setCustomWithoutPreview={setCustomWithoutPreview}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -48,6 +48,23 @@ const baseParams: BuildComplexityRouterConfigParams = {
|
|||
};
|
||||
|
||||
describe("buildComplexityRouterConfig", () => {
|
||||
it("forwards preset references and explicit overrides without materializing absent text on create", () => {
|
||||
const settings = {
|
||||
efficient_profile_preset: "efficient-v1",
|
||||
capable_profile_preset: "capable-v1",
|
||||
harness_preset: "runtime-v1",
|
||||
efficient_profile: "Explicit efficient override",
|
||||
capable_profile: "Explicit capable override",
|
||||
harness: "Explicit harness override",
|
||||
max_quality_gap: 0.05,
|
||||
};
|
||||
const config = buildComplexityRouterConfig({ ...baseParams, classifierType: "llm_v2", llmV2Config: settings });
|
||||
expect(config.llm_v2_config).toEqual(settings);
|
||||
const { efficient_profile: _efficient, capable_profile: _capable, harness: _harness, ...refs } = settings;
|
||||
const refConfig = buildComplexityRouterConfig({ ...baseParams, classifierType: "llm_v2", llmV2Config: refs });
|
||||
expect(JSON.parse(JSON.stringify(refConfig)).llm_v2_config).toEqual(refs);
|
||||
});
|
||||
|
||||
it.each(["capability", "llm_v2", "heuristic"] as const)(
|
||||
"disables the removed overrides only for forecast creates: %s",
|
||||
(classifierType) => {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,13 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import { getForecastConfigError, prepareForecastClassifier } from "./forecast_classifier_config";
|
||||
import {
|
||||
fuseProfileFields,
|
||||
fuseSettingsSchema,
|
||||
getForecastConfigError,
|
||||
prepareForecastClassifier,
|
||||
selectFuseProfile,
|
||||
type FuseSettings,
|
||||
} from "./forecast_classifier_config";
|
||||
import { getKeywordTierRulesError } from "./build_complexity_router_config";
|
||||
import { activeTierRows } from "./tier_rows";
|
||||
import {
|
||||
|
|
@ -32,6 +39,62 @@ const fuse: ComplexityRouterConfigValue = {
|
|||
},
|
||||
};
|
||||
|
||||
describe("Fuse profile presets", () => {
|
||||
const refs: FuseSettings = {
|
||||
efficient_profile_preset: "efficient-v1",
|
||||
capable_profile_preset: "capable-v1",
|
||||
harness_preset: "runtime-v1",
|
||||
max_quality_gap: 0.05,
|
||||
max_output_tokens: 1024,
|
||||
response_format: "json_object",
|
||||
calibration: {
|
||||
version: "fitted-pair",
|
||||
prompt_version: "llm-v2-1",
|
||||
efficient: { slope: 1.1, intercept: -0.1 },
|
||||
capable: { slope: 0.9, intercept: 0.2 },
|
||||
},
|
||||
};
|
||||
|
||||
it.each(fuseProfileFields)("validates both sources of %s without needing catalog availability", (field) => {
|
||||
const presetField = `${field}_preset` as const;
|
||||
expect(fuseSettingsSchema.safeParse(refs).success).toBe(true);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [presetField]: undefined }).success).toBe(false);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [field]: null, [presetField]: null }).success).toBe(false);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [field]: " \n " }).success).toBe(false);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [field]: "a".repeat(4001) }).success).toBe(false);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [field]: "a".repeat(4000) }).success).toBe(true);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [field]: "Override", [presetField]: "" }).success).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ ...fuse.llm_v2_config!, efficient_profile: " Custom solver\n" },
|
||||
refs,
|
||||
{ ...refs, efficient_profile: null, capable_profile: null, harness: null },
|
||||
{ ...fuse.llm_v2_config!, efficient_profile_preset: null, capable_profile_preset: null, harness_preset: null },
|
||||
{ ...refs, efficient_profile: " Explicit override\n", capable_profile: "More budget", harness: "No shell" },
|
||||
])("preserves references, literal overrides and calibration across hydration and unchanged saves: %j", (settings) => {
|
||||
const stored = { ...fuse, llm_v2_config: settings };
|
||||
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
|
||||
expect(hydrated.llm_v2_config).toEqual(settings);
|
||||
expect(getForecastConfigError(hydrated)).toBeNull();
|
||||
const saved = buildUpdatedComplexityRouterConfig(stored, {
|
||||
...hydrated,
|
||||
tiers: { ...hydrated.tiers, SIMPLE: ["arbitrary-new-group"] },
|
||||
});
|
||||
expect(saved.llm_v2_config).toEqual(settings);
|
||||
});
|
||||
|
||||
it.each(fuseProfileFields)("changes only %s ownership on explicit selection", (field) => {
|
||||
const overridden = { ...refs, [field]: "Override" };
|
||||
const selected = selectFuseProfile(overridden, field, "replacement-v2", "Override");
|
||||
expect(selected).toEqual({ ...refs, [field]: undefined, [`${field}_preset`]: "replacement-v2" });
|
||||
expect(JSON.parse(JSON.stringify(selected))).not.toHaveProperty(field);
|
||||
const custom = selectFuseProfile(selected, field, undefined, "Effective preset text");
|
||||
expect(custom).toEqual({ ...refs, [field]: "Effective preset text", [`${field}_preset`]: undefined });
|
||||
expect(JSON.parse(JSON.stringify(custom))).not.toHaveProperty(`${field}_preset`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("forecast classifier configuration", () => {
|
||||
it.each([
|
||||
{ version: "eval", slope: 21, intercept: 0 },
|
||||
|
|
|
|||
|
|
@ -5,7 +5,12 @@ import { tierOrderFor } from "./tier_rows";
|
|||
|
||||
const probability = z.number().finite().min(0).max(1);
|
||||
const version = z.string().trim().min(1).max(512);
|
||||
const profile = z.string().trim().min(1).max(4000);
|
||||
const profile = z
|
||||
.string()
|
||||
.max(4000)
|
||||
.refine((text) => text.trim().length > 0);
|
||||
export const fuseProfileFields = ["efficient_profile", "capable_profile", "harness"] as const;
|
||||
export type FuseProfileField = (typeof fuseProfileFields)[number];
|
||||
const transport = {
|
||||
max_output_tokens: z.number().int().positive().optional(),
|
||||
response_format: z.enum(["json_schema", "json_object"]).optional(),
|
||||
|
|
@ -42,18 +47,36 @@ const fuseCalibrationShape = {
|
|||
const fuseShape = {
|
||||
efficient_tier: z.string().min(1).optional(),
|
||||
capable_tier: z.string().min(1).optional(),
|
||||
efficient_profile: profile,
|
||||
capable_profile: profile,
|
||||
harness: profile,
|
||||
efficient_profile: profile.nullish(),
|
||||
capable_profile: profile.nullish(),
|
||||
harness: profile.nullish(),
|
||||
efficient_profile_preset: z.string().min(1).nullish(),
|
||||
capable_profile_preset: z.string().min(1).nullish(),
|
||||
harness_preset: z.string().min(1).nullish(),
|
||||
max_quality_gap: probability,
|
||||
...transport,
|
||||
calibration: z.object(fuseCalibrationShape).nullable().optional(),
|
||||
};
|
||||
export const fuseSettingsSchema = z.object(fuseShape);
|
||||
export const fuseSettingsSchema = z
|
||||
.object(fuseShape)
|
||||
.refine((settings) =>
|
||||
fuseProfileFields.every((field) => settings[field] != null || settings[`${field}_preset`] != null),
|
||||
);
|
||||
|
||||
export type CapabilitySettings = z.infer<typeof capabilitySettingsSchema>;
|
||||
export type FuseSettings = z.infer<typeof fuseSettingsSchema>;
|
||||
|
||||
export const selectFuseProfile = (
|
||||
value: FuseSettings,
|
||||
field: FuseProfileField,
|
||||
presetId: string | undefined,
|
||||
effectiveText: string,
|
||||
): FuseSettings => ({
|
||||
...value,
|
||||
[field]: presetId === undefined ? effectiveText : undefined,
|
||||
[`${field}_preset`]: presetId,
|
||||
});
|
||||
|
||||
export const isForecastClassifier = (type: ClassifierType): boolean => type === "capability" || type === "llm_v2";
|
||||
|
||||
export const newCapabilitySettings = (): CapabilitySettings => ({
|
||||
|
|
|
|||
82
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
82
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -12571,6 +12571,23 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/public/complexity_router/fuse_presets": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** Get Public Fuse Presets */
|
||||
get: operations["get_public_fuse_presets_public_complexity_router_fuse_presets_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/public/complexity_router/scorer_defaults": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -28665,6 +28682,39 @@ export interface components {
|
|||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/** FuseHarnessPreset */
|
||||
FuseHarnessPreset: {
|
||||
/** Id */
|
||||
id: string;
|
||||
/** Label */
|
||||
label: string;
|
||||
/** Sources */
|
||||
sources: string[];
|
||||
/** Text */
|
||||
text: string;
|
||||
};
|
||||
/** FuseModelPreset */
|
||||
FuseModelPreset: {
|
||||
/** Id */
|
||||
id: string;
|
||||
/** Label */
|
||||
label: string;
|
||||
/** Model */
|
||||
model: string;
|
||||
/** Sources */
|
||||
sources: string[];
|
||||
/** Text */
|
||||
text: string;
|
||||
};
|
||||
/** FusePresetCatalog */
|
||||
FusePresetCatalog: {
|
||||
/** Harnesses */
|
||||
harnesses: components["schemas"]["FuseHarnessPreset"][];
|
||||
/** Models */
|
||||
models: components["schemas"]["FuseModelPreset"][];
|
||||
/** Version */
|
||||
version: string;
|
||||
};
|
||||
/**
|
||||
* GUARDRAIL_DEFINITION_LOCATION
|
||||
* @enum {string}
|
||||
|
|
@ -29648,21 +29698,27 @@ export interface components {
|
|||
LLMV2Config: {
|
||||
calibration?: components["schemas"]["LLMV2Calibration"] | null;
|
||||
/** Capable Profile */
|
||||
capable_profile: string;
|
||||
capable_profile?: string | null;
|
||||
/** Capable Profile Preset */
|
||||
capable_profile_preset?: string | null;
|
||||
/**
|
||||
* Capable Tier
|
||||
* @default REASONING
|
||||
*/
|
||||
capable_tier: string;
|
||||
/** Efficient Profile */
|
||||
efficient_profile: string;
|
||||
efficient_profile?: string | null;
|
||||
/** Efficient Profile Preset */
|
||||
efficient_profile_preset?: string | null;
|
||||
/**
|
||||
* Efficient Tier
|
||||
* @default SIMPLE
|
||||
*/
|
||||
efficient_tier: string;
|
||||
/** Harness */
|
||||
harness: string;
|
||||
harness?: string | null;
|
||||
/** Harness Preset */
|
||||
harness_preset?: string | null;
|
||||
/**
|
||||
* Max Output Tokens
|
||||
* @default 1024
|
||||
|
|
@ -57940,6 +57996,26 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
get_public_fuse_presets_public_complexity_router_fuse_presets_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["FusePresetCatalog"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_complexity_scorer_defaults_public_complexity_router_scorer_defaults_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue