mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
feat(router): add maintained Fuse model and harness presets
This commit is contained in:
Tin Chi Lo
parent
4b368bf066
commit
2fa115db2b
17 changed files with 917 additions and 43 deletions
|
|
@ -23,6 +23,7 @@ from litellm.proxy._types import (
|
|||
)
|
||||
from litellm.proxy.utils import get_custom_url
|
||||
from litellm.repositories.table_repositories import ClaudeCodePluginRepository
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import FusePresetCatalog, get_fuse_presets
|
||||
from litellm.types.agents import AgentCard
|
||||
from litellm.types.mcp import MCPPublicServer
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
|
|
@ -424,6 +425,14 @@ async def get_complexity_scorer_defaults() -> ComplexityScorerDefaults:
|
|||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/public/complexity_router/fuse_presets",
|
||||
response_model=FusePresetCatalog,
|
||||
)
|
||||
async def get_public_fuse_presets() -> FusePresetCatalog:
|
||||
return get_fuse_presets()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/public/litellm_model_cost_map",
|
||||
tags=["public", "model management"],
|
||||
|
|
|
|||
|
|
@ -179,6 +179,51 @@ Configure capability forecasting through YAML or the model-management API.
|
|||
The dashboard preserves its classifier and calibration on an untouched save;
|
||||
it does not provide a capability-card editor
|
||||
|
||||
### Fuse v2 profile presets
|
||||
|
||||
Fuse v2 accepts maintained model and runtime descriptions instead of requiring
|
||||
custom prose for both solvers and the harness. Select profiles explicitly for
|
||||
all deployments behind your configured model groups and their actual settings.
|
||||
Group names do not select profiles automatically
|
||||
|
||||
```yaml
|
||||
complexity_router_config:
|
||||
classifier_type: llm_v2
|
||||
classifier_llm_config:
|
||||
model: your-judge-group
|
||||
tiers:
|
||||
SIMPLE: your-efficient-group
|
||||
REASONING: your-capable-group
|
||||
llm_v2_config:
|
||||
efficient_profile_preset: claude-sonnet-5-v1
|
||||
capable_profile_preset: claude-fable-5-1-v1
|
||||
harness_preset: claude-code-v1
|
||||
max_quality_gap: 0.05
|
||||
```
|
||||
|
||||
`GET /public/complexity_router/fuse_presets` returns the catalog version, model
|
||||
profiles, and runtime descriptions, including source URLs. The bundled catalog
|
||||
is loaded once per process without network requests. Sources are citations only
|
||||
|
||||
Each of `efficient_profile`, `capable_profile`, and `harness` requires either
|
||||
nonblank custom text or its corresponding preset reference. Custom text wins
|
||||
when both are supplied, but an unknown or wrong-kind preset is still rejected.
|
||||
Explicit blank text is invalid even with a valid preset. Custom text remains
|
||||
limited to 4000 characters
|
||||
|
||||
Saved configurations retain preset references and explicit text separately.
|
||||
Preset text is resolved when building the classifier prompt, not copied into
|
||||
stored custom fields. Existing all-custom configurations keep the same prompt.
|
||||
Versioned preset IDs identify immutable content: revised wording receives a new
|
||||
ID, and older referenced entries must remain available
|
||||
|
||||
The runtime presets do not imply a repository, runnable tests, network access,
|
||||
additional tools, or a step, time, or spending budget. mini-SWE-agent describes
|
||||
an agent interface, not a SWE-bench task. Model descriptions summarize provider
|
||||
positioning without solve rates or guaranteed rankings. Wording is an evaluation
|
||||
input, not a calibrated quality claim. Existing Fuse licensing, policy,
|
||||
calibration, and prompt version are unchanged
|
||||
|
||||
### Heuristic v2
|
||||
|
||||
Set `classifier_type: heuristic_v2` to classify with the bundled calibrated
|
||||
|
|
|
|||
100
litellm/router_strategy/complexity_router/fuse_presets.json
Normal file
100
litellm/router_strategy/complexity_router/fuse_presets.json
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
{
|
||||
"version": "2026-09-17-v1",
|
||||
"models": [
|
||||
{
|
||||
"id": "gpt-6-astra-v1",
|
||||
"label": "GPT-6 Astra",
|
||||
"model": "gpt-6-astra",
|
||||
"text": "OpenAI model for demanding end-to-end work, including reasoning, coding, research, and document tasks",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-astra"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-sol-v1",
|
||||
"label": "GPT-5.6 Sol",
|
||||
"model": "gpt-5.6-sol",
|
||||
"text": "OpenAI model for complex professional work, supporting reasoning and tool calling",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-sol"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-luna-v1",
|
||||
"label": "GPT-5.6 Luna",
|
||||
"model": "gpt-5.6-luna",
|
||||
"text": "OpenAI model for high-volume workloads, supporting reasoning and tool calling",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-luna"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-terra-v1",
|
||||
"label": "GPT-5.6 Terra",
|
||||
"model": "gpt-5.6-terra",
|
||||
"text": "OpenAI general-purpose model supporting reasoning, text and image input, and tool calling",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-terra"]
|
||||
},
|
||||
{
|
||||
"id": "claude-haiku-4-5-v1",
|
||||
"label": "Claude Haiku 4.5",
|
||||
"model": "claude-haiku-4-5",
|
||||
"text": "Anthropic latency-focused model supporting text and image input, tool use, and extended thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/haiku-4-5/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-sonnet-5-v1",
|
||||
"label": "Claude Sonnet 5",
|
||||
"model": "claude-sonnet-5",
|
||||
"text": "Anthropic model balancing speed and capability, with adaptive thinking and tool use",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/sonnet-5/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-opus-5-v1",
|
||||
"label": "Claude Opus 5",
|
||||
"model": "claude-opus-5",
|
||||
"text": "Anthropic model for complex agentic coding and enterprise work, with adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/opus-5/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-fable-5-v1",
|
||||
"label": "Claude Fable 5",
|
||||
"model": "claude-fable-5",
|
||||
"text": "Anthropic model for demanding reasoning and long-running agent tasks, with always-on adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/fable-5/introducing-claude-fable-5-and-claude-mythos-5"]
|
||||
},
|
||||
{
|
||||
"id": "claude-fable-5-1-v1",
|
||||
"label": "Claude Fable 5.1",
|
||||
"model": "claude-fable-5-1",
|
||||
"text": "Anthropic model for demanding reasoning, long-running agentic coding, and multistep research, with always-on adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/fable-5-1/overview"]
|
||||
}
|
||||
],
|
||||
"harnesses": [
|
||||
{
|
||||
"id": "unspecified-v1",
|
||||
"label": "Unspecified runtime",
|
||||
"text": "Agent runtime is unspecified. Assess the task using the supplied context without assuming repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://code.claude.com/docs/en/how-claude-code-works", "https://mini-swe-agent.com/latest/faq/"]
|
||||
},
|
||||
{
|
||||
"id": "claude-code-v1",
|
||||
"label": "Claude Code",
|
||||
"text": "Claude Code supplies an agent loop with context management and configured tools. Available actions depend on the session's tools, permissions, and execution environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://code.claude.com/docs/en/how-claude-code-works"]
|
||||
},
|
||||
{
|
||||
"id": "codex-cli-v1",
|
||||
"label": "Codex CLI",
|
||||
"text": "Codex CLI supplies a terminal-based coding agent. File operations, command execution, and integrations depend on the session's tools, permissions, and sandbox. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://learn.chatgpt.com/docs/codex/cli", "https://learn.chatgpt.com/codex/permissions"]
|
||||
},
|
||||
{
|
||||
"id": "opencode-v1",
|
||||
"label": "OpenCode",
|
||||
"text": "OpenCode supplies a configurable agent runtime. Available actions depend on the selected agent, tools, permissions, and execution environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://opencode.ai/docs/agents/"]
|
||||
},
|
||||
{
|
||||
"id": "mini-swe-agent-v1",
|
||||
"label": "mini-SWE-agent",
|
||||
"text": "The standard mini-SWE-agent setup uses a bash-only action interface and separate command executions. Available commands and resources depend on its configured environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget",
|
||||
"sources": ["https://mini-swe-agent.com/latest/faq/"]
|
||||
}
|
||||
]
|
||||
}
|
||||
52
litellm/router_strategy/complexity_router/fuse_presets.py
Normal file
52
litellm/router_strategy/complexity_router/fuse_presets.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
from functools import lru_cache
|
||||
from importlib.resources import files
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StringConstraints
|
||||
|
||||
ProfileText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=4000)]
|
||||
|
||||
|
||||
class FuseModelPreset(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
id: str
|
||||
label: str
|
||||
text: ProfileText
|
||||
sources: tuple[str, ...] = Field(min_length=1)
|
||||
model: str
|
||||
|
||||
|
||||
class FuseHarnessPreset(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
id: str
|
||||
label: str
|
||||
text: ProfileText
|
||||
sources: tuple[str, ...] = Field(min_length=1)
|
||||
|
||||
|
||||
class FusePresetCatalog(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
version: str
|
||||
models: tuple[FuseModelPreset, ...]
|
||||
harnesses: tuple[FuseHarnessPreset, ...]
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_fuse_presets() -> FusePresetCatalog:
|
||||
return FusePresetCatalog.model_validate_json(
|
||||
files(__package__).joinpath("fuse_presets.json").read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
|
||||
def resolve_fuse_profile(text: str | None, preset_id: str | None, kind: Literal["model", "harness"]) -> str | None:
|
||||
if preset_id is None:
|
||||
return text
|
||||
catalog: Final = get_fuse_presets()
|
||||
presets: Final = catalog.models if kind == "model" else catalog.harnesses
|
||||
preset: Final = next((entry for entry in presets if entry.id == preset_id), None)
|
||||
if preset is None:
|
||||
return None
|
||||
return text if text is not None else preset.text
|
||||
|
|
@ -7,15 +7,15 @@ from dataclasses import dataclass
|
|||
from sys import float_info
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StringConstraints, TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StringConstraints, TypeAdapter, model_validator
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.llms.base_llm.base_utils import (
|
||||
type_to_response_format_param, # pyright: ignore[reportUnknownVariableType] # legacy output validated below
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import ProfileText, resolve_fuse_profile
|
||||
|
||||
ShortText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=512)]
|
||||
ProfileText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=4000)]
|
||||
|
||||
|
||||
class _SolverProfile(TypedDict):
|
||||
|
|
@ -139,20 +139,41 @@ class LLMV2Config(BaseModel):
|
|||
|
||||
efficient_tier: str = "SIMPLE"
|
||||
capable_tier: str = "REASONING"
|
||||
efficient_profile: ProfileText
|
||||
capable_profile: ProfileText
|
||||
harness: ProfileText
|
||||
efficient_profile: ProfileText | None = None
|
||||
capable_profile: ProfileText | None = None
|
||||
harness: ProfileText | None = None
|
||||
efficient_profile_preset: str | None = None
|
||||
capable_profile_preset: str | None = None
|
||||
harness_preset: str | None = None
|
||||
max_quality_gap: float = Field(ge=0.0, le=1.0, description="Maximum estimated success loss allowed for efficient.")
|
||||
max_output_tokens: int = Field(default=1024, ge=1)
|
||||
response_format: Literal["json_schema", "json_object"] = "json_schema"
|
||||
calibration: LLMV2Calibration | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_profiles(self) -> LLMV2Config:
|
||||
self._profile_texts()
|
||||
return self
|
||||
|
||||
def _profile_texts(self) -> tuple[str, str, str]:
|
||||
efficient: Final = resolve_fuse_profile(self.efficient_profile, self.efficient_profile_preset, "model")
|
||||
capable: Final = resolve_fuse_profile(self.capable_profile, self.capable_profile_preset, "model")
|
||||
harness: Final = resolve_fuse_profile(self.harness, self.harness_preset, "harness")
|
||||
if efficient is None:
|
||||
raise ValueError("efficient_profile requires text or a known efficient_profile_preset")
|
||||
if capable is None:
|
||||
raise ValueError("capable_profile requires text or a known capable_profile_preset")
|
||||
if harness is None:
|
||||
raise ValueError("harness requires text or a known harness_preset")
|
||||
return efficient, capable, harness
|
||||
|
||||
def system_prompt(self, efficient_model: str, capable_model: str) -> str:
|
||||
efficient, capable, harness = self._profile_texts()
|
||||
profiles: Final[_SolverProfiles] = {
|
||||
"prompt_version": LLM_V2_PROMPT_VERSION,
|
||||
"harness": self.harness,
|
||||
"efficient": {"model": efficient_model, "profile": self.efficient_profile},
|
||||
"capable": {"model": capable_model, "profile": self.capable_profile},
|
||||
"harness": harness,
|
||||
"efficient": {"model": efficient_model, "profile": efficient},
|
||||
"capable": {"model": capable_model, "profile": capable},
|
||||
}
|
||||
schema: Final = (
|
||||
"\n\nResponse JSON schema:\n" + json.dumps(LLMV2Verdict.model_json_schema())
|
||||
|
|
|
|||
|
|
@ -290,6 +290,7 @@ editable-profile = "dev"
|
|||
include = [
|
||||
"litellm/proxy/_experimental/out/**",
|
||||
"litellm/router_strategy/complexity_router/artifacts/*.json",
|
||||
"litellm/router_strategy/complexity_router/fuse_presets.json",
|
||||
"litellm/proxy/client/cli/commands/codex_base_instructions.md",
|
||||
]
|
||||
exclude = [
|
||||
|
|
|
|||
|
|
@ -11,12 +11,23 @@ from fastapi.testclient import TestClient
|
|||
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.public_endpoints import router
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelGroupInfoProxy,
|
||||
)
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
||||
def test_fuse_presets_route_serves_the_shared_catalog_without_authentication() -> None:
|
||||
app: Final = FastAPI()
|
||||
app.include_router(router)
|
||||
client: Final = TestClient(app)
|
||||
response: Final = client.get("/public/complexity_router/fuse_presets")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == get_fuse_presets().model_dump(mode="json")
|
||||
assert client.get("/public/complexity_router/fuse_presets").json() == response.json()
|
||||
|
||||
|
||||
def test_get_supported_providers_returns_enum_values():
|
||||
app_instance = FastAPI()
|
||||
app_instance.include_router(router)
|
||||
|
|
|
|||
43
tests/test_litellm/router_strategy/test_fuse_presets.py
Normal file
43
tests/test_litellm/router_strategy/test_fuse_presets.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
import json
|
||||
from importlib.resources import files
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets, resolve_fuse_profile
|
||||
|
||||
|
||||
def test_catalog_is_loaded_once_and_preserves_bundled_content() -> None:
|
||||
get_fuse_presets.cache_clear()
|
||||
first: Final = get_fuse_presets()
|
||||
second: Final = get_fuse_presets()
|
||||
assert first is second
|
||||
bundled: Final = json.loads(
|
||||
files("litellm.router_strategy.complexity_router").joinpath("fuse_presets.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert first.model_dump(mode="json") == bundled
|
||||
entries: Final = (*first.models, *first.harnesses)
|
||||
assert len({entry.id for entry in entries}) == len(entries)
|
||||
assert len(first.models) == 9
|
||||
assert len(first.harnesses) == 5
|
||||
assert all(entry.sources and all(source.startswith("https://") for source in entry.sources) for entry in entries)
|
||||
|
||||
|
||||
def test_every_catalog_entry_resolves_without_changing_custom_ownership() -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
for entry in catalog.models:
|
||||
assert resolve_fuse_profile(None, entry.id, "model") == entry.text
|
||||
assert resolve_fuse_profile("Custom text", entry.id, "model") == "Custom text"
|
||||
for entry in catalog.harnesses:
|
||||
assert resolve_fuse_profile(None, entry.id, "harness") == entry.text
|
||||
assert resolve_fuse_profile("Custom text", entry.id, "harness") == "Custom text"
|
||||
assert resolve_fuse_profile("Custom text", None, "model") == "Custom text"
|
||||
assert resolve_fuse_profile("Custom text", None, "harness") == "Custom text"
|
||||
|
||||
|
||||
def test_cached_catalog_and_records_cannot_be_modified() -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
for record, field in ((catalog, "version"), (catalog.models[0], "text"), (catalog.harnesses[0], "text")):
|
||||
with pytest.raises(ValidationError, match="frozen"):
|
||||
setattr(record, field, "Changed")
|
||||
|
|
@ -11,8 +11,10 @@ from litellm import ModelResponse, Router
|
|||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, ComplexityTier
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
from litellm.router_strategy.complexity_router.llm_v2 import (
|
||||
LLM_V2_PROMPT_VERSION,
|
||||
LLM_V2_SYSTEM_PROMPT,
|
||||
LLMV2Calibration,
|
||||
LLMV2Config,
|
||||
LLMV2ProbabilityCalibration,
|
||||
|
|
@ -174,6 +176,114 @@ def test_invalid_forecast_settings_are_rejected(overrides: dict[str, object]) ->
|
|||
LLMV2Config.model_validate({**base.model_dump(), **overrides})
|
||||
|
||||
|
||||
def _preset_config(**overrides: object) -> LLMV2Config:
|
||||
catalog: Final = get_fuse_presets()
|
||||
return LLMV2Config.model_validate(
|
||||
{
|
||||
"efficient_profile_preset": catalog.models[0].id,
|
||||
"capable_profile_preset": catalog.models[-1].id,
|
||||
"harness_preset": catalog.harnesses[-1].id,
|
||||
"max_quality_gap": 0.05,
|
||||
**overrides,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_preset_roundtrip_keeps_references_without_materializing_text() -> None:
|
||||
config: Final = _preset_config()
|
||||
serialized: Final = config.model_dump(exclude_none=True)
|
||||
assert serialized["efficient_profile_preset"] == config.efficient_profile_preset
|
||||
assert serialized["capable_profile_preset"] == config.capable_profile_preset
|
||||
assert serialized["harness_preset"] == config.harness_preset
|
||||
assert not {"efficient_profile", "capable_profile", "harness"}.intersection(serialized)
|
||||
assert LLMV2Config.model_validate(config.model_dump()) == config
|
||||
assert LLMV2Config.model_validate_json(config.model_dump_json()) == config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
def test_preset_explicit_override_wins_and_survives_roundtrip(field: str) -> None:
|
||||
config: Final = _preset_config(**{field: " Operator description "})
|
||||
roundtrip: Final = LLMV2Config.model_validate_json(config.model_dump_json())
|
||||
assert roundtrip.model_dump()[field] == "Operator description"
|
||||
assert roundtrip.efficient_profile_preset == config.efficient_profile_preset
|
||||
assert roundtrip.capable_profile_preset == config.capable_profile_preset
|
||||
assert roundtrip.harness_preset == config.harness_preset
|
||||
payload: Final = json.loads(
|
||||
roundtrip.system_prompt("opaque-efficient", "opaque-capable").split("Configured solver profiles:\n")[1]
|
||||
)
|
||||
if field == "harness":
|
||||
assert payload["harness"] == "Operator description"
|
||||
else:
|
||||
assert payload[field.removesuffix("_profile")]["profile"] == "Operator description"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
@pytest.mark.parametrize("invalid", ("", " \n\t", "x" * 4001))
|
||||
def test_preset_does_not_bypass_supplied_text_bounds(field: str, invalid: str) -> None:
|
||||
with pytest.raises(ValidationError, match=field):
|
||||
_preset_config(**{field: invalid})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
@pytest.mark.parametrize("override", (None, "Custom override"))
|
||||
@pytest.mark.parametrize("invalid_id", ("missing-v1", ""))
|
||||
def test_preset_unknown_reference_rejects_even_when_overridden(
|
||||
field: str, override: str | None, invalid_id: str
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match=f"{field}.*preset"):
|
||||
_preset_config(**{field: override, f"{field}_preset": invalid_id})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
def test_preset_missing_text_and_reference_rejects(field: str) -> None:
|
||||
with pytest.raises(ValidationError, match=field):
|
||||
_preset_config(**{f"{field}_preset": None})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
def test_preset_reference_rejects_the_wrong_catalog_kind(field: str) -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
wrong_id: Final = catalog.models[0].id if field == "harness" else catalog.harnesses[0].id
|
||||
with pytest.raises(ValidationError, match=field):
|
||||
_preset_config(**{f"{field}_preset": wrong_id})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ("json_schema", "json_object"))
|
||||
def test_custom_profile_prompt_bytes_are_unchanged(mode: str) -> None:
|
||||
base: Final = _config().llm_v2_config
|
||||
assert base is not None
|
||||
config: Final = LLMV2Config.model_validate({**base.model_dump(), "response_format": mode})
|
||||
old_payload: Final = {
|
||||
"prompt_version": LLM_V2_PROMPT_VERSION,
|
||||
"harness": config.harness,
|
||||
"efficient": {"model": "opaque-efficient", "profile": config.efficient_profile},
|
||||
"capable": {"model": "opaque-capable", "profile": config.capable_profile},
|
||||
}
|
||||
schema: Final = (
|
||||
"\n\nResponse JSON schema:\n" + json.dumps(LLMV2Verdict.model_json_schema()) if mode == "json_object" else ""
|
||||
)
|
||||
assert config.system_prompt("opaque-efficient", "opaque-capable") == (
|
||||
LLM_V2_SYSTEM_PROMPT + "\n\nConfigured solver profiles:\n" + json.dumps(old_payload) + schema
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preset_router_passes_catalog_text_and_opaque_group_names_to_judge() -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
config: Final = _config(llm_v2_config=_preset_config().model_dump())
|
||||
router, client = _router(_verdict().model_dump_json(), config)
|
||||
outcome: Final = await router.aclassify("Complete the supplied task")
|
||||
assert outcome.tier == ComplexityTier.SIMPLE
|
||||
prompt: Final = client.acompletion.call_args.kwargs["messages"][0]["content"]
|
||||
payload: Final = json.loads(prompt.split("Configured solver profiles:\n")[1])
|
||||
assert payload == {
|
||||
"prompt_version": LLM_V2_PROMPT_VERSION,
|
||||
"harness": catalog.harnesses[-1].text,
|
||||
"efficient": {"model": "efficient", "profile": catalog.models[0].text},
|
||||
"capable": {"model": "capable", "profile": catalog.models[-1].text},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_judge_fuses_whole_task_and_keeps_caller_text_out_of_system_prompt() -> None:
|
||||
router, client = _router(_verdict().model_dump_json())
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
carries_complexity_router_settings,
|
||||
classify_strategy_router_model,
|
||||
|
|
@ -171,6 +174,52 @@ def test_validate_accepts_loadable_complexity_config(complexity_router_config):
|
|||
assert validate_complexity_router_config_write(complexity_router_config=complexity_router_config) is None
|
||||
|
||||
|
||||
def _fuse_write_config(profiles: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {
|
||||
"classifier_type": "llm_v2",
|
||||
"classifier_llm_config": {"model": "judge"},
|
||||
"tiers": {"SIMPLE": ["opaque-efficient"], "REASONING": ["opaque-capable"]},
|
||||
"llm_v2_config": {"max_quality_gap": 0.05, **profiles},
|
||||
}
|
||||
|
||||
|
||||
def test_fuse_write_accepts_presets_and_custom_text_with_the_same_entitlement() -> None:
|
||||
catalog: Final = get_fuse_presets()
|
||||
presets: Final = _fuse_write_config(
|
||||
{
|
||||
"efficient_profile_preset": catalog.models[0].id,
|
||||
"capable_profile_preset": catalog.models[-1].id,
|
||||
"harness_preset": catalog.harnesses[0].id,
|
||||
}
|
||||
)
|
||||
custom: Final = _fuse_write_config(
|
||||
{
|
||||
"efficient_profile": catalog.models[0].text,
|
||||
"capable_profile": catalog.models[-1].text,
|
||||
"harness": catalog.harnesses[0].text,
|
||||
}
|
||||
)
|
||||
assert validate_complexity_router_config_write(presets) is None
|
||||
assert validate_complexity_router_config_write(custom) is None
|
||||
assert claimed_capability(presets) is claimed_capability(custom)
|
||||
assert claimed_capability(presets) is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
|
||||
def test_fuse_write_rejects_unknown_preset_even_with_custom_text(field: str) -> None:
|
||||
config: Final = _fuse_write_config(
|
||||
{
|
||||
"efficient_profile": "Custom efficient solver",
|
||||
"capable_profile": "Custom capable solver",
|
||||
"harness": "Custom runtime",
|
||||
f"{field}_preset": "unknown-v1",
|
||||
}
|
||||
)
|
||||
violation: Final = validate_complexity_router_config_write(config)
|
||||
assert violation is not None
|
||||
assert f"{field}_preset" in violation
|
||||
|
||||
|
||||
def test_naming_check_ignores_the_config_entirely():
|
||||
"""The naming contract and the config's contents are separate questions with separate owners;
|
||||
a write may carry a config without naming a model, so neither can stand in for the other."""
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import React, { useState } from "react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
|
||||
import { act, fireEvent, renderWithProviders, screen, testQueryClient, waitFor } from "../../../tests/test-utils";
|
||||
import ClassificationMethodConfig from "./ClassificationMethodConfig";
|
||||
import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs";
|
||||
import ForecastClassifierConfig from "./ForecastClassifierConfig";
|
||||
|
|
@ -37,6 +37,53 @@ const fuseInitial: ComplexityRouterConfigValue = {
|
|||
},
|
||||
};
|
||||
const options = ["judge", "efficient", "capable"].map((model) => ({ value: model, label: model }));
|
||||
const catalog = {
|
||||
version: "catalog-v1",
|
||||
models: [
|
||||
{
|
||||
id: "efficient-v1",
|
||||
label: "Efficient preset",
|
||||
text: "Maintained efficient profile",
|
||||
sources: ["https://example.com/efficient"],
|
||||
model: "efficient-model",
|
||||
},
|
||||
{
|
||||
id: "capable-v1",
|
||||
label: "Capable preset",
|
||||
text: "Maintained capable profile",
|
||||
sources: ["https://example.com/capable"],
|
||||
model: "capable-model",
|
||||
},
|
||||
],
|
||||
harnesses: [
|
||||
{
|
||||
id: "runtime-v1",
|
||||
label: "Runtime preset",
|
||||
text: "Maintained runtime profile",
|
||||
sources: ["https://example.com/runtime"],
|
||||
},
|
||||
],
|
||||
};
|
||||
const presetConfig = {
|
||||
efficient_profile_preset: catalog.models[0].id,
|
||||
capable_profile_preset: catalog.models[1].id,
|
||||
harness_preset: catalog.harnesses[0].id,
|
||||
max_quality_gap: 0.05,
|
||||
};
|
||||
const presetInitial = { ...fuseInitial, llm_v2_config: presetConfig };
|
||||
|
||||
beforeEach(() => {
|
||||
testQueryClient.clear();
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockImplementation(async () => Response.json(catalog)),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
testQueryClient.clear();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
function Form({ initialValue = initial }: { initialValue?: ComplexityRouterConfigValue }) {
|
||||
const [value, setValue] = useState(initialValue);
|
||||
|
|
@ -72,6 +119,118 @@ function Form({ initialValue = initial }: { initialValue?: ComplexityRouterConfi
|
|||
}
|
||||
|
||||
describe("forecast classifier form", () => {
|
||||
it("selects all three maintained presets, previews provenance, and saves only references", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<Form initialValue={fuseInitial} />);
|
||||
await user.click(screen.getByRole("combobox", { name: "Efficient solver profile preset" }));
|
||||
await user.click(await screen.findByRole("option", { name: /^Efficient preset/ }));
|
||||
await user.click(screen.getByRole("combobox", { name: "Capable solver profile preset" }));
|
||||
await user.click(screen.getByRole("option", { name: /^Capable preset/ }));
|
||||
await user.click(screen.getByRole("combobox", { name: "Harness and budget preset" }));
|
||||
await user.click(screen.getByRole("option", { name: /^Runtime preset/ }));
|
||||
expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(catalog.models[0].text);
|
||||
expect(screen.getByLabelText("Efficient solver profile")).toHaveAttribute("readonly");
|
||||
expect(screen.getByLabelText("Capable solver profile")).toHaveValue(catalog.models[1].text);
|
||||
expect(screen.getByLabelText("Harness and budget")).toHaveValue(catalog.harnesses[0].text);
|
||||
expect(screen.getAllByText(`Catalog version: ${catalog.version}`)).toHaveLength(3);
|
||||
expect(screen.getByText(`Model: ${catalog.models[0].model}`)).toBeInTheDocument();
|
||||
expect(screen.getAllByRole("link", { name: "Source 1" }).map((link) => link.getAttribute("href"))).toEqual([
|
||||
catalog.models[0].sources[0],
|
||||
catalog.models[1].sources[0],
|
||||
catalog.harnesses[0].sources[0],
|
||||
]);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
expect(JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config).toEqual(
|
||||
presetConfig,
|
||||
);
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
expect(fetch).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ url: expect.stringMatching(/\/public\/complexity_router\/fuse_presets$/) }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([undefined, null, "Explicit override"])(
|
||||
"copies effective text to Custom and clears only that reference, override=%s",
|
||||
async (override) => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(
|
||||
<Form initialValue={{ ...presetInitial, llm_v2_config: { ...presetConfig, efficient_profile: override } }} />,
|
||||
);
|
||||
const effectiveText = override ?? catalog.models[0].text;
|
||||
await waitFor(() => expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(effectiveText));
|
||||
await user.click(screen.getByRole("combobox", { name: "Efficient solver profile preset" }));
|
||||
await user.click(screen.getByRole("option", { name: "Custom", exact: true }));
|
||||
expect(screen.getByLabelText("Efficient solver profile")).not.toHaveAttribute("readonly");
|
||||
expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(effectiveText);
|
||||
fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Custom budget" } });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
const { efficient_profile_preset: _preset, ...rest } = presetConfig;
|
||||
expect(
|
||||
JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config,
|
||||
).toEqual({
|
||||
...rest,
|
||||
efficient_profile: "Custom budget",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ ...fuseInitial.llm_v2_config!, efficient_profile: catalog.models[0].text },
|
||||
{
|
||||
...presetConfig,
|
||||
efficient_profile: "Explicit override",
|
||||
capable_profile: "Capable override",
|
||||
harness: "Harness override",
|
||||
},
|
||||
])("keeps existing custom ownership and references on an unchanged save: %j", async (settings) => {
|
||||
renderWithProviders(<Form initialValue={{ ...fuseInitial, llm_v2_config: settings }} />);
|
||||
await waitFor(() => expect(screen.queryByText(/Loading profile presets/)).not.toBeInTheDocument());
|
||||
expect(screen.getByRole("combobox", { name: "Efficient solver profile preset" })).toHaveValue("Custom");
|
||||
expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(settings.efficient_profile);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
expect(JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config).toEqual(
|
||||
settings,
|
||||
);
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
"keeps edits and stored IDs while the pending catalog settles, success=%s",
|
||||
async (success) => {
|
||||
let resolveCatalog: (response: Response) => void = () => {};
|
||||
vi.mocked(fetch).mockReturnValue(
|
||||
new Promise<Response>((resolve) => {
|
||||
resolveCatalog = resolve;
|
||||
}),
|
||||
);
|
||||
const settings = { ...presetConfig, efficient_profile: "Original override" };
|
||||
renderWithProviders(<Form initialValue={{ ...fuseInitial, llm_v2_config: settings }} />);
|
||||
expect(screen.getByText(/Loading profile presets/)).toBeInTheDocument();
|
||||
fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Typed while loading" } });
|
||||
await act(async () =>
|
||||
resolveCatalog(success ? Response.json(catalog) : Response.json({ error: "unavailable" }, { status: 503 })),
|
||||
);
|
||||
if (success) await screen.findAllByText(`Catalog version: ${catalog.version}`);
|
||||
else expect(await screen.findByText(/Profile presets could not be loaded/)).toBeInTheDocument();
|
||||
expect(screen.getByLabelText("Efficient solver profile")).toHaveValue("Typed while loading");
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
expect(
|
||||
JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config,
|
||||
).toEqual({ ...settings, efficient_profile: "Typed while loading" });
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps unknown saved IDs visible with unavailable previews rather than replacing them", async () => {
|
||||
const settings = { ...presetConfig, efficient_profile_preset: "unavailable-v8" };
|
||||
renderWithProviders(<Form initialValue={{ ...fuseInitial, llm_v2_config: settings }} />);
|
||||
await screen.findAllByText(`Catalog version: ${catalog.version}`);
|
||||
expect(screen.getByRole("combobox", { name: "Efficient solver profile preset" })).toHaveValue("unavailable-v8");
|
||||
expect(screen.getByText("Preset preview unavailable. The saved reference is preserved")).toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save configuration" }));
|
||||
expect(JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config).toEqual(
|
||||
settings,
|
||||
);
|
||||
});
|
||||
|
||||
it("switches a populated standard router to Capability without saving hidden pools or their overrides", () => {
|
||||
renderWithProviders(
|
||||
<Form
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/component
|
|||
import { ChevronRight } from "lucide-react";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import FuseProfilePresets from "./FuseProfilePresets";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { MultiSelect } from "@/components/shared/MultiSelect";
|
||||
|
|
@ -239,29 +239,7 @@ const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptions
|
|||
</>
|
||||
) : (
|
||||
<>
|
||||
{(["efficient_profile", "capable_profile", "harness"] as const).map((field) => {
|
||||
const label = {
|
||||
efficient_profile: "Efficient solver profile",
|
||||
capable_profile: "Capable solver profile",
|
||||
harness: "Harness and budget",
|
||||
}[field];
|
||||
return (
|
||||
<div key={field} className="space-y-1">
|
||||
<Label htmlFor={`${id}-${field}`}>{label}</Label>
|
||||
<Textarea
|
||||
id={`${id}-${field}`}
|
||||
value={fuse[field]}
|
||||
maxLength={4000}
|
||||
placeholder={
|
||||
field === "harness"
|
||||
? "Tools, execution environment, verification, and budget available to each solver"
|
||||
: "Describe this solver's strengths, limitations, and settings"
|
||||
}
|
||||
onChange={(event) => updateFuse({ ...fuse, [field]: event.target.value })}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<FuseProfilePresets value={fuse} onChange={updateFuse} />
|
||||
<NumberField
|
||||
label="Maximum quality gap"
|
||||
value={fuse.max_quality_gap}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
import React from "react";
|
||||
import { $api } from "@/lib/http/api";
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { fuseProfileFields, selectFuseProfile, type FuseSettings } from "./forecast_classifier_config";
|
||||
|
||||
const catalogQueryOptions = {
|
||||
staleTime: Infinity,
|
||||
gcTime: Infinity,
|
||||
retry: false,
|
||||
refetchOnMount: false,
|
||||
refetchOnWindowFocus: false,
|
||||
refetchOnReconnect: false,
|
||||
};
|
||||
|
||||
export default function FuseProfilePresets({
|
||||
value,
|
||||
onChange,
|
||||
}: {
|
||||
value: FuseSettings;
|
||||
onChange: (value: FuseSettings) => void;
|
||||
}) {
|
||||
const id = React.useId();
|
||||
const { data, isPending, isError } = $api.useQuery(
|
||||
"get",
|
||||
"/public/complexity_router/fuse_presets",
|
||||
{},
|
||||
catalogQueryOptions,
|
||||
);
|
||||
return (
|
||||
<div className="space-y-4 min-w-0">
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Choose profiles that match every deployment in each solver group and its actual settings. Profile selection is
|
||||
independent of routing model names
|
||||
</p>
|
||||
{isPending && (
|
||||
<p role="status" className="text-xs text-muted-foreground">
|
||||
Loading profile presets. Custom editing is available
|
||||
</p>
|
||||
)}
|
||||
{isError && (
|
||||
<p role="status" className="text-xs text-muted-foreground">
|
||||
Profile presets could not be loaded. Saved references are preserved and Custom editing is available
|
||||
</p>
|
||||
)}
|
||||
{fuseProfileFields.map((field) => {
|
||||
const label = {
|
||||
efficient_profile: "Efficient solver profile",
|
||||
capable_profile: "Capable solver profile",
|
||||
harness: "Harness and budget",
|
||||
}[field];
|
||||
const presetId = value[`${field}_preset`];
|
||||
const presets = field === "harness" ? data?.harnesses : data?.models;
|
||||
const preset = presets?.find((entry) => entry.id === presetId);
|
||||
const custom = value[field] != null || presetId == null;
|
||||
const effectiveText = value[field] ?? preset?.text ?? "";
|
||||
return (
|
||||
<div key={field} className="space-y-2 min-w-0">
|
||||
<Label htmlFor={`${id}-${field}-preset`}>{label} preset</Label>
|
||||
<SearchSelect
|
||||
inputId={`${id}-${field}-preset`}
|
||||
aria-label={`${label} preset`}
|
||||
value={custom ? "custom" : presetId}
|
||||
allowClear={false}
|
||||
options={[
|
||||
{ value: "custom", label: "Custom" },
|
||||
...(presets ?? []).map((entry) => ({ value: entry.id, label: entry.label, sublabel: entry.id })),
|
||||
]}
|
||||
onValueChange={(selected) => {
|
||||
if (selected)
|
||||
onChange(
|
||||
selectFuseProfile(value, field, selected === "custom" ? undefined : selected, effectiveText),
|
||||
);
|
||||
}}
|
||||
/>
|
||||
<Textarea
|
||||
aria-label={label}
|
||||
value={effectiveText}
|
||||
readOnly={!custom}
|
||||
maxLength={4000}
|
||||
rows={4}
|
||||
placeholder={
|
||||
field === "harness"
|
||||
? "Tools, execution environment, verification, and budget available to each solver"
|
||||
: "Describe this solver's strengths, limitations, and settings"
|
||||
}
|
||||
onChange={(event) => onChange({ ...value, [field]: event.target.value })}
|
||||
/>
|
||||
{presetId && (
|
||||
<div className="space-y-1 text-xs text-muted-foreground break-words">
|
||||
<p>
|
||||
{custom ? "Custom text overrides preset" : "Preset"}: {presetId}
|
||||
</p>
|
||||
{preset ? (
|
||||
<>
|
||||
<p>Catalog version: {data?.version}</p>
|
||||
{"model" in preset && typeof preset.model === "string" && <p>Model: {preset.model}</p>}
|
||||
<div className="flex flex-wrap gap-x-3 gap-y-1">
|
||||
{preset.sources.map((source, index) => (
|
||||
<a key={source} href={source} target="_blank" rel="noopener noreferrer" className="underline">
|
||||
Source {index + 1}
|
||||
</a>
|
||||
))}
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<p>Preset preview unavailable. The saved reference is preserved</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -48,6 +48,23 @@ const baseParams: BuildComplexityRouterConfigParams = {
|
|||
};
|
||||
|
||||
describe("buildComplexityRouterConfig", () => {
|
||||
it("forwards preset references and explicit overrides without materializing absent text on create", () => {
|
||||
const settings = {
|
||||
efficient_profile_preset: "efficient-v1",
|
||||
capable_profile_preset: "capable-v1",
|
||||
harness_preset: "runtime-v1",
|
||||
efficient_profile: "Explicit efficient override",
|
||||
capable_profile: "Explicit capable override",
|
||||
harness: "Explicit harness override",
|
||||
max_quality_gap: 0.05,
|
||||
};
|
||||
const config = buildComplexityRouterConfig({ ...baseParams, classifierType: "llm_v2", llmV2Config: settings });
|
||||
expect(config.llm_v2_config).toEqual(settings);
|
||||
const { efficient_profile: _efficient, capable_profile: _capable, harness: _harness, ...refs } = settings;
|
||||
const refConfig = buildComplexityRouterConfig({ ...baseParams, classifierType: "llm_v2", llmV2Config: refs });
|
||||
expect(JSON.parse(JSON.stringify(refConfig)).llm_v2_config).toEqual(refs);
|
||||
});
|
||||
|
||||
it.each(["capability", "llm_v2", "heuristic"] as const)(
|
||||
"disables the removed overrides only for forecast creates: %s",
|
||||
(classifierType) => {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,13 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import { getForecastConfigError, prepareForecastClassifier } from "./forecast_classifier_config";
|
||||
import {
|
||||
fuseProfileFields,
|
||||
fuseSettingsSchema,
|
||||
getForecastConfigError,
|
||||
prepareForecastClassifier,
|
||||
selectFuseProfile,
|
||||
type FuseSettings,
|
||||
} from "./forecast_classifier_config";
|
||||
import { getKeywordTierRulesError } from "./build_complexity_router_config";
|
||||
import { activeTierRows } from "./tier_rows";
|
||||
import {
|
||||
|
|
@ -32,6 +39,62 @@ const fuse: ComplexityRouterConfigValue = {
|
|||
},
|
||||
};
|
||||
|
||||
describe("Fuse profile presets", () => {
|
||||
const refs: FuseSettings = {
|
||||
efficient_profile_preset: "efficient-v1",
|
||||
capable_profile_preset: "capable-v1",
|
||||
harness_preset: "runtime-v1",
|
||||
max_quality_gap: 0.05,
|
||||
max_output_tokens: 1024,
|
||||
response_format: "json_object",
|
||||
calibration: {
|
||||
version: "fitted-pair",
|
||||
prompt_version: "llm-v2-1",
|
||||
efficient: { slope: 1.1, intercept: -0.1 },
|
||||
capable: { slope: 0.9, intercept: 0.2 },
|
||||
},
|
||||
};
|
||||
|
||||
it.each(fuseProfileFields)("validates both sources of %s without needing catalog availability", (field) => {
|
||||
const presetField = `${field}_preset` as const;
|
||||
expect(fuseSettingsSchema.safeParse(refs).success).toBe(true);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [presetField]: undefined }).success).toBe(false);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [field]: null, [presetField]: null }).success).toBe(false);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [field]: " \n " }).success).toBe(false);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [field]: "a".repeat(4001) }).success).toBe(false);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [field]: "a".repeat(4000) }).success).toBe(true);
|
||||
expect(fuseSettingsSchema.safeParse({ ...refs, [field]: "Override", [presetField]: "" }).success).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ ...fuse.llm_v2_config!, efficient_profile: " Custom solver\n" },
|
||||
refs,
|
||||
{ ...refs, efficient_profile: null, capable_profile: null, harness: null },
|
||||
{ ...fuse.llm_v2_config!, efficient_profile_preset: null, capable_profile_preset: null, harness_preset: null },
|
||||
{ ...refs, efficient_profile: " Explicit override\n", capable_profile: "More budget", harness: "No shell" },
|
||||
])("preserves references, literal overrides and calibration across hydration and unchanged saves: %j", (settings) => {
|
||||
const stored = { ...fuse, llm_v2_config: settings };
|
||||
const hydrated = hydrateComplexityRouterConfig(stored, undefined);
|
||||
expect(hydrated.llm_v2_config).toEqual(settings);
|
||||
expect(getForecastConfigError(hydrated)).toBeNull();
|
||||
const saved = buildUpdatedComplexityRouterConfig(stored, {
|
||||
...hydrated,
|
||||
tiers: { ...hydrated.tiers, SIMPLE: ["arbitrary-new-group"] },
|
||||
});
|
||||
expect(saved.llm_v2_config).toEqual(settings);
|
||||
});
|
||||
|
||||
it.each(fuseProfileFields)("changes only %s ownership on explicit selection", (field) => {
|
||||
const overridden = { ...refs, [field]: "Override" };
|
||||
const selected = selectFuseProfile(overridden, field, "replacement-v2", "Override");
|
||||
expect(selected).toEqual({ ...refs, [field]: undefined, [`${field}_preset`]: "replacement-v2" });
|
||||
expect(JSON.parse(JSON.stringify(selected))).not.toHaveProperty(field);
|
||||
const custom = selectFuseProfile(selected, field, undefined, "Effective preset text");
|
||||
expect(custom).toEqual({ ...refs, [field]: "Effective preset text", [`${field}_preset`]: undefined });
|
||||
expect(JSON.parse(JSON.stringify(custom))).not.toHaveProperty(`${field}_preset`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("forecast classifier configuration", () => {
|
||||
it.each([
|
||||
{ version: "eval", slope: 21, intercept: 0 },
|
||||
|
|
|
|||
|
|
@ -5,7 +5,12 @@ import { tierOrderFor } from "./tier_rows";
|
|||
|
||||
const probability = z.number().finite().min(0).max(1);
|
||||
const version = z.string().trim().min(1).max(512);
|
||||
const profile = z.string().trim().min(1).max(4000);
|
||||
const profile = z
|
||||
.string()
|
||||
.max(4000)
|
||||
.refine((text) => text.trim().length > 0);
|
||||
export const fuseProfileFields = ["efficient_profile", "capable_profile", "harness"] as const;
|
||||
export type FuseProfileField = (typeof fuseProfileFields)[number];
|
||||
const transport = {
|
||||
max_output_tokens: z.number().int().positive().optional(),
|
||||
response_format: z.enum(["json_schema", "json_object"]).optional(),
|
||||
|
|
@ -42,18 +47,36 @@ const fuseCalibrationShape = {
|
|||
const fuseShape = {
|
||||
efficient_tier: z.string().min(1).optional(),
|
||||
capable_tier: z.string().min(1).optional(),
|
||||
efficient_profile: profile,
|
||||
capable_profile: profile,
|
||||
harness: profile,
|
||||
efficient_profile: profile.nullish(),
|
||||
capable_profile: profile.nullish(),
|
||||
harness: profile.nullish(),
|
||||
efficient_profile_preset: z.string().min(1).nullish(),
|
||||
capable_profile_preset: z.string().min(1).nullish(),
|
||||
harness_preset: z.string().min(1).nullish(),
|
||||
max_quality_gap: probability,
|
||||
...transport,
|
||||
calibration: z.object(fuseCalibrationShape).nullable().optional(),
|
||||
};
|
||||
export const fuseSettingsSchema = z.object(fuseShape);
|
||||
export const fuseSettingsSchema = z
|
||||
.object(fuseShape)
|
||||
.refine((settings) =>
|
||||
fuseProfileFields.every((field) => settings[field] != null || settings[`${field}_preset`] != null),
|
||||
);
|
||||
|
||||
export type CapabilitySettings = z.infer<typeof capabilitySettingsSchema>;
|
||||
export type FuseSettings = z.infer<typeof fuseSettingsSchema>;
|
||||
|
||||
export const selectFuseProfile = (
|
||||
value: FuseSettings,
|
||||
field: FuseProfileField,
|
||||
presetId: string | undefined,
|
||||
effectiveText: string,
|
||||
): FuseSettings => ({
|
||||
...value,
|
||||
[field]: presetId === undefined ? effectiveText : undefined,
|
||||
[`${field}_preset`]: presetId,
|
||||
});
|
||||
|
||||
export const isForecastClassifier = (type: ClassifierType): boolean => type === "capability" || type === "llm_v2";
|
||||
|
||||
export const newCapabilitySettings = (): CapabilitySettings => ({
|
||||
|
|
|
|||
82
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
82
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -12425,6 +12425,23 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/public/complexity_router/fuse_presets": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** Get Public Fuse Presets */
|
||||
get: operations["get_public_fuse_presets_public_complexity_router_fuse_presets_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/public/complexity_router/scorer_defaults": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -28207,6 +28224,39 @@ export interface components {
|
|||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/** FuseHarnessPreset */
|
||||
FuseHarnessPreset: {
|
||||
/** Id */
|
||||
id: string;
|
||||
/** Label */
|
||||
label: string;
|
||||
/** Sources */
|
||||
sources: string[];
|
||||
/** Text */
|
||||
text: string;
|
||||
};
|
||||
/** FuseModelPreset */
|
||||
FuseModelPreset: {
|
||||
/** Id */
|
||||
id: string;
|
||||
/** Label */
|
||||
label: string;
|
||||
/** Model */
|
||||
model: string;
|
||||
/** Sources */
|
||||
sources: string[];
|
||||
/** Text */
|
||||
text: string;
|
||||
};
|
||||
/** FusePresetCatalog */
|
||||
FusePresetCatalog: {
|
||||
/** Harnesses */
|
||||
harnesses: components["schemas"]["FuseHarnessPreset"][];
|
||||
/** Models */
|
||||
models: components["schemas"]["FuseModelPreset"][];
|
||||
/** Version */
|
||||
version: string;
|
||||
};
|
||||
/**
|
||||
* GUARDRAIL_DEFINITION_LOCATION
|
||||
* @enum {string}
|
||||
|
|
@ -29136,21 +29186,27 @@ export interface components {
|
|||
LLMV2Config: {
|
||||
calibration?: components["schemas"]["LLMV2Calibration"] | null;
|
||||
/** Capable Profile */
|
||||
capable_profile: string;
|
||||
capable_profile?: string | null;
|
||||
/** Capable Profile Preset */
|
||||
capable_profile_preset?: string | null;
|
||||
/**
|
||||
* Capable Tier
|
||||
* @default REASONING
|
||||
*/
|
||||
capable_tier: string;
|
||||
/** Efficient Profile */
|
||||
efficient_profile: string;
|
||||
efficient_profile?: string | null;
|
||||
/** Efficient Profile Preset */
|
||||
efficient_profile_preset?: string | null;
|
||||
/**
|
||||
* Efficient Tier
|
||||
* @default SIMPLE
|
||||
*/
|
||||
efficient_tier: string;
|
||||
/** Harness */
|
||||
harness: string;
|
||||
harness?: string | null;
|
||||
/** Harness Preset */
|
||||
harness_preset?: string | null;
|
||||
/**
|
||||
* Max Output Tokens
|
||||
* @default 1024
|
||||
|
|
@ -56944,6 +57000,26 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
get_public_fuse_presets_public_complexity_router_fuse_presets_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["FusePresetCatalog"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_complexity_scorer_defaults_public_complexity_router_scorer_defaults_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue