diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
index 41198887cab..fb29a92e306 100644
--- a/.github/workflows/test-unit.yml
+++ b/.github/workflows/test-unit.yml
@@ -1035,9 +1035,9 @@ jobs:
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
- .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
+ .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
- .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
+ .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@@ -1294,7 +1294,7 @@ jobs:
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
- .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+ .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
@@ -1497,9 +1497,9 @@ jobs:
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
run: |
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
- .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
+ .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
- .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
+ .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
fi
diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py
index 8c402fe9b44..f6bad7f611b 100644
--- a/litellm/litellm_core_utils/sensitive_data_masker.py
+++ b/litellm/litellm_core_utils/sensitive_data_masker.py
@@ -117,7 +117,9 @@ class SensitiveDataMasker:
elif key_is_sensitive and isinstance(item, str):
masked_items.append(self.mask_value(item))
else:
- masked_items.append(item if isinstance(item, (int, float, bool, str, list)) else str(item))
+ masked_items.append(
+ item if item is None or isinstance(item, (int, float, bool, str, list)) else str(item)
+ )
return masked_items
def mask_dict(
@@ -144,7 +146,7 @@ class SensitiveDataMasker:
str_value = str(v) if v is not None else ""
masked_data[k] = self.mask_value(str_value)
else:
- masked_data[k] = v if isinstance(v, (int, float, bool, str, list)) else str(v)
+ masked_data[k] = v if v is None or isinstance(v, (int, float, bool, str, list)) else str(v)
except Exception:
masked_data[k] = ""
diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
index 8f1a08d2964..ca9529ba246 100644
--- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py
+++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
@@ -795,18 +795,21 @@ class AnthropicMessagesHandler(BaseTranslation):
data: dict, # mutable-ok: the request body the adapter reads
*,
keep_cache_control: bool = False,
+ keep_tool_errors: bool = False,
) -> ChatCompletionRequest:
"""Translate Anthropic request to OpenAI chat completion format."""
(
chat_completion_compatible_request,
_tool_name_mapping,
- ) = LiteLLMAnthropicMessagesAdapter(keep_cache_control=keep_cache_control).translate_anthropic_to_openai(
+ ) = LiteLLMAnthropicMessagesAdapter(
+ keep_cache_control=keep_cache_control, keep_tool_errors=keep_tool_errors
+ ).translate_anthropic_to_openai(
anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)),
preserve_midturn_system=True,
)
return chat_completion_compatible_request
- def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None:
+ def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list[AllMessageValues] | None:
"""
Convert Anthropic messages request data to OpenAI-spec structured messages.
@@ -815,7 +818,7 @@ class AnthropicMessagesHandler(BaseTranslation):
messages: Final = data.get("messages")
if messages is None:
return None
- chat_completion_compatible_request: Final = self._translate_to_openai(data)
+ chat_completion_compatible_request: Final = self._translate_to_openai(data, keep_tool_errors=keep_tool_errors)
result: Final = cast(
list[AllMessageValues],
chat_completion_compatible_request.get("messages", []),
diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py
index 43eaa37aba7..5e4fba6d706 100644
--- a/litellm/llms/anthropic/messages/adapters/transformation.py
+++ b/litellm/llms/anthropic/messages/adapters/transformation.py
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
from pydantic import JsonValue, TypeAdapter
import litellm
-from litellm.litellm_core_utils.core_helpers import normalize_drop_params
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part
from litellm.llms.anthropic.messages.utils import (
@@ -434,8 +434,9 @@ _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.va
class LiteLLMAnthropicMessagesAdapter:
- def __init__(self, *, keep_cache_control: bool = False) -> None:
+ def __init__(self, *, keep_cache_control: bool = False, keep_tool_errors: bool = False) -> None:
self.keep_cache_control: Final = keep_cache_control
+ self.keep_tool_errors: Final = keep_tool_errors
### FOR [BETA] `/v1/messages` endpoint support
@@ -615,11 +616,7 @@ class LiteLLMAnthropicMessagesAdapter:
self._add_cache_control_if_applicable(content, doc_obj, model)
new_user_content_list.append(doc_obj)
elif content.get("type") == "tool_result":
- tool_result = ChatCompletionToolMessage(
- role="tool",
- tool_call_id=content.get("tool_use_id", ""),
- content=self._tool_result_content(content.get("content")),
- )
+ tool_result = self._tool_message(content)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result)
@@ -1432,6 +1429,15 @@ class LiteLLMAnthropicMessagesAdapter:
return None
return anthropic_image_source_to_openai_url(image_source)
+ def _tool_message(self, block: object) -> ChatCompletionToolMessage:
+ fields: Final = as_str_mapping(block) or {}
+ is_error: Final = fields.get("is_error")
+ tool_call_id: Final = tool_use_id if isinstance(tool_use_id := fields.get("tool_use_id"), str) else ""
+ content: Final = self._tool_result_content(fields.get("content"))
+ if self.keep_tool_errors and isinstance(is_error, bool):
+ return ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content, is_error=is_error)
+ return ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content)
+
def _tool_result_content(self, raw_content: object) -> ToolResultContent:
if isinstance(raw_content, str):
return raw_content
diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py
index 6399ded9555..71368f4ef61 100644
--- a/litellm/llms/base_llm/guardrail_translation/base_translation.py
+++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py
@@ -257,11 +257,12 @@ class BaseTranslation(ABC):
"""
return None
- def get_structured_messages(self, data: dict) -> list["AllMessageValues"] | None:
+ def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list["AllMessageValues"] | None:
"""
Convert request data to OpenAI-spec structured messages.
- Override in subclasses for format-specific conversion.
+ Override in subclasses for format-specific conversion. ``keep_tool_errors`` copies a provider's
+ tool-failure marker onto tool messages as ``is_error``; only for copies that are never sent to a model.
Returns None if no convertible content is found.
"""
diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py
index c5488bc8a98..26246cfd7b6 100644
--- a/litellm/llms/openai/chat/guardrail_translation/handler.py
+++ b/litellm/llms/openai/chat/guardrail_translation/handler.py
@@ -103,7 +103,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
delivers_ended_stream_rewrites = True
assembles_streamed_response = True
- def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None:
+ def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list[AllMessageValues] | None:
"""
Convert chat completions request data to OpenAI-spec structured messages.
diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index aa4dea196ff..20b7506ae0a 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -777,7 +777,7 @@ class OpenAIResponsesHandler(BaseTranslation):
delivers_ended_stream_rewrites = True
assembles_streamed_response = True
- def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None:
+ def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list[AllMessageValues] | None:
"""
Convert Responses API request data to OpenAI-spec structured messages.
diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index 137d6016402..b50e3955838 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -49950,6 +49950,10 @@
],
"title": "Content"
},
+ "is_error": {
+ "title": "Is Error",
+ "type": "boolean"
+ },
"role": {
"const": "tool",
"title": "Role",
diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py
index 42321fe1b1d..3c733d08a2e 100644
--- a/litellm/proxy/management_endpoints/auto_router_endpoints.py
+++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py
@@ -1112,6 +1112,7 @@ def _is_configured_pre_routing_strategy(llm_router: "Router", router_name: str)
router_registry.plugins("complexity"),
router_registry.plugins("adaptive"),
router_registry.plugins("quality"),
+ router_registry.plugins("switchyard"),
)
)
diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py
index 2c7453ebb1c..b8b926a3057 100644
--- a/litellm/proxy/management_endpoints/model_management_endpoints.py
+++ b/litellm/proxy/management_endpoints/model_management_endpoints.py
@@ -142,6 +142,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import (
count_capability_routers,
gated_capabilities_of,
is_complexity_router_model,
+ strategy_router_dependencies,
validate_complexity_router_config_placement,
validate_complexity_router_config_write,
validate_strategy_router_model_write,
@@ -355,6 +356,22 @@ async def get_db_model(model_id: str, prisma_client: PrismaClient) -> Deployment
return deployment_pydantic_obj
+def _self_routing_violation(
+ model_name: str | None,
+ incoming_params: GenericLiteLLMParams | None,
+ existing_params: GenericLiteLLMParams | None,
+) -> str | None:
+ if not model_name:
+ return None
+ effective_params: Final = {
+ **(_decrypted_litellm_params(existing_params) if existing_params is not None else {}),
+ **(_dump_without_none(incoming_params) if incoming_params is not None else {}),
+ }
+ if any(dependency.model_name == model_name for dependency in strategy_router_dependencies(effective_params)):
+ return f"A strategy router cannot route to itself: {model_name} is listed as one of its own models"
+ return None
+
+
def _strategy_router_write_violation(
incoming_params: GenericLiteLLMParams | None,
existing_params: GenericLiteLLMParams | None,
@@ -374,6 +391,11 @@ def _strategy_router_write_violation(
"""
if incoming_params is None:
return None
+ from litellm.router_strategy.switchyard_router.config import validate_switchyard_config_write
+
+ switchyard_violation: Final = validate_switchyard_config_write(incoming_params.switchyard_router_config)
+ if switchyard_violation is not None:
+ return switchyard_violation
config_violation: Final = validate_complexity_router_config_write(
complexity_router_config=(
_effective_complexity_router_config(incoming_params, existing_params)
@@ -406,8 +428,11 @@ def _strategy_router_write_violation(
def _raise_on_strategy_router_write_violation(
incoming_params: GenericLiteLLMParams | None,
existing_params: GenericLiteLLMParams | None,
+ model_name: str | None = None,
) -> None:
- violation = _strategy_router_write_violation(incoming_params=incoming_params, existing_params=existing_params)
+ violation: Final = _self_routing_violation(model_name, incoming_params, existing_params) or (
+ _strategy_router_write_violation(incoming_params=incoming_params, existing_params=existing_params)
+ )
if violation is None:
return
raise ProxyException(
@@ -1751,6 +1776,7 @@ async def patch_model(
_raise_on_strategy_router_write_violation(
incoming_params=incoming_params,
existing_params=remaining_params,
+ model_name=patch_data.model_name or db_model.model_name,
)
written_team_id: Final = (
@@ -3120,6 +3146,7 @@ async def add_new_model(
_raise_on_strategy_router_write_violation(
incoming_params=model_params.litellm_params,
existing_params=None,
+ model_name=model_params.model_name,
)
_raise_if_rate_limits_required_but_missing(
@@ -3325,6 +3352,7 @@ async def update_model(
_raise_on_strategy_router_write_violation(
incoming_params=model_params.litellm_params,
existing_params=deployment.litellm_params,
+ model_name=model_params.model_name or deployment.model_name,
)
renamed_to: Final = (
model_params.model_name
@@ -4028,6 +4056,7 @@ async def clear_cache() -> ReconcileOutcome:
llm_router.registry().plugins("complexity").pop(model_name, None)
llm_router.registry().plugins("adaptive").pop(model_name, None)
llm_router.registry().plugins("quality").pop(model_name, None)
+ llm_router.registry().plugins("switchyard").pop(model_name, None)
# Reload only DB models. _add_deployment_locked, not add_deployment: this
# coroutine already holds MODEL_RECONCILE_LOCK and asyncio.Lock is not
diff --git a/litellm/router_api.py b/litellm/router_api.py
index 1ade356f494..ca370488f5d 100644
--- a/litellm/router_api.py
+++ b/litellm/router_api.py
@@ -65,6 +65,7 @@ if TYPE_CHECKING:
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
from litellm.router_strategy.quality_router.quality_router import QualityRouter
+ from litellm.router_strategy.switchyard_router.switchyard_router import SwitchyardRouter
from litellm.router_utils.cooldown_cache import CooldownCache
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets as _AttemptedFallbackTargets
from litellm.router_utils.health_state_cache import DeploymentHealthCache
@@ -106,6 +107,7 @@ if TYPE_CHECKING:
| dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]]
| dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]]
| dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]]
+ | dict[str, list[TaggedPreRoutingStrategy[SwitchyardRouter]]]
| list[RoutingPlugin]
| RouterBudgetLimiting
| DualCache
@@ -171,6 +173,7 @@ PluginKind: TypeAlias = Literal[
"complexity",
"quality",
"adaptive",
+ "switchyard",
"routing",
"budget_limiter",
"prompt_caching_pins",
@@ -1106,6 +1109,11 @@ class RouterRegistry:
self, kind: Literal["adaptive"]
) -> "dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]]": ... # mutable-ok: the live dict
+ @overload
+ def plugins(
+ self, kind: Literal["switchyard"]
+ ) -> "dict[str, list[TaggedPreRoutingStrategy[SwitchyardRouter]]]": ... # mutable-ok: the live dict
+
@overload
def plugins(self, kind: Literal["routing"]) -> "list[RoutingPlugin]": ... # mutable-ok: the live list
@@ -1132,6 +1140,8 @@ class RouterRegistry:
return r._core_quality_routers
if kind == "adaptive":
return r._core_adaptive_routers
+ if kind == "switchyard":
+ return r._core_switchyard_routers
if kind == "routing":
return r._core_routing_plugins
if kind == "budget_limiter":
diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py
index 40ae7e0e554..3e222313f32 100644
--- a/litellm/router_rust/core.py
+++ b/litellm/router_rust/core.py
@@ -1177,6 +1177,7 @@ _HOSTED_STATE: Final = frozenset(
"_core_auto_routers",
"_core_complexity_routers",
"_core_quality_routers",
+ "_core_switchyard_routers",
"_core_adaptive_routers",
"_core_routing_plugins",
"_core_deployment_latency_map",
@@ -1722,6 +1723,7 @@ class RustCore(
run_plugins=partial(self._run_plugins, request_kwargs, messages)
if self._hosted._core_routing_plugins # pyright: ignore[reportPrivateUsage] # the live plugin list
else None,
+ call_type=None,
)
def _core_log_retry(
@@ -2715,7 +2717,7 @@ class RustCore(
web_search: Final = not plan.sync and self._picks_async() and is_web_search_request(request)
# The generic handler `_route_capability_pin` names: a call only some providers serve
# drops the others, and a Decisions call carrying media skips the deployments refusing it.
- handler: Final = plan.host_op.rsplit(".", 1)[-1]
+ handler: Final = plan.handler
decisions_media: Final = handler in DECISIONS_MEDIA_CALL_TYPES and bool(
request.get("audio") or request.get("videos")
)
@@ -3113,6 +3115,7 @@ class RustCore(
specific_deployment: object = _NO_INPUT,
*,
pass_through: bool = False,
+ call_type: str | None = None,
) -> object:
"""The host's pre-routing of one pick (router_rust.steps.prerouting): the model group to resolve
instead of the step's ``model``, a custom strategy's deployment, or ``None``. ``pass_through``
@@ -3166,6 +3169,7 @@ class RustCore(
fallback_depth=depth if isinstance(depth, int) else 0,
router_strategy=normalize_strategy(r.routing_strategy),
pass_through=pass_through,
+ call_type=call_type,
enable_tag_filtering=r.enable_tag_filtering,
match_any=r.tag_filtering_match_any,
public_router=public if public is not None else self,
@@ -5266,7 +5270,9 @@ class _Driver:
async def pre_route(self, step: Mapping[str, object]) -> object:
request: Final = self._request
try:
- return await self._core._pre_route(step, request, *self._plan.pick_input(request)) # pyright: ignore[reportPrivateUsage] # see above
+ return await self._core._pre_route( # pyright: ignore[reportPrivateUsage] # see above
+ step, request, *self._plan.pick_input(request), call_type=self._plan.handler
+ )
except Exception as error: # noqa: BLE001 # see select
return self._pick_raised(error, logged=True)
diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py
index e2c621ccb7d..7a5b52761e2 100644
--- a/litellm/router_rust/invoke/endpoints.py
+++ b/litellm/router_rust/invoke/endpoints.py
@@ -244,6 +244,11 @@ class Plan:
# the request (`{**data, name: value, **kwargs}`).
named: tuple[str, ...] = ()
+ @property
+ def handler(self) -> str:
+ """The handler name the host op ends in, which route capabilities and pre-routing strategies key on."""
+ return self.host_op.rsplit(".", 1)[-1]
+
@property
def on_its_own_loop(self) -> bool:
"""``function_with_fallbacks`` runs these sync endpoints' whole walk on a private event loop."""
diff --git a/litellm/router_rust/steps/prerouting.py b/litellm/router_rust/steps/prerouting.py
index 63d15a1a80a..47bd1e69046 100644
--- a/litellm/router_rust/steps/prerouting.py
+++ b/litellm/router_rust/steps/prerouting.py
@@ -197,9 +197,13 @@ class PreRoutingStrategies:
def quality(self) -> StrategyRegistry:
return self._hosted._core_quality_routers # pyright: ignore[reportPrivateUsage] # see above
+ @property
+ def switchyard(self) -> StrategyRegistry:
+ return self._hosted._core_switchyard_routers # pyright: ignore[reportPrivateUsage] # see above
+
def all(self) -> tuple[StrategyRegistry, ...]:
"""In the order ``_select_pre_routing_strategy`` searches them."""
- return (self.auto, self.complexity, self.adaptive, self.quality)
+ return (self.switchyard, self.auto, self.complexity, self.adaptive, self.quality)
def any_registered(self) -> bool:
return any(self.all())
@@ -211,6 +215,7 @@ class PreRoutingStrategies:
self._hosted._core_quality_routers = {} # pyright: ignore[reportPrivateUsage] # see above
self._hosted._core_complexity_routers = {} # pyright: ignore[reportPrivateUsage] # see above
self._hosted._core_auto_routers = {} # pyright: ignore[reportPrivateUsage] # see above
+ self._hosted._core_switchyard_routers = {} # pyright: ignore[reportPrivateUsage] # see above
def register(self, deployment: Deployment) -> None:
"""The end of ``_add_deployment``: an auto, complexity or quality marker registers its router
@@ -222,6 +227,8 @@ class PreRoutingStrategies:
self._init_complexity_router(deployment)
case "quality":
self._init_quality_router(deployment)
+ case "switchyard":
+ self._init_switchyard_router(deployment)
case _:
pass
@@ -231,7 +238,7 @@ class PreRoutingStrategies:
return
model_name: Final = deployment.model_name
tags: Final = _deployment_tags(deployment)
- for registry in (self.auto, self.complexity, self.quality):
+ for registry in (self.auto, self.complexity, self.quality, self.switchyard):
_ = _unregister(registry, model_name, tags)
if _unregister(self.adaptive, model_name, tags):
self._sync_adaptive_router_hooks()
@@ -463,6 +470,32 @@ class PreRoutingStrategies:
)
self._register(self.quality, deployment, router, "Quality-router")
+ def _init_switchyard_router(self, deployment: Deployment) -> None:
+ from litellm.router_strategy.switchyard_router.config import SwitchyardRouterConfig, build_algorithm
+ from litellm.router_strategy.switchyard_router.switchyard_router import SwitchyardRouter
+ from litellm.types.decisions import OpenAIDecisionResponse
+ from litellm.types.utils import ModelResponse
+
+ config: Final = SwitchyardRouterConfig.model_validate(deployment.litellm_params.switchyard_router_config)
+ if deployment.model_name in (config.models.capable, config.models.efficient, config.models.judge):
+ raise ValueError("Switchyard cannot route to itself")
+ ctx: Final = self._ctx
+
+ async def complete_judge(request: Mapping[str, object]) -> ModelResponse:
+ result: Final = await ctx.acompletion(**request) # pyright: ignore[reportAttributeAccessIssue] # the core's RoutingContext
+ if not isinstance(result, ModelResponse):
+ raise TypeError("Switchyard judge must return a non-streaming completion")
+ return result
+
+ async def decide(request: Mapping[str, object]) -> OpenAIDecisionResponse:
+ result: Final = await ctx.adecisions(**request) # pyright: ignore[reportAttributeAccessIssue] # see above
+ if not isinstance(result, OpenAIDecisionResponse):
+ raise TypeError("Switchyard decision judge must return an OpenAI-shaped decision response")
+ return result
+
+ router: Final = SwitchyardRouter(deployment.model_name, config, build_algorithm(config), complete_judge, decide)
+ self._register(self.switchyard, deployment, router, "Switchyard")
+
def _init_adaptive_router(self, deployment: Deployment) -> None:
from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
@@ -741,6 +774,7 @@ class PreRouter:
public_router: object,
run_plugins: Callable[[str], Awaitable[None]] | None = None,
pass_through: bool = False,
+ call_type: str | None = None,
) -> str | None:
"""The start of ``_async_get_available_deployment``: run the hook, record its pick and pin the
tier's litellm_params. The model group to resolve, or ``None`` to resolve ``model`` itself.
@@ -759,6 +793,7 @@ class PreRouter:
match_any=match_any,
public_router=public_router,
run_plugins=run_plugins,
+ call_type=call_type,
)
routed: Final = response.model if response is not None else model
if response is not None:
@@ -782,6 +817,7 @@ class PreRouter:
match_any: bool,
public_router: object,
run_plugins: Callable[[str], Awaitable[None]] | None,
+ call_type: str | None,
) -> "PreRoutingHookResponse | None":
_restore_client_output_ceiling(request_kwargs)
_record_routing_decision(request_kwargs, None)
@@ -861,6 +897,7 @@ class PreRouter:
messages=routing_messages if routing_messages is not None else messages, # pyright: ignore[reportArgumentType] # the request's messages, as given
input=input, # pyright: ignore[reportArgumentType] # the request's input, as given
specific_deployment=False,
+ call_type=call_type,
)
hooked: Final[PreRoutingHookResponse | None] = routed
traced: Final = (
diff --git a/litellm/router_rust/surface/registry.py b/litellm/router_rust/surface/registry.py
index 7c3abdcd45e..820db52d5a1 100644
--- a/litellm/router_rust/surface/registry.py
+++ b/litellm/router_rust/surface/registry.py
@@ -658,6 +658,7 @@ class HostRegistry(HostModelInfo):
self._core_complexity_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same
self._core_adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same
self._core_quality_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same
+ self._core_switchyard_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same
self.strategies: PreRoutingStrategies | None = None
# The router's optional pre-call checks (deployment budgets, prompt caching pins);
# RustCore installs its ``OptionalChecks`` (litellm/router_rust/steps/callbacks.py).
diff --git a/litellm/router_strategy/adaptive_router/adaptive_router.py b/litellm/router_strategy/adaptive_router/adaptive_router.py
index c3a65d6aa91..43869fdf832 100644
--- a/litellm/router_strategy/adaptive_router/adaptive_router.py
+++ b/litellm/router_strategy/adaptive_router/adaptive_router.py
@@ -184,6 +184,7 @@ class AdaptiveRouter:
messages: list[dict[str, object]] | None = None,
input: str | list | None = None,
specific_deployment: bool | None = False,
+ call_type: str | None = None,
) -> PreRoutingHookResponse | None:
"""
Plugin entry point invoked by `Router.async_pre_routing_hook` when the
diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py
index b92c53fa93c..68e8701eb23 100644
--- a/litellm/router_strategy/auto_router/auto_router.py
+++ b/litellm/router_strategy/auto_router/auto_router.py
@@ -204,6 +204,7 @@ class AutoRouter(CustomLogger):
messages: list[dict[str, object]] | None = None,
input: str | list | None = None,
specific_deployment: bool | None = False,
+ call_type: str | None = None,
) -> Optional["PreRoutingHookResponse"]:
"""
This hook is called before the routing decision is made.
diff --git a/litellm/router_strategy/auto_router/auto_router_model_naming.py b/litellm/router_strategy/auto_router/auto_router_model_naming.py
index f5aeccd9305..47daa175c99 100644
--- a/litellm/router_strategy/auto_router/auto_router_model_naming.py
+++ b/litellm/router_strategy/auto_router/auto_router_model_naming.py
@@ -26,7 +26,7 @@ from litellm.router_strategy.complexity_router.config import (
AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/"
-StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"]
+StrategyRouterKind: TypeAlias = Literal["semantic", "complexity", "adaptive", "quality", "switchyard"]
StrategyRouterDependencyRole: TypeAlias = Literal[
"tier", "default", "classifier", "embedding", "evaluation", "compactor"
@@ -43,6 +43,7 @@ class StrategyRouterDependency:
STRATEGY_ROUTER_PARAM_FIELDS: Final[frozenset[str]] = frozenset(
{
+ "switchyard_router_config",
"auto_router_config",
"auto_router_config_path",
"auto_router_default_model",
@@ -57,6 +58,7 @@ STRATEGY_ROUTER_PARAM_FIELDS: Final[frozenset[str]] = frozenset(
)
_REQUIRED_FIELD_GROUPS: Final[Mapping[StrategyRouterKind, tuple[tuple[str, ...], ...]]] = {
+ "switchyard": (("switchyard_router_config",),),
"semantic": (
("auto_router_config", "auto_router_config_path"),
("auto_router_default_model",),
@@ -78,6 +80,8 @@ def classify_strategy_router_model(model: str) -> StrategyRouterKind | None:
if not model.startswith(AUTO_ROUTER_MODEL_PREFIX):
return None
remainder: Final = model[len(AUTO_ROUTER_MODEL_PREFIX) :]
+ if remainder.startswith("switchyard"):
+ return "switchyard"
if remainder.startswith("complexity_router"):
return "complexity"
if remainder.startswith("adaptive_router"):
@@ -137,6 +141,13 @@ def strategy_router_dependencies(
kind: Final = classify_strategy_router_model(str(litellm_params.get("model", "")))
if kind is None:
return ()
+ if kind == "switchyard":
+ models: Final = _mapping(_mapping(litellm_params.get("switchyard_router_config")).get("models"))
+ return (
+ _named(models.get("capable"), "tier")
+ + _named(models.get("efficient"), "tier")
+ + _named(models.get("judge"), "classifier")
+ )
if kind == "semantic":
return _named(litellm_params.get("auto_router_default_model"), "default") + _named(
litellm_params.get("auto_router_embedding_model"), "embedding"
diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py
index 05dd4706b62..f004dfcc38d 100644
--- a/litellm/router_strategy/complexity_router/complexity_router.py
+++ b/litellm/router_strategy/complexity_router/complexity_router.py
@@ -4490,6 +4490,7 @@ class ComplexityRouter(CustomLogger):
messages: list[dict[str, object]] | None = None,
input: str | list | None = None,
specific_deployment: bool | None = False,
+ call_type: str | None = None,
) -> PreRoutingHookResponse | None:
"""
Pre-routing hook called before the routing decision.
diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py
index 2d9ad63e9de..fc8d5ab5fcb 100644
--- a/litellm/router_strategy/quality_router/quality_router.py
+++ b/litellm/router_strategy/quality_router/quality_router.py
@@ -330,6 +330,7 @@ class QualityRouter(CustomLogger):
messages: list[dict[str, object]] | None = None,
input: str | list | None = None,
specific_deployment: bool | None = False,
+ call_type: str | None = None,
) -> Optional["PreRoutingHookResponse"]:
"""Try keyword override first; fall back to complexity-tier routing."""
from litellm.types.router import PreRoutingHookResponse
diff --git a/litellm/router_strategy/switchyard_router/__init__.py b/litellm/router_strategy/switchyard_router/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/litellm/router_strategy/switchyard_router/config.py b/litellm/router_strategy/switchyard_router/config.py
new file mode 100644
index 00000000000..7e0d3ad3a94
--- /dev/null
+++ b/litellm/router_strategy/switchyard_router/config.py
@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
+
+from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag, TypeAdapter, model_validator
+
+if TYPE_CHECKING:
+ from switchyard_rust.libsy import Algorithm, TaskClassifierConfig
+
+
+class LlmClassifierConfig(BaseModel):
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ kind: Literal["llm"] = "llm"
+ base_threshold: float = Field(default=0.5, ge=0, le=1)
+ threshold_step: float = Field(default=0.0, ge=0, le=1)
+ recent_turn_window: int | None = Field(default=None, gt=0)
+ max_output_tokens: int = Field(default=4096, gt=0)
+ response_format_type: Literal["json_schema", "json_object"] = "json_schema"
+ prompt: str | None = None
+
+
+class DecisionClassifierConfig(BaseModel):
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ kind: Literal["decision"]
+ cutoff: float = Field(default=0.6, ge=0, le=1)
+ instructions: str | Mapping[str, object] | None = None
+ evidence: Mapping[str, object] | None = None
+
+
+_KIND: Final = TypeAdapter(Mapping[str, object])
+
+
+def _classifier_kind(value: object) -> str:
+ match value:
+ case LlmClassifierConfig() | DecisionClassifierConfig():
+ return value.kind
+ case Mapping():
+ return str(_KIND.validate_python(value).get("kind", "llm"))
+ case _:
+ return "llm"
+
+
+ClassifierConfig: TypeAlias = Annotated[
+ Annotated[LlmClassifierConfig, Tag("llm")] | Annotated[DecisionClassifierConfig, Tag("decision")],
+ Discriminator(_classifier_kind),
+]
+
+
+class StageOptions(BaseModel):
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ picker: Literal["efficient_first", "capable_first"] = "efficient_first"
+ confidence_threshold: float | None = Field(default=None, ge=0, le=1)
+ recent_window: int | None = Field(default=None, gt=0)
+ tool_semantics: Mapping[str, tuple[str, ...]] | None = None
+ only_on_wrong_signal_escalation: bool = True
+
+
+class StageModels(BaseModel):
+ model_config = ConfigDict(frozen=True, extra="forbid", str_strip_whitespace=True)
+
+ capable: str = Field(min_length=1)
+ efficient: str = Field(min_length=1)
+ judge: str | None = Field(default=None, min_length=1)
+
+ @model_validator(mode="after")
+ def reject_recursive_targets(self) -> StageModels:
+ if any(name.startswith("auto_router/") for name in (self.capable, self.efficient, self.judge or "")):
+ raise ValueError("Switchyard targets must be concrete model groups, not auto-router markers")
+ if self.capable == self.efficient:
+ raise ValueError("The capable and efficient models must be different model groups")
+ return self
+
+
+class SwitchyardRouterConfig(BaseModel):
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ algorithm: Literal["stage_router"] = "stage_router"
+ models: StageModels
+ options: StageOptions = Field(default_factory=StageOptions)
+ classifier: ClassifierConfig | None = None
+
+ @model_validator(mode="after")
+ def validate_judge(self) -> SwitchyardRouterConfig:
+ if (self.classifier is None) != (self.models.judge is None):
+ raise ValueError("A judge model is required exactly when the classifier is enabled")
+ return self
+
+ @property
+ def confidence_threshold(self) -> float:
+ if self.options.confidence_threshold is not None:
+ return self.options.confidence_threshold
+ return 0.7 if self.classifier is not None else 0.5
+
+ def runtime_models(self) -> Mapping[str, tuple[str, ...]]:
+ return {
+ "any": (self.models.capable, self.models.efficient),
+ "capable": (self.models.capable,),
+ "efficient": (self.models.efficient,),
+ **({"judge": (self.models.judge,)} if self.models.judge is not None else {}),
+ }
+
+
+def _task_classifier(config: SwitchyardRouterConfig) -> TaskClassifierConfig | None:
+ from switchyard_rust.libsy import TaskClassifierConfig
+
+ match config.classifier:
+ case None:
+ return None
+ case LlmClassifierConfig() as llm:
+ return TaskClassifierConfig(
+ base_threshold=llm.base_threshold,
+ threshold_step=llm.threshold_step,
+ recent_turn_window=llm.recent_turn_window,
+ max_output_tokens=llm.max_output_tokens,
+ response_format_type=llm.response_format_type,
+ prompt=llm.prompt,
+ )
+ case DecisionClassifierConfig() as decision:
+ return TaskClassifierConfig.decision(
+ cutoff=decision.cutoff,
+ candidates={"capable": config.models.capable, "efficient": config.models.efficient},
+ evidence=dict(decision.evidence or {}),
+ instructions=decision.instructions,
+ )
+
+
+def build_algorithm(config: SwitchyardRouterConfig) -> Algorithm:
+ from switchyard_rust.libsy import LlmFallback, stage_router
+
+ classifier: Final = _task_classifier(config)
+ return stage_router(
+ picker=config.options.picker,
+ confidence_threshold=config.confidence_threshold,
+ recent_window=config.options.recent_window,
+ tool_semantics=dict(config.options.tool_semantics) if config.options.tool_semantics is not None else None,
+ only_on_wrong_signal_escalation=config.options.only_on_wrong_signal_escalation,
+ classifier=LlmFallback(config=classifier) if classifier is not None else None,
+ )
+
+
+def validate_switchyard_config_write(config: Mapping[str, object] | None) -> str | None:
+ if config is None:
+ return None
+ try:
+ build_algorithm(SwitchyardRouterConfig.model_validate(config))
+ except ImportError:
+ return "Switchyard requires the optional dependency. Install litellm[switchyard] (nemo-switchyard==0.4.0a1)."
+ except (ValueError, TypeError) as error:
+ return f"Invalid switchyard_router_config: {error}"
+ return None
diff --git a/litellm/router_strategy/switchyard_router/switchyard_router.py b/litellm/router_strategy/switchyard_router/switchyard_router.py
new file mode 100644
index 00000000000..cb455bab337
--- /dev/null
+++ b/litellm/router_strategy/switchyard_router/switchyard_router.py
@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from typing import TYPE_CHECKING, Final, Literal, TypeAlias, TypeVar
+
+from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
+from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
+from litellm.litellm_core_utils.internal_call_metadata import (
+ effective_turn_off_message_logging,
+ forwarded_internal_call_metadata,
+ parent_session_kwargs,
+)
+from litellm.router_strategy.switchyard_router.config import (
+ DecisionClassifierConfig,
+ LlmClassifierConfig,
+ SwitchyardRouterConfig,
+)
+from litellm.router_strategy.switchyard_router.translation import (
+ chat_messages,
+ decision_request,
+ decision_response,
+ judge_request,
+ judge_response,
+ mapping,
+ request_to_switchyard,
+)
+from litellm.types.decisions import OpenAIDecisionResponse
+from litellm.types.router import PreRoutingHookResponse
+from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, StandardLoggingRoutingDecision
+
+if TYPE_CHECKING:
+ from switchyard_rust.libsy import Algorithm, DecisionCall, ModelCall, RoutingOutcome
+
+JudgeCompletion: TypeAlias = Callable[[Mapping[str, object]], Awaitable[ModelResponse]]
+DecisionCompletion: TypeAlias = Callable[[Mapping[str, object]], Awaitable[OpenAIDecisionResponse]]
+FallbackReason: TypeAlias = Literal[
+ "unsupported_request_type",
+ "unsupported_content",
+ "libsy_error",
+ "unexpected_judge_call",
+ "unexpected_decision_call",
+ "target_outside_config",
+ "no_decision",
+]
+_T = TypeVar("_T")
+_SESSION_HEADER: Final = "x-switchyard-session-id"
+_FORWARDED_HEADER_PREFIXES: Final = ("x-switchyard-", "x-codex-", "x-nemo-relay-", "x-dynamo-")
+_FORWARDED_HEADERS: Final = frozenset(
+ {"x-session-id", "session-id", "thread-id", "x-task-id", "x-request-id", "x-client-request-id"}
+)
+
+
+async def _resolved(value: _T) -> _T:
+ return value
+
+
+_SESSION_ALIASES: Final = frozenset(
+ {_SESSION_HEADER, "x-claude-code-session-id", "x-nemo-relay-session-id", "x-session-id", "session-id"}
+)
+_CODEX_TURN_METADATA_HEADER: Final = "x-codex-turn-metadata"
+
+
+def _forwarded_header(name: str, session_scoped: bool) -> bool:
+ lowered: Final = name.lower()
+ if lowered in _SESSION_ALIASES or (lowered == _CODEX_TURN_METADATA_HEADER and not session_scoped):
+ return False
+ return lowered in _FORWARDED_HEADERS or lowered.startswith(_FORWARDED_HEADER_PREFIXES)
+
+
+def _caller_key_hash(request_kwargs: Mapping[str, object]) -> str:
+ internal: Final = mapping(request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)) or {})
+ key: Final = internal.get("user_api_key_hash")
+ return str(key) if key is not None else ""
+
+
+def _scoped_session(request_kwargs: Mapping[str, object]) -> str | None:
+ session: Final = parent_session_kwargs(request_kwargs).get("litellm_session_id")
+ if not session:
+ return None
+ return hashlib.sha256(f"{_caller_key_hash(request_kwargs)}\0{session}".encode()).hexdigest()
+
+
+def switchyard_headers(request_kwargs: Mapping[str, object]) -> Mapping[str, str]:
+ proxy_request: Final = mapping(request_kwargs.get("proxy_server_request") or {})
+ client: Final = mapping(proxy_request.get("headers") or {})
+ session: Final = _scoped_session(request_kwargs)
+ forwarded: Final = {
+ name.lower(): value
+ for name, value in client.items()
+ if isinstance(value, str) and _forwarded_header(name, session is not None)
+ }
+ return {**forwarded, **({_SESSION_HEADER: session} if session else {})}
+
+
+class SwitchyardRouter:
+ def __init__(
+ self,
+ model_name: str,
+ config: SwitchyardRouterConfig,
+ algorithm: Algorithm,
+ completion: JudgeCompletion,
+ decide: DecisionCompletion,
+ ) -> None:
+ self.model_name = model_name
+ self.config = config
+ self.algorithm = algorithm
+ self.completion = completion
+ self.decide = decide
+
+ def _fallback(
+ self, reason: FallbackReason, messages: Sequence[Mapping[str, object]] | None
+ ) -> PreRoutingHookResponse:
+ return PreRoutingHookResponse(
+ model=self.config.models.capable,
+ messages=[dict(message) for message in messages] if messages is not None else None,
+ routing_decision={
+ "router_type": "switchyard",
+ "router_model_name": self.model_name,
+ "routed_model": self.config.models.capable,
+ "switchyard_algorithm": self.config.algorithm,
+ "switchyard_fallback_reason": reason,
+ },
+ )
+
+ def _internal_call(
+ self,
+ request: Mapping[str, object],
+ caller: Mapping[str, object],
+ bucket: Literal["metadata", "litellm_metadata"],
+ ) -> Mapping[str, object]:
+ caller_metadata: Final = mapping(caller.get(get_metadata_variable_name_from_kwargs(caller)) or {})
+ return {
+ **request,
+ bucket: {
+ **forwarded_internal_call_metadata(caller_metadata, AUTOROUTER_CLASSIFIER_CALL_ORIGIN),
+ INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
+ },
+ "proxy_server_request": {"body": request},
+ "num_retries": 0,
+ "turn_off_message_logging": effective_turn_off_message_logging(caller),
+ **parent_session_kwargs(caller),
+ }
+
+ def _decision(
+ self, pick: str, evidence: object, messages: Sequence[Mapping[str, object]] | None
+ ) -> PreRoutingHookResponse:
+ decision: Final[StandardLoggingRoutingDecision] = {
+ "router_type": "switchyard",
+ "router_model_name": self.model_name,
+ "routed_model": pick,
+ "switchyard_algorithm": self.config.algorithm,
+ "switchyard_evidence": evidence,
+ }
+ return PreRoutingHookResponse(
+ model=pick,
+ messages=[dict(message) for message in messages] if messages is not None else None,
+ routing_decision=decision,
+ )
+
+ async def _answer_judge(self, call: ModelCall, caller: Mapping[str, object]) -> None:
+ try:
+ converted: Final = judge_response(
+ await self.completion(
+ self._internal_call(judge_request(call.request, call.models[0]), caller, "metadata")
+ )
+ )
+ except Exception as error: # noqa: BLE001 # judge/provider failures are inputs to Switchyard's fall-open decision
+ await call.fail(error)
+ else:
+ await call.respond(_resolved(converted))
+
+ async def _answer_decision(self, call: DecisionCall, caller: Mapping[str, object]) -> None:
+ try:
+ answer: Final = decision_response(
+ await self.decide(
+ self._internal_call(decision_request(call.request, call.model), caller, "litellm_metadata")
+ ),
+ call.request,
+ )
+ except Exception as error: # noqa: BLE001 # decision/provider failures are inputs to Switchyard's fall-open decision
+ await call.fail(error)
+ else:
+ await call.respond(_resolved(answer))
+
+ def _outcome(
+ self, outcome: RoutingOutcome, messages: Sequence[Mapping[str, object]] | None
+ ) -> PreRoutingHookResponse:
+ pick: Final = outcome.selected_model_ids[0]
+ if pick not in (self.config.models.capable, self.config.models.efficient):
+ return self._fallback("target_outside_config", messages)
+ metadata: Final = outcome.metadata
+ evidence: Final[object] = metadata.evidence if metadata is not None else None
+ return self._decision(pick, evidence, messages)
+
+ async def async_pre_routing_hook(
+ self,
+ model: str,
+ request_kwargs: Mapping[str, object],
+ messages: Sequence[Mapping[str, object]] | None = None,
+ input: object = None,
+ specific_deployment: bool | None = False,
+ call_type: str | None = None,
+ ) -> PreRoutingHookResponse | None:
+ from switchyard_rust.libsy import LibsyError, Step
+
+ if model != self.model_name:
+ return None
+ try:
+ chat: Final = chat_messages(call_type, model, messages, input, request_kwargs)
+ request: Final = request_to_switchyard(chat) if chat is not None else None
+ except (ValueError, KeyError, TypeError):
+ return self._fallback("unsupported_content", messages)
+ if request is None:
+ return self._fallback("unsupported_request_type", messages)
+ judge: Final = self.config.models.judge
+ try:
+ async for step in self.algorithm.run_stream(
+ request, self.config.runtime_models(), headers=switchyard_headers(request_kwargs)
+ ):
+ match step:
+ case Step.CallModel(call) if isinstance(
+ self.config.classifier, LlmClassifierConfig
+ ) and call.models[:1] == [judge]:
+ await self._answer_judge(call, request_kwargs)
+ case Step.CallDecision(call) if (
+ isinstance(self.config.classifier, DecisionClassifierConfig) and call.model == judge
+ ):
+ await self._answer_decision(call, request_kwargs)
+ case Step.CallModel():
+ return self._fallback("unexpected_judge_call", messages)
+ case Step.CallDecision():
+ return self._fallback("unexpected_decision_call", messages)
+ case Step.Done(outcome):
+ return self._outcome(outcome, messages)
+ except LibsyError:
+ return self._fallback("libsy_error", messages)
+ return self._fallback("no_decision", messages)
diff --git a/litellm/router_strategy/switchyard_router/translation.py b/litellm/router_strategy/switchyard_router/translation.py
new file mode 100644
index 00000000000..246f2ba3e41
--- /dev/null
+++ b/litellm/router_strategy/switchyard_router/translation.py
@@ -0,0 +1,232 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Callable, Mapping, Sequence
+from typing import TYPE_CHECKING, Final, Protocol
+
+from pydantic import TypeAdapter
+
+from litellm.types.decisions import (
+ OpenAIChoiceAnswer,
+ OpenAIDecisionResponse,
+ systemone_choice_key,
+)
+from litellm.types.utils import CallTypes, ModelResponse
+
+if TYPE_CHECKING:
+ from switchyard_rust.libsy import LlmResponse
+
+_MAPPING: Final = TypeAdapter(Mapping[str, object])
+_SEQUENCE: Final = TypeAdapter(tuple[object, ...])
+_CALL_TYPES: Final = frozenset(call_type.value for call_type in CallTypes)
+
+
+def mapping(value: object) -> Mapping[str, object]:
+ return _MAPPING.validate_python(value)
+
+
+def sequence(value: object) -> tuple[object, ...]:
+ return _SEQUENCE.validate_python(value)
+
+
+def text_content(value: object) -> tuple[Mapping[str, object], ...]:
+ if value is None:
+ return ()
+ if isinstance(value, str):
+ return ({"type": "text", "text": value},)
+ blocks: Final = tuple(mapping(block) for block in sequence(value))
+ if any(block.get("type") != "text" or not isinstance(block.get("text"), str) for block in blocks):
+ raise ValueError("Switchyard routing supports text content only; media and audio are unsupported")
+ return tuple({"type": "text", "text": block["text"]} for block in blocks)
+
+
+def tool_call(value: object) -> Mapping[str, object]:
+ call: Final = mapping(value)
+ function: Final = mapping(call.get("function"))
+ arguments: Final = function.get("arguments")
+ if call.get("type") != "function" or not isinstance(arguments, str):
+ raise ValueError("Switchyard requires function calls with JSON arguments")
+ return {
+ "type": "tool_call",
+ "id": call["id"],
+ "name": function["name"],
+ "arguments": _MAPPING.validate_json(arguments),
+ }
+
+
+def message_to_switchyard(value: object) -> Mapping[str, object]:
+ message: Final = mapping(value)
+ if message.get("audio") is not None:
+ raise ValueError("Switchyard routing does not support audio")
+ role: Final = message.get("role")
+ content: Final = text_content(message.get("content"))
+ if role == "tool":
+ return {
+ "role": "tool",
+ "content": (
+ {
+ "type": "tool_result",
+ "tool_call_id": message["tool_call_id"],
+ "content": content,
+ "is_error": message.get("is_error"),
+ },
+ ),
+ }
+ if role not in ("user", "system", "developer", "assistant"):
+ raise ValueError("Switchyard received an unsupported message role")
+ calls: Final = tuple(tool_call(call) for call in sequence(message.get("tool_calls") or ()))
+ return {"role": role, "content": (*content, *calls)}
+
+
+class ChatConverter(Protocol):
+ def get_structured_messages(
+ self,
+ data: dict[str, object], # mutable-ok: matches the request converters' existing dict parameter
+ *,
+ keep_tool_errors: bool = False,
+ ) -> Sequence[object] | None: ...
+
+
+def _instantiate(converter: Callable[[], ChatConverter]) -> ChatConverter:
+ return converter()
+
+
+def chat_messages(
+ call_type: str | None,
+ model: str,
+ messages: Sequence[Mapping[str, object]] | None,
+ input: object,
+ request_kwargs: Mapping[str, object],
+) -> tuple[Mapping[str, object], ...] | None:
+ from litellm.llms import load_guardrail_translation_mappings
+
+ converter: Final = (
+ load_guardrail_translation_mappings().get(CallTypes(call_type)) if call_type in _CALL_TYPES else None
+ )
+ if converter is None:
+ return None
+ request: Final = {
+ **request_kwargs,
+ "model": model,
+ **({"messages": [dict(message) for message in messages]} if messages is not None else {}),
+ **({"input": input} if input is not None else {}),
+ }
+ return tuple(map(mapping, _instantiate(converter).get_structured_messages(request, keep_tool_errors=True) or ()))
+
+
+def request_to_switchyard(messages: Sequence[Mapping[str, object]]) -> Mapping[str, object]:
+ if not messages:
+ raise ValueError("Switchyard requires at least one message")
+ return {
+ "model": "auto",
+ "messages": tuple(message_to_switchyard(message) for message in messages),
+ "stream": False,
+ }
+
+
+def judge_message(value: object) -> Mapping[str, object]:
+ message: Final = mapping(value)
+ blocks: Final = text_content(message.get("content"))
+ return {"role": message["role"], "content": "".join(str(block["text"]) for block in blocks)}
+
+
+def judge_request(request: Mapping[str, object], model: str) -> Mapping[str, object]:
+ output: Final = mapping(request.get("output") or {})
+ sampling: Final = mapping(request.get("sampling") or {})
+ options: Final = {
+ "max_tokens": output.get("max_output_tokens"),
+ "response_format": output.get("response_format"),
+ "temperature": sampling.get("temperature"),
+ "top_p": sampling.get("top_p"),
+ }
+ return {
+ "model": model,
+ "stream": False,
+ "messages": [
+ judge_message(message)
+ for message in (
+ *sequence(request.get("instructions") or ()),
+ *sequence(request.get("messages") or ()),
+ )
+ ],
+ **{key: value for key, value in options.items() if value is not None},
+ }
+
+
+def judge_response(response: ModelResponse) -> LlmResponse.Agg:
+ from switchyard_rust.libsy import LlmResponse
+
+ body: Final = _MAPPING.validate_json(response.model_dump_json(exclude_unset=True))
+ choices: Final = tuple(mapping(choice) for choice in sequence(body["choices"]))
+ return LlmResponse.Agg(
+ {
+ "id": response.id,
+ "model": response.model,
+ "outputs": tuple(message_to_switchyard(choice["message"]) for choice in choices),
+ }
+ )
+
+
+def _decision_text(value: object) -> str:
+ return value if isinstance(value, str) else json.dumps(value)
+
+
+def _decision_question(name: str, question: Mapping[str, object]) -> Mapping[str, object]:
+ kind: Final = mapping(question["kind"])
+ if kind.get("type") != "choice":
+ raise ValueError("Switchyard decision routing asks choice questions only")
+ return {
+ "type": "choice",
+ "name": name,
+ "instructions": _decision_text(question.get("instructions")),
+ "choices": [
+ {
+ "value": option["id"],
+ **({"description": _decision_text(option["description"])} if option.get("description") else {}),
+ }
+ for option in map(mapping, sequence(mapping(kind["data"])["options"]))
+ ],
+ }
+
+
+def decision_request(request: Mapping[str, object], model: str) -> Mapping[str, object]:
+ questions: Final = mapping(request["questions"])
+ return {
+ "model": model,
+ "input": _decision_text(request["context"]),
+ "questions": [_decision_question(name, mapping(question)) for name, question in questions.items()],
+ }
+
+
+def _choice_value(answer: object) -> Mapping[str, object]:
+ if not isinstance(answer, OpenAIChoiceAnswer):
+ raise TypeError("The decision model refused or returned a non-choice answer")
+ return {
+ "type": "choice",
+ "data": {
+ "selected": systemone_choice_key(answer.choice),
+ "probabilities": {systemone_choice_key(p.value): p.probability for p in answer.probabilities},
+ },
+ }
+
+
+def decision_response(response: OpenAIDecisionResponse, request: Mapping[str, object]) -> Mapping[str, object]:
+ names: Final = tuple(mapping(request["questions"]))
+ if len(response.answers) != len(names):
+ raise ValueError("The decision model did not answer every question")
+ return {
+ "id": None,
+ "model": response.model,
+ "answers": {
+ answer.name or name: {
+ "value": _choice_value(answer),
+ "provider_confidence": getattr(answer, "confidence", None),
+ }
+ for name, answer in zip(names, response.answers, strict=True)
+ },
+ "usage": {
+ "input_tokens": response.usage.input_tokens,
+ "output_tokens": response.usage.output_tokens,
+ "total_tokens": response.usage.total_tokens,
+ },
+ }
diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py
index 9f63d27f5f3..c04430322c4 100644
--- a/litellm/types/litellm_params.py
+++ b/litellm/types/litellm_params.py
@@ -207,6 +207,7 @@ class DeploymentOptions:
@dataclass(frozen=True, slots=True, kw_only=True)
class SpecializedRouterOptions:
+ switchyard_router_config: Mapping[str, object] | None = None
auto_router_config_path: str | None = None
auto_router_config: str | None = None
auto_router_default_model: str | None = None
diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py
index c9739eee91d..378899b667d 100644
--- a/litellm/types/llms/openai.py
+++ b/litellm/types/llms/openai.py
@@ -957,6 +957,7 @@ class ChatCompletionToolMessage(TypedDict):
role: Literal["tool"]
content: str | Iterable[ToolMessageContentPart] # writable-ok: Pydantic warns on ReadOnly TypedDict fields
tool_call_id: str
+ is_error: NotRequired[bool] # writable-ok: Pydantic warns on ReadOnly TypedDict fields
class ChatCompletionFunctionMessage(TypedDict):
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 3dcfc694003..c0b182f0f20 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -659,6 +659,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
tag_regex: list[str] | None = None
# auto-router params
+ switchyard_router_config: Mapping[str, object] | None = None
auto_router_config_path: str | None = None
auto_router_config: str | None = None
auto_router_default_model: str | None = None
@@ -1461,6 +1462,7 @@ class PreRoutingStrategy(Protocol):
messages: list[dict[str, Any]] | None = None,
input: "str | list[Any] | None" = None,
specific_deployment: bool | None = False,
+ call_type: str | None = None,
) -> "PreRoutingHookResponse | None": ...
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index b082198d3f2..755a0a8cc7a 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -3382,7 +3382,10 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
"""Per-request provenance for a pre-routing strategy (auto-router) decision."""
router_model_name: str
- router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic"]]
+ router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic", "switchyard"]]
+ switchyard_algorithm: ReadOnly[str]
+ switchyard_evidence: ReadOnly[object]
+ switchyard_fallback_reason: ReadOnly[str]
router_config_id: ReadOnly[str]
router_config_updated_at: ReadOnly[str]
router_config_fingerprint: ReadOnly[str]
@@ -3443,10 +3446,12 @@ class StandardLoggingKubernetesPodRouting(TypedDict):
# so a redacted row stays explainable. `test_every_routing_decision_field_is_classified`
# fails if a field is added to the record without being placed in one set or the other.
PROMPT_QUOTING_ROUTING_DECISION_FIELDS: frozenset[str] = frozenset(
- {"signals", "matched_keyword", "escalation_keyword", "classifier_crux"}
+ {"signals", "matched_keyword", "escalation_keyword", "classifier_crux", "switchyard_evidence"}
)
DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset(
{
+ "switchyard_algorithm",
+ "switchyard_fallback_reason",
"router_model_name",
"router_type",
"router_config_id",
diff --git a/pyproject.toml b/pyproject.toml
index e193eeee93f..02834f604d2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -54,6 +54,7 @@ Documentation = "https://docs.litellm.ai"
# sub-packages and the opentelemetry trio move in lockstep, and grpcio is
# supply-chain-pinned to a vetted, aged release.
[project.optional-dependencies]
+switchyard = ["nemo-switchyard==0.4.0a1"]
proxy = [
"gunicorn>=23.0.0,<24.0",
"uvicorn>=0.33.0,<1.0",
@@ -268,6 +269,7 @@ proxy-dev = [
"opentelemetry-instrumentation-fastapi==0.66b1",
"azure-identity==1.25.2",
"a2a-sdk==1.1.0",
+ "nemo-switchyard==0.4.0a1",
"backports-zstd==1.8.0; python_version < '3.14'",
"httptools>=0.7.1,<0.8",
"semver>=3.0.4,<4",
diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
index 8affe2e136e..b6229dbaec0 100644
--- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
+++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
@@ -3458,3 +3458,41 @@ def test_reading_a_dropped_budget_for_a_guardrail_logs_no_drop_warning(caplog: p
assert messages == [{"role": "user", "content": "hi"}]
assert not [record for record in caplog.records if "Dropping" in record.getMessage()]
+
+
+FAILED_TOOL_RESULT_REQUEST: Final = {
+ "model": "claude-sonnet",
+ "max_tokens": 16,
+ "messages": [
+ {"role": "user", "content": "Rename the helper"},
+ {
+ "role": "assistant",
+ "content": [{"type": "tool_use", "id": "edit0", "name": "Edit", "input": {"file_path": "a.py"}}],
+ },
+ {
+ "role": "user",
+ "content": [
+ {"type": "tool_result", "tool_use_id": "edit0", "content": "String not found", "is_error": True}
+ ],
+ },
+ ],
+}
+
+
+@pytest.mark.parametrize("keep_tool_errors", [True, False])
+def test_structured_messages_carry_tool_failures_only_when_asked(keep_tool_errors: bool) -> None:
+ messages: Final = AnthropicMessagesHandler().get_structured_messages(
+ FAILED_TOOL_RESULT_REQUEST, keep_tool_errors=keep_tool_errors
+ )
+ assert messages is not None
+ (tool_message,) = [message for message in messages if message["role"] == "tool"]
+ assert tool_message.get("is_error") is (True if keep_tool_errors else None)
+
+
+def test_request_sent_to_a_model_never_carries_tool_failure_marker() -> None:
+ from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
+
+ request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
+ anthropic_message_request=FAILED_TOOL_RESULT_REQUEST # pyright: ignore[reportArgumentType] # a JSON request in the Anthropic shape
+ )
+ assert "is_error" not in json.dumps(request["messages"], default=str)
diff --git a/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py b/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py
index 3e3e89f8117..3eb901144a3 100644
--- a/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py
+++ b/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py
@@ -77,9 +77,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
),
],
)
-def test_remove_sensitive_info_from_deployment(
- model_config: dict, expected_config: dict
-):
+def test_remove_sensitive_info_from_deployment(model_config: dict, expected_config: dict):
sanitized_config = remove_sensitive_info_from_deployment(model_config)
assert sanitized_config == expected_config
@@ -110,10 +108,7 @@ def test_remove_sensitive_info_from_deployment_with_excluded_keys():
sanitized_config = remove_sensitive_info_from_deployment(
copy.deepcopy(base_config), excluded_keys={"litellm_credentials_name"}
)
- assert (
- sanitized_config["litellm_params"]["litellm_credentials_name"]
- == "my-credential-name"
- )
+ assert sanitized_config["litellm_params"]["litellm_credentials_name"] == "my-credential-name"
# access_token should still be masked (not in excluded_keys)
assert sanitized_config["litellm_params"]["access_token"] != "token-12345"
@@ -121,3 +116,24 @@ def test_remove_sensitive_info_from_deployment_with_excluded_keys():
# api_key should still be removed (popped) regardless of excluded_keys
assert "api_key" not in sanitized_config["litellm_params"]
+
+
+def test_remove_sensitive_info_from_deployment_keeps_nested_nulls():
+ router_config = {
+ "models": {"capable": "strong", "efficient": "fast", "judge": None},
+ "options": {"confidence_threshold": None, "tool_semantics": None},
+ "fallbacks": [None, "fast"],
+ }
+ deployment = {
+ "model_name": "route",
+ "litellm_params": {
+ "model": "auto_router/switchyard",
+ "api_key": "sk-sensitive-key-123",
+ "switchyard_router_config": copy.deepcopy(router_config),
+ },
+ }
+
+ sanitized = remove_sensitive_info_from_deployment(deployment)
+
+ assert "api_key" not in sanitized["litellm_params"]
+ assert sanitized["litellm_params"]["switchyard_router_config"] == router_config
diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py
index 32c9535b4c8..2fdb77af8f9 100644
--- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py
+++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py
@@ -1085,7 +1085,7 @@ class TestClearCache:
Test that clear_cache successfully clears router model caches and reloads models.
"""
mock_router = MagicMock()
- plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
+ plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = ["openai/gpt-4o", "openai/gpt-4o-mini"]
@@ -1125,7 +1125,7 @@ class TestClearCache:
# entries are auto_router/* deployments (so their router-map entries should be
# cleared for reload); the config-defined router is preserved.
mock_router = MagicMock()
- plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
+ plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = [
{
@@ -1209,7 +1209,7 @@ class TestClearCache:
)
mock_router = MagicMock()
- plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
+ plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = [
{
@@ -1270,7 +1270,7 @@ class TestClearCachePreservesConfigRouters:
)
mock_router = MagicMock()
- plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
+ plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = [
{
@@ -1315,7 +1315,7 @@ class TestClearCachePreservesConfigRouters:
from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache
mock_router = MagicMock()
- plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
+ plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = [
{
@@ -1346,15 +1346,15 @@ class TestClearCachePreservesConfigRouters:
assert "shared-name" in plugins["complexity"]
@pytest.mark.asyncio
- async def test_db_quality_and_adaptive_routers_are_evicted(self):
- """The auto_router/ prefix also covers quality_router/ and adaptive_router/. Their
+ async def test_db_quality_adaptive_and_switchyard_routers_are_evicted(self):
+ """The auto_router/ prefix also covers quality_router/, adaptive_router/ and switchyard. Their
registry entries must be popped too, or reload's init raises 'already exists'
(quality) or leaves a stale entry (adaptive).
"""
from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache
mock_router = MagicMock()
- plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
+ plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = [
{
@@ -1367,11 +1367,17 @@ class TestClearCachePreservesConfigRouters:
"model_info": {"id": "db-a", "db_model": True},
"litellm_params": {"model": "auto_router/adaptive_router/a1"},
},
+ {
+ "model_name": "s1",
+ "model_info": {"id": "db-s", "db_model": True},
+ "litellm_params": {"model": "auto_router/switchyard"},
+ },
]
plugins["auto"] = {}
plugins["complexity"] = {}
plugins["quality"] = {"q1": MagicMock()}
plugins["adaptive"] = {"a1": MagicMock()}
+ plugins["switchyard"] = {"s1": MagicMock()}
mock_config = MagicMock()
mock_config.add_deployment_locked = AsyncMock(
@@ -1389,6 +1395,7 @@ class TestClearCachePreservesConfigRouters:
assert "q1" not in plugins["quality"]
assert "a1" not in plugins["adaptive"]
+ assert "s1" not in plugins["switchyard"]
class TestDeleteModelClearsRouterRegistry:
@@ -1503,7 +1510,7 @@ class TestDeleteModelClearsRouterRegistry:
mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row)
mock_router = MagicMock()
- plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
+ plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.apply.return_value = ApplyResult(
version=1,
@@ -11431,6 +11438,98 @@ async def test_add_new_model_reports_a_database_outage_during_the_insert_as_503(
assert (exc_info.value.code, exc_info.value.type) == ("503", "no_db_connection")
+@pytest.mark.parametrize("invalid", [
+ {"algorithm": "random"},
+ {"classifier": {}},
+ {"options": {"capable_system_prompt": "ignored"}},
+])
+def test_switchyard_write_rejects_unloadable_configuration(invalid: dict[str, object]) -> None:
+ from litellm.proxy.management_endpoints.model_management_endpoints import _strategy_router_write_violation
+ from litellm.types.router import updateLiteLLMParams
+
+ violation: Final = _strategy_router_write_violation(
+ incoming_params=updateLiteLLMParams(
+ model="auto_router/switchyard",
+ switchyard_router_config={"models": {"capable": "strong", "efficient": "fast"}, **invalid},
+ ),
+ existing_params=None,
+ )
+ assert violation is not None
+ assert "Invalid switchyard_router_config" in violation
+
+
+SWITCHYARD_PARAMS: Final = {
+ "model": "auto_router/switchyard",
+ "switchyard_router_config": {"models": {"capable": "strong", "efficient": "fast"}},
+}
+
+
+@pytest.mark.parametrize(
+ ("model_name", "incoming", "existing"),
+ [
+ ("strong", SWITCHYARD_PARAMS, None),
+ ("fast", {"model": "auto_router/switchyard"}, SWITCHYARD_PARAMS),
+ ("fast", None, SWITCHYARD_PARAMS),
+ (
+ "smart",
+ {
+ "model": "auto_router/complexity_router",
+ "complexity_router_config": {"tiers": {"SIMPLE": "smart", "COMPLEX": "strong"}},
+ },
+ None,
+ ),
+ ],
+)
+def test_strategy_router_that_lists_itself_is_rejected_before_saving(
+ model_name: str, incoming: dict[str, object] | None, existing: dict[str, object] | None
+) -> None:
+ from litellm.proxy._types import ProxyException
+ from litellm.proxy.management_endpoints.model_management_endpoints import (
+ _raise_on_strategy_router_write_violation,
+ )
+ from litellm.types.router import updateLiteLLMParams
+
+ with pytest.raises(ProxyException) as exc_info:
+ _raise_on_strategy_router_write_violation(
+ incoming_params=updateLiteLLMParams(**incoming) if incoming is not None else None,
+ existing_params=updateLiteLLMParams(**existing) if existing is not None else None,
+ model_name=model_name,
+ )
+ assert exc_info.value.code == "400"
+ assert model_name in exc_info.value.message
+
+
+def test_strategy_router_with_other_targets_saves() -> None:
+ from litellm.proxy.management_endpoints.model_management_endpoints import _self_routing_violation
+ from litellm.types.router import updateLiteLLMParams
+
+ assert _self_routing_violation("route", updateLiteLLMParams(**SWITCHYARD_PARAMS), None) is None
+
+
+def test_self_routing_is_caught_when_the_stored_model_is_encrypted(monkeypatch: pytest.MonkeyPatch) -> None:
+ from litellm.proxy._types import ProxyException
+ from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
+ from litellm.proxy.management_endpoints.model_management_endpoints import (
+ _raise_on_strategy_router_write_violation,
+ )
+ from litellm.types.router import updateLiteLLMParams
+
+ monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key-for-encryption")
+ stored: Final = updateLiteLLMParams(
+ **{**SWITCHYARD_PARAMS, "model": encrypt_value_helper("auto_router/switchyard")}
+ )
+ assert stored.model != "auto_router/switchyard"
+ with pytest.raises(ProxyException) as exc_info:
+ _raise_on_strategy_router_write_violation(
+ incoming_params=updateLiteLLMParams(
+ switchyard_router_config={"models": {"capable": "route", "efficient": "fast"}}
+ ),
+ existing_params=stored,
+ model_name="route",
+ )
+ assert exc_info.value.code == "400"
+
+
@pytest.mark.asyncio
async def test_rename_rewrites_only_in_a_transaction_holding_every_locked_teams_advisory_lock():
"""A team that starts listing the old name between the advisory snapshot and the row lock must not be
diff --git a/tests/unit/router_strategy/switchyard_router/__init__.py b/tests/unit/router_strategy/switchyard_router/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/unit/router_strategy/switchyard_router/test_config.py b/tests/unit/router_strategy/switchyard_router/test_config.py
new file mode 100644
index 00000000000..c24023ddb79
--- /dev/null
+++ b/tests/unit/router_strategy/switchyard_router/test_config.py
@@ -0,0 +1,100 @@
+from typing import Final
+
+import pytest
+from pydantic import ValidationError
+
+from litellm.router_strategy.switchyard_router.config import SwitchyardRouterConfig, validate_switchyard_config_write
+
+
+@pytest.mark.parametrize("classifier", [None, {}])
+def test_default_confidence_depends_on_classifier(classifier: dict[str, object] | None) -> None:
+ config: Final = SwitchyardRouterConfig.model_validate(
+ {
+ "models": {
+ "capable": "strong",
+ "efficient": "fast",
+ **({"judge": "judge"} if classifier is not None else {}),
+ },
+ "classifier": classifier,
+ }
+ )
+ assert config.confidence_threshold == (0.7 if classifier is not None else 0.5)
+
+
+@pytest.mark.parametrize("classifier", [None, {}])
+def test_explicit_confidence_is_preserved(classifier: dict[str, object] | None) -> None:
+ config: Final = SwitchyardRouterConfig.model_validate(
+ {
+ "models": {
+ "capable": "strong",
+ "efficient": "fast",
+ **({"judge": "judge"} if classifier is not None else {}),
+ },
+ "classifier": classifier,
+ "options": {"confidence_threshold": 0.63, "picker": "capable_first"},
+ }
+ )
+ assert config.confidence_threshold == 0.63
+
+
+@pytest.mark.parametrize(
+ "option", ["escalation_note", "deescalation_note", "capable_system_prompt", "efficient_system_prompt", "unknown"]
+)
+def test_request_rewrite_and_unknown_options_are_rejected(option: str) -> None:
+ with pytest.raises(ValidationError, match=option):
+ SwitchyardRouterConfig.model_validate(
+ {"models": {"capable": "strong", "efficient": "fast"}, "options": {option: "ignored"}}
+ )
+
+
+@pytest.mark.parametrize(
+ "bad",
+ [
+ {"algorithm": "random"},
+ {"classifier": {}},
+ {"options": {"confidence_threshold": 2}},
+ {"models": {"capable": "", "efficient": "fast"}},
+ {"models": {"capable": "same", "efficient": "same"}},
+ {"models": {"capable": "strong", "efficient": "fast", "judge": "j"}, "classifier": {"session_affinity": True}},
+ {"models": {"capable": "strong", "efficient": "fast", "judge": "judge"}},
+ {"models": {"capable": "strong", "efficient": "fast", "judge": "j"}, "classifier": {"kind": "other"}},
+ {
+ "models": {"capable": "strong", "efficient": "fast", "judge": "j"},
+ "classifier": {"kind": "decision", "cutoff": 1.5},
+ },
+ {
+ "models": {"capable": "strong", "efficient": "fast", "judge": "j"},
+ "classifier": {"kind": "decision", "base_threshold": 0.5},
+ },
+ {
+ "models": {"capable": "strong", "efficient": "fast", "judge": "j"},
+ "classifier": {"kind": "llm", "cutoff": 0.5},
+ },
+ ],
+)
+def test_invalid_config_cannot_be_saved(bad: dict[str, object]) -> None:
+ violation: Final = validate_switchyard_config_write({"models": {"capable": "strong", "efficient": "fast"}, **bad})
+ assert violation is not None
+ assert "Invalid switchyard_router_config" in violation
+
+
+def test_missing_package_rejects_save_with_install_instructions() -> None:
+ import sys
+ from unittest.mock import patch
+
+ with patch.dict(sys.modules, {"switchyard_rust.libsy": None}):
+ violation: Final = validate_switchyard_config_write({"models": {"capable": "strong", "efficient": "fast"}})
+ assert violation is not None
+ assert "litellm[switchyard]" in violation
+
+
+@pytest.mark.parametrize(
+ "classifier", [{"kind": "decision"}, {"kind": "decision", "cutoff": 0.4, "evidence": {"a": 1}}]
+)
+def test_decision_classifier_config_can_be_saved(classifier: dict[str, object]) -> None:
+ assert (
+ validate_switchyard_config_write(
+ {"models": {"capable": "strong", "efficient": "fast", "judge": "jev"}, "classifier": classifier}
+ )
+ is None
+ )
diff --git a/tests/unit/router_strategy/switchyard_router/test_switchyard_router.py b/tests/unit/router_strategy/switchyard_router/test_switchyard_router.py
new file mode 100644
index 00000000000..8a26b4610db
--- /dev/null
+++ b/tests/unit/router_strategy/switchyard_router/test_switchyard_router.py
@@ -0,0 +1,438 @@
+import asyncio
+import json
+from itertools import chain
+from collections.abc import AsyncIterator, Mapping
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.router_api import ChangeSet
+from litellm.router_strategy.switchyard_router.config import SwitchyardRouterConfig, build_algorithm
+from litellm.router_strategy.switchyard_router.switchyard_router import SwitchyardRouter
+from litellm.types.decisions import OpenAIDecisionResponse
+from litellm.types.router import Deployment
+from litellm.types.utils import ModelResponse
+
+libsy = pytest.importorskip("switchyard_rust.libsy")
+
+HELLO: Final = [{"role": "user", "content": "Hello"}]
+
+
+async def unexpected_judge(request: Mapping[str, object]) -> ModelResponse:
+ raise AssertionError("This route must not call an LLM judge")
+
+
+async def unexpected_decision(request: Mapping[str, object]) -> OpenAIDecisionResponse:
+ raise AssertionError("This route must not call a decision model")
+
+
+def config(
+ *, classifier: Mapping[str, object] | None = None, picker: str = "efficient_first", threshold: float | None = None
+) -> SwitchyardRouterConfig:
+ return SwitchyardRouterConfig.model_validate(
+ {
+ "models": {
+ "capable": "strong",
+ "efficient": "fast",
+ **({"judge": "judge"} if classifier is not None else {}),
+ },
+ "options": {"picker": picker, **({"confidence_threshold": threshold} if threshold is not None else {})},
+ **({"classifier": classifier} if classifier is not None else {}),
+ }
+ )
+
+
+def router(
+ route_config: SwitchyardRouterConfig,
+ *,
+ completion=unexpected_judge,
+ decide=unexpected_decision,
+ algorithm: object = None,
+) -> SwitchyardRouter:
+ return SwitchyardRouter("route", route_config, algorithm or build_algorithm(route_config), completion, decide)
+
+
+async def route(
+ route_config: SwitchyardRouterConfig,
+ messages: object,
+ *,
+ call_type: str | None = "acompletion",
+ request_kwargs: Mapping[str, object] | None = None,
+ **router_kwargs: object,
+) -> tuple[str, Mapping[str, object]]:
+ result: Final = await router(route_config, **router_kwargs).async_pre_routing_hook( # pyright: ignore[reportArgumentType] # test doubles
+ "route", request_kwargs or {}, messages, call_type=call_type
+ )
+ assert result is not None
+ assert result.routing_decision is not None
+ return result.model, result.routing_decision
+
+
+def anthropic_turn(call_id: str, tool: str, result: str, marked: bool) -> tuple[dict[str, object], ...]:
+ error: Final = {"is_error": True} if marked else {}
+ return (
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": "Next step", "signature": "sig"},
+ {"type": "tool_use", "id": call_id, "name": tool, "input": {"file_path": "a.py"}},
+ ],
+ },
+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": call_id, "content": result, **error}]},
+ )
+
+
+def responses_turn(call_id: str, tool: str, result: str, marked: bool) -> tuple[dict[str, object], ...]:
+ return (
+ {"type": "function_call", "call_id": call_id, "name": tool, "arguments": '{"file_path":"a.py"}'},
+ {"type": "function_call_output", "call_id": call_id, "output": result},
+ )
+
+
+def chat_turn(call_id: str, tool: str, result: str, marked: bool) -> tuple[dict[str, object], ...]:
+ call: Final = {"id": call_id, "type": "function", "function": {"name": tool, "arguments": '{"file_path":"a.py"}'}}
+ return (
+ {"role": "assistant", "content": None, "tool_calls": [call]},
+ {"role": "tool", "tool_call_id": call_id, "content": result},
+ )
+
+
+TURN_FORMATS: Final = {"anthropic_messages": anthropic_turn, "aresponses": responses_turn, "acompletion": chat_turn}
+
+
+def transcript(endpoint: str, tool: str, results: tuple[str, ...], *, marked: bool = False) -> list[dict[str, object]]:
+ turn: Final = TURN_FORMATS[endpoint]
+ turns: Final = chain.from_iterable(
+ turn(f"call{index}", tool, result, marked) for index, result in enumerate(results)
+ )
+ return [{"role": "user", "content": "Fix the helper"}, *turns]
+
+
+def llm_router(
+ *, route_config: Mapping[str, object], judge_fails: bool = False, num_retries: int = 0
+) -> litellm.Router:
+ def deployment(name: str, reply: object) -> Mapping[str, object]:
+ return {
+ "model_name": name,
+ "litellm_params": {"model": f"openai/{name}", "api_key": "k", "mock_response": reply},
+ }
+
+ return litellm.Router(
+ num_retries=num_retries,
+ model_list=[
+ deployment("strong", "strong"),
+ deployment("fast", "fast"),
+ *((deployment("judge", Exception("judge down")),) if judge_fails else ()),
+ switchyard_deployment(route_config),
+ ],
+ )
+
+
+def switchyard_deployment(route_config: Mapping[str, object]) -> dict[str, object]:
+ return {
+ "model_name": "route",
+ "litellm_params": {"model": "auto_router/switchyard", "switchyard_router_config": route_config},
+ "model_info": {"id": "route-id"},
+ }
+
+
+@pytest.mark.asyncio
+async def test_critical_tool_error_selects_capable_without_rewriting_messages() -> None:
+ messages: Final = transcript("acompletion", "bash", ("CUDA out of memory",))
+ result: Final = await router(config()).async_pre_routing_hook("route", {}, messages, call_type="acompletion")
+ assert result is not None
+ assert result.model == "strong"
+ assert result.messages == messages
+
+
+async def routed_reply(proxy: litellm.Router, endpoint: str, edits: tuple[str, ...]) -> object:
+ match endpoint:
+ case "anthropic_messages":
+ message: Final = await proxy.aanthropic_messages(
+ model="route", messages=transcript(endpoint, "Edit", edits), max_tokens=16
+ )
+ return message["content"][0]["text"] # pyright: ignore[reportIndexIssue] # non-streaming response
+ case "aresponses":
+ response: Final = await proxy.aresponses(model="route", input=transcript(endpoint, "Edit", edits))
+ return response.output[0].content[0].text # pyright: ignore[reportAttributeAccessIssue] # non-streaming response
+ case _:
+ chat: Final = await proxy.acompletion(model="route", messages=transcript(endpoint, "Edit", edits))
+ return chat.choices[0].message.content # pyright: ignore[reportAttributeAccessIssue] # non-streaming response
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("endpoint", ["acompletion", "anthropic_messages", "aresponses"])
+async def test_router_routes_agent_traffic_from_every_endpoint(endpoint: str) -> None:
+ proxy: Final = llm_router(route_config={"models": {"capable": "strong", "efficient": "fast"}})
+ assert await routed_reply(proxy, endpoint, ("Edit applied",) * 3) == "fast"
+
+
+@pytest.mark.asyncio
+async def test_router_follows_an_edited_switchyard_router_and_forgets_a_deleted_one() -> None:
+ proxy: Final = llm_router(route_config={"models": {"capable": "strong", "efficient": "fast"}})
+ assert await routed_reply(proxy, "acompletion", ("Edit applied",) * 3) == "fast"
+
+ swapped: Final = switchyard_deployment({"models": {"capable": "fast", "efficient": "strong"}})
+ _ = proxy.apply(ChangeSet(upsert=[Deployment(**swapped)]))
+ assert await routed_reply(proxy, "acompletion", ("Edit applied",) * 3) == "strong"
+
+ _ = proxy.apply(ChangeSet(delete=["route-id"]))
+ with pytest.raises(litellm.BadRequestError):
+ await proxy.acompletion(model="route", messages=HELLO)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("marked", "expected"), [(True, "strong"), (False, "fast")])
+async def test_anthropic_tool_failure_marker_reaches_switchyard(marked: bool, expected: str) -> None:
+ failures: Final = transcript("anthropic_messages", "Edit", ("String to replace not found.",) * 4, marked=marked)
+ model, decision = await route(config(), failures, call_type="anthropic_messages")
+ assert model == expected
+ assert "switchyard_fallback_reason" not in decision
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("request_kwargs", "expected"),
+ [
+ ({"litellm_session_id": "s1"}, ["strong", "strong", "strong", "fast"]),
+ (
+ {"proxy_server_request": {"headers": {"X-Session-Id": "s1", "session-id": "s1"}}},
+ ["strong", "fast", "fast", "fast"],
+ ),
+ ({}, ["strong", "fast", "fast", "fast"]),
+ ],
+)
+async def test_capable_hold_follows_the_session(request_kwargs: Mapping[str, object], expected: list[str]) -> None:
+ switchyard: Final = router(config())
+ turns: Final = (("CUDA out of memory",), *((("All tests passed",) * count) for count in (2, 3, 4)))
+ picks: Final = [await pick(switchyard, request_kwargs, results) for results in turns]
+ assert picks == expected
+
+
+async def pick(switchyard: SwitchyardRouter, request_kwargs: Mapping[str, object], results: tuple[str, ...]) -> str:
+ result: Final = await switchyard.async_pre_routing_hook(
+ "route", request_kwargs, transcript("acompletion", "bash", results), call_type="acompletion"
+ )
+ assert result is not None
+ return result.model
+
+
+@pytest.mark.asyncio
+async def test_a_session_hold_never_crosses_api_keys() -> None:
+ switchyard: Final = router(config())
+ owner: Final = {"litellm_session_id": "shared", "metadata": {"user_api_key_hash": "owner"}}
+ other: Final = {"litellm_session_id": "shared", "metadata": {"user_api_key_hash": "other"}}
+ assert await pick(switchyard, owner, ("CUDA out of memory",)) == "strong"
+ assert await pick(switchyard, other, ("All tests passed",) * 2) == "fast"
+ assert await pick(switchyard, owner, ("All tests passed",) * 2) == "strong"
+
+
+@pytest.mark.asyncio
+async def test_a_client_metadata_key_hash_cannot_join_another_keys_session() -> None:
+ switchyard: Final = router(config())
+ owner: Final = {"litellm_session_id": "shared", "litellm_metadata": {"user_api_key_hash": "owner"}}
+ spoofer: Final = {
+ "litellm_session_id": "shared",
+ "metadata": {"user_api_key_hash": "owner"},
+ "litellm_metadata": {"user_api_key_hash": "spoofer"},
+ }
+ assert await pick(switchyard, owner, ("CUDA out of memory",)) == "strong"
+ assert await pick(switchyard, spoofer, ("All tests passed",) * 2) == "fast"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("p_solve", "expected"), [(0.0, "strong"), (1.0, "fast")])
+async def test_llm_judge_verdict_selects_the_target(p_solve: float, expected: str) -> None:
+ async def judge(request: Mapping[str, object]) -> ModelResponse:
+ assert "autorouter_classifier" in str(request["metadata"])
+ verdict: Final = {"crux": "c", "primary_rule": "none", "capability_boundary": "unmatched", "p_solve": p_solve}
+ return ModelResponse(choices=[{"message": {"role": "assistant", "content": json.dumps(verdict)}}])
+
+ model, _ = await route(config(classifier={}), HELLO, completion=judge)
+ assert model == expected
+
+
+def decision_answer(advantage: float) -> OpenAIDecisionResponse:
+ return OpenAIDecisionResponse.model_validate(
+ {
+ "model": "jev",
+ "answers": [
+ {
+ "type": "choice",
+ "name": "route",
+ "choice": "advantage",
+ "probabilities": [
+ {"value": "advantage", "probability": advantage},
+ {"value": "no_advantage", "probability": 1 - advantage},
+ ],
+ }
+ ],
+ "usage": {"input_tokens": 1, "output_tokens": 0, "total_tokens": 1},
+ }
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("parent_bucket", ["metadata", "litellm_metadata"])
+@pytest.mark.parametrize("classifier", [{}, {"kind": "decision"}])
+async def test_judge_calls_carry_the_callers_tags_and_origin_where_their_endpoint_reads_them(
+ parent_bucket: str, classifier: Mapping[str, object]
+) -> None:
+ seen: Final[list[Mapping[str, object]]] = [] # mutable-ok: collects the judge request
+
+ async def judge(request: Mapping[str, object]) -> ModelResponse:
+ seen.append(request)
+ verdict: Final = {"crux": "c", "primary_rule": "none", "capability_boundary": "unmatched", "p_solve": 1.0}
+ return ModelResponse(choices=[{"message": {"role": "assistant", "content": json.dumps(verdict)}}])
+
+ async def decide(request: Mapping[str, object]) -> OpenAIDecisionResponse:
+ seen.append(request)
+ return decision_answer(0.0)
+
+ caller: Final = {parent_bucket: {"tags": ["eu-only"], "inherited_tags": ["team-eu"]}}
+ await route(
+ config(classifier=classifier, threshold=1.0), HELLO, completion=judge, decide=decide, request_kwargs=caller
+ )
+ (request,) = seen
+ read_by_endpoint: Final = "litellm_metadata" if classifier else "metadata"
+ assert set(request) & {"metadata", "litellm_metadata"} == {read_by_endpoint}
+ metadata: Final = request[read_by_endpoint]
+ assert metadata["tags"] == ["eu-only"] # pyright: ignore[reportIndexIssue] # the judge request's metadata dict
+ assert metadata["inherited_tags"] == ["team-eu"] # pyright: ignore[reportIndexIssue] # same
+ assert metadata["internal_call_origin"] == "autorouter_classifier" # pyright: ignore[reportIndexIssue] # same
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("advantage", "expected"), [(0.61, "strong"), (0.6, "fast")])
+async def test_decision_model_advantage_must_exceed_the_cutoff(advantage: float, expected: str) -> None:
+ async def decide(request: Mapping[str, object]) -> OpenAIDecisionResponse:
+ (question,) = request["questions"] # pyright: ignore[reportGeneralTypeIssues] # the decisions request shape
+ assert {choice["value"] for choice in question["choices"]} == {"advantage", "no_advantage"}
+ return decision_answer(advantage)
+
+ model, decision = await route(
+ config(classifier={"kind": "decision", "cutoff": 0.6}, threshold=1.0), HELLO, decide=decide
+ )
+ assert model == expected
+ assert decision["switchyard_evidence"]["score"] == advantage
+
+
+async def judge_down(request: Mapping[str, object]) -> ModelResponse:
+ raise ConnectionError("judge down")
+
+
+async def decision_refused(request: Mapping[str, object]) -> OpenAIDecisionResponse:
+ return OpenAIDecisionResponse.model_validate(
+ {
+ "model": "jev",
+ "answers": [{"type": "refusal", "name": "route"}],
+ "usage": {"input_tokens": 1, "output_tokens": 0, "total_tokens": 1},
+ }
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("classifier", "router_kwargs", "picker", "expected"),
+ [
+ ({}, {"completion": judge_down}, "efficient_first", "fast"),
+ ({}, {"completion": judge_down}, "capable_first", "strong"),
+ ({"kind": "decision"}, {"decide": decision_refused}, "efficient_first", "fast"),
+ ],
+)
+async def test_classifier_failure_falls_open_to_the_picker_default(
+ classifier: Mapping[str, object], router_kwargs: Mapping[str, object], picker: str, expected: str
+) -> None:
+ model, decision = await route(config(classifier=classifier, picker=picker, threshold=1.0), HELLO, **router_kwargs)
+ assert model == expected
+ assert "switchyard_fallback_reason" not in decision
+
+
+class EndsWithoutDecision:
+ async def _steps(self) -> AsyncIterator[object]:
+ for step in ():
+ yield step
+
+ def run_stream(self, *args: object, **kwargs: object) -> AsyncIterator[object]:
+ return self._steps()
+
+
+class RaisesLibsyError:
+ async def _steps(self) -> AsyncIterator[object]:
+ raise libsy.LibsyError("algorithm failed")
+ yield
+
+ def run_stream(self, *args: object, **kwargs: object) -> AsyncIterator[object]:
+ return self._steps()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("route_config", "messages", "call_type", "algorithm", "reason"),
+ [
+ (
+ config(),
+ [{"role": "user", "content": [{"type": "input_audio"}]}],
+ "acompletion",
+ None,
+ "unsupported_content",
+ ),
+ (config(), HELLO, None, None, "unsupported_request_type"),
+ (config(), HELLO, "acompletion", EndsWithoutDecision(), "no_decision"),
+ (config(), HELLO, "acompletion", RaisesLibsyError(), "libsy_error"),
+ (
+ config(classifier={}, threshold=1.0),
+ HELLO,
+ "acompletion",
+ build_algorithm(config(classifier={"kind": "decision"}, threshold=1.0)),
+ "unexpected_decision_call",
+ ),
+ (
+ config(classifier={"kind": "decision"}, threshold=1.0),
+ HELLO,
+ "acompletion",
+ build_algorithm(config(classifier={}, threshold=1.0)),
+ "unexpected_judge_call",
+ ),
+ ],
+)
+async def test_anything_switchyard_cannot_route_goes_to_capable(
+ route_config: SwitchyardRouterConfig, messages: object, call_type: str | None, algorithm: object, reason: str
+) -> None:
+ model, decision = await route(route_config, messages, call_type=call_type, algorithm=algorithm)
+ assert model == "strong"
+ assert decision["switchyard_fallback_reason"] == reason
+
+
+@pytest.mark.asyncio
+async def test_failing_judge_is_called_once_even_when_the_router_retries(monkeypatch: pytest.MonkeyPatch) -> None:
+ attempts: Final[list[str]] = []
+
+ class CountAttempts(CustomLogger):
+ def log_pre_api_call(self, model: str, messages: object, kwargs: dict[str, object]) -> None:
+ attempts.append(model)
+
+ monkeypatch.setattr(litellm, "callbacks", [CountAttempts()])
+ proxy: Final = llm_router(
+ route_config={
+ "models": {"capable": "strong", "efficient": "fast", "judge": "judge"},
+ "classifier": {},
+ "options": {"confidence_threshold": 1.0},
+ },
+ judge_fails=True,
+ num_retries=3,
+ )
+ await proxy.acompletion(model="route", messages=HELLO)
+ assert attempts.count("judge") == 1
+
+
+@pytest.mark.asyncio
+async def test_judge_cancellation_propagates() -> None:
+ async def cancelled(request: Mapping[str, object]) -> ModelResponse:
+ raise asyncio.CancelledError()
+
+ with pytest.raises(asyncio.CancelledError):
+ await route(config(classifier={}), HELLO, completion=cancelled)
diff --git a/tests/unit/router_strategy/test_router_tag_routing.py b/tests/unit/router_strategy/test_router_tag_routing.py
index 0d3876c64a3..e916e5b235d 100644
--- a/tests/unit/router_strategy/test_router_tag_routing.py
+++ b/tests/unit/router_strategy/test_router_tag_routing.py
@@ -3023,7 +3023,7 @@ class _RewriteToTierStrategy:
self.rewrite_to = rewrite_to
async def async_pre_routing_hook(
- self, model, request_kwargs, messages=None, input=None, specific_deployment=False
+ self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None
):
from litellm.types.router import PreRoutingHookResponse
diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py
index 1b98cd4b4b6..ca61fcd0905 100644
--- a/tests/unit/test_router/test_router.py
+++ b/tests/unit/test_router/test_router.py
@@ -9006,7 +9006,7 @@ class TestConsumedRequestTagsStamp:
self.rewrite_to = rewrite_to
async def async_pre_routing_hook(
- self, model, request_kwargs, messages=None, input=None, specific_deployment=False
+ self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None
):
from litellm.types.router import PreRoutingHookResponse
@@ -9097,7 +9097,7 @@ class TestClaudeCodeSubagentSessionRouterBinding:
self.routed_model = routed_model
async def async_pre_routing_hook(
- self, model, request_kwargs, messages=None, input=None, specific_deployment=False
+ self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None
):
from litellm.types.router import PreRoutingHookResponse
@@ -10151,7 +10151,7 @@ class TestModelGroupAliasReachesPreRoutingStrategies:
class _RewriteStrategy:
async def async_pre_routing_hook(
- self, model, request_kwargs, messages=None, input=None, specific_deployment=False
+ self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None
):
from litellm.types.router import PreRoutingHookResponse
@@ -10268,7 +10268,7 @@ class TestTeamPublicNameReachesPreRoutingStrategies:
self.rewrite_to = rewrite_to
async def async_pre_routing_hook(
- self, model, request_kwargs, messages=None, input=None, specific_deployment=False
+ self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None
):
from litellm.types.router import PreRoutingHookResponse
@@ -10505,7 +10505,7 @@ class TestAutoRouterCompressionDecoupling:
self.received_messages: list[dict] | None = None
async def async_pre_routing_hook(
- self, model, request_kwargs, messages=None, input=None, specific_deployment=False
+ self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None
):
from litellm.types.router import PreRoutingHookResponse
diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py
index d71791b6f11..aae5dfe6472 100644
--- a/tests/unit/types/test_litellm_params.py
+++ b/tests/unit/types/test_litellm_params.py
@@ -168,6 +168,7 @@ OPTION_NAMES: Final = (
"tag_regex",
"max_file_size_mb",
"silent_model",
+ "switchyard_router_config",
"auto_router_config_path",
"auto_router_config",
"auto_router_default_model",
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts
index f9304bab0d2..d13bc3d3181 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts
@@ -109,6 +109,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment {
model?: string | null;
base_model?: string | null;
custom_llm_provider?: string | null;
+ switchyard_router_config?: unknown;
complexity_router_config?: unknown;
complexity_router_default_model?: string | null;
auto_router_config?: unknown;
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts
index 2cdc13606bb..0fe320a318a 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts
@@ -87,6 +87,14 @@ const configManaged = (label: string, config: Record): Presenta
/** How each strategy renders itself, given its own config object. */
const PRESENTERS: Record) => Presentation> = {
+ switchyard: (config) => ({
+ typeLabel: "Switchyard",
+ targets: dedupe(
+ [asRecord(config.models).capable, asRecord(config.models).efficient].filter(
+ (model): model is string => typeof model === "string",
+ ),
+ ),
+ }),
complexity: (config) => ({
typeLabel: complexityTypeLabel(config),
targets: dedupe(Object.values(asRecord(config.tiers)).flatMap(normalizeTierModels)),
@@ -131,7 +139,8 @@ export const toAutoRouterRow = (
canDelete: canDelete && mayActOnRow,
editBlockedReason,
createdAt: info.created_at ?? undefined,
- defaultModel: (params[strategy.defaultModelKey] as string | null | undefined) ?? null,
+ defaultModel:
+ strategy.kind === "switchyard" ? null : (params[strategy.defaultModelKey] as string | null | undefined) ?? null,
deployment,
...PRESENTERS[strategy.kind](asRecord(params[strategy.configKey])),
};
diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx
index bd8b4eb9854..0dd80854312 100644
--- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx
@@ -135,6 +135,7 @@ interface AutoRouterClassifierTabsProps {
children: React.ReactNode;
judgeModel?: React.ReactNode;
decisionModel?: React.ReactNode;
+ switchyard?: { selected: boolean; onSelect: () => void; fields: React.ReactNode };
}
const AutoRouterClassifierTabs: React.FC = ({
@@ -143,6 +144,7 @@ const AutoRouterClassifierTabs: React.FC = ({
children,
judgeModel,
decisionModel,
+ switchyard,
}) => {
const { t } = useTranslation();
const id = useId();
@@ -159,14 +161,16 @@ const AutoRouterClassifierTabs: React.FC = ({
jev: "jev",
custom: "custom",
};
- const family = familyByType[classifierType];
+ const family = switchyard?.selected ? "switchyard" : familyByType[classifierType];
const hasCustomTiers = Boolean(value.custom_tier_set);
const changeType = (next: ClassifierType) => {
if (next !== classifierType) onChange(transitionClassifierType(value, next));
};
const changeFamily = (next: unknown) => {
+ if (next === "switchyard") return switchyard?.onSelect();
if (next === family) return;
+ if (next === familyByType[classifierType]) return onChange(value);
if (next === "heuristics") changeType("heuristic");
if (next === "llm") changeType("llm");
if (next === "jev") changeType("jev");
@@ -207,7 +211,11 @@ const AutoRouterClassifierTabs: React.FC = ({
-
+
{[
{
value: "heuristics",
@@ -220,6 +228,9 @@ const AutoRouterClassifierTabs: React.FC = ({
label: t("addModel.autoRouterClassifierTabs.families.decisionModel"),
description: t("addModel.autoRouterClassifierTabs.families.decisionModelDescription"),
},
+ ...(switchyard
+ ? [{ value: "switchyard", label: "Switchyard", description: "Route by the conversation’s tool history." }]
+ : []),
].map((option) => (
+ {family === "switchyard" && switchyard?.fields}
{family === "jev" &&
(decisionModel ?? (
@@ -355,7 +367,7 @@ const AutoRouterClassifierTabs: React.FC
= ({
)}
- {hasCustomTiers && (
+ {family !== "switchyard" && hasCustomTiers && (
{t("addModel.autoRouterClassifierTabs.restoreStandardTiers")}
)}
{availability.data?.error && !availability.isChecking && (
@@ -372,7 +384,7 @@ const AutoRouterClassifierTabs: React.FC = ({
)}
- {children}
+ {family !== "switchyard" && children}
);
};
diff --git a/ui/litellm-dashboard/src/components/add_model/BlockedReasonTooltip.tsx b/ui/litellm-dashboard/src/components/add_model/BlockedReasonTooltip.tsx
new file mode 100644
index 00000000000..59588cf478f
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/BlockedReasonTooltip.tsx
@@ -0,0 +1,17 @@
+import React from "react";
+import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
+
+const BlockedReasonTooltip: React.FC<{ reason: string | null; children: React.ReactElement }> = ({
+ reason,
+ children,
+}) =>
+ reason === null ? (
+ children
+ ) : (
+
+
+ {reason}
+
+ );
+
+export default BlockedReasonTooltip;
diff --git a/ui/litellm-dashboard/src/components/add_model/SwitchyardConfigFields.tsx b/ui/litellm-dashboard/src/components/add_model/SwitchyardConfigFields.tsx
new file mode 100644
index 00000000000..2043e392928
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/SwitchyardConfigFields.tsx
@@ -0,0 +1,342 @@
+"use client";
+
+import { useId, useState } from "react";
+import { Input } from "@/components/ui/input";
+import { Textarea } from "@/components/ui/textarea";
+import { Slider } from "@/components/ui/slider";
+import { Switch } from "@/components/ui/switch";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+import ModelChoiceCombobox, { type ModelChoice } from "./ModelChoiceCombobox";
+import {
+ stageConfidence,
+ toggleClassifier,
+ setClassifierKind,
+ parseToolSemantics,
+ type SwitchyardConfig,
+} from "./switchyard_config";
+
+interface Props {
+ value: SwitchyardConfig;
+ onChange: (value: SwitchyardConfig) => void;
+ models: ModelChoice[];
+ decisionModels: ModelChoice[];
+ onToolSemanticsError: (error: string | null) => void;
+}
+
+const isStructured = (instructions: unknown): instructions is Record =>
+ typeof instructions === "object" && instructions !== null;
+const instructionsText = (instructions: string | Record | null | undefined): string =>
+ isStructured(instructions) ? JSON.stringify(instructions, null, 2) : instructions ?? "";
+
+export default function SwitchyardConfigFields({
+ value,
+ onChange,
+ models,
+ decisionModels,
+ onToolSemanticsError,
+}: Props) {
+ const id = useId();
+ const [toolSemantics, setToolSemantics] = useState(
+ value.options.tool_semantics ? JSON.stringify(value.options.tool_semantics, null, 2) : "",
+ );
+ const classifier = value.classifier;
+ const llm = classifier?.kind === "llm" ? classifier : undefined;
+ const decision = classifier?.kind === "decision" ? classifier : undefined;
+ const setOption = (key: "recent_window", text: string) =>
+ onChange({ ...value, options: { ...value.options, [key]: text === "" ? undefined : Number(text) } });
+ const setClassifierNumber = (
+ key: "base_threshold" | "threshold_step" | "max_output_tokens" | "recent_turn_window",
+ text: string,
+ ) => {
+ if (llm) onChange({ ...value, classifier: { ...llm, [key]: text === "" ? undefined : Number(text) } });
+ };
+ const changeToolSemantics = (text: string) => {
+ setToolSemantics(text);
+ try {
+ const parsed = parseToolSemantics(text);
+ onChange({ ...value, options: { ...value.options, tool_semantics: parsed } });
+ onToolSemanticsError(null);
+ } catch {
+ onToolSemanticsError("Tool semantics must be JSON mapping observe, mutate, plan, or new to arrays of tool names");
+ }
+ };
+ return (
+
+
+
+
+ Heuristic
+
+
+ Stage Router
+
+
+
+
+
+ {value.options.picker === "capable_first" && (
+
+ Switchyard has not benchmarked this bias, so it has no calibrated confidence thresholds.
+
+ )}
+
+ {(["capable", "efficient"] as const).map((tier) => (
+
+
+ onChange({ ...value, models: { ...value.models, [tier]: model ?? "" } })}
+ choices={models}
+ placeholder="Select a model"
+ ariaInvalid={undefined}
+ ariaDescribedBy={undefined}
+ />
+
+ ))}
+
+
+
+
+
+
+ Ask a judge model when the tool history does not give a confident routing decision.
+
+
+
onChange(toggleClassifier(value, enabled))}
+ />
+
+ {classifier && (
+
+
+
+
+
+
+
+ onChange({ ...value, models: { ...value.models, judge: judge ?? "" } })}
+ choices={decision ? decisionModels : models}
+ placeholder={decision ? "Select a decision model" : "Select a judge model"}
+ ariaInvalid={undefined}
+ ariaDescribedBy={undefined}
+ />
+
+
+ )}
+
+
+
+
+ Confidence threshold
+
+ {stageConfidence(value).toFixed(2)}
+
+
+ onChange({
+ ...value,
+ options: { ...value.options, confidence_threshold: Array.isArray(next) ? next[0] : next },
+ })
+ }
+ />
+
+ Below this confidence, use {classifier ? "the judge model" : "the bias’s default model"}. Critical errors can
+ still select the capable model.
+
+
+
+ Advanced settings
+
+
Stage
+
+
+ setOption("recent_window", event.target.value)}
+ />
+
+
+
+
+ {decision && (
+ <>
+
Decision model
+
+
+
+ onChange({ ...value, classifier: { ...decision, cutoff: Number(event.target.value) } })
+ }
+ />
+
+ Route to the capable model when the decision model’s advantage probability is above this value.
+
+
+
+
+
+ >
+ )}
+ {llm && (
+ <>
+
Classifier
+
+ {(
+ [
+ ["base_threshold", "Capability threshold"],
+ ["threshold_step", "Threshold step"],
+ ["max_output_tokens", "Maximum output tokens"],
+ ["recent_turn_window", "Recent turn window"],
+ ] as const
+ ).map(([key, label]) => (
+
+
+ setClassifierNumber(key, event.target.value)}
+ />
+
+ ))}
+
+
+
+
+
+
+
+
+ >
+ )}
+
+
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/add_model/SwitchyardRouterForm.tsx b/ui/litellm-dashboard/src/components/add_model/SwitchyardRouterForm.tsx
new file mode 100644
index 00000000000..445ab8120c0
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/SwitchyardRouterForm.tsx
@@ -0,0 +1,176 @@
+"use client";
+
+import { useId, useState } from "react";
+import { useQuery } from "@tanstack/react-query";
+import { Input } from "@/components/ui/input";
+import { Button } from "@/components/ui/button";
+import { fetchAvailableModels } from "@/components/llm_calls/fetch_models";
+import { modelPatchUpdateCall } from "@/components/networking";
+import {
+ emptySwitchyardConfig,
+ isDecisionModelMode,
+ switchyardConfigSchema,
+ switchyardRouterPayload,
+ switchyardSaveError,
+ type SwitchyardConfig,
+} from "./switchyard_config";
+import SwitchyardConfigFields from "./SwitchyardConfigFields";
+import { handleAddSwitchyardSubmit, type SwitchyardModelInfo } from "./handle_add_auto_router_submit";
+
+interface FieldsProps {
+ accessToken: string;
+ routerName: string;
+ value: SwitchyardConfig;
+ onChange: (value: SwitchyardConfig) => void;
+ onToolSemanticsError: (error: string | null) => void;
+}
+
+export function SwitchyardModelFields({ accessToken, routerName, value, onChange, onToolSemanticsError }: FieldsProps) {
+ const { data: models = [], isError } = useQuery({
+ queryKey: ["availableModels", "switchyard", accessToken],
+ queryFn: () => fetchAvailableModels(accessToken),
+ enabled: Boolean(accessToken),
+ });
+ const routable = models.filter(
+ (model) => model.model_group !== routerName && !model.model_group.startsWith("auto_router/"),
+ );
+ const toChoice = (model: { model_group: string }) => ({ value: model.model_group, label: model.model_group });
+ return (
+
+
+ Route between a capable and an efficient model using the conversation’s tool history.
+
+ {isError && (
+
+ Unable to load available models.
+
+ )}
+
!isDecisionModelMode(model.mode)).map(toChoice)}
+ decisionModels={routable.filter((model) => isDecisionModelMode(model.mode)).map(toChoice)}
+ onToolSemanticsError={onToolSemanticsError}
+ />
+
+ );
+}
+
+interface StageSeed {
+ capable?: string;
+ efficient?: string;
+}
+
+export function useSwitchyardOption(accessToken: string, routerName: string, enabled: boolean) {
+ const [selected, setSelected] = useState(false);
+ const [config, setConfig] = useState(emptySwitchyardConfig);
+ const [toolSemanticsError, setToolSemanticsError] = useState(null);
+ const select = (seed: StageSeed) => {
+ setSelected(true);
+ setToolSemanticsError(null);
+ setConfig((current) => ({
+ ...current,
+ models: {
+ ...current.models,
+ capable: current.models.capable || seed.capable || "",
+ efficient: current.models.efficient || seed.efficient || "",
+ },
+ }));
+ };
+ return {
+ selected,
+ deselect: () => setSelected(false),
+ option: (seed: StageSeed) =>
+ enabled
+ ? {
+ selected,
+ onSelect: () => select(seed),
+ fields: (
+
+ ),
+ }
+ : undefined,
+ submit: (name: string, modelInfo: SwitchyardModelInfo, resetForm: () => void, onSuccess: () => void) => {
+ const draft = { name, config, toolSemanticsError, modelInfo };
+ return handleAddSwitchyardSubmit(draft, accessToken, resetForm, onSuccess);
+ },
+ };
+}
+
+interface Props {
+ accessToken: string;
+ onSuccess: (saved: { model_name: string; switchyard_router_config: SwitchyardConfig }) => void;
+ initial: { name: string; modelId: string; config: unknown };
+}
+
+export default function SwitchyardRouterForm({ accessToken, onSuccess, initial }: Props) {
+ const id = useId();
+ const parsed = switchyardConfigSchema.safeParse(initial.config);
+ const [config, setConfig] = useState(parsed.success ? parsed.data : emptySwitchyardConfig());
+ const [name, setName] = useState(initial.name);
+ const [error, setError] = useState(
+ parsed.success ? null : "The stored Switchyard configuration is invalid. Correct it before saving.",
+ );
+ const [toolError, setToolError] = useState(null);
+ const [saving, setSaving] = useState(false);
+ const save = async () => {
+ const invalid = switchyardSaveError(name, config);
+ if (invalid) {
+ setError(invalid);
+ return;
+ }
+ setSaving(true);
+ setError(null);
+ try {
+ const payload = switchyardRouterPayload(name, config);
+ await modelPatchUpdateCall(accessToken, payload, initial.modelId);
+ onSuccess({
+ model_name: payload.model_name,
+ switchyard_router_config: payload.litellm_params.switchyard_router_config,
+ });
+ } catch (failure) {
+ setError(failure instanceof Error ? failure.message : "Unable to save Switchyard router");
+ } finally {
+ setSaving(false);
+ }
+ };
+ return (
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx
index d4c74785107..f348fce9eb4 100644
--- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx
@@ -19,7 +19,7 @@ import userEvent from "@testing-library/user-event";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import AddAutoRouterTab from "./add_auto_router_tab";
import { toast } from "@/lib/toast";
-import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
+import { handleAddAutoRouterSubmit, handleAddSwitchyardSubmit } from "./handle_add_auto_router_submit";
import { getMissingTiersError } from "./build_complexity_router_config";
import { getSubmitBlockedReason } from "./add_auto_router_tab";
import { buildModelAvailability } from "@/lib/autorouter_presets";
@@ -145,6 +145,7 @@ vi.mock("@/app/(dashboard)/hooks/models/useModels", async (importOriginal) => {
vi.mock("./handle_add_auto_router_submit", () => ({
handleAddAutoRouterSubmit: vi.fn(),
+ handleAddSwitchyardSubmit: vi.fn(),
}));
// Kept real by default so the "mandatory field" test still sees genuine tier validation; one
@@ -179,7 +180,7 @@ vi.mock("../common_components/team_dropdown", () => ({
const renderAdvanced: typeof renderWithProviders = (...args) => {
const result = renderWithProviders(...args);
- fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true }));
+ fireEvent.click(screen.getByRole("button", { name: "Customize tiers and keywords", exact: true }));
return result;
};
@@ -202,6 +203,64 @@ describe("AddAutoRouterTab", () => {
vi.mocked(modelCostMap).mockResolvedValue({});
});
+ it("swaps the tiers for the Switchyard fields and saves Switchyard through the same form", async () => {
+ renderAdvanced();
+ fireEvent.click(screen.getByRole("radio", { name: "Switchyard", exact: true }));
+ expect(screen.getByText("Stage Router", { exact: true })).toBeVisible();
+ expect(screen.queryByText("Define your tiers")).not.toBeInTheDocument();
+ expect(screen.getByTestId("auto-router-test-connect-btn")).not.toBeVisible();
+
+ fireEvent.click(screen.getByRole("radio", { name: "Heuristics", exact: true }));
+ expect(screen.getByText("Define your tiers")).toBeVisible();
+ expect(screen.queryByText("Stage Router", { exact: true })).not.toBeInTheDocument();
+
+ fireEvent.click(screen.getByRole("radio", { name: "Switchyard", exact: true }));
+ fireEvent.change(screen.getByLabelText(/Auto Router Name/), { target: { value: "stage-test" } });
+ fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
+ await waitFor(() => expect(handleAddSwitchyardSubmit).toHaveBeenCalledOnce());
+ expect(vi.mocked(handleAddSwitchyardSubmit).mock.calls[0][0].name).toBe("stage-test");
+ expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
+ });
+
+ it("starts Switchyard from the quick setup models and creates it for the selected team", async () => {
+ mockFetchAvailableModels.mockResolvedValue(
+ ["claude-haiku-5-5", "claude-opus-5-5"].map((model_group) => ({ model_group, mode: "chat" })),
+ );
+ const user = userEvent.setup();
+ renderWithProviders(
+ ,
+ );
+ fireEvent.click(screen.getByRole("radio", { name: "Switchyard", exact: true }));
+ fireEvent.change(screen.getByLabelText(/Auto Router Name/), { target: { value: "stage-test" } });
+ fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
+ await waitFor(() => expect(screen.getByLabelText(/Auto Router Name/)).toHaveValue("stage-test"));
+ expect(handleAddSwitchyardSubmit).not.toHaveBeenCalled();
+
+ fireEvent.click(screen.getByRole("radio", { name: "Heuristics", exact: true }));
+ await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1");
+ expect(await screen.findByRole("combobox", { name: "Everyday model" })).toHaveTextContent("claude-haiku-5-5");
+ fireEvent.click(screen.getByRole("radio", { name: "Switchyard", exact: true }));
+ fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
+ await waitFor(() => expect(handleAddSwitchyardSubmit).toHaveBeenCalledOnce());
+ expect(vi.mocked(handleAddSwitchyardSubmit).mock.calls[0][0]).toMatchObject({
+ config: { models: { capable: "claude-opus-5-5", efficient: "claude-haiku-5-5" } },
+ modelInfo: { team_id: "team-1" },
+ });
+ });
+
+ it("forgets a tool semantics error once Switchyard is reopened from the saved settings", async () => {
+ renderAdvanced();
+ fireEvent.click(screen.getByRole("radio", { name: "Switchyard", exact: true }));
+ fireEvent.change(screen.getByLabelText("Tool semantics"), { target: { value: "{not json" } });
+ fireEvent.click(screen.getByRole("radio", { name: "Heuristics", exact: true }));
+ fireEvent.click(screen.getByRole("radio", { name: "Switchyard", exact: true }));
+ expect(screen.getByLabelText("Tool semantics")).toHaveValue("");
+ fireEvent.change(screen.getByLabelText(/Auto Router Name/), { target: { value: "stage-test" } });
+ fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
+ await waitFor(() => expect(handleAddSwitchyardSubmit).toHaveBeenCalledOnce());
+ expect(vi.mocked(handleAddSwitchyardSubmit).mock.calls[0][0].toolSemanticsError).toBeNull();
+ });
+
it("creates the goal-selected latest OpenAI pair using its Bedrock deployment aliases", async () => {
mockFetchAvailableModels.mockResolvedValue(
["small", "middle-old", "middle", "large"].map((model_group) => ({ model_group, mode: "chat" })),
@@ -322,6 +381,25 @@ describe("AddAutoRouterTab", () => {
},
);
+ it("shows every strategy up front and carries the quick setup models into the one picked", async () => {
+ mockFetchAvailableModels.mockResolvedValue(
+ ["claude-haiku-5-5", "claude-opus-5-5"].map((model_group) => ({ model_group, mode: "chat" })),
+ );
+ renderWithProviders();
+ expect(await screen.findByRole("combobox", { name: "Everyday model" })).toHaveTextContent("claude-haiku-5-5");
+ expect(screen.getByRole("radio", { name: "Heuristics", exact: true })).toBeChecked();
+
+ fireEvent.click(screen.getByRole("radio", { name: "Switchyard", exact: true }));
+ expect(screen.queryByRole("combobox", { name: "Everyday model" })).not.toBeInTheDocument();
+ fireEvent.click(screen.getByRole("radio", { name: "Heuristics", exact: true }));
+ expect(screen.getByRole("combobox", { name: "Everyday model" })).toHaveTextContent("claude-haiku-5-5");
+
+ fireEvent.click(screen.getByRole("radio", { name: "LLM", exact: true }));
+ expect(screen.getByRole("combobox", { name: "Judge model" })).toBeInTheDocument();
+ expectTierModel("Simple", "claude-haiku-5-5");
+ expectTierModel("Reasoning", "claude-opus-5-5");
+ });
+
it("carries Responses model choices from simple setup into Advanced tiers", async () => {
const models = ["gpt-6-luna", "gpt-6-astra"];
mockFetchAvailableModels.mockResolvedValue(models.map((model_group) => ({ model_group, mode: "responses" })));
@@ -338,7 +416,7 @@ describe("AddAutoRouterTab", () => {
);
expect(screen.getByRole("combobox", { name: "Hard tasks model" })).toHaveTextContent("gpt-6-astra");
expect(screen.queryByText(/Two deployed models are needed/)).not.toBeInTheDocument();
- fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true }));
+ fireEvent.click(screen.getByRole("button", { name: "Customize tiers and keywords", exact: true }));
expectTierModel("Simple", "gpt-6-luna");
expectTierModel("Reasoning", "gpt-6-astra");
});
@@ -357,7 +435,7 @@ describe("AddAutoRouterTab", () => {
fireEvent.click(screen.getByRole("button", { name: "Cheapest", exact: true }));
expect(screen.getByRole("combobox", { name: "Everyday model" })).toHaveTextContent("custom-fast");
expect(screen.getByRole("combobox", { name: "Hard tasks model" })).toHaveTextContent("claude-opus-5-5");
- fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true }));
+ fireEvent.click(screen.getByRole("button", { name: "Customize tiers and keywords", exact: true }));
expectTierModel("Simple", "custom-fast");
expectTierModel("Reasoning", "claude-opus-5-5");
});
@@ -392,7 +470,7 @@ describe("AddAutoRouterTab", () => {
renderWithProviders();
expect(await screen.findByText(/Two deployed models are needed/)).toBeVisible();
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
- fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true }));
+ fireEvent.click(screen.getByRole("button", { name: "Customize tiers and keywords", exact: true }));
expect(screen.getByText("Define your tiers")).toBeVisible();
});
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
index 0fdc9a5f68f..3abca6f9547 100644
--- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
@@ -5,6 +5,8 @@ import ClassifierPrimarySettings from "./ClassifierPrimarySettings";
import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs";
import { getForecastConfigError, isForecastClassifier } from "../add_model/forecast_classifier_config";
import React, { useEffect, useState } from "react";
+import BlockedReasonTooltip from "./BlockedReasonTooltip";
+import { useSwitchyardOption } from "./SwitchyardRouterForm";
import { useTranslation } from "react-i18next";
import { useQuery } from "@tanstack/react-query";
import { useWatch } from "react-hook-form";
@@ -166,19 +168,6 @@ const EMPTY_FORM_VALUES: AddAutoRouterFormValues = {
const teamScopePayload = (requiresTeamScope: boolean, teamId: string | null): { team_id?: string } =>
requiresTeamScope && teamId ? { team_id: teamId } : {};
-const BlockedReasonTooltip: React.FC<{ reason: string | null; children: React.ReactElement }> = ({
- reason,
- children,
-}) =>
- reason === null ? (
- children
- ) : (
-
-
- {reason}
-
- );
-
const AddAutoRouterTab: React.FC = ({
handleOk,
accessToken,
@@ -191,6 +180,7 @@ const AddAutoRouterTab: React.FC = ({
const requiresTeamScope = createScope === "team-required";
const form = useZodForm(autoRouterSchema(requiresTeamScope), { defaultValues: EMPTY_FORM_VALUES });
const watchedName = useWatch({ control: form.control, name: "auto_router_name" });
+ const switchyard = useSwitchyardOption(accessToken, watchedName, isProxyAdminRole(userRole));
const watchedTeamId = useWatch({ control: form.control, name: "team_id" });
const actor = { userRole, userID: userId ?? null, isViewOnly: false };
const isMemberManaged =
@@ -341,11 +331,8 @@ const AddAutoRouterTab: React.FC = ({
setAdvanced(true);
};
- // A preset's models can only be trusted against a successfully loaded list. Selection and the
- // greyed-out state derive from this one function, so a preset that cannot be selected can never
- // have been applied: while loading we withhold selection rather than let a caller pick a preset
- // whose models we cannot yet verify, and a failed fetch leaves every preset unverifiable. This
- // makes the load-race (pick during loading, then discover a missing model) unrepresentable.
+ // Withhold preset selection until its models are verified against a successfully loaded list.
+ // Reuse this check for application and disabled state to prevent load races.
const presetAvailability = React.useCallback(
(preset: AutoRouterPreset): PresetAvailability => {
if (modelsLoading) return { kind: "loading" };
@@ -444,8 +431,15 @@ const AddAutoRouterTab: React.FC = ({
? t("addModel.addAutoRouterTab.selectTeamToSeeModels")
: simpleBlockedReason ?? submitBlockedReason ?? routerAvailability.saveBlockedReason;
const handleClassifierChange = (config: ComplexityRouterConfigValue) => {
- setSelectedPreset(undefined);
+ switchyard.deselect();
+ if (advanced || effectiveClassifierType(config) === effectiveClassifierType(complexityRouterConfig)) {
+ setSelectedPreset(undefined);
+ setComplexityRouterConfig(config);
+ return;
+ }
setComplexityRouterConfig(config);
+ setPrefillAttempted(simpleConfig !== null);
+ setAdvanced(true);
};
const jevRequestParams =
effectiveClassifierType(complexityRouterConfig) === "jev"
@@ -523,6 +517,23 @@ const AddAutoRouterTab: React.FC = ({
await handleAddAutoRouterSubmit(submitValues, accessToken, () => form.reset(EMPTY_FORM_VALUES), handleOk);
};
+ const submitSwitchyard = async (name: string) => {
+ if (requiresTeamScope && !(await form.trigger(["auto_router_name", "team_id"]))) {
+ toast.fromError(t("addModel.addAutoRouterTab.requiredFieldsMissing"));
+ return;
+ }
+ const accessGroups = isMemberManaged ? undefined : form.getValues("model_access_group");
+ await switchyard.submit(
+ name,
+ {
+ ...teamScopePayload(requiresTeamScope, form.getValues("team_id")),
+ ...(accessGroups?.length ? { access_groups: accessGroups } : {}),
+ },
+ () => form.reset(EMPTY_FORM_VALUES),
+ handleOk,
+ );
+ };
+
const handleAutoRouterSubmit = async () => {
if (isSubmitting) return;
const name = form.getValues("auto_router_name");
@@ -535,7 +546,7 @@ const AddAutoRouterTab: React.FC = ({
setIsSubmitting(true);
try {
- await submitRecommendedRouter(name);
+ await (switchyard.selected ? submitSwitchyard(name) : submitRecommendedRouter(name));
} finally {
setIsSubmitting(false);
}
@@ -672,63 +683,31 @@ const AddAutoRouterTab: React.FC = ({
)}
- {!advanced && (
- setSolverOverrides({})}
- goal={goal}
- family={family}
- models={solverModels}
- everyday={everyday}
- hard={hard}
- waitingForTeam={requiresTeamScope && !watchedTeamId}
- loading={modelsLoading}
- error={inventoryUnavailable}
- notice={
- Object.keys(solverOverrides).length > 0
- ? t("addModel.addAutoRouterTab.choicesKept")
- : recommended?.notice ?? null
- }
- onGoalChange={setGoal}
- onFamilyChange={(next) => {
- setFamily(next);
- setSolverOverrides({});
- }}
- onModelChange={(role, model) =>
- setSolverOverrides({ everyday: everyday?.name, hard: hard?.name, [role]: model })
- }
- onAdvanced={openAdvanced}
- onRetry={() => {
- void refetchModels();
- void refetchDeployments();
- }}
- />
- )}
- {advanced && (
-
- }
- judgeModel={
- model.mode !== "embedding")
- .map((model) => ({ value: model.model_group, label: model.model_group }))}
- showValidationErrors={showValidationErrors}
- />
- }
- >
+
+ }
+ judgeModel={
+ model.mode !== "embedding")
+ .map((model) => ({ value: model.model_group, label: model.model_group }))}
+ showValidationErrors={showValidationErrors}
+ />
+ }
+ >
+ {advanced ? (
{modelsUnverifiable && (
@@ -740,21 +719,40 @@ const AddAutoRouterTab: React.FC = ({
)}
{configurationForm}
-
- )}
- {!advanced && routerAvailability.isError && (
-
- {t("addModel.addAutoRouterTab.availabilityCheckFailed")}{" "}
-
-
- )}
- {!advanced && routerAvailability.data?.error && (
-
- {routerAvailability.data.error}
-
- )}
+ ) : (
+ setSolverOverrides({})}
+ goal={goal}
+ family={family}
+ models={solverModels}
+ everyday={everyday}
+ hard={hard}
+ waitingForTeam={requiresTeamScope && !watchedTeamId}
+ loading={modelsLoading}
+ error={inventoryUnavailable}
+ notice={
+ Object.keys(solverOverrides).length > 0
+ ? t("addModel.addAutoRouterTab.choicesKept")
+ : recommended?.notice ?? null
+ }
+ onGoalChange={setGoal}
+ onFamilyChange={(next) => {
+ setFamily(next);
+ setSolverOverrides({});
+ }}
+ onModelChange={(role, model) =>
+ setSolverOverrides({ everyday: everyday?.name, hard: hard?.name, [role]: model })
+ }
+ onAdvanced={openAdvanced}
+ onRetry={() => {
+ void refetchModels();
+ void refetchDeployments();
+ }}
+ />
+ )}
+
= ({
type="button"
variant="outline"
data-testid="auto-router-test-routing-btn"
+ hidden={switchyard.selected}
disabled={submitBlockedReason !== null || isSubmitting}
onClick={() => setIsRoutingTestVisible(true)}
>
@@ -785,16 +784,17 @@ const AddAutoRouterTab: React.FC = ({
type="button"
variant="outline"
data-testid="auto-router-test-connect-btn"
+ hidden={switchyard.selected}
onClick={handleTestConnection}
disabled={isTestingConnection}
>
{isTestingConnection && }
{t("addModel.addAutoRouterTab.testConnection")}
-
+