diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 41198887cab..fb29a92e306 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -1035,9 +1035,9 @@ jobs: run: | diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))" @@ -1294,7 +1294,7 @@ jobs: - name: Install dependencies if: steps.changes.outputs.decision != 'skip' run: | - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard - name: Cache Prisma binaries if: steps.changes.outputs.decision != 'skip' @@ -1497,9 +1497,9 @@ jobs: RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }} run: | if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so fi diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 8c402fe9b44..f6bad7f611b 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -117,7 +117,9 @@ class SensitiveDataMasker: elif key_is_sensitive and isinstance(item, str): masked_items.append(self.mask_value(item)) else: - masked_items.append(item if isinstance(item, (int, float, bool, str, list)) else str(item)) + masked_items.append( + item if item is None or isinstance(item, (int, float, bool, str, list)) else str(item) + ) return masked_items def mask_dict( @@ -144,7 +146,7 @@ class SensitiveDataMasker: str_value = str(v) if v is not None else "" masked_data[k] = self.mask_value(str_value) else: - masked_data[k] = v if isinstance(v, (int, float, bool, str, list)) else str(v) + masked_data[k] = v if v is None or isinstance(v, (int, float, bool, str, list)) else str(v) except Exception: masked_data[k] = "" diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 8f1a08d2964..ca9529ba246 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -795,18 +795,21 @@ class AnthropicMessagesHandler(BaseTranslation): data: dict, # mutable-ok: the request body the adapter reads *, keep_cache_control: bool = False, + keep_tool_errors: bool = False, ) -> ChatCompletionRequest: """Translate Anthropic request to OpenAI chat completion format.""" ( chat_completion_compatible_request, _tool_name_mapping, - ) = LiteLLMAnthropicMessagesAdapter(keep_cache_control=keep_cache_control).translate_anthropic_to_openai( + ) = LiteLLMAnthropicMessagesAdapter( + keep_cache_control=keep_cache_control, keep_tool_errors=keep_tool_errors + ).translate_anthropic_to_openai( anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)), preserve_midturn_system=True, ) return chat_completion_compatible_request - def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: + def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list[AllMessageValues] | None: """ Convert Anthropic messages request data to OpenAI-spec structured messages. @@ -815,7 +818,7 @@ class AnthropicMessagesHandler(BaseTranslation): messages: Final = data.get("messages") if messages is None: return None - chat_completion_compatible_request: Final = self._translate_to_openai(data) + chat_completion_compatible_request: Final = self._translate_to_openai(data, keep_tool_errors=keep_tool_errors) result: Final = cast( list[AllMessageValues], chat_completion_compatible_request.get("messages", []), diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 43eaa37aba7..5e4fba6d706 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import JsonValue, TypeAdapter import litellm -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part from litellm.llms.anthropic.messages.utils import ( @@ -434,8 +434,9 @@ _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.va class LiteLLMAnthropicMessagesAdapter: - def __init__(self, *, keep_cache_control: bool = False) -> None: + def __init__(self, *, keep_cache_control: bool = False, keep_tool_errors: bool = False) -> None: self.keep_cache_control: Final = keep_cache_control + self.keep_tool_errors: Final = keep_tool_errors ### FOR [BETA] `/v1/messages` endpoint support @@ -615,11 +616,7 @@ class LiteLLMAnthropicMessagesAdapter: self._add_cache_control_if_applicable(content, doc_obj, model) new_user_content_list.append(doc_obj) elif content.get("type") == "tool_result": - tool_result = ChatCompletionToolMessage( - role="tool", - tool_call_id=content.get("tool_use_id", ""), - content=self._tool_result_content(content.get("content")), - ) + tool_result = self._tool_message(content) self._add_cache_control_if_applicable(content, tool_result, model) tool_message_list.append(tool_result) @@ -1432,6 +1429,15 @@ class LiteLLMAnthropicMessagesAdapter: return None return anthropic_image_source_to_openai_url(image_source) + def _tool_message(self, block: object) -> ChatCompletionToolMessage: + fields: Final = as_str_mapping(block) or {} + is_error: Final = fields.get("is_error") + tool_call_id: Final = tool_use_id if isinstance(tool_use_id := fields.get("tool_use_id"), str) else "" + content: Final = self._tool_result_content(fields.get("content")) + if self.keep_tool_errors and isinstance(is_error, bool): + return ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content, is_error=is_error) + return ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content) + def _tool_result_content(self, raw_content: object) -> ToolResultContent: if isinstance(raw_content, str): return raw_content diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 6399ded9555..71368f4ef61 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -257,11 +257,12 @@ class BaseTranslation(ABC): """ return None - def get_structured_messages(self, data: dict) -> list["AllMessageValues"] | None: + def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list["AllMessageValues"] | None: """ Convert request data to OpenAI-spec structured messages. - Override in subclasses for format-specific conversion. + Override in subclasses for format-specific conversion. ``keep_tool_errors`` copies a provider's + tool-failure marker onto tool messages as ``is_error``; only for copies that are never sent to a model. Returns None if no convertible content is found. """ diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index c5488bc8a98..26246cfd7b6 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -103,7 +103,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): delivers_ended_stream_rewrites = True assembles_streamed_response = True - def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: + def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list[AllMessageValues] | None: """ Convert chat completions request data to OpenAI-spec structured messages. diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index aa4dea196ff..20b7506ae0a 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -777,7 +777,7 @@ class OpenAIResponsesHandler(BaseTranslation): delivers_ended_stream_rewrites = True assembles_streamed_response = True - def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: + def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list[AllMessageValues] | None: """ Convert Responses API request data to OpenAI-spec structured messages. diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 137d6016402..b50e3955838 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -49950,6 +49950,10 @@ ], "title": "Content" }, + "is_error": { + "title": "Is Error", + "type": "boolean" + }, "role": { "const": "tool", "title": "Role", diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 42321fe1b1d..3c733d08a2e 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -1112,6 +1112,7 @@ def _is_configured_pre_routing_strategy(llm_router: "Router", router_name: str) router_registry.plugins("complexity"), router_registry.plugins("adaptive"), router_registry.plugins("quality"), + router_registry.plugins("switchyard"), ) ) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 2c7453ebb1c..b8b926a3057 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -142,6 +142,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import ( count_capability_routers, gated_capabilities_of, is_complexity_router_model, + strategy_router_dependencies, validate_complexity_router_config_placement, validate_complexity_router_config_write, validate_strategy_router_model_write, @@ -355,6 +356,22 @@ async def get_db_model(model_id: str, prisma_client: PrismaClient) -> Deployment return deployment_pydantic_obj +def _self_routing_violation( + model_name: str | None, + incoming_params: GenericLiteLLMParams | None, + existing_params: GenericLiteLLMParams | None, +) -> str | None: + if not model_name: + return None + effective_params: Final = { + **(_decrypted_litellm_params(existing_params) if existing_params is not None else {}), + **(_dump_without_none(incoming_params) if incoming_params is not None else {}), + } + if any(dependency.model_name == model_name for dependency in strategy_router_dependencies(effective_params)): + return f"A strategy router cannot route to itself: {model_name} is listed as one of its own models" + return None + + def _strategy_router_write_violation( incoming_params: GenericLiteLLMParams | None, existing_params: GenericLiteLLMParams | None, @@ -374,6 +391,11 @@ def _strategy_router_write_violation( """ if incoming_params is None: return None + from litellm.router_strategy.switchyard_router.config import validate_switchyard_config_write + + switchyard_violation: Final = validate_switchyard_config_write(incoming_params.switchyard_router_config) + if switchyard_violation is not None: + return switchyard_violation config_violation: Final = validate_complexity_router_config_write( complexity_router_config=( _effective_complexity_router_config(incoming_params, existing_params) @@ -406,8 +428,11 @@ def _strategy_router_write_violation( def _raise_on_strategy_router_write_violation( incoming_params: GenericLiteLLMParams | None, existing_params: GenericLiteLLMParams | None, + model_name: str | None = None, ) -> None: - violation = _strategy_router_write_violation(incoming_params=incoming_params, existing_params=existing_params) + violation: Final = _self_routing_violation(model_name, incoming_params, existing_params) or ( + _strategy_router_write_violation(incoming_params=incoming_params, existing_params=existing_params) + ) if violation is None: return raise ProxyException( @@ -1751,6 +1776,7 @@ async def patch_model( _raise_on_strategy_router_write_violation( incoming_params=incoming_params, existing_params=remaining_params, + model_name=patch_data.model_name or db_model.model_name, ) written_team_id: Final = ( @@ -3120,6 +3146,7 @@ async def add_new_model( _raise_on_strategy_router_write_violation( incoming_params=model_params.litellm_params, existing_params=None, + model_name=model_params.model_name, ) _raise_if_rate_limits_required_but_missing( @@ -3325,6 +3352,7 @@ async def update_model( _raise_on_strategy_router_write_violation( incoming_params=model_params.litellm_params, existing_params=deployment.litellm_params, + model_name=model_params.model_name or deployment.model_name, ) renamed_to: Final = ( model_params.model_name @@ -4028,6 +4056,7 @@ async def clear_cache() -> ReconcileOutcome: llm_router.registry().plugins("complexity").pop(model_name, None) llm_router.registry().plugins("adaptive").pop(model_name, None) llm_router.registry().plugins("quality").pop(model_name, None) + llm_router.registry().plugins("switchyard").pop(model_name, None) # Reload only DB models. _add_deployment_locked, not add_deployment: this # coroutine already holds MODEL_RECONCILE_LOCK and asyncio.Lock is not diff --git a/litellm/router_api.py b/litellm/router_api.py index 1ade356f494..ca370488f5d 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -65,6 +65,7 @@ if TYPE_CHECKING: from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter from litellm.router_strategy.quality_router.quality_router import QualityRouter + from litellm.router_strategy.switchyard_router.switchyard_router import SwitchyardRouter from litellm.router_utils.cooldown_cache import CooldownCache from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets as _AttemptedFallbackTargets from litellm.router_utils.health_state_cache import DeploymentHealthCache @@ -106,6 +107,7 @@ if TYPE_CHECKING: | dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] | dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] | dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] + | dict[str, list[TaggedPreRoutingStrategy[SwitchyardRouter]]] | list[RoutingPlugin] | RouterBudgetLimiting | DualCache @@ -171,6 +173,7 @@ PluginKind: TypeAlias = Literal[ "complexity", "quality", "adaptive", + "switchyard", "routing", "budget_limiter", "prompt_caching_pins", @@ -1106,6 +1109,11 @@ class RouterRegistry: self, kind: Literal["adaptive"] ) -> "dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]]": ... # mutable-ok: the live dict + @overload + def plugins( + self, kind: Literal["switchyard"] + ) -> "dict[str, list[TaggedPreRoutingStrategy[SwitchyardRouter]]]": ... # mutable-ok: the live dict + @overload def plugins(self, kind: Literal["routing"]) -> "list[RoutingPlugin]": ... # mutable-ok: the live list @@ -1132,6 +1140,8 @@ class RouterRegistry: return r._core_quality_routers if kind == "adaptive": return r._core_adaptive_routers + if kind == "switchyard": + return r._core_switchyard_routers if kind == "routing": return r._core_routing_plugins if kind == "budget_limiter": diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index 40ae7e0e554..3e222313f32 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -1177,6 +1177,7 @@ _HOSTED_STATE: Final = frozenset( "_core_auto_routers", "_core_complexity_routers", "_core_quality_routers", + "_core_switchyard_routers", "_core_adaptive_routers", "_core_routing_plugins", "_core_deployment_latency_map", @@ -1722,6 +1723,7 @@ class RustCore( run_plugins=partial(self._run_plugins, request_kwargs, messages) if self._hosted._core_routing_plugins # pyright: ignore[reportPrivateUsage] # the live plugin list else None, + call_type=None, ) def _core_log_retry( @@ -2715,7 +2717,7 @@ class RustCore( web_search: Final = not plan.sync and self._picks_async() and is_web_search_request(request) # The generic handler `_route_capability_pin` names: a call only some providers serve # drops the others, and a Decisions call carrying media skips the deployments refusing it. - handler: Final = plan.host_op.rsplit(".", 1)[-1] + handler: Final = plan.handler decisions_media: Final = handler in DECISIONS_MEDIA_CALL_TYPES and bool( request.get("audio") or request.get("videos") ) @@ -3113,6 +3115,7 @@ class RustCore( specific_deployment: object = _NO_INPUT, *, pass_through: bool = False, + call_type: str | None = None, ) -> object: """The host's pre-routing of one pick (router_rust.steps.prerouting): the model group to resolve instead of the step's ``model``, a custom strategy's deployment, or ``None``. ``pass_through`` @@ -3166,6 +3169,7 @@ class RustCore( fallback_depth=depth if isinstance(depth, int) else 0, router_strategy=normalize_strategy(r.routing_strategy), pass_through=pass_through, + call_type=call_type, enable_tag_filtering=r.enable_tag_filtering, match_any=r.tag_filtering_match_any, public_router=public if public is not None else self, @@ -5266,7 +5270,9 @@ class _Driver: async def pre_route(self, step: Mapping[str, object]) -> object: request: Final = self._request try: - return await self._core._pre_route(step, request, *self._plan.pick_input(request)) # pyright: ignore[reportPrivateUsage] # see above + return await self._core._pre_route( # pyright: ignore[reportPrivateUsage] # see above + step, request, *self._plan.pick_input(request), call_type=self._plan.handler + ) except Exception as error: # noqa: BLE001 # see select return self._pick_raised(error, logged=True) diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index e2c621ccb7d..7a5b52761e2 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -244,6 +244,11 @@ class Plan: # the request (`{**data, name: value, **kwargs}`). named: tuple[str, ...] = () + @property + def handler(self) -> str: + """The handler name the host op ends in, which route capabilities and pre-routing strategies key on.""" + return self.host_op.rsplit(".", 1)[-1] + @property def on_its_own_loop(self) -> bool: """``function_with_fallbacks`` runs these sync endpoints' whole walk on a private event loop.""" diff --git a/litellm/router_rust/steps/prerouting.py b/litellm/router_rust/steps/prerouting.py index 63d15a1a80a..47bd1e69046 100644 --- a/litellm/router_rust/steps/prerouting.py +++ b/litellm/router_rust/steps/prerouting.py @@ -197,9 +197,13 @@ class PreRoutingStrategies: def quality(self) -> StrategyRegistry: return self._hosted._core_quality_routers # pyright: ignore[reportPrivateUsage] # see above + @property + def switchyard(self) -> StrategyRegistry: + return self._hosted._core_switchyard_routers # pyright: ignore[reportPrivateUsage] # see above + def all(self) -> tuple[StrategyRegistry, ...]: """In the order ``_select_pre_routing_strategy`` searches them.""" - return (self.auto, self.complexity, self.adaptive, self.quality) + return (self.switchyard, self.auto, self.complexity, self.adaptive, self.quality) def any_registered(self) -> bool: return any(self.all()) @@ -211,6 +215,7 @@ class PreRoutingStrategies: self._hosted._core_quality_routers = {} # pyright: ignore[reportPrivateUsage] # see above self._hosted._core_complexity_routers = {} # pyright: ignore[reportPrivateUsage] # see above self._hosted._core_auto_routers = {} # pyright: ignore[reportPrivateUsage] # see above + self._hosted._core_switchyard_routers = {} # pyright: ignore[reportPrivateUsage] # see above def register(self, deployment: Deployment) -> None: """The end of ``_add_deployment``: an auto, complexity or quality marker registers its router @@ -222,6 +227,8 @@ class PreRoutingStrategies: self._init_complexity_router(deployment) case "quality": self._init_quality_router(deployment) + case "switchyard": + self._init_switchyard_router(deployment) case _: pass @@ -231,7 +238,7 @@ class PreRoutingStrategies: return model_name: Final = deployment.model_name tags: Final = _deployment_tags(deployment) - for registry in (self.auto, self.complexity, self.quality): + for registry in (self.auto, self.complexity, self.quality, self.switchyard): _ = _unregister(registry, model_name, tags) if _unregister(self.adaptive, model_name, tags): self._sync_adaptive_router_hooks() @@ -463,6 +470,32 @@ class PreRoutingStrategies: ) self._register(self.quality, deployment, router, "Quality-router") + def _init_switchyard_router(self, deployment: Deployment) -> None: + from litellm.router_strategy.switchyard_router.config import SwitchyardRouterConfig, build_algorithm + from litellm.router_strategy.switchyard_router.switchyard_router import SwitchyardRouter + from litellm.types.decisions import OpenAIDecisionResponse + from litellm.types.utils import ModelResponse + + config: Final = SwitchyardRouterConfig.model_validate(deployment.litellm_params.switchyard_router_config) + if deployment.model_name in (config.models.capable, config.models.efficient, config.models.judge): + raise ValueError("Switchyard cannot route to itself") + ctx: Final = self._ctx + + async def complete_judge(request: Mapping[str, object]) -> ModelResponse: + result: Final = await ctx.acompletion(**request) # pyright: ignore[reportAttributeAccessIssue] # the core's RoutingContext + if not isinstance(result, ModelResponse): + raise TypeError("Switchyard judge must return a non-streaming completion") + return result + + async def decide(request: Mapping[str, object]) -> OpenAIDecisionResponse: + result: Final = await ctx.adecisions(**request) # pyright: ignore[reportAttributeAccessIssue] # see above + if not isinstance(result, OpenAIDecisionResponse): + raise TypeError("Switchyard decision judge must return an OpenAI-shaped decision response") + return result + + router: Final = SwitchyardRouter(deployment.model_name, config, build_algorithm(config), complete_judge, decide) + self._register(self.switchyard, deployment, router, "Switchyard") + def _init_adaptive_router(self, deployment: Deployment) -> None: from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook @@ -741,6 +774,7 @@ class PreRouter: public_router: object, run_plugins: Callable[[str], Awaitable[None]] | None = None, pass_through: bool = False, + call_type: str | None = None, ) -> str | None: """The start of ``_async_get_available_deployment``: run the hook, record its pick and pin the tier's litellm_params. The model group to resolve, or ``None`` to resolve ``model`` itself. @@ -759,6 +793,7 @@ class PreRouter: match_any=match_any, public_router=public_router, run_plugins=run_plugins, + call_type=call_type, ) routed: Final = response.model if response is not None else model if response is not None: @@ -782,6 +817,7 @@ class PreRouter: match_any: bool, public_router: object, run_plugins: Callable[[str], Awaitable[None]] | None, + call_type: str | None, ) -> "PreRoutingHookResponse | None": _restore_client_output_ceiling(request_kwargs) _record_routing_decision(request_kwargs, None) @@ -861,6 +897,7 @@ class PreRouter: messages=routing_messages if routing_messages is not None else messages, # pyright: ignore[reportArgumentType] # the request's messages, as given input=input, # pyright: ignore[reportArgumentType] # the request's input, as given specific_deployment=False, + call_type=call_type, ) hooked: Final[PreRoutingHookResponse | None] = routed traced: Final = ( diff --git a/litellm/router_rust/surface/registry.py b/litellm/router_rust/surface/registry.py index 7c3abdcd45e..820db52d5a1 100644 --- a/litellm/router_rust/surface/registry.py +++ b/litellm/router_rust/surface/registry.py @@ -658,6 +658,7 @@ class HostRegistry(HostModelInfo): self._core_complexity_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same self._core_adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same self._core_quality_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same + self._core_switchyard_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same self.strategies: PreRoutingStrategies | None = None # The router's optional pre-call checks (deployment budgets, prompt caching pins); # RustCore installs its ``OptionalChecks`` (litellm/router_rust/steps/callbacks.py). diff --git a/litellm/router_strategy/adaptive_router/adaptive_router.py b/litellm/router_strategy/adaptive_router/adaptive_router.py index c3a65d6aa91..43869fdf832 100644 --- a/litellm/router_strategy/adaptive_router/adaptive_router.py +++ b/litellm/router_strategy/adaptive_router/adaptive_router.py @@ -184,6 +184,7 @@ class AdaptiveRouter: messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + call_type: str | None = None, ) -> PreRoutingHookResponse | None: """ Plugin entry point invoked by `Router.async_pre_routing_hook` when the diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index b92c53fa93c..68e8701eb23 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -204,6 +204,7 @@ class AutoRouter(CustomLogger): messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + call_type: str | None = None, ) -> Optional["PreRoutingHookResponse"]: """ This hook is called before the routing decision is made. diff --git a/litellm/router_strategy/auto_router/auto_router_model_naming.py b/litellm/router_strategy/auto_router/auto_router_model_naming.py index f5aeccd9305..47daa175c99 100644 --- a/litellm/router_strategy/auto_router/auto_router_model_naming.py +++ b/litellm/router_strategy/auto_router/auto_router_model_naming.py @@ -26,7 +26,7 @@ from litellm.router_strategy.complexity_router.config import ( AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/" -StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"] +StrategyRouterKind: TypeAlias = Literal["semantic", "complexity", "adaptive", "quality", "switchyard"] StrategyRouterDependencyRole: TypeAlias = Literal[ "tier", "default", "classifier", "embedding", "evaluation", "compactor" @@ -43,6 +43,7 @@ class StrategyRouterDependency: STRATEGY_ROUTER_PARAM_FIELDS: Final[frozenset[str]] = frozenset( { + "switchyard_router_config", "auto_router_config", "auto_router_config_path", "auto_router_default_model", @@ -57,6 +58,7 @@ STRATEGY_ROUTER_PARAM_FIELDS: Final[frozenset[str]] = frozenset( ) _REQUIRED_FIELD_GROUPS: Final[Mapping[StrategyRouterKind, tuple[tuple[str, ...], ...]]] = { + "switchyard": (("switchyard_router_config",),), "semantic": ( ("auto_router_config", "auto_router_config_path"), ("auto_router_default_model",), @@ -78,6 +80,8 @@ def classify_strategy_router_model(model: str) -> StrategyRouterKind | None: if not model.startswith(AUTO_ROUTER_MODEL_PREFIX): return None remainder: Final = model[len(AUTO_ROUTER_MODEL_PREFIX) :] + if remainder.startswith("switchyard"): + return "switchyard" if remainder.startswith("complexity_router"): return "complexity" if remainder.startswith("adaptive_router"): @@ -137,6 +141,13 @@ def strategy_router_dependencies( kind: Final = classify_strategy_router_model(str(litellm_params.get("model", ""))) if kind is None: return () + if kind == "switchyard": + models: Final = _mapping(_mapping(litellm_params.get("switchyard_router_config")).get("models")) + return ( + _named(models.get("capable"), "tier") + + _named(models.get("efficient"), "tier") + + _named(models.get("judge"), "classifier") + ) if kind == "semantic": return _named(litellm_params.get("auto_router_default_model"), "default") + _named( litellm_params.get("auto_router_embedding_model"), "embedding" diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 05dd4706b62..f004dfcc38d 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -4490,6 +4490,7 @@ class ComplexityRouter(CustomLogger): messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + call_type: str | None = None, ) -> PreRoutingHookResponse | None: """ Pre-routing hook called before the routing decision. diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py index 2d9ad63e9de..fc8d5ab5fcb 100644 --- a/litellm/router_strategy/quality_router/quality_router.py +++ b/litellm/router_strategy/quality_router/quality_router.py @@ -330,6 +330,7 @@ class QualityRouter(CustomLogger): messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + call_type: str | None = None, ) -> Optional["PreRoutingHookResponse"]: """Try keyword override first; fall back to complexity-tier routing.""" from litellm.types.router import PreRoutingHookResponse diff --git a/litellm/router_strategy/switchyard_router/__init__.py b/litellm/router_strategy/switchyard_router/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/router_strategy/switchyard_router/config.py b/litellm/router_strategy/switchyard_router/config.py new file mode 100644 index 00000000000..7e0d3ad3a94 --- /dev/null +++ b/litellm/router_strategy/switchyard_router/config.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag, TypeAdapter, model_validator + +if TYPE_CHECKING: + from switchyard_rust.libsy import Algorithm, TaskClassifierConfig + + +class LlmClassifierConfig(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["llm"] = "llm" + base_threshold: float = Field(default=0.5, ge=0, le=1) + threshold_step: float = Field(default=0.0, ge=0, le=1) + recent_turn_window: int | None = Field(default=None, gt=0) + max_output_tokens: int = Field(default=4096, gt=0) + response_format_type: Literal["json_schema", "json_object"] = "json_schema" + prompt: str | None = None + + +class DecisionClassifierConfig(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["decision"] + cutoff: float = Field(default=0.6, ge=0, le=1) + instructions: str | Mapping[str, object] | None = None + evidence: Mapping[str, object] | None = None + + +_KIND: Final = TypeAdapter(Mapping[str, object]) + + +def _classifier_kind(value: object) -> str: + match value: + case LlmClassifierConfig() | DecisionClassifierConfig(): + return value.kind + case Mapping(): + return str(_KIND.validate_python(value).get("kind", "llm")) + case _: + return "llm" + + +ClassifierConfig: TypeAlias = Annotated[ + Annotated[LlmClassifierConfig, Tag("llm")] | Annotated[DecisionClassifierConfig, Tag("decision")], + Discriminator(_classifier_kind), +] + + +class StageOptions(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + picker: Literal["efficient_first", "capable_first"] = "efficient_first" + confidence_threshold: float | None = Field(default=None, ge=0, le=1) + recent_window: int | None = Field(default=None, gt=0) + tool_semantics: Mapping[str, tuple[str, ...]] | None = None + only_on_wrong_signal_escalation: bool = True + + +class StageModels(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", str_strip_whitespace=True) + + capable: str = Field(min_length=1) + efficient: str = Field(min_length=1) + judge: str | None = Field(default=None, min_length=1) + + @model_validator(mode="after") + def reject_recursive_targets(self) -> StageModels: + if any(name.startswith("auto_router/") for name in (self.capable, self.efficient, self.judge or "")): + raise ValueError("Switchyard targets must be concrete model groups, not auto-router markers") + if self.capable == self.efficient: + raise ValueError("The capable and efficient models must be different model groups") + return self + + +class SwitchyardRouterConfig(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + algorithm: Literal["stage_router"] = "stage_router" + models: StageModels + options: StageOptions = Field(default_factory=StageOptions) + classifier: ClassifierConfig | None = None + + @model_validator(mode="after") + def validate_judge(self) -> SwitchyardRouterConfig: + if (self.classifier is None) != (self.models.judge is None): + raise ValueError("A judge model is required exactly when the classifier is enabled") + return self + + @property + def confidence_threshold(self) -> float: + if self.options.confidence_threshold is not None: + return self.options.confidence_threshold + return 0.7 if self.classifier is not None else 0.5 + + def runtime_models(self) -> Mapping[str, tuple[str, ...]]: + return { + "any": (self.models.capable, self.models.efficient), + "capable": (self.models.capable,), + "efficient": (self.models.efficient,), + **({"judge": (self.models.judge,)} if self.models.judge is not None else {}), + } + + +def _task_classifier(config: SwitchyardRouterConfig) -> TaskClassifierConfig | None: + from switchyard_rust.libsy import TaskClassifierConfig + + match config.classifier: + case None: + return None + case LlmClassifierConfig() as llm: + return TaskClassifierConfig( + base_threshold=llm.base_threshold, + threshold_step=llm.threshold_step, + recent_turn_window=llm.recent_turn_window, + max_output_tokens=llm.max_output_tokens, + response_format_type=llm.response_format_type, + prompt=llm.prompt, + ) + case DecisionClassifierConfig() as decision: + return TaskClassifierConfig.decision( + cutoff=decision.cutoff, + candidates={"capable": config.models.capable, "efficient": config.models.efficient}, + evidence=dict(decision.evidence or {}), + instructions=decision.instructions, + ) + + +def build_algorithm(config: SwitchyardRouterConfig) -> Algorithm: + from switchyard_rust.libsy import LlmFallback, stage_router + + classifier: Final = _task_classifier(config) + return stage_router( + picker=config.options.picker, + confidence_threshold=config.confidence_threshold, + recent_window=config.options.recent_window, + tool_semantics=dict(config.options.tool_semantics) if config.options.tool_semantics is not None else None, + only_on_wrong_signal_escalation=config.options.only_on_wrong_signal_escalation, + classifier=LlmFallback(config=classifier) if classifier is not None else None, + ) + + +def validate_switchyard_config_write(config: Mapping[str, object] | None) -> str | None: + if config is None: + return None + try: + build_algorithm(SwitchyardRouterConfig.model_validate(config)) + except ImportError: + return "Switchyard requires the optional dependency. Install litellm[switchyard] (nemo-switchyard==0.4.0a1)." + except (ValueError, TypeError) as error: + return f"Invalid switchyard_router_config: {error}" + return None diff --git a/litellm/router_strategy/switchyard_router/switchyard_router.py b/litellm/router_strategy/switchyard_router/switchyard_router.py new file mode 100644 index 00000000000..cb455bab337 --- /dev/null +++ b/litellm/router_strategy/switchyard_router/switchyard_router.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import hashlib +from collections.abc import Awaitable, Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, TypeVar + +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs +from litellm.litellm_core_utils.internal_call_metadata import ( + effective_turn_off_message_logging, + forwarded_internal_call_metadata, + parent_session_kwargs, +) +from litellm.router_strategy.switchyard_router.config import ( + DecisionClassifierConfig, + LlmClassifierConfig, + SwitchyardRouterConfig, +) +from litellm.router_strategy.switchyard_router.translation import ( + chat_messages, + decision_request, + decision_response, + judge_request, + judge_response, + mapping, + request_to_switchyard, +) +from litellm.types.decisions import OpenAIDecisionResponse +from litellm.types.router import PreRoutingHookResponse +from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, StandardLoggingRoutingDecision + +if TYPE_CHECKING: + from switchyard_rust.libsy import Algorithm, DecisionCall, ModelCall, RoutingOutcome + +JudgeCompletion: TypeAlias = Callable[[Mapping[str, object]], Awaitable[ModelResponse]] +DecisionCompletion: TypeAlias = Callable[[Mapping[str, object]], Awaitable[OpenAIDecisionResponse]] +FallbackReason: TypeAlias = Literal[ + "unsupported_request_type", + "unsupported_content", + "libsy_error", + "unexpected_judge_call", + "unexpected_decision_call", + "target_outside_config", + "no_decision", +] +_T = TypeVar("_T") +_SESSION_HEADER: Final = "x-switchyard-session-id" +_FORWARDED_HEADER_PREFIXES: Final = ("x-switchyard-", "x-codex-", "x-nemo-relay-", "x-dynamo-") +_FORWARDED_HEADERS: Final = frozenset( + {"x-session-id", "session-id", "thread-id", "x-task-id", "x-request-id", "x-client-request-id"} +) + + +async def _resolved(value: _T) -> _T: + return value + + +_SESSION_ALIASES: Final = frozenset( + {_SESSION_HEADER, "x-claude-code-session-id", "x-nemo-relay-session-id", "x-session-id", "session-id"} +) +_CODEX_TURN_METADATA_HEADER: Final = "x-codex-turn-metadata" + + +def _forwarded_header(name: str, session_scoped: bool) -> bool: + lowered: Final = name.lower() + if lowered in _SESSION_ALIASES or (lowered == _CODEX_TURN_METADATA_HEADER and not session_scoped): + return False + return lowered in _FORWARDED_HEADERS or lowered.startswith(_FORWARDED_HEADER_PREFIXES) + + +def _caller_key_hash(request_kwargs: Mapping[str, object]) -> str: + internal: Final = mapping(request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)) or {}) + key: Final = internal.get("user_api_key_hash") + return str(key) if key is not None else "" + + +def _scoped_session(request_kwargs: Mapping[str, object]) -> str | None: + session: Final = parent_session_kwargs(request_kwargs).get("litellm_session_id") + if not session: + return None + return hashlib.sha256(f"{_caller_key_hash(request_kwargs)}\0{session}".encode()).hexdigest() + + +def switchyard_headers(request_kwargs: Mapping[str, object]) -> Mapping[str, str]: + proxy_request: Final = mapping(request_kwargs.get("proxy_server_request") or {}) + client: Final = mapping(proxy_request.get("headers") or {}) + session: Final = _scoped_session(request_kwargs) + forwarded: Final = { + name.lower(): value + for name, value in client.items() + if isinstance(value, str) and _forwarded_header(name, session is not None) + } + return {**forwarded, **({_SESSION_HEADER: session} if session else {})} + + +class SwitchyardRouter: + def __init__( + self, + model_name: str, + config: SwitchyardRouterConfig, + algorithm: Algorithm, + completion: JudgeCompletion, + decide: DecisionCompletion, + ) -> None: + self.model_name = model_name + self.config = config + self.algorithm = algorithm + self.completion = completion + self.decide = decide + + def _fallback( + self, reason: FallbackReason, messages: Sequence[Mapping[str, object]] | None + ) -> PreRoutingHookResponse: + return PreRoutingHookResponse( + model=self.config.models.capable, + messages=[dict(message) for message in messages] if messages is not None else None, + routing_decision={ + "router_type": "switchyard", + "router_model_name": self.model_name, + "routed_model": self.config.models.capable, + "switchyard_algorithm": self.config.algorithm, + "switchyard_fallback_reason": reason, + }, + ) + + def _internal_call( + self, + request: Mapping[str, object], + caller: Mapping[str, object], + bucket: Literal["metadata", "litellm_metadata"], + ) -> Mapping[str, object]: + caller_metadata: Final = mapping(caller.get(get_metadata_variable_name_from_kwargs(caller)) or {}) + return { + **request, + bucket: { + **forwarded_internal_call_metadata(caller_metadata, AUTOROUTER_CLASSIFIER_CALL_ORIGIN), + INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN, + }, + "proxy_server_request": {"body": request}, + "num_retries": 0, + "turn_off_message_logging": effective_turn_off_message_logging(caller), + **parent_session_kwargs(caller), + } + + def _decision( + self, pick: str, evidence: object, messages: Sequence[Mapping[str, object]] | None + ) -> PreRoutingHookResponse: + decision: Final[StandardLoggingRoutingDecision] = { + "router_type": "switchyard", + "router_model_name": self.model_name, + "routed_model": pick, + "switchyard_algorithm": self.config.algorithm, + "switchyard_evidence": evidence, + } + return PreRoutingHookResponse( + model=pick, + messages=[dict(message) for message in messages] if messages is not None else None, + routing_decision=decision, + ) + + async def _answer_judge(self, call: ModelCall, caller: Mapping[str, object]) -> None: + try: + converted: Final = judge_response( + await self.completion( + self._internal_call(judge_request(call.request, call.models[0]), caller, "metadata") + ) + ) + except Exception as error: # noqa: BLE001 # judge/provider failures are inputs to Switchyard's fall-open decision + await call.fail(error) + else: + await call.respond(_resolved(converted)) + + async def _answer_decision(self, call: DecisionCall, caller: Mapping[str, object]) -> None: + try: + answer: Final = decision_response( + await self.decide( + self._internal_call(decision_request(call.request, call.model), caller, "litellm_metadata") + ), + call.request, + ) + except Exception as error: # noqa: BLE001 # decision/provider failures are inputs to Switchyard's fall-open decision + await call.fail(error) + else: + await call.respond(_resolved(answer)) + + def _outcome( + self, outcome: RoutingOutcome, messages: Sequence[Mapping[str, object]] | None + ) -> PreRoutingHookResponse: + pick: Final = outcome.selected_model_ids[0] + if pick not in (self.config.models.capable, self.config.models.efficient): + return self._fallback("target_outside_config", messages) + metadata: Final = outcome.metadata + evidence: Final[object] = metadata.evidence if metadata is not None else None + return self._decision(pick, evidence, messages) + + async def async_pre_routing_hook( + self, + model: str, + request_kwargs: Mapping[str, object], + messages: Sequence[Mapping[str, object]] | None = None, + input: object = None, + specific_deployment: bool | None = False, + call_type: str | None = None, + ) -> PreRoutingHookResponse | None: + from switchyard_rust.libsy import LibsyError, Step + + if model != self.model_name: + return None + try: + chat: Final = chat_messages(call_type, model, messages, input, request_kwargs) + request: Final = request_to_switchyard(chat) if chat is not None else None + except (ValueError, KeyError, TypeError): + return self._fallback("unsupported_content", messages) + if request is None: + return self._fallback("unsupported_request_type", messages) + judge: Final = self.config.models.judge + try: + async for step in self.algorithm.run_stream( + request, self.config.runtime_models(), headers=switchyard_headers(request_kwargs) + ): + match step: + case Step.CallModel(call) if isinstance( + self.config.classifier, LlmClassifierConfig + ) and call.models[:1] == [judge]: + await self._answer_judge(call, request_kwargs) + case Step.CallDecision(call) if ( + isinstance(self.config.classifier, DecisionClassifierConfig) and call.model == judge + ): + await self._answer_decision(call, request_kwargs) + case Step.CallModel(): + return self._fallback("unexpected_judge_call", messages) + case Step.CallDecision(): + return self._fallback("unexpected_decision_call", messages) + case Step.Done(outcome): + return self._outcome(outcome, messages) + except LibsyError: + return self._fallback("libsy_error", messages) + return self._fallback("no_decision", messages) diff --git a/litellm/router_strategy/switchyard_router/translation.py b/litellm/router_strategy/switchyard_router/translation.py new file mode 100644 index 00000000000..246f2ba3e41 --- /dev/null +++ b/litellm/router_strategy/switchyard_router/translation.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Final, Protocol + +from pydantic import TypeAdapter + +from litellm.types.decisions import ( + OpenAIChoiceAnswer, + OpenAIDecisionResponse, + systemone_choice_key, +) +from litellm.types.utils import CallTypes, ModelResponse + +if TYPE_CHECKING: + from switchyard_rust.libsy import LlmResponse + +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_SEQUENCE: Final = TypeAdapter(tuple[object, ...]) +_CALL_TYPES: Final = frozenset(call_type.value for call_type in CallTypes) + + +def mapping(value: object) -> Mapping[str, object]: + return _MAPPING.validate_python(value) + + +def sequence(value: object) -> tuple[object, ...]: + return _SEQUENCE.validate_python(value) + + +def text_content(value: object) -> tuple[Mapping[str, object], ...]: + if value is None: + return () + if isinstance(value, str): + return ({"type": "text", "text": value},) + blocks: Final = tuple(mapping(block) for block in sequence(value)) + if any(block.get("type") != "text" or not isinstance(block.get("text"), str) for block in blocks): + raise ValueError("Switchyard routing supports text content only; media and audio are unsupported") + return tuple({"type": "text", "text": block["text"]} for block in blocks) + + +def tool_call(value: object) -> Mapping[str, object]: + call: Final = mapping(value) + function: Final = mapping(call.get("function")) + arguments: Final = function.get("arguments") + if call.get("type") != "function" or not isinstance(arguments, str): + raise ValueError("Switchyard requires function calls with JSON arguments") + return { + "type": "tool_call", + "id": call["id"], + "name": function["name"], + "arguments": _MAPPING.validate_json(arguments), + } + + +def message_to_switchyard(value: object) -> Mapping[str, object]: + message: Final = mapping(value) + if message.get("audio") is not None: + raise ValueError("Switchyard routing does not support audio") + role: Final = message.get("role") + content: Final = text_content(message.get("content")) + if role == "tool": + return { + "role": "tool", + "content": ( + { + "type": "tool_result", + "tool_call_id": message["tool_call_id"], + "content": content, + "is_error": message.get("is_error"), + }, + ), + } + if role not in ("user", "system", "developer", "assistant"): + raise ValueError("Switchyard received an unsupported message role") + calls: Final = tuple(tool_call(call) for call in sequence(message.get("tool_calls") or ())) + return {"role": role, "content": (*content, *calls)} + + +class ChatConverter(Protocol): + def get_structured_messages( + self, + data: dict[str, object], # mutable-ok: matches the request converters' existing dict parameter + *, + keep_tool_errors: bool = False, + ) -> Sequence[object] | None: ... + + +def _instantiate(converter: Callable[[], ChatConverter]) -> ChatConverter: + return converter() + + +def chat_messages( + call_type: str | None, + model: str, + messages: Sequence[Mapping[str, object]] | None, + input: object, + request_kwargs: Mapping[str, object], +) -> tuple[Mapping[str, object], ...] | None: + from litellm.llms import load_guardrail_translation_mappings + + converter: Final = ( + load_guardrail_translation_mappings().get(CallTypes(call_type)) if call_type in _CALL_TYPES else None + ) + if converter is None: + return None + request: Final = { + **request_kwargs, + "model": model, + **({"messages": [dict(message) for message in messages]} if messages is not None else {}), + **({"input": input} if input is not None else {}), + } + return tuple(map(mapping, _instantiate(converter).get_structured_messages(request, keep_tool_errors=True) or ())) + + +def request_to_switchyard(messages: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + if not messages: + raise ValueError("Switchyard requires at least one message") + return { + "model": "auto", + "messages": tuple(message_to_switchyard(message) for message in messages), + "stream": False, + } + + +def judge_message(value: object) -> Mapping[str, object]: + message: Final = mapping(value) + blocks: Final = text_content(message.get("content")) + return {"role": message["role"], "content": "".join(str(block["text"]) for block in blocks)} + + +def judge_request(request: Mapping[str, object], model: str) -> Mapping[str, object]: + output: Final = mapping(request.get("output") or {}) + sampling: Final = mapping(request.get("sampling") or {}) + options: Final = { + "max_tokens": output.get("max_output_tokens"), + "response_format": output.get("response_format"), + "temperature": sampling.get("temperature"), + "top_p": sampling.get("top_p"), + } + return { + "model": model, + "stream": False, + "messages": [ + judge_message(message) + for message in ( + *sequence(request.get("instructions") or ()), + *sequence(request.get("messages") or ()), + ) + ], + **{key: value for key, value in options.items() if value is not None}, + } + + +def judge_response(response: ModelResponse) -> LlmResponse.Agg: + from switchyard_rust.libsy import LlmResponse + + body: Final = _MAPPING.validate_json(response.model_dump_json(exclude_unset=True)) + choices: Final = tuple(mapping(choice) for choice in sequence(body["choices"])) + return LlmResponse.Agg( + { + "id": response.id, + "model": response.model, + "outputs": tuple(message_to_switchyard(choice["message"]) for choice in choices), + } + ) + + +def _decision_text(value: object) -> str: + return value if isinstance(value, str) else json.dumps(value) + + +def _decision_question(name: str, question: Mapping[str, object]) -> Mapping[str, object]: + kind: Final = mapping(question["kind"]) + if kind.get("type") != "choice": + raise ValueError("Switchyard decision routing asks choice questions only") + return { + "type": "choice", + "name": name, + "instructions": _decision_text(question.get("instructions")), + "choices": [ + { + "value": option["id"], + **({"description": _decision_text(option["description"])} if option.get("description") else {}), + } + for option in map(mapping, sequence(mapping(kind["data"])["options"])) + ], + } + + +def decision_request(request: Mapping[str, object], model: str) -> Mapping[str, object]: + questions: Final = mapping(request["questions"]) + return { + "model": model, + "input": _decision_text(request["context"]), + "questions": [_decision_question(name, mapping(question)) for name, question in questions.items()], + } + + +def _choice_value(answer: object) -> Mapping[str, object]: + if not isinstance(answer, OpenAIChoiceAnswer): + raise TypeError("The decision model refused or returned a non-choice answer") + return { + "type": "choice", + "data": { + "selected": systemone_choice_key(answer.choice), + "probabilities": {systemone_choice_key(p.value): p.probability for p in answer.probabilities}, + }, + } + + +def decision_response(response: OpenAIDecisionResponse, request: Mapping[str, object]) -> Mapping[str, object]: + names: Final = tuple(mapping(request["questions"])) + if len(response.answers) != len(names): + raise ValueError("The decision model did not answer every question") + return { + "id": None, + "model": response.model, + "answers": { + answer.name or name: { + "value": _choice_value(answer), + "provider_confidence": getattr(answer, "confidence", None), + } + for name, answer in zip(names, response.answers, strict=True) + }, + "usage": { + "input_tokens": response.usage.input_tokens, + "output_tokens": response.usage.output_tokens, + "total_tokens": response.usage.total_tokens, + }, + } diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 9f63d27f5f3..c04430322c4 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -207,6 +207,7 @@ class DeploymentOptions: @dataclass(frozen=True, slots=True, kw_only=True) class SpecializedRouterOptions: + switchyard_router_config: Mapping[str, object] | None = None auto_router_config_path: str | None = None auto_router_config: str | None = None auto_router_default_model: str | None = None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index c9739eee91d..378899b667d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -957,6 +957,7 @@ class ChatCompletionToolMessage(TypedDict): role: Literal["tool"] content: str | Iterable[ToolMessageContentPart] # writable-ok: Pydantic warns on ReadOnly TypedDict fields tool_call_id: str + is_error: NotRequired[bool] # writable-ok: Pydantic warns on ReadOnly TypedDict fields class ChatCompletionFunctionMessage(TypedDict): diff --git a/litellm/types/router.py b/litellm/types/router.py index 3dcfc694003..c0b182f0f20 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -659,6 +659,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): tag_regex: list[str] | None = None # auto-router params + switchyard_router_config: Mapping[str, object] | None = None auto_router_config_path: str | None = None auto_router_config: str | None = None auto_router_default_model: str | None = None @@ -1461,6 +1462,7 @@ class PreRoutingStrategy(Protocol): messages: list[dict[str, Any]] | None = None, input: "str | list[Any] | None" = None, specific_deployment: bool | None = False, + call_type: str | None = None, ) -> "PreRoutingHookResponse | None": ... diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b082198d3f2..755a0a8cc7a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3382,7 +3382,10 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" router_model_name: str - router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic"]] + router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic", "switchyard"]] + switchyard_algorithm: ReadOnly[str] + switchyard_evidence: ReadOnly[object] + switchyard_fallback_reason: ReadOnly[str] router_config_id: ReadOnly[str] router_config_updated_at: ReadOnly[str] router_config_fingerprint: ReadOnly[str] @@ -3443,10 +3446,12 @@ class StandardLoggingKubernetesPodRouting(TypedDict): # so a redacted row stays explainable. `test_every_routing_decision_field_is_classified` # fails if a field is added to the record without being placed in one set or the other. PROMPT_QUOTING_ROUTING_DECISION_FIELDS: frozenset[str] = frozenset( - {"signals", "matched_keyword", "escalation_keyword", "classifier_crux"} + {"signals", "matched_keyword", "escalation_keyword", "classifier_crux", "switchyard_evidence"} ) DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset( { + "switchyard_algorithm", + "switchyard_fallback_reason", "router_model_name", "router_type", "router_config_id", diff --git a/pyproject.toml b/pyproject.toml index e193eeee93f..02834f604d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ Documentation = "https://docs.litellm.ai" # sub-packages and the opentelemetry trio move in lockstep, and grpcio is # supply-chain-pinned to a vetted, aged release. [project.optional-dependencies] +switchyard = ["nemo-switchyard==0.4.0a1"] proxy = [ "gunicorn>=23.0.0,<24.0", "uvicorn>=0.33.0,<1.0", @@ -268,6 +269,7 @@ proxy-dev = [ "opentelemetry-instrumentation-fastapi==0.66b1", "azure-identity==1.25.2", "a2a-sdk==1.1.0", + "nemo-switchyard==0.4.0a1", "backports-zstd==1.8.0; python_version < '3.14'", "httptools>=0.7.1,<0.8", "semver>=3.0.4,<4", diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 8affe2e136e..b6229dbaec0 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -3458,3 +3458,41 @@ def test_reading_a_dropped_budget_for_a_guardrail_logs_no_drop_warning(caplog: p assert messages == [{"role": "user", "content": "hi"}] assert not [record for record in caplog.records if "Dropping" in record.getMessage()] + + +FAILED_TOOL_RESULT_REQUEST: Final = { + "model": "claude-sonnet", + "max_tokens": 16, + "messages": [ + {"role": "user", "content": "Rename the helper"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "edit0", "name": "Edit", "input": {"file_path": "a.py"}}], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "edit0", "content": "String not found", "is_error": True} + ], + }, + ], +} + + +@pytest.mark.parametrize("keep_tool_errors", [True, False]) +def test_structured_messages_carry_tool_failures_only_when_asked(keep_tool_errors: bool) -> None: + messages: Final = AnthropicMessagesHandler().get_structured_messages( + FAILED_TOOL_RESULT_REQUEST, keep_tool_errors=keep_tool_errors + ) + assert messages is not None + (tool_message,) = [message for message in messages if message["role"] == "tool"] + assert tool_message.get("is_error") is (True if keep_tool_errors else None) + + +def test_request_sent_to_a_model_never_carries_tool_failure_marker() -> None: + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter + + request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request=FAILED_TOOL_RESULT_REQUEST # pyright: ignore[reportArgumentType] # a JSON request in the Anthropic shape + ) + assert "is_error" not in json.dumps(request["messages"], default=str) diff --git a/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py b/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py index 3e3e89f8117..3eb901144a3 100644 --- a/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py +++ b/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py @@ -77,9 +77,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ), ], ) -def test_remove_sensitive_info_from_deployment( - model_config: dict, expected_config: dict -): +def test_remove_sensitive_info_from_deployment(model_config: dict, expected_config: dict): sanitized_config = remove_sensitive_info_from_deployment(model_config) assert sanitized_config == expected_config @@ -110,10 +108,7 @@ def test_remove_sensitive_info_from_deployment_with_excluded_keys(): sanitized_config = remove_sensitive_info_from_deployment( copy.deepcopy(base_config), excluded_keys={"litellm_credentials_name"} ) - assert ( - sanitized_config["litellm_params"]["litellm_credentials_name"] - == "my-credential-name" - ) + assert sanitized_config["litellm_params"]["litellm_credentials_name"] == "my-credential-name" # access_token should still be masked (not in excluded_keys) assert sanitized_config["litellm_params"]["access_token"] != "token-12345" @@ -121,3 +116,24 @@ def test_remove_sensitive_info_from_deployment_with_excluded_keys(): # api_key should still be removed (popped) regardless of excluded_keys assert "api_key" not in sanitized_config["litellm_params"] + + +def test_remove_sensitive_info_from_deployment_keeps_nested_nulls(): + router_config = { + "models": {"capable": "strong", "efficient": "fast", "judge": None}, + "options": {"confidence_threshold": None, "tool_semantics": None}, + "fallbacks": [None, "fast"], + } + deployment = { + "model_name": "route", + "litellm_params": { + "model": "auto_router/switchyard", + "api_key": "sk-sensitive-key-123", + "switchyard_router_config": copy.deepcopy(router_config), + }, + } + + sanitized = remove_sensitive_info_from_deployment(deployment) + + assert "api_key" not in sanitized["litellm_params"] + assert sanitized["litellm_params"]["switchyard_router_config"] == router_config diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 32c9535b4c8..2fdb77af8f9 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -1085,7 +1085,7 @@ class TestClearCache: Test that clear_cache successfully clears router model caches and reloads models. """ mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = ["openai/gpt-4o", "openai/gpt-4o-mini"] @@ -1125,7 +1125,7 @@ class TestClearCache: # entries are auto_router/* deployments (so their router-map entries should be # cleared for reload); the config-defined router is preserved. mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1209,7 +1209,7 @@ class TestClearCache: ) mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1270,7 +1270,7 @@ class TestClearCachePreservesConfigRouters: ) mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1315,7 +1315,7 @@ class TestClearCachePreservesConfigRouters: from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1346,15 +1346,15 @@ class TestClearCachePreservesConfigRouters: assert "shared-name" in plugins["complexity"] @pytest.mark.asyncio - async def test_db_quality_and_adaptive_routers_are_evicted(self): - """The auto_router/ prefix also covers quality_router/ and adaptive_router/. Their + async def test_db_quality_adaptive_and_switchyard_routers_are_evicted(self): + """The auto_router/ prefix also covers quality_router/, adaptive_router/ and switchyard. Their registry entries must be popped too, or reload's init raises 'already exists' (quality) or leaves a stale entry (adaptive). """ from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1367,11 +1367,17 @@ class TestClearCachePreservesConfigRouters: "model_info": {"id": "db-a", "db_model": True}, "litellm_params": {"model": "auto_router/adaptive_router/a1"}, }, + { + "model_name": "s1", + "model_info": {"id": "db-s", "db_model": True}, + "litellm_params": {"model": "auto_router/switchyard"}, + }, ] plugins["auto"] = {} plugins["complexity"] = {} plugins["quality"] = {"q1": MagicMock()} plugins["adaptive"] = {"a1": MagicMock()} + plugins["switchyard"] = {"s1": MagicMock()} mock_config = MagicMock() mock_config.add_deployment_locked = AsyncMock( @@ -1389,6 +1395,7 @@ class TestClearCachePreservesConfigRouters: assert "q1" not in plugins["quality"] assert "a1" not in plugins["adaptive"] + assert "s1" not in plugins["switchyard"] class TestDeleteModelClearsRouterRegistry: @@ -1503,7 +1510,7 @@ class TestDeleteModelClearsRouterRegistry: mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row) mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.apply.return_value = ApplyResult( version=1, @@ -11431,6 +11438,98 @@ async def test_add_new_model_reports_a_database_outage_during_the_insert_as_503( assert (exc_info.value.code, exc_info.value.type) == ("503", "no_db_connection") +@pytest.mark.parametrize("invalid", [ + {"algorithm": "random"}, + {"classifier": {}}, + {"options": {"capable_system_prompt": "ignored"}}, +]) +def test_switchyard_write_rejects_unloadable_configuration(invalid: dict[str, object]) -> None: + from litellm.proxy.management_endpoints.model_management_endpoints import _strategy_router_write_violation + from litellm.types.router import updateLiteLLMParams + + violation: Final = _strategy_router_write_violation( + incoming_params=updateLiteLLMParams( + model="auto_router/switchyard", + switchyard_router_config={"models": {"capable": "strong", "efficient": "fast"}, **invalid}, + ), + existing_params=None, + ) + assert violation is not None + assert "Invalid switchyard_router_config" in violation + + +SWITCHYARD_PARAMS: Final = { + "model": "auto_router/switchyard", + "switchyard_router_config": {"models": {"capable": "strong", "efficient": "fast"}}, +} + + +@pytest.mark.parametrize( + ("model_name", "incoming", "existing"), + [ + ("strong", SWITCHYARD_PARAMS, None), + ("fast", {"model": "auto_router/switchyard"}, SWITCHYARD_PARAMS), + ("fast", None, SWITCHYARD_PARAMS), + ( + "smart", + { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": {"SIMPLE": "smart", "COMPLEX": "strong"}}, + }, + None, + ), + ], +) +def test_strategy_router_that_lists_itself_is_rejected_before_saving( + model_name: str, incoming: dict[str, object] | None, existing: dict[str, object] | None +) -> None: + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _raise_on_strategy_router_write_violation, + ) + from litellm.types.router import updateLiteLLMParams + + with pytest.raises(ProxyException) as exc_info: + _raise_on_strategy_router_write_violation( + incoming_params=updateLiteLLMParams(**incoming) if incoming is not None else None, + existing_params=updateLiteLLMParams(**existing) if existing is not None else None, + model_name=model_name, + ) + assert exc_info.value.code == "400" + assert model_name in exc_info.value.message + + +def test_strategy_router_with_other_targets_saves() -> None: + from litellm.proxy.management_endpoints.model_management_endpoints import _self_routing_violation + from litellm.types.router import updateLiteLLMParams + + assert _self_routing_violation("route", updateLiteLLMParams(**SWITCHYARD_PARAMS), None) is None + + +def test_self_routing_is_caught_when_the_stored_model_is_encrypted(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy._types import ProxyException + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _raise_on_strategy_router_write_violation, + ) + from litellm.types.router import updateLiteLLMParams + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key-for-encryption") + stored: Final = updateLiteLLMParams( + **{**SWITCHYARD_PARAMS, "model": encrypt_value_helper("auto_router/switchyard")} + ) + assert stored.model != "auto_router/switchyard" + with pytest.raises(ProxyException) as exc_info: + _raise_on_strategy_router_write_violation( + incoming_params=updateLiteLLMParams( + switchyard_router_config={"models": {"capable": "route", "efficient": "fast"}} + ), + existing_params=stored, + model_name="route", + ) + assert exc_info.value.code == "400" + + @pytest.mark.asyncio async def test_rename_rewrites_only_in_a_transaction_holding_every_locked_teams_advisory_lock(): """A team that starts listing the old name between the advisory snapshot and the row lock must not be diff --git a/tests/unit/router_strategy/switchyard_router/__init__.py b/tests/unit/router_strategy/switchyard_router/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/router_strategy/switchyard_router/test_config.py b/tests/unit/router_strategy/switchyard_router/test_config.py new file mode 100644 index 00000000000..c24023ddb79 --- /dev/null +++ b/tests/unit/router_strategy/switchyard_router/test_config.py @@ -0,0 +1,100 @@ +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.router_strategy.switchyard_router.config import SwitchyardRouterConfig, validate_switchyard_config_write + + +@pytest.mark.parametrize("classifier", [None, {}]) +def test_default_confidence_depends_on_classifier(classifier: dict[str, object] | None) -> None: + config: Final = SwitchyardRouterConfig.model_validate( + { + "models": { + "capable": "strong", + "efficient": "fast", + **({"judge": "judge"} if classifier is not None else {}), + }, + "classifier": classifier, + } + ) + assert config.confidence_threshold == (0.7 if classifier is not None else 0.5) + + +@pytest.mark.parametrize("classifier", [None, {}]) +def test_explicit_confidence_is_preserved(classifier: dict[str, object] | None) -> None: + config: Final = SwitchyardRouterConfig.model_validate( + { + "models": { + "capable": "strong", + "efficient": "fast", + **({"judge": "judge"} if classifier is not None else {}), + }, + "classifier": classifier, + "options": {"confidence_threshold": 0.63, "picker": "capable_first"}, + } + ) + assert config.confidence_threshold == 0.63 + + +@pytest.mark.parametrize( + "option", ["escalation_note", "deescalation_note", "capable_system_prompt", "efficient_system_prompt", "unknown"] +) +def test_request_rewrite_and_unknown_options_are_rejected(option: str) -> None: + with pytest.raises(ValidationError, match=option): + SwitchyardRouterConfig.model_validate( + {"models": {"capable": "strong", "efficient": "fast"}, "options": {option: "ignored"}} + ) + + +@pytest.mark.parametrize( + "bad", + [ + {"algorithm": "random"}, + {"classifier": {}}, + {"options": {"confidence_threshold": 2}}, + {"models": {"capable": "", "efficient": "fast"}}, + {"models": {"capable": "same", "efficient": "same"}}, + {"models": {"capable": "strong", "efficient": "fast", "judge": "j"}, "classifier": {"session_affinity": True}}, + {"models": {"capable": "strong", "efficient": "fast", "judge": "judge"}}, + {"models": {"capable": "strong", "efficient": "fast", "judge": "j"}, "classifier": {"kind": "other"}}, + { + "models": {"capable": "strong", "efficient": "fast", "judge": "j"}, + "classifier": {"kind": "decision", "cutoff": 1.5}, + }, + { + "models": {"capable": "strong", "efficient": "fast", "judge": "j"}, + "classifier": {"kind": "decision", "base_threshold": 0.5}, + }, + { + "models": {"capable": "strong", "efficient": "fast", "judge": "j"}, + "classifier": {"kind": "llm", "cutoff": 0.5}, + }, + ], +) +def test_invalid_config_cannot_be_saved(bad: dict[str, object]) -> None: + violation: Final = validate_switchyard_config_write({"models": {"capable": "strong", "efficient": "fast"}, **bad}) + assert violation is not None + assert "Invalid switchyard_router_config" in violation + + +def test_missing_package_rejects_save_with_install_instructions() -> None: + import sys + from unittest.mock import patch + + with patch.dict(sys.modules, {"switchyard_rust.libsy": None}): + violation: Final = validate_switchyard_config_write({"models": {"capable": "strong", "efficient": "fast"}}) + assert violation is not None + assert "litellm[switchyard]" in violation + + +@pytest.mark.parametrize( + "classifier", [{"kind": "decision"}, {"kind": "decision", "cutoff": 0.4, "evidence": {"a": 1}}] +) +def test_decision_classifier_config_can_be_saved(classifier: dict[str, object]) -> None: + assert ( + validate_switchyard_config_write( + {"models": {"capable": "strong", "efficient": "fast", "judge": "jev"}, "classifier": classifier} + ) + is None + ) diff --git a/tests/unit/router_strategy/switchyard_router/test_switchyard_router.py b/tests/unit/router_strategy/switchyard_router/test_switchyard_router.py new file mode 100644 index 00000000000..8a26b4610db --- /dev/null +++ b/tests/unit/router_strategy/switchyard_router/test_switchyard_router.py @@ -0,0 +1,438 @@ +import asyncio +import json +from itertools import chain +from collections.abc import AsyncIterator, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs +from litellm.integrations.custom_logger import CustomLogger +from litellm.router_api import ChangeSet +from litellm.router_strategy.switchyard_router.config import SwitchyardRouterConfig, build_algorithm +from litellm.router_strategy.switchyard_router.switchyard_router import SwitchyardRouter +from litellm.types.decisions import OpenAIDecisionResponse +from litellm.types.router import Deployment +from litellm.types.utils import ModelResponse + +libsy = pytest.importorskip("switchyard_rust.libsy") + +HELLO: Final = [{"role": "user", "content": "Hello"}] + + +async def unexpected_judge(request: Mapping[str, object]) -> ModelResponse: + raise AssertionError("This route must not call an LLM judge") + + +async def unexpected_decision(request: Mapping[str, object]) -> OpenAIDecisionResponse: + raise AssertionError("This route must not call a decision model") + + +def config( + *, classifier: Mapping[str, object] | None = None, picker: str = "efficient_first", threshold: float | None = None +) -> SwitchyardRouterConfig: + return SwitchyardRouterConfig.model_validate( + { + "models": { + "capable": "strong", + "efficient": "fast", + **({"judge": "judge"} if classifier is not None else {}), + }, + "options": {"picker": picker, **({"confidence_threshold": threshold} if threshold is not None else {})}, + **({"classifier": classifier} if classifier is not None else {}), + } + ) + + +def router( + route_config: SwitchyardRouterConfig, + *, + completion=unexpected_judge, + decide=unexpected_decision, + algorithm: object = None, +) -> SwitchyardRouter: + return SwitchyardRouter("route", route_config, algorithm or build_algorithm(route_config), completion, decide) + + +async def route( + route_config: SwitchyardRouterConfig, + messages: object, + *, + call_type: str | None = "acompletion", + request_kwargs: Mapping[str, object] | None = None, + **router_kwargs: object, +) -> tuple[str, Mapping[str, object]]: + result: Final = await router(route_config, **router_kwargs).async_pre_routing_hook( # pyright: ignore[reportArgumentType] # test doubles + "route", request_kwargs or {}, messages, call_type=call_type + ) + assert result is not None + assert result.routing_decision is not None + return result.model, result.routing_decision + + +def anthropic_turn(call_id: str, tool: str, result: str, marked: bool) -> tuple[dict[str, object], ...]: + error: Final = {"is_error": True} if marked else {} + return ( + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Next step", "signature": "sig"}, + {"type": "tool_use", "id": call_id, "name": tool, "input": {"file_path": "a.py"}}, + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": call_id, "content": result, **error}]}, + ) + + +def responses_turn(call_id: str, tool: str, result: str, marked: bool) -> tuple[dict[str, object], ...]: + return ( + {"type": "function_call", "call_id": call_id, "name": tool, "arguments": '{"file_path":"a.py"}'}, + {"type": "function_call_output", "call_id": call_id, "output": result}, + ) + + +def chat_turn(call_id: str, tool: str, result: str, marked: bool) -> tuple[dict[str, object], ...]: + call: Final = {"id": call_id, "type": "function", "function": {"name": tool, "arguments": '{"file_path":"a.py"}'}} + return ( + {"role": "assistant", "content": None, "tool_calls": [call]}, + {"role": "tool", "tool_call_id": call_id, "content": result}, + ) + + +TURN_FORMATS: Final = {"anthropic_messages": anthropic_turn, "aresponses": responses_turn, "acompletion": chat_turn} + + +def transcript(endpoint: str, tool: str, results: tuple[str, ...], *, marked: bool = False) -> list[dict[str, object]]: + turn: Final = TURN_FORMATS[endpoint] + turns: Final = chain.from_iterable( + turn(f"call{index}", tool, result, marked) for index, result in enumerate(results) + ) + return [{"role": "user", "content": "Fix the helper"}, *turns] + + +def llm_router( + *, route_config: Mapping[str, object], judge_fails: bool = False, num_retries: int = 0 +) -> litellm.Router: + def deployment(name: str, reply: object) -> Mapping[str, object]: + return { + "model_name": name, + "litellm_params": {"model": f"openai/{name}", "api_key": "k", "mock_response": reply}, + } + + return litellm.Router( + num_retries=num_retries, + model_list=[ + deployment("strong", "strong"), + deployment("fast", "fast"), + *((deployment("judge", Exception("judge down")),) if judge_fails else ()), + switchyard_deployment(route_config), + ], + ) + + +def switchyard_deployment(route_config: Mapping[str, object]) -> dict[str, object]: + return { + "model_name": "route", + "litellm_params": {"model": "auto_router/switchyard", "switchyard_router_config": route_config}, + "model_info": {"id": "route-id"}, + } + + +@pytest.mark.asyncio +async def test_critical_tool_error_selects_capable_without_rewriting_messages() -> None: + messages: Final = transcript("acompletion", "bash", ("CUDA out of memory",)) + result: Final = await router(config()).async_pre_routing_hook("route", {}, messages, call_type="acompletion") + assert result is not None + assert result.model == "strong" + assert result.messages == messages + + +async def routed_reply(proxy: litellm.Router, endpoint: str, edits: tuple[str, ...]) -> object: + match endpoint: + case "anthropic_messages": + message: Final = await proxy.aanthropic_messages( + model="route", messages=transcript(endpoint, "Edit", edits), max_tokens=16 + ) + return message["content"][0]["text"] # pyright: ignore[reportIndexIssue] # non-streaming response + case "aresponses": + response: Final = await proxy.aresponses(model="route", input=transcript(endpoint, "Edit", edits)) + return response.output[0].content[0].text # pyright: ignore[reportAttributeAccessIssue] # non-streaming response + case _: + chat: Final = await proxy.acompletion(model="route", messages=transcript(endpoint, "Edit", edits)) + return chat.choices[0].message.content # pyright: ignore[reportAttributeAccessIssue] # non-streaming response + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["acompletion", "anthropic_messages", "aresponses"]) +async def test_router_routes_agent_traffic_from_every_endpoint(endpoint: str) -> None: + proxy: Final = llm_router(route_config={"models": {"capable": "strong", "efficient": "fast"}}) + assert await routed_reply(proxy, endpoint, ("Edit applied",) * 3) == "fast" + + +@pytest.mark.asyncio +async def test_router_follows_an_edited_switchyard_router_and_forgets_a_deleted_one() -> None: + proxy: Final = llm_router(route_config={"models": {"capable": "strong", "efficient": "fast"}}) + assert await routed_reply(proxy, "acompletion", ("Edit applied",) * 3) == "fast" + + swapped: Final = switchyard_deployment({"models": {"capable": "fast", "efficient": "strong"}}) + _ = proxy.apply(ChangeSet(upsert=[Deployment(**swapped)])) + assert await routed_reply(proxy, "acompletion", ("Edit applied",) * 3) == "strong" + + _ = proxy.apply(ChangeSet(delete=["route-id"])) + with pytest.raises(litellm.BadRequestError): + await proxy.acompletion(model="route", messages=HELLO) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("marked", "expected"), [(True, "strong"), (False, "fast")]) +async def test_anthropic_tool_failure_marker_reaches_switchyard(marked: bool, expected: str) -> None: + failures: Final = transcript("anthropic_messages", "Edit", ("String to replace not found.",) * 4, marked=marked) + model, decision = await route(config(), failures, call_type="anthropic_messages") + assert model == expected + assert "switchyard_fallback_reason" not in decision + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_kwargs", "expected"), + [ + ({"litellm_session_id": "s1"}, ["strong", "strong", "strong", "fast"]), + ( + {"proxy_server_request": {"headers": {"X-Session-Id": "s1", "session-id": "s1"}}}, + ["strong", "fast", "fast", "fast"], + ), + ({}, ["strong", "fast", "fast", "fast"]), + ], +) +async def test_capable_hold_follows_the_session(request_kwargs: Mapping[str, object], expected: list[str]) -> None: + switchyard: Final = router(config()) + turns: Final = (("CUDA out of memory",), *((("All tests passed",) * count) for count in (2, 3, 4))) + picks: Final = [await pick(switchyard, request_kwargs, results) for results in turns] + assert picks == expected + + +async def pick(switchyard: SwitchyardRouter, request_kwargs: Mapping[str, object], results: tuple[str, ...]) -> str: + result: Final = await switchyard.async_pre_routing_hook( + "route", request_kwargs, transcript("acompletion", "bash", results), call_type="acompletion" + ) + assert result is not None + return result.model + + +@pytest.mark.asyncio +async def test_a_session_hold_never_crosses_api_keys() -> None: + switchyard: Final = router(config()) + owner: Final = {"litellm_session_id": "shared", "metadata": {"user_api_key_hash": "owner"}} + other: Final = {"litellm_session_id": "shared", "metadata": {"user_api_key_hash": "other"}} + assert await pick(switchyard, owner, ("CUDA out of memory",)) == "strong" + assert await pick(switchyard, other, ("All tests passed",) * 2) == "fast" + assert await pick(switchyard, owner, ("All tests passed",) * 2) == "strong" + + +@pytest.mark.asyncio +async def test_a_client_metadata_key_hash_cannot_join_another_keys_session() -> None: + switchyard: Final = router(config()) + owner: Final = {"litellm_session_id": "shared", "litellm_metadata": {"user_api_key_hash": "owner"}} + spoofer: Final = { + "litellm_session_id": "shared", + "metadata": {"user_api_key_hash": "owner"}, + "litellm_metadata": {"user_api_key_hash": "spoofer"}, + } + assert await pick(switchyard, owner, ("CUDA out of memory",)) == "strong" + assert await pick(switchyard, spoofer, ("All tests passed",) * 2) == "fast" + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("p_solve", "expected"), [(0.0, "strong"), (1.0, "fast")]) +async def test_llm_judge_verdict_selects_the_target(p_solve: float, expected: str) -> None: + async def judge(request: Mapping[str, object]) -> ModelResponse: + assert "autorouter_classifier" in str(request["metadata"]) + verdict: Final = {"crux": "c", "primary_rule": "none", "capability_boundary": "unmatched", "p_solve": p_solve} + return ModelResponse(choices=[{"message": {"role": "assistant", "content": json.dumps(verdict)}}]) + + model, _ = await route(config(classifier={}), HELLO, completion=judge) + assert model == expected + + +def decision_answer(advantage: float) -> OpenAIDecisionResponse: + return OpenAIDecisionResponse.model_validate( + { + "model": "jev", + "answers": [ + { + "type": "choice", + "name": "route", + "choice": "advantage", + "probabilities": [ + {"value": "advantage", "probability": advantage}, + {"value": "no_advantage", "probability": 1 - advantage}, + ], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 0, "total_tokens": 1}, + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("parent_bucket", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize("classifier", [{}, {"kind": "decision"}]) +async def test_judge_calls_carry_the_callers_tags_and_origin_where_their_endpoint_reads_them( + parent_bucket: str, classifier: Mapping[str, object] +) -> None: + seen: Final[list[Mapping[str, object]]] = [] # mutable-ok: collects the judge request + + async def judge(request: Mapping[str, object]) -> ModelResponse: + seen.append(request) + verdict: Final = {"crux": "c", "primary_rule": "none", "capability_boundary": "unmatched", "p_solve": 1.0} + return ModelResponse(choices=[{"message": {"role": "assistant", "content": json.dumps(verdict)}}]) + + async def decide(request: Mapping[str, object]) -> OpenAIDecisionResponse: + seen.append(request) + return decision_answer(0.0) + + caller: Final = {parent_bucket: {"tags": ["eu-only"], "inherited_tags": ["team-eu"]}} + await route( + config(classifier=classifier, threshold=1.0), HELLO, completion=judge, decide=decide, request_kwargs=caller + ) + (request,) = seen + read_by_endpoint: Final = "litellm_metadata" if classifier else "metadata" + assert set(request) & {"metadata", "litellm_metadata"} == {read_by_endpoint} + metadata: Final = request[read_by_endpoint] + assert metadata["tags"] == ["eu-only"] # pyright: ignore[reportIndexIssue] # the judge request's metadata dict + assert metadata["inherited_tags"] == ["team-eu"] # pyright: ignore[reportIndexIssue] # same + assert metadata["internal_call_origin"] == "autorouter_classifier" # pyright: ignore[reportIndexIssue] # same + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("advantage", "expected"), [(0.61, "strong"), (0.6, "fast")]) +async def test_decision_model_advantage_must_exceed_the_cutoff(advantage: float, expected: str) -> None: + async def decide(request: Mapping[str, object]) -> OpenAIDecisionResponse: + (question,) = request["questions"] # pyright: ignore[reportGeneralTypeIssues] # the decisions request shape + assert {choice["value"] for choice in question["choices"]} == {"advantage", "no_advantage"} + return decision_answer(advantage) + + model, decision = await route( + config(classifier={"kind": "decision", "cutoff": 0.6}, threshold=1.0), HELLO, decide=decide + ) + assert model == expected + assert decision["switchyard_evidence"]["score"] == advantage + + +async def judge_down(request: Mapping[str, object]) -> ModelResponse: + raise ConnectionError("judge down") + + +async def decision_refused(request: Mapping[str, object]) -> OpenAIDecisionResponse: + return OpenAIDecisionResponse.model_validate( + { + "model": "jev", + "answers": [{"type": "refusal", "name": "route"}], + "usage": {"input_tokens": 1, "output_tokens": 0, "total_tokens": 1}, + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("classifier", "router_kwargs", "picker", "expected"), + [ + ({}, {"completion": judge_down}, "efficient_first", "fast"), + ({}, {"completion": judge_down}, "capable_first", "strong"), + ({"kind": "decision"}, {"decide": decision_refused}, "efficient_first", "fast"), + ], +) +async def test_classifier_failure_falls_open_to_the_picker_default( + classifier: Mapping[str, object], router_kwargs: Mapping[str, object], picker: str, expected: str +) -> None: + model, decision = await route(config(classifier=classifier, picker=picker, threshold=1.0), HELLO, **router_kwargs) + assert model == expected + assert "switchyard_fallback_reason" not in decision + + +class EndsWithoutDecision: + async def _steps(self) -> AsyncIterator[object]: + for step in (): + yield step + + def run_stream(self, *args: object, **kwargs: object) -> AsyncIterator[object]: + return self._steps() + + +class RaisesLibsyError: + async def _steps(self) -> AsyncIterator[object]: + raise libsy.LibsyError("algorithm failed") + yield + + def run_stream(self, *args: object, **kwargs: object) -> AsyncIterator[object]: + return self._steps() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route_config", "messages", "call_type", "algorithm", "reason"), + [ + ( + config(), + [{"role": "user", "content": [{"type": "input_audio"}]}], + "acompletion", + None, + "unsupported_content", + ), + (config(), HELLO, None, None, "unsupported_request_type"), + (config(), HELLO, "acompletion", EndsWithoutDecision(), "no_decision"), + (config(), HELLO, "acompletion", RaisesLibsyError(), "libsy_error"), + ( + config(classifier={}, threshold=1.0), + HELLO, + "acompletion", + build_algorithm(config(classifier={"kind": "decision"}, threshold=1.0)), + "unexpected_decision_call", + ), + ( + config(classifier={"kind": "decision"}, threshold=1.0), + HELLO, + "acompletion", + build_algorithm(config(classifier={}, threshold=1.0)), + "unexpected_judge_call", + ), + ], +) +async def test_anything_switchyard_cannot_route_goes_to_capable( + route_config: SwitchyardRouterConfig, messages: object, call_type: str | None, algorithm: object, reason: str +) -> None: + model, decision = await route(route_config, messages, call_type=call_type, algorithm=algorithm) + assert model == "strong" + assert decision["switchyard_fallback_reason"] == reason + + +@pytest.mark.asyncio +async def test_failing_judge_is_called_once_even_when_the_router_retries(monkeypatch: pytest.MonkeyPatch) -> None: + attempts: Final[list[str]] = [] + + class CountAttempts(CustomLogger): + def log_pre_api_call(self, model: str, messages: object, kwargs: dict[str, object]) -> None: + attempts.append(model) + + monkeypatch.setattr(litellm, "callbacks", [CountAttempts()]) + proxy: Final = llm_router( + route_config={ + "models": {"capable": "strong", "efficient": "fast", "judge": "judge"}, + "classifier": {}, + "options": {"confidence_threshold": 1.0}, + }, + judge_fails=True, + num_retries=3, + ) + await proxy.acompletion(model="route", messages=HELLO) + assert attempts.count("judge") == 1 + + +@pytest.mark.asyncio +async def test_judge_cancellation_propagates() -> None: + async def cancelled(request: Mapping[str, object]) -> ModelResponse: + raise asyncio.CancelledError() + + with pytest.raises(asyncio.CancelledError): + await route(config(classifier={}), HELLO, completion=cancelled) diff --git a/tests/unit/router_strategy/test_router_tag_routing.py b/tests/unit/router_strategy/test_router_tag_routing.py index 0d3876c64a3..e916e5b235d 100644 --- a/tests/unit/router_strategy/test_router_tag_routing.py +++ b/tests/unit/router_strategy/test_router_tag_routing.py @@ -3023,7 +3023,7 @@ class _RewriteToTierStrategy: self.rewrite_to = rewrite_to async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 1b98cd4b4b6..ca61fcd0905 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -9006,7 +9006,7 @@ class TestConsumedRequestTagsStamp: self.rewrite_to = rewrite_to async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse @@ -9097,7 +9097,7 @@ class TestClaudeCodeSubagentSessionRouterBinding: self.routed_model = routed_model async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse @@ -10151,7 +10151,7 @@ class TestModelGroupAliasReachesPreRoutingStrategies: class _RewriteStrategy: async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse @@ -10268,7 +10268,7 @@ class TestTeamPublicNameReachesPreRoutingStrategies: self.rewrite_to = rewrite_to async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse @@ -10505,7 +10505,7 @@ class TestAutoRouterCompressionDecoupling: self.received_messages: list[dict] | None = None async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index d71791b6f11..aae5dfe6472 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -168,6 +168,7 @@ OPTION_NAMES: Final = ( "tag_regex", "max_file_size_mb", "silent_model", + "switchyard_router_config", "auto_router_config_path", "auto_router_config", "auto_router_default_model", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index f9304bab0d2..d13bc3d3181 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -109,6 +109,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { model?: string | null; base_model?: string | null; custom_llm_provider?: string | null; + switchyard_router_config?: unknown; complexity_router_config?: unknown; complexity_router_default_model?: string | null; auto_router_config?: unknown; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts index 2cdc13606bb..0fe320a318a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts @@ -87,6 +87,14 @@ const configManaged = (label: string, config: Record): Presenta /** How each strategy renders itself, given its own config object. */ const PRESENTERS: Record) => Presentation> = { + switchyard: (config) => ({ + typeLabel: "Switchyard", + targets: dedupe( + [asRecord(config.models).capable, asRecord(config.models).efficient].filter( + (model): model is string => typeof model === "string", + ), + ), + }), complexity: (config) => ({ typeLabel: complexityTypeLabel(config), targets: dedupe(Object.values(asRecord(config.tiers)).flatMap(normalizeTierModels)), @@ -131,7 +139,8 @@ export const toAutoRouterRow = ( canDelete: canDelete && mayActOnRow, editBlockedReason, createdAt: info.created_at ?? undefined, - defaultModel: (params[strategy.defaultModelKey] as string | null | undefined) ?? null, + defaultModel: + strategy.kind === "switchyard" ? null : (params[strategy.defaultModelKey] as string | null | undefined) ?? null, deployment, ...PRESENTERS[strategy.kind](asRecord(params[strategy.configKey])), }; diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index bd8b4eb9854..0dd80854312 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -135,6 +135,7 @@ interface AutoRouterClassifierTabsProps { children: React.ReactNode; judgeModel?: React.ReactNode; decisionModel?: React.ReactNode; + switchyard?: { selected: boolean; onSelect: () => void; fields: React.ReactNode }; } const AutoRouterClassifierTabs: React.FC = ({ @@ -143,6 +144,7 @@ const AutoRouterClassifierTabs: React.FC = ({ children, judgeModel, decisionModel, + switchyard, }) => { const { t } = useTranslation(); const id = useId(); @@ -159,14 +161,16 @@ const AutoRouterClassifierTabs: React.FC = ({ jev: "jev", custom: "custom", }; - const family = familyByType[classifierType]; + const family = switchyard?.selected ? "switchyard" : familyByType[classifierType]; const hasCustomTiers = Boolean(value.custom_tier_set); const changeType = (next: ClassifierType) => { if (next !== classifierType) onChange(transitionClassifierType(value, next)); }; const changeFamily = (next: unknown) => { + if (next === "switchyard") return switchyard?.onSelect(); if (next === family) return; + if (next === familyByType[classifierType]) return onChange(value); if (next === "heuristics") changeType("heuristic"); if (next === "llm") changeType("llm"); if (next === "jev") changeType("jev"); @@ -207,7 +211,11 @@ const AutoRouterClassifierTabs: React.FC = ({ - + {[ { value: "heuristics", @@ -220,6 +228,9 @@ const AutoRouterClassifierTabs: React.FC = ({ label: t("addModel.autoRouterClassifierTabs.families.decisionModel"), description: t("addModel.autoRouterClassifierTabs.families.decisionModelDescription"), }, + ...(switchyard + ? [{ value: "switchyard", label: "Switchyard", description: "Route by the conversation’s tool history." }] + : []), ].map((option) => ( + {family === "switchyard" && switchyard?.fields} {family === "jev" && (decisionModel ?? (
@@ -355,7 +367,7 @@ const AutoRouterClassifierTabs: React.FC = ({

)} - {hasCustomTiers && ( + {family !== "switchyard" && hasCustomTiers && (

{t("addModel.autoRouterClassifierTabs.restoreStandardTiers")}

)} {availability.data?.error && !availability.isChecking && ( @@ -372,7 +384,7 @@ const AutoRouterClassifierTabs: React.FC = ({

)} - {children} + {family !== "switchyard" && children} ); }; diff --git a/ui/litellm-dashboard/src/components/add_model/BlockedReasonTooltip.tsx b/ui/litellm-dashboard/src/components/add_model/BlockedReasonTooltip.tsx new file mode 100644 index 00000000000..59588cf478f --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/BlockedReasonTooltip.tsx @@ -0,0 +1,17 @@ +import React from "react"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; + +const BlockedReasonTooltip: React.FC<{ reason: string | null; children: React.ReactElement }> = ({ + reason, + children, +}) => + reason === null ? ( + children + ) : ( + + + {reason} + + ); + +export default BlockedReasonTooltip; diff --git a/ui/litellm-dashboard/src/components/add_model/SwitchyardConfigFields.tsx b/ui/litellm-dashboard/src/components/add_model/SwitchyardConfigFields.tsx new file mode 100644 index 00000000000..2043e392928 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/SwitchyardConfigFields.tsx @@ -0,0 +1,342 @@ +"use client"; + +import { useId, useState } from "react"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { Slider } from "@/components/ui/slider"; +import { Switch } from "@/components/ui/switch"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import ModelChoiceCombobox, { type ModelChoice } from "./ModelChoiceCombobox"; +import { + stageConfidence, + toggleClassifier, + setClassifierKind, + parseToolSemantics, + type SwitchyardConfig, +} from "./switchyard_config"; + +interface Props { + value: SwitchyardConfig; + onChange: (value: SwitchyardConfig) => void; + models: ModelChoice[]; + decisionModels: ModelChoice[]; + onToolSemanticsError: (error: string | null) => void; +} + +const isStructured = (instructions: unknown): instructions is Record => + typeof instructions === "object" && instructions !== null; +const instructionsText = (instructions: string | Record | null | undefined): string => + isStructured(instructions) ? JSON.stringify(instructions, null, 2) : instructions ?? ""; + +export default function SwitchyardConfigFields({ + value, + onChange, + models, + decisionModels, + onToolSemanticsError, +}: Props) { + const id = useId(); + const [toolSemantics, setToolSemantics] = useState( + value.options.tool_semantics ? JSON.stringify(value.options.tool_semantics, null, 2) : "", + ); + const classifier = value.classifier; + const llm = classifier?.kind === "llm" ? classifier : undefined; + const decision = classifier?.kind === "decision" ? classifier : undefined; + const setOption = (key: "recent_window", text: string) => + onChange({ ...value, options: { ...value.options, [key]: text === "" ? undefined : Number(text) } }); + const setClassifierNumber = ( + key: "base_threshold" | "threshold_step" | "max_output_tokens" | "recent_turn_window", + text: string, + ) => { + if (llm) onChange({ ...value, classifier: { ...llm, [key]: text === "" ? undefined : Number(text) } }); + }; + const changeToolSemantics = (text: string) => { + setToolSemantics(text); + try { + const parsed = parseToolSemantics(text); + onChange({ ...value, options: { ...value.options, tool_semantics: parsed } }); + onToolSemanticsError(null); + } catch { + onToolSemanticsError("Tool semantics must be JSON mapping observe, mutate, plan, or new to arrays of tool names"); + } + }; + return ( +
+
+
+ + Heuristic + +

+ Stage Router +

+
+
+ + + {value.options.picker === "capable_first" && ( +

+ Switchyard has not benchmarked this bias, so it has no calibrated confidence thresholds. +

+ )} +
+ {(["capable", "efficient"] as const).map((tier) => ( +
+ + onChange({ ...value, models: { ...value.models, [tier]: model ?? "" } })} + choices={models} + placeholder="Select a model" + ariaInvalid={undefined} + ariaDescribedBy={undefined} + /> +
+ ))} +
+
+
+
+ +

+ Ask a judge model when the tool history does not give a confident routing decision. +

+
+ onChange(toggleClassifier(value, enabled))} + /> +
+ {classifier && ( +
+
+ + +
+
+ + onChange({ ...value, models: { ...value.models, judge: judge ?? "" } })} + choices={decision ? decisionModels : models} + placeholder={decision ? "Select a decision model" : "Select a judge model"} + ariaInvalid={undefined} + ariaDescribedBy={undefined} + /> +
+
+ )} +
+
+
+ + Confidence threshold + + {stageConfidence(value).toFixed(2)} +
+ + onChange({ + ...value, + options: { ...value.options, confidence_threshold: Array.isArray(next) ? next[0] : next }, + }) + } + /> +

+ Below this confidence, use {classifier ? "the judge model" : "the bias’s default model"}. Critical errors can + still select the capable model. +

+
+
+ Advanced settings +
+

Stage

+
+ + setOption("recent_window", event.target.value)} + /> +
+
+ +