diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 2fe4130a310..e049c62d28f 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -166,9 +166,11 @@ from litellm.utils import ( def _rust_responses_websocket_enabled( custom_llm_provider: str | None, ) -> bool: - from litellm.rust_bridge.configuration import rust_enabled + from litellm.rust_bridge.catalog import Context, Delivery, Route, decision + from litellm.rust_bridge.configuration import Decision - return custom_llm_provider == "openai" and rust_enabled() + context: Final = Context(Route.RESPONSES, provider=custom_llm_provider, delivery=Delivery.WEBSOCKET) + return decision(context) is not Decision.PYTHON from .http_handler import get_shared_realtime_ssl_context @@ -2454,11 +2456,10 @@ class BaseLLMHTTPHandler: request_body: dict, timeout: float | httpx.Timeout | None, ) -> AnthropicMessagesResponse | None: - if custom_llm_provider not in ("azure_ai", "anthropic"): - return None - from litellm.rust_bridge.configuration import rust_enabled + from litellm.rust_bridge.catalog import Context, Route, decision + from litellm.rust_bridge.configuration import Decision - if not rust_enabled(): + if decision(Context(Route.MESSAGES, provider=custom_llm_provider, model=model)) is Decision.PYTHON: return None if has_agentic_hook: return None diff --git a/litellm/ocr/input.py b/litellm/ocr/input.py index bcb448371c4..a58c7246128 100644 --- a/litellm/ocr/input.py +++ b/litellm/ocr/input.py @@ -5,7 +5,8 @@ from typing import Final, Literal, Protocol, cast # noqa: TID251 # native call from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.configuration import rust_ocr_enabled +from litellm.rust_bridge.catalog import Context, Route, decision +from litellm.rust_bridge.configuration import Decision class FileReader(Protocol): @@ -64,10 +65,15 @@ _MIME_TYPE: Final = NativeBinding( ), ) _PYTHON_MAX_FILE_BYTES: Final = 50 * 1024 * 1024 +_OCR_HELPERS: Final = Context(Route.OCR) + + +def _native_helpers_selected() -> bool: + return decision(_OCR_HELPERS) is not Decision.PYTHON def get_mime_type(file_path: str) -> str: - native: Final = _MIME_TYPE.load() if rust_ocr_enabled() else None + native: Final = _MIME_TYPE.load() if _native_helpers_selected() else None if native is None: from litellm.ocr import legacy @@ -76,14 +82,14 @@ def get_mime_type(file_path: str) -> str: def get_max_file_bytes() -> int: - limit: Final = _MAX_FILE_BYTES.load() if rust_ocr_enabled() else None + limit: Final = _MAX_FILE_BYTES.load() if _native_helpers_selected() else None if limit is None: return _PYTHON_MAX_FILE_BYTES return limit def convert_file_document_to_url_document(document: FileDocument) -> dict[str, str]: - native: Final = _FILE_DOCUMENT.load() if rust_ocr_enabled() else None + native: Final = _FILE_DOCUMENT.load() if _native_helpers_selected() else None if native is None: from litellm.ocr import legacy @@ -94,7 +100,7 @@ def convert_file_document_to_url_document(document: FileDocument) -> dict[str, s def convert_upload_to_url_document( file_content: bytes, filename: str | None, content_type: str | None ) -> dict[str, str]: - native: Final = _UPLOAD_DOCUMENT.load() if rust_ocr_enabled() else None + native: Final = _UPLOAD_DOCUMENT.load() if _native_helpers_selected() else None if native is None: from litellm.ocr import legacy diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 382c5d6aae4..faec3092d2b 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -6,10 +6,10 @@ import httpx from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.ocr import legacy from litellm.ocr.input import convert_file_document_to_url_document, get_mime_type -from litellm.rust_bridge.bindings import native_exception_types -from litellm.rust_bridge.configuration import rust_ocr_enabled +from litellm.rust_bridge.catalog import Context, Route from litellm.rust_bridge.ocr import LiteLLMOcrRequest -from litellm.rust_bridge.ocr_lifecycle import select +from litellm.rust_bridge.ocr_lifecycle import NATIVE_OCR_LIFECYCLE, NativeOcrLifecycle +from litellm.rust_bridge.runtime import arun, run __all__ = ("aocr", "convert_file_document_to_url_document", "get_mime_type", "ocr") @@ -48,36 +48,36 @@ def ocr( **kwargs: object, # kwargs-ok: preserve the public OCR call shape ) -> OCRResponse | Coroutine[object, object, OCRResponse]: request: Final = _public_request("ocr", args, kwargs) - native: Final = select(request) if rust_ocr_enabled() else None - if native is not None: - try: - return cast( # cast-ok: False selects the synchronous result - OCRResponse, native(request, args, kwargs, False) - ) - except _decline_types(): - pass fallback: Final = cast( # cast-ok: forward the original call shape through the legacy @client decorator Callable[..., OCRResponse | Coroutine[object, object, OCRResponse]], legacy.ocr ) - return fallback(*args, **kwargs) + if request.kwargs.get("aocr"): + return fallback(*args, **kwargs) + return run( + _context(request), + binding=NATIVE_OCR_LIFECYCLE, + native=lambda hook: cast( # cast-ok: False selects the synchronous result + OCRResponse, hook(request, args, kwargs, False) + ), + python=lambda: fallback(*args, **kwargs), + ) async def aocr(*args: object, **kwargs: object) -> OCRResponse: # kwargs-ok: preserve the public OCR call shape request: Final = _public_request("aocr", args, kwargs) - native: Final = select(request) if rust_ocr_enabled() else None - if native is not None: - try: - return await cast( # cast-ok: True selects the asynchronous result - Awaitable[OCRResponse], native(request, args, kwargs, True) - ) - except _decline_types(): - pass fallback: Final = cast( # cast-ok: forward the original call shape through the legacy @client decorator Callable[..., Awaitable[OCRResponse]], legacy.aocr ) - return await fallback(*args, **kwargs) + + async def native(hook: NativeOcrLifecycle) -> OCRResponse: + return await cast( # cast-ok: True selects the asynchronous result + Awaitable[OCRResponse], hook(request, args, kwargs, True) + ) + + return await arun( + _context(request), binding=NATIVE_OCR_LIFECYCLE, native=native, python=lambda: fallback(*args, **kwargs) + ) -def _decline_types() -> tuple[type[BaseException], ...]: - exception_types: Final = native_exception_types() - return (exception_types[0],) if exception_types is not None else () +def _context(request: LiteLLMOcrRequest) -> Context: + return Context(Route.OCR, provider=request.custom_llm_provider, model=request.model) diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py new file mode 100644 index 00000000000..04d413beed2 --- /dev/null +++ b/litellm/rust_bridge/catalog.py @@ -0,0 +1,102 @@ +"""Declarative Rust/Python selection matrix for every public LiteLLM route. + +Rules are static data matched top to bottom; the first match wins and a +context with no matching rule stays on Python. Whether the Rust core can serve +a specific request body is not decided here: that is Rust admission, which +signals ``RustBridgeDeclined`` before any provider I/O. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum, StrEnum, auto +from typing import Final, TypeAlias + +from litellm.rust_bridge.configuration import Decision, Rollout +from litellm.rust_bridge.configuration import decision as _decision + + +class Route(StrEnum): + CHAT_COMPLETIONS = "chat_completions" + MESSAGES = "messages" + RESPONSES = "responses" + EMBEDDING = "embedding" + RERANK = "rerank" + IMAGE_GENERATION = "image_generation" + IMAGE_EDIT = "image_edit" + SPEECH = "speech" + TRANSCRIPTION = "transcription" + MODERATION = "moderation" + OCR = "ocr" + + +class Delivery(Enum): + COMPLETED = auto() + STREAMING = auto() + WEBSOCKET = auto() + + +@dataclass(frozen=True, slots=True) +class Context: + route: Route + provider: str | None = None + model: str | None = None + delivery: Delivery = Delivery.COMPLETED + + +@dataclass(frozen=True, slots=True) +class Rule: + route: Route + rollout: Rollout + providers: frozenset[str] | None = None + models: frozenset[str] | None = None + deliveries: frozenset[Delivery] | None = None + + def matches(self, context: Context) -> bool: + return ( + context.route is self.route + and (self.providers is None or context.provider in self.providers) + and (self.models is None or context.model in self.models) + and (self.deliveries is None or context.delivery in self.deliveries) + ) + + +Rules: TypeAlias = tuple[Rule, ...] + +_COMPLETED: Final = frozenset({Delivery.COMPLETED}) + +RULES: Final[Rules] = ( + Rule(Route.OCR, Rollout.RUST_OPT_OUT), + Rule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), + Rule(Route.TRANSCRIPTION, Rollout.PYTHON_ONLY), + Rule( + Route.CHAT_COMPLETIONS, + Rollout.RUST_OPT_IN, + providers=frozenset({"anthropic", "bedrock"}), + deliveries=_COMPLETED, + ), + Rule(Route.CHAT_COMPLETIONS, Rollout.PYTHON_ONLY), + Rule(Route.MESSAGES, Rollout.RUST_OPT_IN, providers=frozenset({"anthropic", "azure_ai"})), + Rule(Route.MESSAGES, Rollout.PYTHON_ONLY), + Rule( + Route.RESPONSES, + Rollout.RUST_OPT_IN, + providers=frozenset({"openai"}), + deliveries=frozenset({Delivery.WEBSOCKET}), + ), + Rule(Route.RESPONSES, Rollout.PYTHON_ONLY), + Rule(Route.EMBEDDING, Rollout.PYTHON_ONLY), + Rule(Route.RERANK, Rollout.PYTHON_ONLY), + Rule(Route.IMAGE_GENERATION, Rollout.PYTHON_ONLY), + Rule(Route.IMAGE_EDIT, Rollout.PYTHON_ONLY), + Rule(Route.SPEECH, Rollout.PYTHON_ONLY), + Rule(Route.MODERATION, Rollout.PYTHON_ONLY), +) + + +def rollout(context: Context, rules: Rules = RULES) -> Rollout: + return next((rule.rollout for rule in rules if rule.matches(context)), Rollout.PYTHON_ONLY) + + +def decision(context: Context, rules: Rules = RULES) -> Decision: + return _decision(rollout(context, rules)) diff --git a/litellm/rust_bridge/chat_completions.py b/litellm/rust_bridge/chat_completions.py index 674bd8847f7..1e03806f38c 100644 --- a/litellm/rust_bridge/chat_completions.py +++ b/litellm/rust_bridge/chat_completions.py @@ -26,7 +26,8 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo convert_to_model_response_object, ) from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned -from litellm.rust_bridge.configuration import rust_enabled +from litellm.rust_bridge.catalog import Context, Delivery, Route, decision +from litellm.rust_bridge.configuration import Decision from litellm.rust_bridge.loader import get_native_bridge from litellm.rust_bridge.timeouts import timeout_to_seconds from litellm.types.utils import ModelResponse @@ -34,10 +35,6 @@ from litellm.types.utils import ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -# Providers whose `/chat/completions` deployments the Rust core can serve. A -# provider outside this set never reaches the bridge. -RUST_CHAT_COMPLETIONS_PROVIDERS: Final = frozenset({"anthropic", "bedrock"}) - # `litellm_params` values are `object`, so validate the one this module reads # rather than narrowing an unparameterized `Mapping` and typing the result Any. _LITELLM_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -243,11 +240,13 @@ def rust_chat_completions_accepts( capability gate answers the second half; it resolves no credentials and performs no I/O. """ - if custom_llm_provider not in RUST_CHAT_COMPLETIONS_PROVIDERS: - return False - if stream: - return False - if not rust_enabled(): + context: Final = Context( + Route.CHAT_COMPLETIONS, + provider=custom_llm_provider, + model=model, + delivery=Delivery.STREAMING if stream else Delivery.COMPLETED, + ) + if decision(context) is Decision.PYTHON: return False if _litellm_metadata_reaches_the_provider(custom_llm_provider, litellm_params): verbose_logger.debug("Rust chat completions declined (litellm metadata user_id); using the Python path") diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index ff2e389a6bb..f7a7e53ad8d 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -1,13 +1,26 @@ from __future__ import annotations import os +from enum import Enum, auto from typing import Final -DEFAULT_RUST_ENABLED: Final = False _TRUE_ENV_VALUES: Final = frozenset({"1", "true", "yes", "on"}) _GLOBAL_ENV_NAME: Final = "LITELLM_RUST" +class Rollout(Enum): + PYTHON_ONLY = auto() + RUST_OPT_IN = auto() + RUST_OPT_OUT = auto() + RUST_REQUIRED = auto() + + +class Decision(Enum): + PYTHON = auto() + RUST_WITH_FALLBACK = auto() + RUST_REQUIRED = auto() + + class _RustConfiguration: def __init__(self) -> None: self.override: bool | None = None @@ -22,44 +35,47 @@ def _parse_env_bool(value: str | None) -> bool | None: return value.strip().lower() in _TRUE_ENV_VALUES -def resolve_rust_enabled( +def decide( + rollout: Rollout, *, process_override: bool | None, environment_override: bool | None, - release_default: bool = DEFAULT_RUST_ENABLED, -) -> bool: - if process_override is not None: - return process_override - if environment_override is not None: - return environment_override - return release_default +) -> Decision: + match rollout: + case Rollout.PYTHON_ONLY: + return Decision.PYTHON + case Rollout.RUST_REQUIRED: + return Decision.RUST_REQUIRED + case Rollout.RUST_OPT_IN | Rollout.RUST_OPT_OUT: + switch: Final = ( + process_override + if process_override is not None + else environment_override + if environment_override is not None + else rollout is Rollout.RUST_OPT_OUT + ) + return Decision.RUST_WITH_FALLBACK if switch else Decision.PYTHON -def rust_enabled() -> bool: - return resolve_rust_enabled( +def decision(rollout: Rollout) -> Decision: + return decide( + rollout, process_override=_CONFIGURATION.override, environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)), ) -def rust_ocr_enabled() -> bool: - environment: Final = _parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)) - if environment is False: - return False - return resolve_rust_enabled( - process_override=_CONFIGURATION.override, - environment_override=environment, - release_default=True, - ) +def rust_enabled() -> bool: + return decision(Rollout.RUST_OPT_IN) is not Decision.PYTHON def reset_rust_configuration() -> None: _CONFIGURATION.override = None -def rust(enabled: bool) -> None: +def rust(enabled: bool | None) -> None: """Set the process override for optional Rust paths. - Rust-only paths, including Bedrock transcription, are not controlled by this switch. + ``PYTHON_ONLY`` and ``RUST_REQUIRED`` routes in the catalog ignore this switch. """ _CONFIGURATION.override = enabled diff --git a/litellm/rust_bridge/ocr_lifecycle.py b/litellm/rust_bridge/ocr_lifecycle.py index 5ca584e1c11..4161007cce4 100644 --- a/litellm/rust_bridge/ocr_lifecycle.py +++ b/litellm/rust_bridge/ocr_lifecycle.py @@ -40,12 +40,6 @@ def _binding(value: object) -> NativeOcrLifecycle | None: NATIVE_OCR_LIFECYCLE: Final = NativeBinding("_ocr_lifecycle", validate=_binding) -def select(request: LiteLLMOcrRequest) -> NativeOcrLifecycle | None: - if request.kwargs.get("aocr"): - return None - return NATIVE_OCR_LIFECYCLE.load() - - def arguments(request: LiteLLMOcrRequest) -> Mapping[str, object]: return request.kwargs diff --git a/litellm/rust_bridge/runtime.py b/litellm/rust_bridge/runtime.py index d411673439f..46b7c99f3bc 100644 --- a/litellm/rust_bridge/runtime.py +++ b/litellm/rust_bridge/runtime.py @@ -2,21 +2,17 @@ from __future__ import annotations from collections.abc import Awaitable, Callable from dataclasses import dataclass -from enum import Enum -from typing import Final, Generic, NoReturn, TypeAlias, TypeVar +from typing import Final, Generic, NoReturn, TypeAlias, TypeVar, assert_never from litellm.exceptions import APIError -from litellm.rust_bridge.bindings import native_exception_types +from litellm.rust_bridge.bindings import NativeBinding, native_exception_types +from litellm.rust_bridge.catalog import RULES, Context, Rules, decision +from litellm.rust_bridge.configuration import Decision NativeT = TypeVar("NativeT") ResultT = TypeVar("ResultT") -class FallbackMode(Enum): - PYTHON = "python" - RUST_REQUIRED = "rust_required" - - @dataclass(frozen=True, slots=True) class RustHandled(Generic[ResultT]): value: ResultT @@ -42,36 +38,68 @@ class BridgeErrorContext: model: str -def invoke( +def run( + context: Context, *, - native_call: Callable[[], NativeT] | None, - fallback: Callable[[], ResultT], - adapt: Callable[[NativeT], ResultT], - mode: FallbackMode, - context: BridgeErrorContext, + binding: NativeBinding[NativeT], + native: Callable[[NativeT], ResultT], + python: Callable[[], ResultT], + rules: Rules = RULES, ) -> ResultT: - result: Final = attempt(native_call=native_call, adapt=adapt, context=context) - if isinstance(result, RustHandled): - return result.value - if mode is FallbackMode.PYTHON: - return fallback() - _raise_required(result, context) + selected: Final = decision(context, rules) + match selected: + case Decision.PYTHON: + return python() + case Decision.RUST_WITH_FALLBACK | Decision.RUST_REQUIRED: + loaded: Final = binding.load() + result: Final = attempt( + native_call=None if loaded is None else lambda: native(loaded), + adapt=_identity, + context=_error_context(context), + ) + if isinstance(result, RustHandled): + return result.value + if selected is Decision.RUST_REQUIRED: + _raise_required(result, _error_context(context)) + return python() + case _: + assert_never(selected) -async def ainvoke( +async def arun( + context: Context, *, - native_call: Callable[[], Awaitable[NativeT]] | None, - fallback: Callable[[], Awaitable[ResultT]], - adapt: Callable[[NativeT], ResultT], - mode: FallbackMode, - context: BridgeErrorContext, + binding: NativeBinding[NativeT], + native: Callable[[NativeT], Awaitable[ResultT]], + python: Callable[[], Awaitable[ResultT]], + rules: Rules = RULES, ) -> ResultT: - result: Final = await aattempt(native_call=native_call, adapt=adapt, context=context) - if isinstance(result, RustHandled): - return result.value - if mode is FallbackMode.PYTHON: - return await fallback() - _raise_required(result, context) + selected: Final = decision(context, rules) + match selected: + case Decision.PYTHON: + return await python() + case Decision.RUST_WITH_FALLBACK | Decision.RUST_REQUIRED: + loaded: Final = binding.load() + result: Final = await aattempt( + native_call=None if loaded is None else lambda: native(loaded), + adapt=_identity, + context=_error_context(context), + ) + if isinstance(result, RustHandled): + return result.value + if selected is Decision.RUST_REQUIRED: + _raise_required(result, _error_context(context)) + return await python() + case _: + assert_never(selected) + + +def _identity(value: ResultT) -> ResultT: + return value + + +def _error_context(context: Context) -> BridgeErrorContext: + return BridgeErrorContext(route=context.route.value, provider=context.provider or "", model=context.model or "") def attempt( diff --git a/tests/test_litellm/ocr/test_legacy.py b/tests/test_litellm/ocr/test_legacy.py index 4b0b78f5a0f..8b87690aedb 100644 --- a/tests/test_litellm/ocr/test_legacy.py +++ b/tests/test_litellm/ocr/test_legacy.py @@ -1,4 +1,3 @@ -import importlib from collections.abc import AsyncGenerator from datetime import datetime from io import BytesIO @@ -16,7 +15,7 @@ from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUs from litellm.llms.custom_httpx import llm_http_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.ocr.legacy import _prepare_ocr_request -from litellm.rust_bridge import bindings, configuration +from litellm.rust_bridge import bindings, configuration, runtime from litellm.rust_bridge.ocr_lifecycle import NATIVE_OCR_LIFECYCLE @@ -61,8 +60,7 @@ async def test_python_request_response_and_callbacks( if dispatch != "disabled": monkeypatch.setenv("LITELLM_RUST", "1") NATIVE_OCR_LIFECYCLE.override(Mock(side_effect=Declined()) if dispatch == "declined" else None) - main: Final = importlib.import_module("litellm.ocr.main") - monkeypatch.setattr(main, "native_exception_types", lambda: (Declined, RuntimeError)) + monkeypatch.setattr(runtime, "native_exception_types", lambda: (Declined, RuntimeError)) logger: Final = Mock(spec=CustomLogger) monkeypatch.setattr(litellm, "input_callback", [logger]) arguments: Final = { diff --git a/tests/test_litellm/rust_bridge/test_catalog.py b/tests/test_litellm/rust_bridge/test_catalog.py new file mode 100644 index 00000000000..98d6f83bd63 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_catalog.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from typing import Final + +import pytest + +from litellm.rust_bridge import catalog +from litellm.rust_bridge.catalog import Context, Delivery, Route, Rule +from litellm.rust_bridge.configuration import Rollout + + +def test_every_route_has_an_explicit_default_rule() -> None: + declared: Final = frozenset( + rule.route for rule in catalog.RULES if rule.providers is None and rule.deliveries is None + ) + assert declared == frozenset(Route) + + +@pytest.mark.parametrize( + ("context", "expected"), + ( + (Context(Route.OCR), Rollout.RUST_OPT_OUT), + (Context(Route.OCR, provider="mistral", model="mistral-ocr-latest"), Rollout.RUST_OPT_OUT), + (Context(Route.TRANSCRIPTION, provider="bedrock"), Rollout.RUST_REQUIRED), + (Context(Route.TRANSCRIPTION, provider="openai"), Rollout.PYTHON_ONLY), + (Context(Route.TRANSCRIPTION), Rollout.PYTHON_ONLY), + (Context(Route.CHAT_COMPLETIONS, provider="anthropic"), Rollout.RUST_OPT_IN), + (Context(Route.CHAT_COMPLETIONS, provider="bedrock"), Rollout.RUST_OPT_IN), + (Context(Route.CHAT_COMPLETIONS, provider="anthropic", delivery=Delivery.STREAMING), Rollout.PYTHON_ONLY), + (Context(Route.CHAT_COMPLETIONS, provider="openai"), Rollout.PYTHON_ONLY), + (Context(Route.MESSAGES, provider="anthropic"), Rollout.RUST_OPT_IN), + (Context(Route.MESSAGES, provider="azure_ai"), Rollout.RUST_OPT_IN), + (Context(Route.MESSAGES, provider="bedrock"), Rollout.PYTHON_ONLY), + (Context(Route.RESPONSES, provider="openai", delivery=Delivery.WEBSOCKET), Rollout.RUST_OPT_IN), + (Context(Route.RESPONSES, provider="openai"), Rollout.PYTHON_ONLY), + (Context(Route.RESPONSES, provider="azure", delivery=Delivery.WEBSOCKET), Rollout.PYTHON_ONLY), + (Context(Route.EMBEDDING, provider="openai"), Rollout.PYTHON_ONLY), + ), +) +def test_shipped_rules(context: Context, expected: Rollout) -> None: + assert catalog.rollout(context) is expected + + +def test_first_matching_rule_wins() -> None: + rules: Final = ( + Rule(Route.EMBEDDING, Rollout.RUST_REQUIRED, providers=frozenset({"openai"}), models=frozenset({"m"})), + Rule(Route.EMBEDDING, Rollout.RUST_OPT_IN, providers=frozenset({"openai"})), + Rule(Route.EMBEDDING, Rollout.PYTHON_ONLY), + ) + + assert catalog.rollout(Context(Route.EMBEDDING, provider="openai", model="m"), rules) is Rollout.RUST_REQUIRED + assert catalog.rollout(Context(Route.EMBEDDING, provider="openai", model="other"), rules) is Rollout.RUST_OPT_IN + assert catalog.rollout(Context(Route.EMBEDDING, provider="cohere", model="m"), rules) is Rollout.PYTHON_ONLY + assert catalog.rollout(Context(Route.RERANK, provider="openai", model="m"), rules) is Rollout.PYTHON_ONLY diff --git a/tests/test_litellm/rust_bridge/test_configuration.py b/tests/test_litellm/rust_bridge/test_configuration.py index 08fa3bfc053..4fa5b6d834d 100644 --- a/tests/test_litellm/rust_bridge/test_configuration.py +++ b/tests/test_litellm/rust_bridge/test_configuration.py @@ -22,48 +22,56 @@ def _isolated_configuration( # pyright: ignore[reportUnusedFunction] # pytest configuration.reset_rust_configuration() +Rollout: Final = configuration.Rollout +Decision: Final = configuration.Decision + + @pytest.mark.parametrize( - ("process", "environment", "release_default", "expected"), + ("rollout", "process", "environment", "expected"), ( - (False, True, True, False), - (True, False, False, True), - (None, False, True, False), - (None, True, False, True), - (None, None, False, False), - (None, None, True, True), + (Rollout.PYTHON_ONLY, True, True, Decision.PYTHON), + (Rollout.RUST_REQUIRED, False, False, Decision.RUST_REQUIRED), + (Rollout.RUST_OPT_IN, None, None, Decision.PYTHON), + (Rollout.RUST_OPT_IN, None, True, Decision.RUST_WITH_FALLBACK), + (Rollout.RUST_OPT_IN, True, False, Decision.RUST_WITH_FALLBACK), + (Rollout.RUST_OPT_IN, False, True, Decision.PYTHON), + (Rollout.RUST_OPT_OUT, None, None, Decision.RUST_WITH_FALLBACK), + (Rollout.RUST_OPT_OUT, None, False, Decision.PYTHON), + (Rollout.RUST_OPT_OUT, False, True, Decision.PYTHON), + (Rollout.RUST_OPT_OUT, True, False, Decision.RUST_WITH_FALLBACK), ), ) -def test_resolution_precedence( +def test_decide_precedence( + rollout: configuration.Rollout, process: bool | None, environment: bool | None, - release_default: bool, - expected: bool, + expected: configuration.Decision, ) -> None: - assert ( - configuration.resolve_rust_enabled( - process_override=process, - environment_override=environment, - release_default=release_default, - ) - is expected - ) + assert configuration.decide(rollout, process_override=process, environment_override=environment) is expected -def test_release_default_remains_disabled() -> None: - assert configuration.DEFAULT_RUST_ENABLED is False +def test_release_default_keeps_opt_in_routes_on_python() -> None: + assert configuration.decision(Rollout.RUST_OPT_IN) is Decision.PYTHON + assert configuration.decision(Rollout.RUST_OPT_OUT) is Decision.RUST_WITH_FALLBACK assert configuration.rust_enabled() is False - assert configuration.rust_ocr_enabled() is True -@pytest.mark.parametrize("process", [None, False, True]) -@pytest.mark.parametrize("environment", [None, "0", "1", "off"]) -def test_ocr_configuration(monkeypatch: pytest.MonkeyPatch, process: bool | None, environment: str | None) -> None: +@pytest.mark.parametrize("process", (None, False, True)) +@pytest.mark.parametrize("environment", (None, "0", "1", "off")) +def test_opt_out_route_configuration( + monkeypatch: pytest.MonkeyPatch, process: bool | None, environment: str | None +) -> None: if environment is not None: monkeypatch.setenv("LITELLM_RUST", environment) if process is not None: configuration.rust(process) - assert configuration.rust_ocr_enabled() is (environment not in {"0", "off"} and process is not False) + expected: Final = ( + Decision.RUST_WITH_FALLBACK + if process is True or (process is None and environment not in frozenset({"0", "off"})) + else Decision.PYTHON + ) + assert configuration.decision(Rollout.RUST_OPT_OUT) is expected def test_process_override_wins_over_environment(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_litellm/rust_bridge/test_ocr_lifecycle.py b/tests/test_litellm/rust_bridge/test_ocr_lifecycle.py index 501a4e986c0..c9c469168ce 100644 --- a/tests/test_litellm/rust_bridge/test_ocr_lifecycle.py +++ b/tests/test_litellm/rust_bridge/test_ocr_lifecycle.py @@ -7,7 +7,7 @@ import pytest import litellm from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.ocr import legacy -from litellm.rust_bridge import bindings, configuration +from litellm.rust_bridge import bindings, configuration, runtime from litellm.rust_bridge.ocr import LiteLLMOcrRequest from litellm.rust_bridge.ocr_lifecycle import NATIVE_OCR_LIFECYCLE @@ -143,7 +143,7 @@ def test_public_missing_required_argument_error_does_not_depend_on_native_select @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", [False, True]) -@pytest.mark.parametrize("enabled", [False, True, None]) +@pytest.mark.parametrize("enabled", [False, None]) async def test_environment_opt_out_never_loads_native( monkeypatch: pytest.MonkeyPatch, asynchronous: bool, enabled: bool | None ) -> None: @@ -196,6 +196,10 @@ class Declined(Exception): pass +class Upstream(Exception): + pass + + @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize("declined", [False, True]) @@ -205,10 +209,7 @@ async def test_only_native_declines_replay_on_legacy( failure: Final = Declined("unsupported") if declined else RuntimeError("provider already called") native: Final = AsyncMock(side_effect=failure) if asynchronous else Mock(side_effect=failure) NATIVE_OCR_LIFECYCLE.override(native) - import importlib - - main: Final = importlib.import_module("litellm.ocr.main") - monkeypatch.setattr(main, "native_exception_types", lambda: (Declined, RuntimeError)) + monkeypatch.setattr(runtime, "native_exception_types", lambda: (Declined, Upstream)) response: Final = OCRResponse(pages=[], model="mistral-ocr-latest") fallback: Final = AsyncMock(return_value=response) if asynchronous else Mock(return_value=response) monkeypatch.setattr(legacy, "aocr" if asynchronous else "ocr", fallback) diff --git a/tests/test_litellm/rust_bridge/test_runtime.py b/tests/test_litellm/rust_bridge/test_runtime.py index b0fa510069b..ee3950f8455 100644 --- a/tests/test_litellm/rust_bridge/test_runtime.py +++ b/tests/test_litellm/rust_bridge/test_runtime.py @@ -1,11 +1,15 @@ from __future__ import annotations +from collections.abc import Generator from types import SimpleNamespace +from typing import Final, Protocol import pytest from litellm.exceptions import APIError -from litellm.rust_bridge import bindings, runtime +from litellm.rust_bridge import bindings, configuration, runtime +from litellm.rust_bridge.catalog import Context, Route, Rule +from litellm.rust_bridge.configuration import Rollout class RustBridgeDeclined(Exception): @@ -17,79 +21,203 @@ class RustUpstreamError(Exception): @pytest.fixture(autouse=True) -def native_exceptions(monkeypatch: pytest.MonkeyPatch) -> None: - native = SimpleNamespace( +def native_exceptions(monkeypatch: pytest.MonkeyPatch) -> Generator[None]: + native: Final = SimpleNamespace( RustBridgeDeclined=RustBridgeDeclined, RustUpstreamError=RustUpstreamError, ) monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) + monkeypatch.delenv("LITELLM_RUST", raising=False) + configuration.reset_rust_configuration() + yield + configuration.reset_rust_configuration() -def context() -> runtime.BridgeErrorContext: - return runtime.BridgeErrorContext(route="messages", provider="anthropic", model="model") +class NativeFn(Protocol): + def __call__(self) -> str: ... -def test_invoke_tags_native_decline_before_running_fallback() -> None: - calls: list[str] = [] +CONTEXT: Final = Context(Route.MESSAGES, provider="anthropic", model="model") +RUST: Final = "rust" +PYTHON: Final = "python" - def decline() -> object: - calls.append("rust") - raise RustBridgeDeclined("unsupported") - value = runtime.invoke( - native_call=decline, - fallback=lambda: calls.append("python") or "fallback", - adapt=str, - mode=runtime.FallbackMode.PYTHON, - context=context(), +def binding(native: NativeFn | None) -> bindings.NativeBinding[NativeFn]: + bound: Final[bindings.NativeBinding[NativeFn]] = bindings.NativeBinding("_messages", validate=lambda _: None) + bound.override(native) + return bound + + +def rules(rollout: Rollout) -> tuple[Rule, ...]: + return (Rule(Route.MESSAGES, rollout, providers=frozenset({"anthropic"})),) + + +class Recorder: + def __init__(self, native_effect: BaseException | None = None) -> None: + self._native_effect: Final = native_effect + self.calls: tuple[str, ...] = () + + def rust(self) -> str: + self.calls = (*self.calls, RUST) + if self._native_effect is not None: + raise self._native_effect + return RUST + + def python(self) -> str: + self.calls = (*self.calls, PYTHON) + return PYTHON + + +def recorder(native_effect: BaseException | None = None) -> Recorder: + return Recorder(native_effect) + + +def run(rollout: Rollout, calls: Recorder, *, native_missing: bool = False, context: Context = CONTEXT) -> str: + return runtime.run( + context, + binding=binding(None if native_missing else calls.rust), + native=lambda fn: fn(), + python=calls.python, + rules=rules(rollout), ) - assert value == "fallback" - assert calls == ["rust", "python"] + +@pytest.mark.parametrize( + ("rollout", "switch", "expected"), + ( + (Rollout.PYTHON_ONLY, None, (PYTHON,)), + (Rollout.PYTHON_ONLY, True, (PYTHON,)), + (Rollout.RUST_OPT_IN, None, (PYTHON,)), + (Rollout.RUST_OPT_IN, True, (RUST,)), + (Rollout.RUST_OPT_OUT, None, (RUST,)), + (Rollout.RUST_OPT_OUT, False, (PYTHON,)), + (Rollout.RUST_REQUIRED, None, (RUST,)), + (Rollout.RUST_REQUIRED, False, (RUST,)), + ), +) +def test_rollout_and_switch_select_native_or_python( + rollout: Rollout, switch: bool | None, expected: tuple[str, ...] +) -> None: + calls: Final = recorder() + if switch is not None: + configuration.rust(switch) + + assert run(rollout, calls) == expected[-1] + assert calls.calls == expected -def test_invoke_translates_upstream_without_fallback() -> None: - def fail() -> object: - raise RustUpstreamError(429, "rate limited") +def test_environment_switch_enables_opt_in_route(monkeypatch: pytest.MonkeyPatch) -> None: + calls: Final = recorder() + monkeypatch.setenv("LITELLM_RUST", "1") + + assert run(Rollout.RUST_OPT_IN, calls) == "rust" + assert calls.calls == (RUST,) + + +def test_context_outside_rule_stays_on_python() -> None: + calls: Final = recorder() + configuration.rust(True) + + assert run(Rollout.RUST_REQUIRED, calls, context=Context(Route.MESSAGES, provider="openai")) == "python" + assert run(Rollout.RUST_REQUIRED, calls, context=Context(Route.EMBEDDING, provider="anthropic")) == "python" + assert calls.calls == (PYTHON, PYTHON) + + +def test_native_decline_falls_back_to_python_once() -> None: + calls: Final = recorder(RustBridgeDeclined("unsupported")) + + assert run(Rollout.RUST_OPT_OUT, calls) == "python" + assert calls.calls == (RUST, PYTHON) + + +def test_unavailable_native_falls_back_to_python() -> None: + calls: Final = recorder() + + assert run(Rollout.RUST_OPT_OUT, calls, native_missing=True) == "python" + assert calls.calls == (PYTHON,) + + +def test_upstream_error_maps_to_api_error_without_fallback() -> None: + calls: Final = recorder(RustUpstreamError(429, "rate limited")) with pytest.raises(APIError, match="rate limited") as caught: - runtime.invoke( - native_call=fail, - fallback=lambda: pytest.fail("fallback must not run"), - adapt=str, - mode=runtime.FallbackMode.PYTHON, - context=context(), - ) + run(Rollout.RUST_OPT_OUT, calls) assert caught.value.status_code == 429 + assert calls.calls == (RUST,) + + +def test_other_native_errors_propagate_without_fallback() -> None: + failure: Final = ValueError("admitted") + calls: Final = recorder(failure) + + with pytest.raises(ValueError, match="admitted") as caught: + run(Rollout.RUST_OPT_OUT, calls) + + assert caught.value is failure + assert calls.calls == (RUST,) + + +def test_required_route_rejects_unavailable_bridge() -> None: + calls: Final = recorder() + + with pytest.raises(RuntimeError, match="Rust messages bridge is unavailable"): + run(Rollout.RUST_REQUIRED, calls, native_missing=True) + + assert PYTHON not in calls.calls + + +def test_required_route_rejects_native_decline() -> None: + calls: Final = recorder(RustBridgeDeclined("unsupported")) + + with pytest.raises(RuntimeError, match="declined the request: unsupported"): + run(Rollout.RUST_REQUIRED, calls) + + assert PYTHON not in calls.calls @pytest.mark.asyncio -async def test_ainvoke_handles_native_success() -> None: - async def native() -> int: - return 3 +@pytest.mark.parametrize( + ("native_effect", "native_missing", "expected"), + ( + (None, False, (RUST,)), + (RustBridgeDeclined("unsupported"), False, (RUST, PYTHON)), + (None, True, (PYTHON,)), + ), +) +async def test_arun_mirrors_sync_fallback( + native_effect: BaseException | None, native_missing: bool, expected: tuple[str, ...] +) -> None: + calls: Final = recorder(native_effect) - async def fallback() -> str: - pytest.fail("fallback must not run") + async def native(fn: NativeFn) -> str: + return fn() - assert ( - await runtime.ainvoke( - native_call=native, - fallback=fallback, - adapt=str, - mode=runtime.FallbackMode.PYTHON, - context=context(), - ) - == "3" + async def python() -> str: + return calls.python() + + result: Final = await runtime.arun( + CONTEXT, + binding=binding(None if native_missing else calls.rust), + native=native, + python=python, + rules=rules(Rollout.RUST_OPT_OUT), ) + assert result == expected[-1] + assert calls.calls == expected + + +@pytest.mark.asyncio +async def test_arun_required_route_rejects_unavailable_bridge() -> None: + async def python() -> str: + pytest.fail("fallback must not run") -def test_required_mode_rejects_unavailable_bridge() -> None: with pytest.raises(RuntimeError, match="is unavailable"): - runtime.invoke( - native_call=None, - fallback=lambda: pytest.fail("fallback must not run"), - adapt=str, - mode=runtime.FallbackMode.RUST_REQUIRED, - context=context(), + await runtime.arun( + CONTEXT, + binding=binding(None), + native=lambda fn: python(), + python=python, + rules=rules(Rollout.RUST_REQUIRED), )