From 5eaf835c241f45c97b770b46772f4e6677f47042 Mon Sep 17 00:00:00 2001 From: Delweng Date: Mon, 12 Oct 2026 12:44:13 +0800 Subject: [PATCH] feat(rust_bridge): restore the Rust opt-in for token counting routes Token counter and tokenizer were shipped as RUST_OPT_IN in #42174 and moved back to PYTHON_ONLY the same day in #42517, whose actual bug was Messages dropping the Anthropic beta inference header. The tokenizer rollback was precautionary, bundled with the Messages fix, and never revisited. The native implementation and its parity tests have been green since, and the spend-tracking caller already falls back to Python counting on RuntimeError and ValueError. RUST_OPT_IN stays default-off, so nothing changes for callers that do not set LITELLM_RUST. Signed-off-by: Delweng --- litellm/rust_bridge/catalog.py | 4 ++-- tests/unit/rust_bridge/test_catalog.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index dda9ed0b566..0b09c6ab2d7 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -73,8 +73,8 @@ RULES: Final[Rules] = ( RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN, providers=frozenset({"anthropic", "deepseek", "vertex_ai"})), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY), RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), - RouteRule(Route.TOKEN_COUNTER, Rollout.PYTHON_ONLY), - RouteRule(Route.TOKENIZER, Rollout.PYTHON_ONLY), + RouteRule(Route.TOKEN_COUNTER, Rollout.RUST_OPT_IN), + RouteRule(Route.TOKENIZER, Rollout.RUST_OPT_IN), RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), ) diff --git a/tests/unit/rust_bridge/test_catalog.py b/tests/unit/rust_bridge/test_catalog.py index 1ba95507d03..2621ad50146 100644 --- a/tests/unit/rust_bridge/test_catalog.py +++ b/tests/unit/rust_bridge/test_catalog.py @@ -43,11 +43,14 @@ def test_shipped_decisions( if environment is not None: monkeypatch.setenv("LITELLM_RUST", environment) context: Final = RouteContext(route, provider=provider, model="test-model") + opt_in_route: Final = route in (Route.TOKEN_COUNTER, Route.TOKENIZER) or ( + route is Route.MESSAGES and provider in ("anthropic", "deepseek", "vertex_ai") + ) if route is Route.OCR or (route is Route.TRANSCRIPTION and provider == "bedrock"): assert catalog.rollout(context) is Rollout.RUST_REQUIRED assert catalog.decision(context) is Decision.RUST_REQUIRED - elif route is Route.MESSAGES and provider in ("anthropic", "deepseek", "vertex_ai"): + elif opt_in_route: assert catalog.rollout(context) is Rollout.RUST_OPT_IN opted_in: Final = environment == "1" or (environment is None and process is True) assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if opted_in else Decision.PYTHON)