scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-23 05:34:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 24

feat(rust_bridge): restore the Rust opt-in for token counting routes

Browse source 
Token counter and tokenizer were shipped as RUST_OPT_IN in #42174 and
moved back to PYTHON_ONLY the same day in #42517, whose actual bug was
Messages dropping the Anthropic beta inference header. The tokenizer
rollback was precautionary, bundled with the Messages fix, and never
revisited. The native implementation and its parity tests have been
green since, and the spend-tracking caller already falls back to Python
counting on RuntimeError and ValueError.

RUST_OPT_IN stays default-off, so nothing changes for callers that do
not set LITELLM_RUST.

Signed-off-by: Delweng <delweng@gmail.com>
This commit is contained in:
Delweng 2026-10-12 12:44:13 +08:00
parent 102619d47a
commit 5eaf835c24
2 changed files with 6 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/rust_bridge/catalog.py
View file

@ -73,8 +73,8 @@ RULES: Final[Rules] = (
RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN, providers=frozenset({"anthropic", "deepseek", "vertex_ai"})),
RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY),
RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY),
RouteRule(Route.TOKEN_COUNTER, Rollout.PYTHON_ONLY),
RouteRule(Route.TOKENIZER, Rollout.PYTHON_ONLY),
RouteRule(Route.TOKEN_COUNTER, Rollout.RUST_OPT_IN),
RouteRule(Route.TOKENIZER, Rollout.RUST_OPT_IN),
RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})),
)

5
tests/unit/rust_bridge/test_catalog.py
View file

@ -43,11 +43,14 @@ def test_shipped_decisions(
if environment is not None:
monkeypatch.setenv("LITELLM_RUST", environment)
context: Final = RouteContext(route, provider=provider, model="test-model")
opt_in_route: Final = route in (Route.TOKEN_COUNTER, Route.TOKENIZER) or (
route is Route.MESSAGES and provider in ("anthropic", "deepseek", "vertex_ai")
)
if route is Route.OCR or (route is Route.TRANSCRIPTION and provider == "bedrock"):
assert catalog.rollout(context) is Rollout.RUST_REQUIRED
assert catalog.decision(context) is Decision.RUST_REQUIRED
elif route is Route.MESSAGES and provider in ("anthropic", "deepseek", "vertex_ai"):
elif opt_in_route:
assert catalog.rollout(context) is Rollout.RUST_OPT_IN
opted_in: Final = environment == "1" or (environment is None and process is True)
assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if opted_in else Decision.PYTHON)