mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
10413796c6
* test(rust): group cache tests under cache/ and fold test_ocr.py into ocr/ The two failure cases in test_ocr.py duplicated the upstream-500 and timeout rows of PUBLIC_FAILURES, so only the file-input encoding case moves to ocr/test_requests.py Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(rust): split the response cache suite into one file per backend test_response_cache.py grew to 2400 lines. Each backend now has its own file, shared fixtures live in cache/conftest.py and shared helpers in support/cache.py. The helpers alias the private native test handles once, dropping the per-call reportPrivateUsage hits Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * split tokenizer test * test(core): consolidate route integration tests under tests/ with rstest and wiremock Moves the public-API OCR route tests out of src/ocr/route.rs and document.rs into tests/ocr/, split per provider plus lifecycle, machine, and document tests, merging the duplicated pairs. Messages, audio transcription, and chat completions share one wiremock-based upstream and recording secret source in tests/support, and gain table-driven cases for auth, routing, upstream errors, streaming, and declines. Tests of litellm-llms items move to that crate. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(messages): keep the stream relay test independent of the stream head contents The stream head carries no headers on main, so the relay test asserts the open-then-deliver order and the relayed body instead of header hand-off. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> --------- Co-authored-by: Yujong Lee <yujong@berri.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
24 lines
903 B
Python
24 lines
903 B
Python
from typing import Final
|
|
|
|
import pytest
|
|
|
|
from litellm.rust_bridge import _native
|
|
from litellm.utils import claude_json_str
|
|
|
|
pytestmark = pytest.mark.requires_rust_extension
|
|
|
|
|
|
def test_huggingface_codec_skips_special_tokens() -> None:
|
|
tokenizer: Final = _native.Tokenizer.from_json(claude_json_str)
|
|
encoded: Final = tokenizer.encode("<SOS>hello<EOT>")
|
|
|
|
assert "<SOS>" in tokenizer.decode(encoded, skip_special_tokens=False)
|
|
assert tokenizer.decode(encoded, skip_special_tokens=True) == "hello"
|
|
|
|
|
|
def test_huggingface_codec_rejects_tiktoken_only_calls() -> None:
|
|
tokenizer: Final = _native.Tokenizer.from_json(claude_json_str)
|
|
with pytest.raises(ValueError, match="requires a tiktoken encoding"):
|
|
tokenizer.token_byte_values()
|
|
with pytest.raises(ValueError, match="requires a Hugging Face tokenizer"):
|
|
_native.Tokenizer.from_tiktoken("cl100k_base").get_vocab()
|