mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
10413796c6
* test(rust): group cache tests under cache/ and fold test_ocr.py into ocr/ The two failure cases in test_ocr.py duplicated the upstream-500 and timeout rows of PUBLIC_FAILURES, so only the file-input encoding case moves to ocr/test_requests.py Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(rust): split the response cache suite into one file per backend test_response_cache.py grew to 2400 lines. Each backend now has its own file, shared fixtures live in cache/conftest.py and shared helpers in support/cache.py. The helpers alias the private native test handles once, dropping the per-call reportPrivateUsage hits Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * split tokenizer test * test(core): consolidate route integration tests under tests/ with rstest and wiremock Moves the public-API OCR route tests out of src/ocr/route.rs and document.rs into tests/ocr/, split per provider plus lifecycle, machine, and document tests, merging the duplicated pairs. Messages, audio transcription, and chat completions share one wiremock-based upstream and recording secret source in tests/support, and gain table-driven cases for auth, routing, upstream errors, streaming, and declines. Tests of litellm-llms items move to that crate. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test(messages): keep the stream relay test independent of the stream head contents The stream head carries no headers on main, so the relay test asserts the open-then-deliver order and the relayed body instead of header hand-off. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> --------- Co-authored-by: Yujong Lee <yujong@berri.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
53 lines
2.1 KiB
Python
53 lines
2.1 KiB
Python
from typing import Final
|
|
|
|
import pytest
|
|
import tiktoken
|
|
|
|
from litellm.rust_bridge import _native
|
|
|
|
pytestmark = pytest.mark.requires_rust_extension
|
|
|
|
|
|
def test_tiktoken_codec_round_trips_and_counts() -> None:
|
|
tokenizer: Final = _native.Tokenizer.from_tiktoken("cl100k_base")
|
|
encoded: Final = tokenizer.encode("hello world")
|
|
|
|
assert tokenizer.name == "cl100k_base"
|
|
assert tokenizer.count("hello world") == len(encoded)
|
|
assert tokenizer.decode(encoded) == "hello world"
|
|
|
|
|
|
def test_tiktoken_codec_keeps_the_requested_encoding_name() -> None:
|
|
assert _native.Tokenizer.from_tiktoken("gpt2").name == "gpt2"
|
|
assert _native.Tokenizer.from_tiktoken("r50k_base").name == "r50k_base"
|
|
assert _native.Tokenizer.from_tiktoken("gpt2").encode("hi") == _native.Tokenizer.from_tiktoken("r50k_base").encode(
|
|
"hi"
|
|
)
|
|
|
|
|
|
def test_tiktoken_codec_exposes_its_vocabulary() -> None:
|
|
reference: Final = tiktoken.get_encoding("cl100k_base")
|
|
tokenizer: Final = _native.Tokenizer.from_tiktoken("cl100k_base")
|
|
|
|
assert tokenizer.special_tokens() == reference._special_tokens
|
|
assert tokenizer.max_token_value() == reference.max_token_value
|
|
assert tokenizer.token_byte_values() == reference.token_byte_values()
|
|
assert tokenizer.encode_single_token(b"hello") == reference.encode_single_token("hello")
|
|
assert tokenizer.is_special_token(reference.eot_token) and not tokenizer.is_special_token(0)
|
|
with pytest.raises(KeyError):
|
|
tokenizer.encode_single_token(b"<|not-a-token|>")
|
|
|
|
|
|
def test_unknown_tiktoken_encoding_raises_value_error() -> None:
|
|
with pytest.raises(ValueError, match="unsupported tokenizer"):
|
|
_native.Tokenizer.from_tiktoken("unknown-encoding")
|
|
|
|
|
|
def test_tiktoken_codec_decodes_truncated_unicode_like_python() -> None:
|
|
reference: Final = tiktoken.get_encoding("cl100k_base")
|
|
tokenizer: Final = _native.Tokenizer.from_tiktoken(reference.name)
|
|
encoded: Final = reference.encode("🙂漢字")
|
|
|
|
assert tuple(tokenizer.decode(encoded[:end]) for end in range(1, len(encoded) + 1)) == tuple(
|
|
reference.decode(encoded[:end]) for end in range(1, len(encoded) + 1)
|
|
)
|