scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

Merge pull request #21423 from BerriAI/fix/token-counter-test-isolation

Browse source 
fix(token-counter): fix test isolation and encode() return type normalization
This commit is contained in:
jquinter 2026-02-17 21:22:05 -03:00 • committed by GitHub
commit 59171bd76d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 7 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
tests/test_litellm/litellm_core_utils/test_token_counter.py
View file

@ -582,14 +582,15 @@ class TestTokenizerSelection(unittest.TestCase):
@patch("litellm.utils._return_huggingface_tokenizer")
def test_disable_hf_tokenizer_download(self, mock_return_huggingface_tokenizer):
# Use pytest.MonkeyPatch() directly instead of fixture
monkeypatch = pytest.MonkeyPatch()
monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", True)
result = _select_tokenizer_helper("grok-32r22r")
mock_return_huggingface_tokenizer.assert_not_called()
assert result["type"] == "openai_tokenizer"
assert result["tokenizer"] == encoding
try:
result = _select_tokenizer_helper("grok-32r22r")
mock_return_huggingface_tokenizer.assert_not_called()
assert result["type"] == "openai_tokenizer"
assert result["tokenizer"] == encoding
finally:
monkeypatch.undo()
@pytest.mark.parametrize(