scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 85

fix(utils.py): fix select tokenizer for custom tokenizer

Browse source
This commit is contained in:
Krrish Dholakia 2025-01-06 21:32:10 -08:00
parent 7133cf5b74
commit c14057c4b6
1 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/utils.py
View file

@ -1251,12 +1251,12 @@ def _select_tokenizer(
model: str, custom_tokenizer: Optional[CustomHuggingfaceTokenizer] = None
):
if custom_tokenizer is not None:
custom_tokenizer = Tokenizer.from_pretrained(
custom_tokenizer["identifier"],
_tokenizer = create_pretrained_tokenizer(
identifier=custom_tokenizer["identifier"],
revision=custom_tokenizer["revision"],
auth_token=custom_tokenizer["auth_token"],
)
return {"type": "huggingface_tokenizer", "tokenizer": custom_tokenizer}
return {"type": "huggingface_tokenizer", "tokenizer": _tokenizer}
return _select_tokenizer_helper(model=model)