From c14057c4b603e2b9e7477889daa8c93c681ed682 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 6 Jan 2025 21:32:10 -0800 Subject: [PATCH] fix(utils.py): fix select tokenizer for custom tokenizer --- litellm/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 2ab06ba89ad..572294de630 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1251,12 +1251,12 @@ def _select_tokenizer( model: str, custom_tokenizer: Optional[CustomHuggingfaceTokenizer] = None ): if custom_tokenizer is not None: - custom_tokenizer = Tokenizer.from_pretrained( - custom_tokenizer["identifier"], + _tokenizer = create_pretrained_tokenizer( + identifier=custom_tokenizer["identifier"], revision=custom_tokenizer["revision"], auth_token=custom_tokenizer["auth_token"], ) - return {"type": "huggingface_tokenizer", "tokenizer": custom_tokenizer} + return {"type": "huggingface_tokenizer", "tokenizer": _tokenizer} return _select_tokenizer_helper(model=model)