diff --git a/litellm/utils.py b/litellm/utils.py
index 2ab06ba89ad..572294de630 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -1251,12 +1251,12 @@ def _select_tokenizer(
     model: str, custom_tokenizer: Optional[CustomHuggingfaceTokenizer] = None
 ):
     if custom_tokenizer is not None:
-        custom_tokenizer = Tokenizer.from_pretrained(
-            custom_tokenizer["identifier"],
+        _tokenizer = create_pretrained_tokenizer(
+            identifier=custom_tokenizer["identifier"],
             revision=custom_tokenizer["revision"],
             auth_token=custom_tokenizer["auth_token"],
         )
-        return {"type": "huggingface_tokenizer", "tokenizer": custom_tokenizer}
+        return {"type": "huggingface_tokenizer", "tokenizer": _tokenizer}
     return _select_tokenizer_helper(model=model)