diff --git a/litellm/utils.py b/litellm/utils.py index ac26e08640d..bbdb280cace 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -29,9 +29,11 @@ from dataclasses import ( dataclass, field, ) # for storing API inputs, outputs, and metadata -import pkg_resources +#import pkg_resources +from importlib import resources -filename = pkg_resources.resource_filename(__name__, "llms/tokenizers") +# filename = pkg_resources.resource_filename(__name__, "llms/tokenizers") +filename = str(resources.files().joinpath("llms/tokenizers")) os.environ[ "TIKTOKEN_CACHE_DIR" ] = filename # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071