diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index 970dd512a40..9bb1f75392a 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -1,4 +1,5 @@ import os +import tempfile from typing import Final import litellm @@ -19,12 +20,25 @@ except (ImportError, AttributeError): # This keeps tiktoken fully offline-capable by default (see #1071). custom_cache_dir: Final = os.getenv("CUSTOM_TIKTOKEN_CACHE_DIR") if custom_cache_dir: - # If the user opts into a custom cache dir, ensure it exists. - os.makedirs(custom_cache_dir, exist_ok=True) cache_dir = custom_cache_dir else: cache_dir = filename +# Ensure cache_dir exists and is writable. +# If the target directory cannot be created or is read-only (e.g. read-only site-packages), +# fall back to a private user-writable temporary directory to avoid FileNotFoundError / PermissionError. +try: + os.makedirs(cache_dir, exist_ok=True) + probe_file = os.path.join(cache_dir, f".probe_{os.getpid()}") + try: + with open(probe_file, "xb") as f: + f.write(b"") + os.remove(probe_file) + except (OSError, PermissionError, FileExistsError): + cache_dir = tempfile.mkdtemp(prefix="litellm_tokenizers_") +except Exception: + cache_dir = tempfile.mkdtemp(prefix="litellm_tokenizers_") + os.environ["TIKTOKEN_CACHE_DIR"] = ( cache_dir # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071 ) @@ -44,6 +58,14 @@ for attempt in range(_max_retries): encoding = tiktoken.get_encoding("cl100k_base") break except (FileExistsError, OSError): + # On early failures, switch to a private fallback cache directory if original cache directory had issues + current_cache = os.environ.get("TIKTOKEN_CACHE_DIR", "") + if "litellm_tokenizers_" not in current_cache: + try: + fallback_dir: Final = tempfile.mkdtemp(prefix="litellm_tokenizers_") + os.environ["TIKTOKEN_CACHE_DIR"] = fallback_dir + except Exception: + pass if attempt == _max_retries - 1: # Last attempt, re-raise the exception raise diff --git a/pyproject.toml b/pyproject.toml index 47bed23c093..ed1bbc4fabc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -336,6 +336,7 @@ features = ["extension-module"] profile = "release" editable-profile = "dev" include = [ + "litellm/litellm_core_utils/tokenizers/**", "litellm/proxy/_experimental/out/**", "litellm/router_strategy/complexity_router/artifacts/*.json", "litellm/router_strategy/complexity_router/fuse_presets.json", diff --git a/tests/windows_tests/test_default_encoding_dir.py b/tests/windows_tests/test_default_encoding_dir.py new file mode 100644 index 00000000000..db9fc216f8a --- /dev/null +++ b/tests/windows_tests/test_default_encoding_dir.py @@ -0,0 +1,59 @@ +import importlib +import os +import shutil +import sys +import tempfile +import types +from typing import Final + +# Ensure _native stub exists if running in an uncompiled source environment +if "litellm.rust_bridge._native" not in sys.modules: + mock_native = types.ModuleType("litellm.rust_bridge._native") + sys.modules["litellm.rust_bridge._native"] = mock_native + + +def test_default_encoding_directory_created() -> None: + """Verify that default_encoding ensures the cache directory exists and is usable.""" + test_cache_dir: Final = os.path.join(tempfile.gettempdir(), "test_litellm_encoding_cache") + if os.path.exists(test_cache_dir): + shutil.rmtree(test_cache_dir, ignore_errors=True) + + original_env: Final = os.environ.get("CUSTOM_TIKTOKEN_CACHE_DIR") + try: + os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = test_cache_dir + import litellm.litellm_core_utils.default_encoding as default_encoding + + importlib.reload(default_encoding) + + assert os.path.exists(test_cache_dir), "Cache dir should be created automatically" + assert os.environ.get("TIKTOKEN_CACHE_DIR") == test_cache_dir + assert default_encoding.encoding is not None + finally: + if original_env is not None: + os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = original_env + else: + os.environ.pop("CUSTOM_TIKTOKEN_CACHE_DIR", None) + if os.path.exists(test_cache_dir): + shutil.rmtree(test_cache_dir, ignore_errors=True) + + +def test_default_encoding_fallback_on_uncreatable_dir() -> None: + """Verify fallback to secure temporary directory when configured cache directory cannot be created.""" + invalid_dir: Final = "Z:\\non_existent_drive_for_test_12345\\tokenizers" + original_env: Final = os.environ.get("CUSTOM_TIKTOKEN_CACHE_DIR") + try: + os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = invalid_dir + import litellm.litellm_core_utils.default_encoding as default_encoding + + importlib.reload(default_encoding) + + active_cache: Final = os.environ.get("TIKTOKEN_CACHE_DIR") + assert active_cache is not None + assert active_cache != invalid_dir, "Should have fallen back to a valid temp cache" + assert os.path.exists(active_cache), "Fallback cache directory must exist" + assert default_encoding.encoding is not None + finally: + if original_env is not None: + os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = original_env + else: + os.environ.pop("CUSTOM_TIKTOKEN_CACHE_DIR", None)