This commit is contained in:
shahmanager 2026-11-06 04:59:36 +00:00 • committed by GitHub
commit 2b40177398
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 84 additions and 2 deletions

View file

@ -1,4 +1,5 @@
import os
import tempfile
from typing import Final
import litellm
@ -19,12 +20,25 @@ except (ImportError, AttributeError):
# This keeps tiktoken fully offline-capable by default (see #1071).
custom_cache_dir: Final = os.getenv("CUSTOM_TIKTOKEN_CACHE_DIR")
if custom_cache_dir:
# If the user opts into a custom cache dir, ensure it exists.
os.makedirs(custom_cache_dir, exist_ok=True)
cache_dir = custom_cache_dir
else:
cache_dir = filename
# Ensure cache_dir exists and is writable.
# If the target directory cannot be created or is read-only (e.g. read-only site-packages),
# fall back to a private user-writable temporary directory to avoid FileNotFoundError / PermissionError.
try:
os.makedirs(cache_dir, exist_ok=True)
probe_file = os.path.join(cache_dir, f".probe_{os.getpid()}")
try:
with open(probe_file, "xb") as f:
f.write(b"")
os.remove(probe_file)
except (OSError, PermissionError, FileExistsError):
cache_dir = tempfile.mkdtemp(prefix="litellm_tokenizers_")
except Exception:
cache_dir = tempfile.mkdtemp(prefix="litellm_tokenizers_")
os.environ["TIKTOKEN_CACHE_DIR"] = (
cache_dir # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071
)
@ -44,6 +58,14 @@ for attempt in range(_max_retries):
encoding = tiktoken.get_encoding("cl100k_base")
break
except (FileExistsError, OSError):
# On early failures, switch to a private fallback cache directory if original cache directory had issues
current_cache = os.environ.get("TIKTOKEN_CACHE_DIR", "")
if "litellm_tokenizers_" not in current_cache:
try:
fallback_dir: Final = tempfile.mkdtemp(prefix="litellm_tokenizers_")
os.environ["TIKTOKEN_CACHE_DIR"] = fallback_dir
except Exception:
pass
if attempt == _max_retries - 1:
# Last attempt, re-raise the exception
raise

View file

@ -336,6 +336,7 @@ features = ["extension-module"]
profile = "release"
editable-profile = "dev"
include = [
"litellm/litellm_core_utils/tokenizers/**",
"litellm/proxy/_experimental/out/**",
"litellm/router_strategy/complexity_router/artifacts/*.json",
"litellm/router_strategy/complexity_router/fuse_presets.json",

View file

@ -0,0 +1,59 @@
import importlib
import os
import shutil
import sys
import tempfile
import types
from typing import Final
# Ensure _native stub exists if running in an uncompiled source environment
if "litellm.rust_bridge._native" not in sys.modules:
mock_native = types.ModuleType("litellm.rust_bridge._native")
sys.modules["litellm.rust_bridge._native"] = mock_native
def test_default_encoding_directory_created() -> None:
"""Verify that default_encoding ensures the cache directory exists and is usable."""
test_cache_dir: Final = os.path.join(tempfile.gettempdir(), "test_litellm_encoding_cache")
if os.path.exists(test_cache_dir):
shutil.rmtree(test_cache_dir, ignore_errors=True)
original_env: Final = os.environ.get("CUSTOM_TIKTOKEN_CACHE_DIR")
try:
os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = test_cache_dir
import litellm.litellm_core_utils.default_encoding as default_encoding
importlib.reload(default_encoding)
assert os.path.exists(test_cache_dir), "Cache dir should be created automatically"
assert os.environ.get("TIKTOKEN_CACHE_DIR") == test_cache_dir
assert default_encoding.encoding is not None
finally:
if original_env is not None:
os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = original_env
else:
os.environ.pop("CUSTOM_TIKTOKEN_CACHE_DIR", None)
if os.path.exists(test_cache_dir):
shutil.rmtree(test_cache_dir, ignore_errors=True)
def test_default_encoding_fallback_on_uncreatable_dir() -> None:
"""Verify fallback to secure temporary directory when configured cache directory cannot be created."""
invalid_dir: Final = "Z:\\non_existent_drive_for_test_12345\\tokenizers"
original_env: Final = os.environ.get("CUSTOM_TIKTOKEN_CACHE_DIR")
try:
os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = invalid_dir
import litellm.litellm_core_utils.default_encoding as default_encoding
importlib.reload(default_encoding)
active_cache: Final = os.environ.get("TIKTOKEN_CACHE_DIR")
assert active_cache is not None
assert active_cache != invalid_dir, "Should have fallen back to a valid temp cache"
assert os.path.exists(active_cache), "Fallback cache directory must exist"
assert default_encoding.encoding is not None
finally:
if original_env is not None:
os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = original_env
else:
os.environ.pop("CUSTOM_TIKTOKEN_CACHE_DIR", None)