mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 2d74975379 into c5da69cd13
This commit is contained in:
commit
c920e23696
3 changed files with 84 additions and 2 deletions
|
|
@ -1,4 +1,5 @@
|
|||
import os
|
||||
import tempfile
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -19,12 +20,25 @@ except (ImportError, AttributeError):
|
|||
# This keeps tiktoken fully offline-capable by default (see #1071).
|
||||
custom_cache_dir: Final = os.getenv("CUSTOM_TIKTOKEN_CACHE_DIR")
|
||||
if custom_cache_dir:
|
||||
# If the user opts into a custom cache dir, ensure it exists.
|
||||
os.makedirs(custom_cache_dir, exist_ok=True)
|
||||
cache_dir = custom_cache_dir
|
||||
else:
|
||||
cache_dir = filename
|
||||
|
||||
# Ensure cache_dir exists and is writable.
|
||||
# If the target directory cannot be created or is read-only (e.g. read-only site-packages),
|
||||
# fall back to a private user-writable temporary directory to avoid FileNotFoundError / PermissionError.
|
||||
try:
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
probe_file = os.path.join(cache_dir, f".probe_{os.getpid()}")
|
||||
try:
|
||||
with open(probe_file, "xb") as f:
|
||||
f.write(b"")
|
||||
os.remove(probe_file)
|
||||
except (OSError, PermissionError, FileExistsError):
|
||||
cache_dir = tempfile.mkdtemp(prefix="litellm_tokenizers_")
|
||||
except Exception:
|
||||
cache_dir = tempfile.mkdtemp(prefix="litellm_tokenizers_")
|
||||
|
||||
os.environ["TIKTOKEN_CACHE_DIR"] = (
|
||||
cache_dir # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071
|
||||
)
|
||||
|
|
@ -44,6 +58,14 @@ for attempt in range(_max_retries):
|
|||
encoding = tiktoken.get_encoding("cl100k_base")
|
||||
break
|
||||
except (FileExistsError, OSError):
|
||||
# On early failures, switch to a private fallback cache directory if original cache directory had issues
|
||||
current_cache = os.environ.get("TIKTOKEN_CACHE_DIR", "")
|
||||
if "litellm_tokenizers_" not in current_cache:
|
||||
try:
|
||||
fallback_dir: Final = tempfile.mkdtemp(prefix="litellm_tokenizers_")
|
||||
os.environ["TIKTOKEN_CACHE_DIR"] = fallback_dir
|
||||
except Exception:
|
||||
pass
|
||||
if attempt == _max_retries - 1:
|
||||
# Last attempt, re-raise the exception
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -334,6 +334,7 @@ features = ["extension-module"]
|
|||
profile = "release"
|
||||
editable-profile = "dev"
|
||||
include = [
|
||||
"litellm/litellm_core_utils/tokenizers/**",
|
||||
"litellm/proxy/_experimental/out/**",
|
||||
"litellm/router_strategy/complexity_router/artifacts/*.json",
|
||||
"litellm/router_strategy/complexity_router/fuse_presets.json",
|
||||
|
|
|
|||
59
tests/windows_tests/test_default_encoding_dir.py
Normal file
59
tests/windows_tests/test_default_encoding_dir.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import importlib
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
from typing import Final
|
||||
|
||||
# Ensure _native stub exists if running in an uncompiled source environment
|
||||
if "litellm.rust_bridge._native" not in sys.modules:
|
||||
mock_native = types.ModuleType("litellm.rust_bridge._native")
|
||||
sys.modules["litellm.rust_bridge._native"] = mock_native
|
||||
|
||||
|
||||
def test_default_encoding_directory_created() -> None:
|
||||
"""Verify that default_encoding ensures the cache directory exists and is usable."""
|
||||
test_cache_dir: Final = os.path.join(tempfile.gettempdir(), "test_litellm_encoding_cache")
|
||||
if os.path.exists(test_cache_dir):
|
||||
shutil.rmtree(test_cache_dir, ignore_errors=True)
|
||||
|
||||
original_env: Final = os.environ.get("CUSTOM_TIKTOKEN_CACHE_DIR")
|
||||
try:
|
||||
os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = test_cache_dir
|
||||
import litellm.litellm_core_utils.default_encoding as default_encoding
|
||||
|
||||
importlib.reload(default_encoding)
|
||||
|
||||
assert os.path.exists(test_cache_dir), "Cache dir should be created automatically"
|
||||
assert os.environ.get("TIKTOKEN_CACHE_DIR") == test_cache_dir
|
||||
assert default_encoding.encoding is not None
|
||||
finally:
|
||||
if original_env is not None:
|
||||
os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = original_env
|
||||
else:
|
||||
os.environ.pop("CUSTOM_TIKTOKEN_CACHE_DIR", None)
|
||||
if os.path.exists(test_cache_dir):
|
||||
shutil.rmtree(test_cache_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def test_default_encoding_fallback_on_uncreatable_dir() -> None:
|
||||
"""Verify fallback to secure temporary directory when configured cache directory cannot be created."""
|
||||
invalid_dir: Final = "Z:\\non_existent_drive_for_test_12345\\tokenizers"
|
||||
original_env: Final = os.environ.get("CUSTOM_TIKTOKEN_CACHE_DIR")
|
||||
try:
|
||||
os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = invalid_dir
|
||||
import litellm.litellm_core_utils.default_encoding as default_encoding
|
||||
|
||||
importlib.reload(default_encoding)
|
||||
|
||||
active_cache: Final = os.environ.get("TIKTOKEN_CACHE_DIR")
|
||||
assert active_cache is not None
|
||||
assert active_cache != invalid_dir, "Should have fallen back to a valid temp cache"
|
||||
assert os.path.exists(active_cache), "Fallback cache directory must exist"
|
||||
assert default_encoding.encoding is not None
|
||||
finally:
|
||||
if original_env is not None:
|
||||
os.environ["CUSTOM_TIKTOKEN_CACHE_DIR"] = original_env
|
||||
else:
|
||||
os.environ.pop("CUSTOM_TIKTOKEN_CACHE_DIR", None)
|
||||
Loading…
Add table
Reference in a new issue