scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 152

cleanup

Browse source
This commit is contained in:
Ishaan Jaffer 2026-02-10 15:16:41 -08:00
parent 276d728112
commit 5c22296315
3 changed files with 20 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
docs/my-website/blog/model_cost_map_incident/index.md
View file

@ -77,9 +77,8 @@ A contributor PR introduced an extra `{` bracket, producing invalid JSON. The re
| 1 | CI validation on `model_prices_and_context_window.json` | ✅ Done | [PR #20605](https://github.com/BerriAI/litellm/pull/20605) |
| 2 | Warning log on fallback to local backup | ✅ Done | [`get_model_cost_map.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py) |
| 3 | `GetModelCostMap` class with integrity validation helpers | ✅ Done | [`get_model_cost_map.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py) |
| 4 | Validation constants (`MODEL_COST_MAP_MIN_MODEL_COUNT`, `MODEL_COST_MAP_MAX_SHRINK_PERCENT`) | ✅ Done | [`constants.py`](https://github.com/BerriAI/litellm/blob/main/litellm/constants.py) |
| 5 | Resilience test suite (bad hosted map, bad backup, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py) |
| 6 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py) |
| 4 | Resilience test suite (bad hosted map, bad backup, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py) |
| 5 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py) |
Enterprises that require zero external dependencies at import time can set `LITELLM_LOCAL_MODEL_COST_MAP=True` to skip the GitHub fetch entirely.

6
litellm/constants.py
View file

@ -53,9 +53,9 @@ DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
MODEL_COST_MAP_MIN_MODEL_COUNT = int(
os.getenv("MODEL_COST_MAP_MIN_MODEL_COUNT", 50)
) # Minimum number of models a fetched cost map must contain to be considered valid
MODEL_COST_MAP_MAX_SHRINK_PERCENT = float(
os.getenv("MODEL_COST_MAP_MAX_SHRINK_PERCENT", 0.5)
) # Maximum allowed shrinkage vs local backup (0.5 = reject if fetched map is <50% of backup)
MODEL_COST_MAP_MAX_SHRINK_RATIO = float(
os.getenv("MODEL_COST_MAP_MAX_SHRINK_RATIO", 0.5)
) # Maximum allowed shrinkage ratio vs local backup (0.5 = reject if fetched map is <50% of backup)
DEFAULT_IMAGE_WIDTH = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300))
DEFAULT_IMAGE_HEIGHT = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300))
# Maximum size for image URL downloads in MB (default 50MB, set to 0 to disable limit)

31
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -16,7 +16,7 @@ import httpx
from litellm import verbose_logger
from litellm.constants import (
MODEL_COST_MAP_MAX_SHRINK_PERCENT,
MODEL_COST_MAP_MAX_SHRINK_RATIO,
MODEL_COST_MAP_MIN_MODEL_COUNT,
)
@ -25,10 +25,9 @@ class GetModelCostMap:
"""
Handles fetching, validating, and loading the model cost map.
The backup model count is cached on first access so that validation
never needs to parse the full backup JSON — only the count is needed
for the shrinkage check. The full backup is only loaded when it must
be *returned* as a fallback.
Only the backup model *count* is cached (a single int). The full
backup dict is never held in memory — it is only parsed when it
needs to be *returned* as a fallback.
"""
_backup_model_count: int = -1 # -1 = not yet loaded
@ -45,7 +44,7 @@ class GetModelCostMap:
@classmethod
def _get_backup_model_count(cls) -> int:
"""Return the number of models in the local backup (cached)."""
"""Return the number of models in the local backup (cached int)."""
if cls._backup_model_count < 0:
backup = cls.load_local_model_cost_map()
cls._backup_model_count = len(backup)
@ -77,7 +76,7 @@ class GetModelCostMap:
fetched_map: dict,
backup_model_count: int,
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
max_shrink_pct: float = MODEL_COST_MAP_MAX_SHRINK_PERCENT,
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
) -> bool:
"""Check 2: model count has not reduced significantly vs backup."""
fetched_count = len(fetched_map)
@ -92,7 +91,7 @@ class GetModelCostMap:
)
return False
if backup_model_count > 0 and fetched_count < backup_model_count * max_shrink_pct:
if backup_model_count > 0 and fetched_count < backup_model_count * max_shrink_ratio:
verbose_logger.warning(
"LiteLLM: Fetched model cost map shrank significantly "
"(fetched=%d, backup=%d, threshold=%.0f%%). "
@ -100,7 +99,7 @@ class GetModelCostMap:
"Falling back to local backup.",
fetched_count,
backup_model_count,
max_shrink_pct * 100,
max_shrink_ratio * 100,
)
return False
@ -112,7 +111,7 @@ class GetModelCostMap:
fetched_map: dict,
backup_model_count: int,
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
max_shrink_pct: float = MODEL_COST_MAP_MAX_SHRINK_PERCENT,
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
) -> bool:
"""
Validate the integrity of a fetched model cost map.
@ -122,7 +121,7 @@ class GetModelCostMap:
Checks:
1. ``_check_is_valid_dict`` -- fetched map is a non-empty dict.
2. ``_check_model_count_not_reduced`` -- model count meets minimum
and has not shrunk >``max_shrink_pct`` vs backup.
and has not shrunk >``max_shrink_ratio`` vs backup.
Returns True if all checks pass, False otherwise.
"""
@ -133,7 +132,7 @@ class GetModelCostMap:
fetched_map=fetched_map,
backup_model_count=backup_model_count,
min_model_count=min_model_count,
max_shrink_pct=max_shrink_pct,
max_shrink_ratio=max_shrink_ratio,
):
return False
@ -160,9 +159,9 @@ def get_model_cost_map(url: str) -> dict:
2. Otherwise fetches from ``url``, validates integrity, and falls back
to the local backup on any failure.
The backup model count is cached in ``GetModelCostMap`` so validation
only costs a cheap integer comparison — the full backup JSON is only
parsed when it needs to be *returned* as a fallback.
Only the backup model count is cached (a single int) for validation.
The full backup dict is only parsed when it must be *returned* as a
fallback — it is never held in memory long-term.
"""
# Note: can't use get_secret_bool here — this runs during litellm.__init__
# before litellm._key_management_settings is set.
@ -180,7 +179,7 @@ def get_model_cost_map(url: str) -> dict:
)
return GetModelCostMap.load_local_model_cost_map()
# Validate fetched JSON integrity — uses cached backup count, no file I/O
# Validate using cached count (cheap int comparison, no file I/O)
if not GetModelCostMap.validate_model_cost_map(
fetched_map=content,
backup_model_count=GetModelCostMap._get_backup_model_count(),