scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-26 01:12:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 99

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_e2e_uv_migration_and_all_proxy_models_test

Browse source
This commit is contained in:
ryan-crabbe-berri 2026-05-20 11:54:52 -07:00
commit 7a3b5d1879
6 changed files with 626 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

227
litellm/model_prices_and_context_window_backup.json
View file

@ -1448,6 +1448,35 @@
"supports_native_structured_output": true,
"supports_minimal_reasoning_effort": true
},
"jp.anthropic.claude-sonnet-4-6": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_max_reasoning_effort": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true,
"supports_minimal_reasoning_effort": true
},
"anthropic.claude-sonnet-4-20250514-v1:0": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
@ -9602,6 +9631,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@ -9795,6 +9825,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -9828,6 +9859,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -9861,6 +9893,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -9895,6 +9928,7 @@
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -14883,7 +14917,65 @@
"mode": "chat",
"output_cost_per_reasoning_token": 1.5e-06,
"output_cost_per_token": 1.5e-06,
"source": "https://ai.google.dev/gemini-api/docs/models",
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_audio_output": false,
"supports_code_execution": true,
"supports_file_search": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"supports_service_tier": true
},
"gemini-3.1-flash-lite": {
"cache_read_input_token_cost": 4.5e-08,
"cache_read_input_token_cost_per_audio_token": 9e-08,
"input_cost_per_audio_token": 9e-07,
"input_cost_per_token": 4.5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_pdf_size_mb": 30,
"max_tokens": 65536,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 2.7e-06,
"output_cost_per_token": 2.7e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
@ -16987,6 +17079,66 @@
"web_search_billing_unit": "per_query",
"supports_service_tier": true
},
"gemini/gemini-3.1-flash-lite": {
"cache_read_input_token_cost": 4.5e-08,
"cache_read_input_token_cost_per_audio_token": 9e-08,
"input_cost_per_audio_token": 9e-07,
"input_cost_per_token": 4.5e-07,
"litellm_provider": "gemini",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_pdf_size_mb": 30,
"max_tokens": 65536,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 2.7e-06,
"output_cost_per_token": 2.7e-06,
"rpm": 15,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_audio_output": false,
"supports_code_execution": true,
"supports_file_search": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 250000,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"supports_service_tier": true
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_audio_token": 1e-06,
@ -24285,6 +24437,21 @@
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/ministral-8b-2512": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 1.5e-07,
"source": "https://mistral.ai/pricing",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/mistral-tiny": {
"input_cost_per_token": 2.5e-07,
"litellm_provider": "mistral",
@ -33605,6 +33772,64 @@
},
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.1-flash-lite": {
"cache_read_input_token_cost": 4.5e-08,
"cache_read_input_token_cost_per_audio_token": 9e-08,
"input_cost_per_audio_token": 9e-07,
"input_cost_per_token": 4.5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_pdf_size_mb": 30,
"max_tokens": 65536,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 2.7e-06,
"output_cost_per_token": 2.7e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_audio_output": false,
"supports_code_execution": true,
"supports_file_search": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"supports_service_tier": true
},
"vertex_ai/deep-research-pro-preview-12-2025": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,

27
litellm/proxy/db/spend_counter_reseed.py
View file

@ -178,15 +178,28 @@ class SpendCounterReseed:
if db_spend is None:
return None
# Warm even when 0 so subsequent reads hit cache, not DB.
#
# Seed via SET NX (cross-pod safe): only one pod initializes the
# Redis key with db_spend; concurrent seeders read the winner's
# value. INCRBYFLOAT-of-db_spend from N pods would multiply the
# counter (N x db_spend) and trigger spurious budget alerts.
current_value: float = float(db_spend)
try:
if spend_counter_cache.redis_cache is not None:
current_value = (
await spend_counter_cache.redis_cache.async_increment(
key=counter_key,
value=db_spend,
refresh_ttl=True,
)
seeded = await spend_counter_cache.redis_cache.async_set_cache(
key=counter_key,
value=db_spend,
nx=True,
)
if seeded:
current_value = float(db_spend)
else:
cached = await spend_counter_cache.redis_cache.async_get_cache(
key=counter_key
)
current_value = (
float(cached) if cached is not None else float(db_spend)
)
spend_counter_cache.in_memory_cache.set_cache(
key=counter_key,
value=current_value,
@ -202,7 +215,7 @@ class SpendCounterReseed:
)
if require_cache_warm:
raise
return db_spend
return current_value
@staticmethod
async def window_from_spend_logs(

191
model_prices_and_context_window.json
View file

@ -14957,6 +14957,64 @@
"web_search_billing_unit": "per_query",
"supports_service_tier": true
},
"gemini-3.1-flash-lite": {
"cache_read_input_token_cost": 4.5e-08,
"cache_read_input_token_cost_per_audio_token": 9e-08,
"input_cost_per_audio_token": 9e-07,
"input_cost_per_token": 4.5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_pdf_size_mb": 30,
"max_tokens": 65536,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 2.7e-06,
"output_cost_per_token": 2.7e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_audio_output": false,
"supports_code_execution": true,
"supports_file_search": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"supports_service_tier": true
},
"deep-research-pro-preview-12-2025": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,
@ -17021,6 +17079,66 @@
"web_search_billing_unit": "per_query",
"supports_service_tier": true
},
"gemini/gemini-3.1-flash-lite": {
"cache_read_input_token_cost": 4.5e-08,
"cache_read_input_token_cost_per_audio_token": 9e-08,
"input_cost_per_audio_token": 9e-07,
"input_cost_per_token": 4.5e-07,
"litellm_provider": "gemini",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_pdf_size_mb": 30,
"max_tokens": 65536,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 2.7e-06,
"output_cost_per_token": 2.7e-06,
"rpm": 15,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_audio_output": false,
"supports_code_execution": true,
"supports_file_search": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 250000,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"supports_service_tier": true
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_audio_token": 1e-06,
@ -24319,6 +24437,21 @@
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/ministral-8b-2512": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "mistral",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 1.5e-07,
"source": "https://mistral.ai/pricing",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"mistral/mistral-tiny": {
"input_cost_per_token": 2.5e-07,
"litellm_provider": "mistral",
@ -33639,6 +33772,64 @@
},
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3.1-flash-lite": {
"cache_read_input_token_cost": 4.5e-08,
"cache_read_input_token_cost_per_audio_token": 9e-08,
"input_cost_per_audio_token": 9e-07,
"input_cost_per_token": 4.5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_audio_length_hours": 8.4,
"max_audio_per_prompt": 1,
"max_images_per_prompt": 3000,
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_pdf_size_mb": 30,
"max_tokens": 65536,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"output_cost_per_reasoning_token": 2.7e-06,
"output_cost_per_token": 2.7e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_audio_output": false,
"supports_code_execution": true,
"supports_file_search": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"supports_service_tier": true
},
"vertex_ai/deep-research-pro-preview-12-2025": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,

14
tests/local_testing/conftest.py
View file

@ -22,6 +22,20 @@ sys.path.insert(
) # Adds the parent directory to the system path
import litellm
# ``litellm.model_cost`` is loaded at import time from the URL pinned to
# ``main`` (``LITELLM_MODEL_COST_MAP_URL``). The in-tree backup ships with
# this branch and can include pricing entries that main has not yet picked
# up (e.g. an upstream provider rotates a model id and the test cassette
# records the new name). Backfill any entries that are missing from the
# remote-fetched map so cost-calculator lookups in tests succeed against
# the cassette state the branch is being tested with.
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
_local_cost_map = GetModelCostMap.load_local_model_cost_map()
for _k, _v in _local_cost_map.items():
litellm.model_cost.setdefault(_k, _v)
del _local_cost_map
from tests._vcr_conftest_common import ( # noqa: E402,F401
VerboseReporterState,
_pin_multipart_boundary,

167
tests/test_litellm/proxy/test_proxy_server.py
View file

@ -5708,6 +5708,7 @@ async def test_init_and_increment_spend_counter_reseeds_from_db_on_counter_miss(
fake_redis = AsyncMock()
fake_redis.async_increment = AsyncMock(side_effect=record_increment)
fake_redis.async_get_cache = AsyncMock(return_value=None) # counter missing
fake_redis.async_set_cache = AsyncMock(return_value=True) # SET NX wins
counter_cache.redis_cache = fake_redis
# Prisma returns spend=42.0 (authoritative) while the stale cached
@ -5744,16 +5745,131 @@ async def test_init_and_increment_spend_counter_reseeds_from_db_on_counter_miss(
fake_prisma.db.litellm_teamtable.find_unique.assert_awaited_once_with(
where={"team_id": "team-9"}
)
# Two increments keyed on the counter: seed ($42) then request ($1.50).
# Seed uses SET NX with db_spend (42) — cross-pod safe, no INCR of 42.
# Only the per-request delta (1.5) goes through INCRBYFLOAT.
fake_redis.async_set_cache.assert_awaited_once_with(
key="spend:team:team-9", value=42.0, nx=True
)
writes = [(c["key"], c["value"]) for c in recorded_increments]
assert ("spend:team:team-9", 42.0) in writes
assert ("spend:team:team-9", 1.5) in writes
assert writes == [("spend:team:team-9", 1.5)]
finally:
ps.user_api_key_cache = orig_user
ps.spend_counter_cache = orig_counter
ps.prisma_client = orig_prisma
@pytest.mark.asyncio
async def test_primary_spend_counter_redis_concurrent_seed_does_not_double_seed():
"""Two pods both observing a missing Redis counter must not both
INCRBYFLOAT the full DB spend. SpendCounterReseed.coalesced uses SET NX
so the loser reads the winner's value; final Redis = db_spend, not
2 * db_spend.
The per-counter asyncio.Lock is per-process, so it does NOT coordinate
across pods. We simulate two pods by patching _get_lock to return a
fresh lock per call (each "pod" has its own lock registry in real life).
"""
from litellm.caching.dual_cache import DualCache
from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed
counter_key = "spend:team:team-concurrent-seed"
redis_store: dict = {}
db_read_count = 0
set_results: list = []
get_after_set_count = 0
set_completed_count = 0
async def redis_set_cache(key, value, nx=False, **_):
# Yield BEFORE the membership check so two concurrent callers
# interleave the way real atomic Redis SET NX does: the first
# to resume runs check + write atomically and wins; the second
# resumes after the key exists and loses. Yielding *after* the
# check would let both callers pass the empty-store check before
# either writes, so neither would ever lose.
await asyncio.sleep(0)
if nx and key in redis_store:
set_results.append(False)
return False
redis_store[key] = float(value)
set_results.append(True)
nonlocal set_completed_count
set_completed_count += 1
return True
async def redis_get_cache(key):
# Track reads that happen after at least one SET NX has completed
# — those are the loser-path fallback reads we want to verify.
if set_completed_count > 0:
nonlocal get_after_set_count
get_after_set_count += 1
return redis_store.get(key)
fake_redis = AsyncMock()
fake_redis.async_get_cache = AsyncMock(side_effect=redis_get_cache)
fake_redis.async_set_cache = AsyncMock(side_effect=redis_set_cache)
async def slow_find_unique(**_):
nonlocal db_read_count
db_read_count += 1
# Both pods read DB before either's SET NX lands.
await asyncio.sleep(0)
row = MagicMock()
row.spend = 506.0
return row
fake_prisma = MagicMock()
fake_prisma.db.litellm_teamtable.find_unique = AsyncMock(
side_effect=slow_find_unique
)
pod_a = DualCache()
pod_a.redis_cache = fake_redis
pod_b = DualCache()
pod_b.redis_cache = fake_redis
# Each "pod" has its own per-process lock registry. Patch _get_lock to
# always return a fresh lock so the two coalesced calls do not serialize
# via one in-process lock (which is what would happen across pods).
async def fresh_lock(_counter_key):
return asyncio.Lock()
with patch.object(SpendCounterReseed, "_get_lock", side_effect=fresh_lock):
results = await asyncio.gather(
SpendCounterReseed.coalesced(
prisma_client=fake_prisma,
spend_counter_cache=pod_a,
counter_key=counter_key,
),
SpendCounterReseed.coalesced(
prisma_client=fake_prisma,
spend_counter_cache=pod_b,
counter_key=counter_key,
),
)
assert all(r == 506.0 for r in results), results
assert redis_store[counter_key] == pytest.approx(506.0), redis_store
# Both pods read the DB and both attempted SET NX; exactly one wrote
# (winner) and one was rejected (loser).
assert db_read_count == 2
assert fake_redis.async_set_cache.await_count == 2
nx_writes = [
call
for call in fake_redis.async_set_cache.await_args_list
if call.kwargs.get("nx") is True
]
assert len(nx_writes) == 2
assert sorted(set_results) == [False, True], (
f"expected exactly one SET NX winner and one loser, got {set_results}"
)
# Loser path executed: after the winner's SET NX returned True, the
# losing coalesced() call falls back to async_get_cache to read the
# winner's value rather than re-seeding.
assert get_after_set_count >= 1, (
"loser branch (else: read back winner's value) was never exercised"
)
@pytest.mark.asyncio
async def test_reseed_spend_from_db_user_and_org_prefixes():
"""User and org counters reseed from their own DB tables.
@ -5877,9 +5993,16 @@ async def test_init_spend_counter_redis_clean_miss_skips_stale_in_memory():
redis_store[key] = (redis_store.get(key) or 0.0) + value
return redis_store[key]
async def redis_set_cache(key, value, nx=False, **_):
if nx and key in redis_store:
return False
redis_store[key] = float(value)
return True
fake_redis = AsyncMock()
fake_redis.async_get_cache = AsyncMock(return_value=None)
fake_redis.async_increment = AsyncMock(side_effect=redis_increment)
fake_redis.async_set_cache = AsyncMock(side_effect=redis_set_cache)
counter_cache.redis_cache = fake_redis
db_row = MagicMock()
@ -5907,6 +6030,7 @@ async def test_init_spend_counter_redis_clean_miss_skips_stale_in_memory():
fake_prisma.db.litellm_teamtable.find_unique.assert_awaited_once_with(
where={"team_id": "team-stale-local"}
)
# Seed via SET NX (42) + delta via INCRBYFLOAT (1.5) = 43.5.
assert redis_store[counter_key] == pytest.approx(43.5)
assert counter_cache.in_memory_cache.get_cache(
key=counter_key
@ -6297,14 +6421,14 @@ async def test_get_current_spend_reseeds_from_db_when_counter_missing():
from litellm.proxy.proxy_server import get_current_spend
counter_cache = DualCache()
recorded_warms: list = []
recorded_seeds: list = []
async def record_increment(key, value, ttl=None, **kwargs):
recorded_warms.append({"key": key, "value": value})
return value
async def record_set_cache(key, value, nx=False, **kwargs):
recorded_seeds.append({"key": key, "value": value, "nx": nx})
return True
fake_redis = AsyncMock()
fake_redis.async_increment = AsyncMock(side_effect=record_increment)
fake_redis.async_set_cache = AsyncMock(side_effect=record_set_cache)
fake_redis.async_get_cache = AsyncMock(return_value=None)
counter_cache.redis_cache = fake_redis
@ -6329,9 +6453,9 @@ async def test_get_current_spend_reseeds_from_db_when_counter_missing():
f"expected DB reseed to return 362.0, got {spend} "
f"(fallback would have returned 30.0 and caused bypass)"
)
# Counter warmed so subsequent reads are fast
assert ("spend:team_member:user-1:team-1", 362.0) in [
(w["key"], w["value"]) for w in recorded_warms
# Counter warmed via SET NX so subsequent reads are fast.
assert ("spend:team_member:user-1:team-1", 362.0, True) in [
(s["key"], s["value"], s["nx"]) for s in recorded_seeds
]
assert counter_cache.in_memory_cache.get_cache(
key="spend:team_member:user-1:team-1"
@ -6408,8 +6532,15 @@ async def test_get_current_spend_coalesces_concurrent_reseeds():
redis_store[key] = (redis_store.get(key) or 0.0) + value
return redis_store[key]
async def redis_set_cache(key, value, nx=False, **_):
if nx and key in redis_store:
return False
redis_store[key] = float(value)
return True
fake_redis.async_get_cache = AsyncMock(side_effect=redis_get)
fake_redis.async_increment = AsyncMock(side_effect=redis_increment)
fake_redis.async_set_cache = AsyncMock(side_effect=redis_set_cache)
counter_cache.redis_cache = fake_redis
fake_prisma = MagicMock()
@ -6516,9 +6647,16 @@ async def test_concurrent_read_and_write_paths_share_one_db_query():
redis_store[key] = (redis_store.get(key) or 0.0) + value
return redis_store[key]
async def redis_set_cache(key, value, nx=False, **_):
if nx and key in redis_store:
return False
redis_store[key] = float(value)
return True
fake_redis = AsyncMock()
fake_redis.async_get_cache = AsyncMock(side_effect=redis_get)
fake_redis.async_increment = AsyncMock(side_effect=redis_increment)
fake_redis.async_set_cache = AsyncMock(side_effect=redis_set_cache)
counter_cache.redis_cache = fake_redis
fake_prisma = MagicMock()
@ -6621,9 +6759,16 @@ async def test_reseed_warms_cache_even_on_zero_db_spend():
redis_store[key] = (redis_store.get(key) or 0.0) + value
return redis_store[key]
async def redis_set_cache(key, value, nx=False, **_):
if nx and key in redis_store:
return False
redis_store[key] = float(value)
return True
fake_redis = AsyncMock()
fake_redis.async_get_cache = AsyncMock(side_effect=redis_get)
fake_redis.async_increment = AsyncMock(side_effect=redis_increment)
fake_redis.async_set_cache = AsyncMock(side_effect=redis_set_cache)
counter_cache.redis_cache = fake_redis
db_call_count = 0

19
tests/test_litellm/test_cost_calculator.py
View file

@ -2059,6 +2059,25 @@ def test_openrouter_gemini_3_1_flash_lite_preview_pricing():
assert model_info["max_output_tokens"] == 65536
def test_gemini_3_1_flash_lite_pricing():
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
for model_name in (
"gemini-3.1-flash-lite",
"gemini/gemini-3.1-flash-lite",
"vertex_ai/gemini-3.1-flash-lite",
):
model_info = litellm.model_cost.get(model_name)
assert model_info is not None, f"Missing model pricing entry: {model_name}"
assert model_info["input_cost_per_token"] == 4.5e-07
assert model_info["input_cost_per_audio_token"] == 9e-07
assert model_info["output_cost_per_token"] == 2.7e-06
assert model_info["output_cost_per_reasoning_token"] == 2.7e-06
assert model_info["cache_read_input_token_cost"] == 4.5e-08
assert model_info["max_input_tokens"] == 1048576
def test_custom_pricing_applies_cache_read_input_cost():
"""
Bug 1 reproduction: custom_cost_per_token with cache_read_input_token_cost