scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-05 02:41:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 49

fix(models): flag prompt caching on Vertex and Azure AI Grok rows

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-09-16 19:17:45 +00:00
parent 7734e3e186
commit 8caf2cb61f
3 changed files with 74 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/model_prices_and_context_window_backup.json
View file

@ -11833,6 +11833,7 @@
"output_cost_per_token": 2.5e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
@ -11851,6 +11852,7 @@
"output_cost_per_token": 2.5e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
@ -50027,6 +50029,7 @@
"output_cost_per_token": 5e-07,
"source": "https://docs.x.ai/developers/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
@ -50043,6 +50046,7 @@
"output_cost_per_token": 5e-07,
"source": "https://docs.x.ai/developers/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -50063,6 +50067,7 @@
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
@ -50082,6 +50087,7 @@
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -50102,6 +50108,7 @@
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -50121,6 +50128,7 @@
"output_cost_per_token_above_200k_tokens": 1.2e-05,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,

8
model_prices_and_context_window.json
View file

@ -11833,6 +11833,7 @@
"output_cost_per_token": 2.5e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
@ -11851,6 +11852,7 @@
"output_cost_per_token": 2.5e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
@ -50027,6 +50029,7 @@
"output_cost_per_token": 5e-07,
"source": "https://docs.x.ai/developers/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
@ -50043,6 +50046,7 @@
"output_cost_per_token": 5e-07,
"source": "https://docs.x.ai/developers/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -50063,6 +50067,7 @@
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
@ -50082,6 +50087,7 @@
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -50102,6 +50108,7 @@
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -50121,6 +50128,7 @@
"output_cost_per_token_above_200k_tokens": 1.2e-05,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,

58
tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py Normal file
View file

@ -0,0 +1,58 @@
from pathlib import Path
from typing import Final
import pytest
from pydantic import TypeAdapter
from litellm import get_model_info
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.utils import supports_prompt_caching
REPO_ROOT: Final = Path(__file__).parents[2]
MODEL: Final = "vertex_ai/xai/grok-4.6"
GROK_KEY_PREFIXES: Final = ("vertex_ai/xai/grok-", "azure_ai/grok-", "xai/grok-")
COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
MAIN_COST_MAP: Final = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_COST_MAP: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
def _cost_map(path: Path) -> dict[str, dict[str, object]]:
return COST_MAP_ADAPTER.validate_json(path.read_bytes())
def test_grok_entries_with_cache_read_price_advertise_prompt_caching() -> None:
cost_map = _cost_map(MAIN_COST_MAP)
cached_grok_entries = tuple(
(key, entry)
for key, entry in cost_map.items()
if key.startswith(GROK_KEY_PREFIXES) and entry.get("cache_read_input_token_cost")
)
assert cached_grok_entries, "expected at least one grok entry with a cache read price"
missing_flag = tuple(key for key, entry in cached_grok_entries if entry.get("supports_prompt_caching") is not True)
assert missing_flag == (), (
f"grok entries with cache_read_input_token_cost missing supports_prompt_caching: {missing_flag}"
)
@pytest.mark.usefixtures("local_model_cost_map")
def test_vertex_ai_grok_4_6_supports_prompt_caching_via_get_model_info() -> None:
routed_model, provider, _, _ = get_llm_provider(model=MODEL)
assert (routed_model, provider) == ("xai/grok-4.6", "vertex_ai")
info = get_model_info(model=routed_model, custom_llm_provider=provider)
assert info["litellm_provider"] == "vertex_ai"
assert info.get("supports_prompt_caching") is True
assert supports_prompt_caching(model=MODEL) is True
def test_vertex_ai_grok_entries_source_and_backup_match() -> None:
main_map = _cost_map(MAIN_COST_MAP)
backup_map = _cost_map(BACKUP_COST_MAP)
vertex_grok_keys = tuple(key for key in main_map if key.startswith("vertex_ai/xai/grok-"))
assert vertex_grok_keys, "expected at least one vertex_ai/xai/grok- entry"
mismatched = tuple(key for key in vertex_grok_keys if backup_map.get(key) != main_map[key])
assert mismatched == (), f"vertex grok entries differ between source and backup: {mismatched}"