diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d901068b06d..2423fb4469b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -19498,106 +19498,6 @@ }, "web_search_billing_unit": "per_query" }, - "gemini-3.1-flash-lite-image": { - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, - "gemini/gemini-3.1-flash-lite-image": { - "rpm": 1000, - "tpm": 4000000, - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_batches": 1.25e-07, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "output_cost_per_token_batches": 7.5e-07, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_prompt_caching": false, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, - "vertex_ai/gemini-3.1-flash-lite-image": { - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, "gemini-3.1-flash-image": { "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -19675,6 +19575,44 @@ }, "web_search_billing_unit": "per_query" }, + "gemini-3.1-flash-lite-image": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_video_input": true, + "supports_vision": true + }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, @@ -21505,6 +21443,42 @@ }, "web_search_billing_unit": "per_query" }, + "gemini/gemini-3.1-flash-lite-image": { + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "gemini", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "rpm": 1000, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_vision": true, + "tpm": 4000000 + }, "gemini/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -41066,6 +41040,44 @@ "supports_reasoning": false, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" }, + "vertex_ai/gemini-3.1-flash-lite-image": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_video_input": true, + "supports_vision": true + }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d901068b06d..2423fb4469b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -19498,106 +19498,6 @@ }, "web_search_billing_unit": "per_query" }, - "gemini-3.1-flash-lite-image": { - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, - "gemini/gemini-3.1-flash-lite-image": { - "rpm": 1000, - "tpm": 4000000, - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_batches": 1.25e-07, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "output_cost_per_token_batches": 7.5e-07, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_prompt_caching": false, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, - "vertex_ai/gemini-3.1-flash-lite-image": { - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, "gemini-3.1-flash-image": { "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -19675,6 +19575,44 @@ }, "web_search_billing_unit": "per_query" }, + "gemini-3.1-flash-lite-image": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_video_input": true, + "supports_vision": true + }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, @@ -21505,6 +21443,42 @@ }, "web_search_billing_unit": "per_query" }, + "gemini/gemini-3.1-flash-lite-image": { + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "gemini", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "rpm": 1000, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_vision": true, + "tpm": 4000000 + }, "gemini/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -41066,6 +41040,44 @@ "supports_reasoning": false, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" }, + "vertex_ai/gemini-3.1-flash-lite-image": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_video_input": true, + "supports_vision": true + }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index f66056a54e2..0cc1701b8a7 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1909,6 +1909,7 @@ def test_service_tier_ultrafast_fallback_pricing(): [ "gemini-3-pro-image-preview", "gemini-3.1-flash-image-preview", + "gemini-3.1-flash-lite-image", ], ) def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): diff --git a/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py b/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py index 67d6b9e76cf..276f54c116a 100644 --- a/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py +++ b/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py @@ -1,147 +1,284 @@ -"""Pricing entry for ``gemini-3.1-flash-lite-image`` (Google's Nano Banana 2 Lite). - -Google publishes: $0.25/1M input, $1.50/1M text output, and $30/1M image-output -tokens for the Lite image model (https://cloud.google.com/vertex-ai/generative-ai/pricing). -A 1K image is ~1120 output image tokens => ~$0.0336 / image. - -Without this entry, ``completion_cost`` raises "model isn't mapped yet" and Vertex -generateContent pass-through cost tracking silently logs $0. These tests pin the -values in both the primary price map and the ``litellm/`` backup, and verify -``get_model_info`` / ``completion_cost`` surface them. -""" - import json -import os +from pathlib import Path + +import pytest import litellm from litellm import completion_cost -from litellm.types.utils import CompletionTokensDetailsWrapper, ModelResponse, Usage +from litellm.cost_calculator import cost_per_token +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.llms.gemini.image_generation.cost_calculator import ( + cost_calculator as gemini_image_generation_cost_calculator, +) +from litellm.llms.vertex_ai.image_generation.cost_calculator import ( + cost_calculator as vertex_image_generation_cost_calculator, +) +from litellm.types.utils import ( + CompletionTokensDetailsWrapper, + ImageObject, + ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, +) -VARIANTS = [ - "gemini-3.1-flash-lite-image", - "gemini/gemini-3.1-flash-lite-image", - "vertex_ai/gemini-3.1-flash-lite-image", -] +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" -EXPECTED = { - "input_cost_per_token": 2.5e-07, - "output_cost_per_token": 1.5e-06, - "output_cost_per_image_token": 3e-05, +UNPREFIXED = "gemini-3.1-flash-lite-image" +GEMINI = "gemini/gemini-3.1-flash-lite-image" +VERTEX = "vertex_ai/gemini-3.1-flash-lite-image" +ALL_KEYS = (UNPREFIXED, GEMINI, VERTEX) + +INPUT_COST = 2.5e-07 +INPUT_COST_BATCHES = 1.25e-07 +OUTPUT_TEXT_COST = 1.5e-06 +OUTPUT_TEXT_COST_BATCHES = 7.5e-07 +OUTPUT_IMAGE_TOKEN_COST = 3e-05 +OUTPUT_COST_PER_1K_IMAGE = 0.0336 +INPUT_COST_PER_IMAGE = 0.00028 +CACHE_READ_COST = 2.5e-08 +MAX_INPUT_TOKENS = 65536 +MAX_OUTPUT_TOKENS = 4096 +TOKENS_PER_1K_IMAGE = 1120 + +SHARED_FIELDS = { "mode": "image_generation", -} - -EXPECTED_CAPABILITIES = { - "max_output_tokens": 4096, - "max_tokens": 4096, + "input_cost_per_token": INPUT_COST, + "input_cost_per_token_batches": INPUT_COST_BATCHES, + "input_cost_per_image": INPUT_COST_PER_IMAGE, + "output_cost_per_token": OUTPUT_TEXT_COST, + "output_cost_per_token_batches": OUTPUT_TEXT_COST_BATCHES, + "output_cost_per_image": OUTPUT_COST_PER_1K_IMAGE, + "output_cost_per_image_token": OUTPUT_IMAGE_TOKEN_COST, + "max_input_tokens": MAX_INPUT_TOKENS, + "max_output_tokens": MAX_OUTPUT_TOKENS, + "max_tokens": MAX_OUTPUT_TOKENS, + "supported_endpoints": ["/v1/chat/completions", "/v1/completions", "/v1/batch"], + "supported_output_modalities": ["text", "image"], + "supports_reasoning": False, "supports_response_schema": False, - "supports_reasoning": True, + "supports_system_messages": True, + "supports_vision": True, } -EXPECTED_PER_ROUTE = { - "gemini-3.1-flash-lite-image": { - "supports_prompt_caching": True, - "supports_function_calling": False, - }, - "vertex_ai/gemini-3.1-flash-lite-image": { - "supports_prompt_caching": True, - "supports_function_calling": False, - }, - "gemini/gemini-3.1-flash-lite-image": { - "supports_prompt_caching": False, +VERTEX_ROUTE_FIELDS = { + "litellm_provider": "vertex_ai-language-models", + "cache_read_input_token_cost": CACHE_READ_COST, + "supported_modalities": ["text", "image", "video"], + "supports_function_calling": False, + "supports_pdf_input": True, + "supports_prompt_caching": True, + "supports_video_input": True, +} + +PER_ROUTE_FIELDS = { + UNPREFIXED: VERTEX_ROUTE_FIELDS, + VERTEX: VERTEX_ROUTE_FIELDS, + GEMINI: { + "litellm_provider": "gemini", + "supported_modalities": ["text", "image"], "supports_function_calling": True, - "input_cost_per_token_batches": 1.25e-07, - "output_cost_per_token_batches": 7.5e-07, + "supports_prompt_caching": False, + "rpm": 1000, + "tpm": 4000000, }, } +GROUNDING_FIELDS = ( + "supports_web_search", + "search_context_cost_per_query", + "web_search_billing_unit", +) -def _load_json(path: str) -> dict: + +def _load(path: Path) -> dict: with open(path, encoding="utf-8") as f: return json.load(f) -def _backup_path() -> str: - return os.path.join( - os.path.dirname(litellm.__file__), - "model_prices_and_context_window_backup.json", +@pytest.fixture +def local_model_cost_map(monkeypatch): + original_model_cost = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + try: + yield + finally: + litellm.model_cost = original_model_cost + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize("model", ALL_KEYS) +@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup")) +def test_published_prices_are_registered(model: str, path: Path): + info = _load(path).get(model) + assert info is not None, f"{model} missing from {path.name}" + for field, value in SHARED_FIELDS.items(): + assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}" + + +@pytest.mark.parametrize("model", ALL_KEYS) +@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup")) +def test_per_route_capabilities_match_model_cards(model: str, path: Path): + info = _load(path)[model] + for field, value in PER_ROUTE_FIELDS[model].items(): + assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}" + + +@pytest.mark.parametrize("model", ALL_KEYS) +@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup")) +def test_grounding_fields_absent(model: str, path: Path): + info = _load(path)[model] + for field in GROUNDING_FIELDS: + assert field not in info, f"{model} should not define {field}" + + +@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup")) +def test_ai_studio_route_has_no_implicit_cache_price(path: Path): + assert "cache_read_input_token_cost" not in _load(path)[GEMINI] + + +@pytest.mark.parametrize("model", ALL_KEYS) +def test_backup_matches_main(model: str): + assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model) + + +def test_one_k_image_price_matches_official_token_math(): + assert TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST == pytest.approx(OUTPUT_COST_PER_1K_IMAGE) + assert TOKENS_PER_1K_IMAGE * INPUT_COST == pytest.approx(INPUT_COST_PER_IMAGE) + + +def test_gemini_prefix_routes_to_gemini(): + routed_model, provider, _, _ = get_llm_provider(model=GEMINI) + assert routed_model == UNPREFIXED + assert provider == "gemini" + + +def test_vertex_prefix_routes_to_vertex(): + routed_model, provider, _, _ = get_llm_provider(model=VERTEX) + assert routed_model == UNPREFIXED + assert provider == "vertex_ai" + + +def test_get_model_info_reports_published_costs(local_model_cost_map): + info = litellm.get_model_info(UNPREFIXED) + assert info["input_cost_per_token"] == INPUT_COST + assert info["output_cost_per_token"] == OUTPUT_TEXT_COST + assert info["cache_read_input_token_cost"] == CACHE_READ_COST + + +@pytest.mark.parametrize("model", ALL_KEYS) +def test_reasoning_params_are_not_offered_on_an_image_endpoint(model: str, local_model_cost_map): + assert litellm.supports_reasoning(model) is False + + +def test_text_token_cost(local_model_cost_map): + prompt_cost, text_completion_cost = cost_per_token( + model=GEMINI, prompt_tokens=1000, completion_tokens=500 + ) + assert prompt_cost == pytest.approx(1000 * INPUT_COST) + assert text_completion_cost == pytest.approx(500 * OUTPUT_TEXT_COST) + + +def test_completion_cost_bills_one_k_image(local_model_cost_map): + response = ModelResponse() + response.model = UNPREFIXED + response.usage = Usage( + prompt_tokens=7, + completion_tokens=TOKENS_PER_1K_IMAGE, + total_tokens=7 + TOKENS_PER_1K_IMAGE, + completion_tokens_details=CompletionTokensDetailsWrapper( + image_tokens=TOKENS_PER_1K_IMAGE, text_tokens=0 + ), + ) + billed = completion_cost( + completion_response=response, + model=UNPREFIXED, + custom_llm_provider="vertex_ai", + ) + expected = TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST + 7 * INPUT_COST + assert billed == pytest.approx(expected) + + +def test_image_tokens_are_not_billed_as_text(local_model_cost_map): + usage = Usage( + completion_tokens=1345, + prompt_tokens=10, + total_tokens=1355, + completion_tokens_details=CompletionTokensDetailsWrapper( + accepted_prediction_tokens=None, + audio_tokens=None, + reasoning_tokens=225, + rejected_prediction_tokens=None, + text_tokens=0, + image_tokens=TOKENS_PER_1K_IMAGE, + ), + prompt_tokens_details=PromptTokensDetailsWrapper( + audio_tokens=None, cached_tokens=None, text_tokens=10, image_tokens=None + ), + ) + + _, image_completion_cost = generic_cost_per_token( + model=UNPREFIXED, + usage=usage, + custom_llm_provider="vertex_ai", + ) + + expected_completion_cost = ( + TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST + 225 * OUTPUT_TEXT_COST + ) + bugged_text_only_cost = 1345 * OUTPUT_TEXT_COST + assert image_completion_cost > bugged_text_only_cost * 2 + assert image_completion_cost == pytest.approx(expected_completion_cost) + + +def _one_k_image_response() -> ImageResponse: + return ImageResponse( + data=[ImageObject(b64_json="img1")], + usage=ImageUsage( + input_tokens=50 + TOKENS_PER_1K_IMAGE, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=50, + image_tokens=TOKENS_PER_1K_IMAGE, + ), + output_tokens=TOKENS_PER_1K_IMAGE, + total_tokens=50 + TOKENS_PER_1K_IMAGE + TOKENS_PER_1K_IMAGE, + ), ) -def _main_path() -> str: - return os.path.join( - os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json" +def test_gemini_image_generation_uses_token_pricing(local_model_cost_map): + cost = gemini_image_generation_cost_calculator( + model=GEMINI, image_response=_one_k_image_response() ) + expected = ( + 50 + TOKENS_PER_1K_IMAGE + ) * INPUT_COST + TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST + assert cost == pytest.approx(expected) + assert cost != OUTPUT_COST_PER_1K_IMAGE -class TestGeminiFlashLiteImagePricingData: - """Both price maps must carry Google's published Nano Banana 2 Lite costs.""" - - def test_present_in_both_maps(self): - main = _load_json(_main_path()) - backup = _load_json(_backup_path()) - for key in VARIANTS: - for label, data in (("main", main), ("backup", backup)): - assert key in data, f"{key} missing from {label} JSON" - entry = data[key] - for field, value in EXPECTED.items(): - assert entry[field] == value, f"{key} {field} in {label}: {entry.get(field)} != {value}" - - def test_capabilities_match_model_cards(self): - main = _load_json(_main_path()) - backup = _load_json(_backup_path()) - for key in VARIANTS: - expected = {**EXPECTED_CAPABILITIES, **EXPECTED_PER_ROUTE[key]} - for label, data in (("main", main), ("backup", backup)): - entry = data[key] - for field, value in expected.items(): - assert entry[field] == value, f"{key} {field} in {label}: {entry.get(field)} != {value}" - - def test_grounding_fields_absent(self): - """Grounding with Google Search is unsupported on Lite, so no search pricing.""" - for path in (_main_path(), _backup_path()): - data = _load_json(path) - for key in VARIANTS: - for field in ( - "supports_web_search", - "search_context_cost_per_query", - "web_search_billing_unit", - ): - assert field not in data[key], f"{key} should not define {field}" - - def test_image_output_pricing_consistent(self): - """1120 image-output tokens * output_cost_per_image_token == output_cost_per_image.""" - backup = _load_json(_backup_path()) - entry = backup["gemini-3.1-flash-lite-image"] - assert round(1120 * entry["output_cost_per_image_token"], 6) == entry["output_cost_per_image"] +def test_vertex_image_generation_uses_token_pricing(local_model_cost_map): + cost = vertex_image_generation_cost_calculator( + model=UNPREFIXED, image_response=_one_k_image_response() + ) + expected = ( + 50 + TOKENS_PER_1K_IMAGE + ) * INPUT_COST + TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST + assert cost == pytest.approx(expected) -class TestGeminiFlashLiteImageModelInfo: - """``get_model_info`` and ``completion_cost`` must report the new costs.""" - - def test_get_model_info_and_cost(self): - original = litellm.model_cost - try: - litellm.model_cost = _load_json(_backup_path()) - info = litellm.get_model_info("gemini-3.1-flash-lite-image") - assert info["input_cost_per_token"] == EXPECTED["input_cost_per_token"] - assert info["output_cost_per_token"] == EXPECTED["output_cost_per_token"] - - resp = ModelResponse() - resp.model = "gemini-3.1-flash-lite-image" - resp.usage = Usage( - prompt_tokens=7, - completion_tokens=1120, - total_tokens=1127, - completion_tokens_details=CompletionTokensDetailsWrapper( - image_tokens=1120, text_tokens=0 - ), - ) - cost = completion_cost( - completion_response=resp, - model="gemini-3.1-flash-lite-image", - custom_llm_provider="vertex_ai", - ) - expected_cost = 1120 * 3e-05 + 7 * 2.5e-07 - assert abs(cost - expected_cost) < 1e-6, f"unexpected cost {cost}" - finally: - litellm.model_cost = original +def test_vertex_image_generation_falls_back_to_flat_image_price(local_model_cost_map): + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] + ) + cost = vertex_image_generation_cost_calculator( + model=UNPREFIXED, image_response=image_response + ) + assert cost == pytest.approx(2 * OUTPUT_COST_PER_1K_IMAGE) diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index cb7023e6c12..6114d1d8aba 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -11,6 +11,7 @@ import pytest REPO_ROOT = Path(__file__).parents[2] GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py" PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json" @@ -118,6 +119,31 @@ def test_schema_accepts_cache_creation_cost_inside_a_pricing_tier(committed_sche assert validator.is_valid({"some-model": entry}) +def find_duplicate_keys(path: Path) -> list[str]: + duplicates: list[str] = [] + + def record_duplicates(pairs): + seen: set[str] = set() + for key, _ in pairs: + if key in seen: + duplicates.append(key) + seen.add(key) + return dict(pairs) + + json.loads(path.read_text(), object_pairs_hook=record_duplicates) + return duplicates + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_price_map_has_no_duplicate_keys(path: Path): + assert find_duplicate_keys(path) == [], ( + f"{path.name} defines the same key twice; JSON parsers keep only the last " + "occurrence, so the earlier entry's fields are silently dropped. This is what " + "a clean text merge of two branches that both added a model looks like: " + "deduplicate the keys into one entry" + ) + + DATED_VARIANT = re.compile(r"^(.*?)-(\d{4}-\d{2}-\d{2})$") SERVICE_TIER_SUFFIXES = ("_flex", "_priority") diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 43100b10aeb..bd23ca11fbe 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4391,11 +4391,13 @@ class TestVertexEmbeddingEncodingFormat: "vertex_ai/gemini-3-pro-image-preview", "vertex_ai/gemini-3.1-flash-image", "vertex_ai/gemini-3.1-flash-image-preview", + "vertex_ai/gemini-3.1-flash-lite-image", "gemini/gemini-2.5-flash-image", "gemini/gemini-3-pro-image", "gemini/gemini-3-pro-image-preview", "gemini/gemini-3.1-flash-image", "gemini/gemini-3.1-flash-image-preview", + "gemini/gemini-3.1-flash-lite-image", ], ) def test_gemini_image_models_do_not_support_reasoning(