scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-19 00:01:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 88

Merge pull request #41763 from BerriAI/litellm_drop_cost_map_pinning_tests

Browse source
This commit is contained in:
kerry-berri 2026-09-17 22:11:32 -07:00 • committed by GitHub
commit e8d30efe99
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
67 changed files with 16 additions and 2595 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/test-unit.yml vendored
View file

@ -213,7 +213,6 @@ jobs:
test-path: >-
tests/local_testing/test_cache_preset_key.py
tests/local_testing/test_caching_handler.py
tests/local_testing/test_prompt_caching.py
tests/local_testing/test_responses_stream_cache_keys.py
tests/local_testing/test_unit_test_caching.py
workers: 2

180
tests/litellm_utils_tests/test_utils.py
View file

@ -22,11 +22,7 @@ from litellm.litellm_core_utils.duration_parser import (
)
from litellm.utils import (
check_valid_key,
create_pretrained_tokenizer,
create_tokenizer,
function_to_dict,
get_llm_provider,
get_max_tokens,
get_supported_openai_params,
get_token_count,
get_valid_models,
@ -500,74 +496,6 @@ def test_function_to_dict():
# test_function_to_dict()
@pytest.mark.parametrize(
"model, expected_bool",
[
("gpt-3.5-turbo", True),
("azure/gpt-4-1106-preview", True),
("groq/gemma-7b-it", True),
("gemini/gemini-2.5-flash", True),
],
)
def test_supports_function_calling(model, expected_bool):
try:
assert litellm.supports_function_calling(model=model) == expected_bool
except Exception as e:
pytest.fail(f"Error occurred: {e}")
@pytest.mark.parametrize(
"model, expected_bool",
[
("gpt-4o-mini-search-preview", True),
("openai/gpt-4o-mini-search-preview", True),
("gpt-4o-search-preview", True),
("openai/gpt-4o-search-preview", True),
("groq/deepseek-r1-distill-llama-70b", False),
("groq/llama-3.3-70b-versatile", False),
("codestral/codestral-latest", False),
],
)
def test_supports_web_search(model, expected_bool):
try:
assert litellm.supports_web_search(model=model) == expected_bool
except Exception as e:
pytest.fail(f"Error occurred: {e}")
@pytest.mark.parametrize(
"model, expected_bool",
[
("openai/o3-mini", True),
("o3-mini", True),
("xai/grok-3-mini-beta", True),
("xai/grok-3-mini-fast-beta", True),
("xai/grok-2", False),
("gpt-3.5-turbo", False),
],
)
def test_supports_reasoning(model, expected_bool):
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
try:
assert litellm.supports_reasoning(model=model) == expected_bool
except Exception as e:
pytest.fail(f"Error occurred: {e}")
def test_get_max_token_unit_test():
"""
More complete testing in `test_completion_cost.py`
"""
model = "bedrock/anthropic.claude-3-haiku-20240307-v1:0"
max_tokens = get_max_tokens(
model
) # Returns a number instead of throwing an Exception
assert isinstance(max_tokens, int)
def test_get_supported_openai_params() -> None:
# Mapped provider
assert isinstance(get_supported_openai_params("gpt-4"), list)
@ -1041,73 +969,6 @@ def test_parse_content_for_reasoning(content, expected_reasoning, expected_conte
)
@pytest.mark.parametrize(
"model, expected_bool",
[
("vertex_ai/gemini-2.5-pro", True),
("gemini/gemini-2.5-pro", True),
("predibase/llama3-8b-instruct", True),
("databricks/databricks-meta-llama-3-1-70b-instruct", True),
("gpt-3.5-turbo", False),
("groq/llama-3.3-70b-versatile", False),
],
)
def test_supports_response_schema(model, expected_bool):
"""
Unit tests for 'supports_response_schema' helper function.
Should be true for gemini-2.5-pro on google ai studio / vertex ai AND predibase models
Should be false otherwise
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
from litellm.utils import supports_response_schema
response = supports_response_schema(model=model, custom_llm_provider=None)
assert expected_bool == response
@pytest.mark.parametrize(
"model, expected_bool",
[
("gpt-3.5-turbo", True),
("gpt-4", True),
("command-nightly", False),
("gemini-2.5-pro", True),
],
)
def test_supports_function_calling_v2(model, expected_bool):
"""
Unit test for 'supports_function_calling' helper function.
"""
from litellm.utils import supports_function_calling
response = supports_function_calling(model=model, custom_llm_provider=None)
assert expected_bool == response
@pytest.mark.parametrize(
"model, expected_bool",
[
("gpt-4o", True),
("gpt-3.5-turbo", False),
("claude-sonnet-4-6", True),
("gemini-2.5-flash", True),
("command-nightly", False),
],
)
def test_supports_vision(model, expected_bool):
"""
Unit test for 'supports_vision' helper function.
"""
from litellm.utils import supports_vision
response = supports_vision(model=model, custom_llm_provider=None)
assert expected_bool == response
def test_usage_object_null_tokens():
"""
Unit test.
@ -1146,7 +1007,6 @@ def test_is_base64_encoded():
clear=True,
)
def test_async_http_handler(mock_async_client):
import httpx
import ssl
timeout = 120
@ -1221,20 +1081,6 @@ def test_async_http_handler_force_ipv4(mock_async_client):
litellm.force_ipv4 = False
@pytest.mark.parametrize(
"model, expected_bool", [("gpt-3.5-turbo", False), ("gpt-4o-audio-preview", True)]
)
def test_supports_audio_input(model, expected_bool):
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
from litellm.utils import supports_audio_input, supports_audio_output
supports_pc = supports_audio_input(model=model)
assert supports_pc == expected_bool
def test_is_base64_encoded_2():
from litellm.utils import is_base64_encoded
@ -1569,23 +1415,6 @@ def test_token_counter_with_image_url_with_detail_high():
assert _tokens == DEFAULT_IMAGE_TOKEN_COUNT + 7
def test_fireworks_ai_vision_capability_from_cost_map(monkeypatch):
"""
Fireworks deprecated document inlining on 2025-06-30, so vision/PDF support is
no longer hardcoded to True for every Fireworks model. Capabilities are read
from the model cost map: unmapped models no longer advertise vision or PDF
support, while mapped VLMs still do.
"""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
from litellm.utils import supports_pdf_input, supports_vision
assert supports_vision("fireworks_ai/llama-3.1-8b-instruct") is False
assert supports_pdf_input("fireworks_ai/llama-3.1-8b-instruct") is False
assert supports_vision("fireworks_ai/minimax-m3") is True
def test_logprobs_type():
from litellm.types.utils import Logprobs
@ -1728,21 +1557,12 @@ def test_get_valid_models_default(monkeypatch):
Prevent regression for existing usage.
"""
from litellm.utils import get_valid_models
import litellm
monkeypatch.setenv("FIREWORKS_API_KEY", "sk-1234")
valid_models = get_valid_models()
assert len(valid_models) > 0
def test_supports_vision_gemini():
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
from litellm.utils import supports_vision
assert supports_vision("gemini-2.5-pro") is True
def test_pick_cheapest_chat_model_from_llm_provider():
from litellm.litellm_core_utils.llm_request_utils import (
pick_cheapest_chat_models_from_llm_provider,

7
tests/llm_translation/test_azure_o_series.py
View file

@ -1,15 +1,12 @@
import json
import os
from datetime import datetime
from unittest.mock import AsyncMock, patch, MagicMock
from unittest.mock import patch
import httpx
import pytest
import litellm
from litellm import Choices, Message, ModelResponse
from litellm import ModelResponse
from base_llm_unit_tests import BaseLLMChatTest, BaseOSeriesModelsTest

32
tests/llm_translation/test_lambda_ai.py
View file

@ -102,35 +102,3 @@ async def test_lambda_ai_completion_call():
raise
def test_lambda_ai_model_list_populated():
"""Test that lambda_ai_models list is populated correctly"""
# Ensure we're using local model cost map and repopulate models
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
# Clear and repopulate all model lists after reloading model_cost
litellm.lambda_ai_models = set()
litellm.add_known_models()
# This should be populated by the add_known_models function
assert (
len(litellm.lambda_ai_models) > 0
), "lambda_ai_models list should not be empty"
# Check that all models in the list are Lambda AI models
for model in litellm.lambda_ai_models:
assert model.startswith(
"lambda_ai/"
), f"Model {model} should start with 'lambda_ai/'"
# Check some expected models are in the list
expected_models = [
"lambda_ai/llama3.1-8b-instruct",
"lambda_ai/hermes3-405b",
"lambda_ai/deepseek-v3-0324",
]
for model in expected_models:
assert (
model in litellm.lambda_ai_models
), f"{model} should be in lambda_ai_models list"

45
tests/llm_translation/test_perplexity_reasoning.py
View file

@ -1,4 +1,3 @@
import json
import os
from unittest.mock import patch, MagicMock
@ -136,50 +135,6 @@ class TestPerplexityReasoning:
== "This is a test response from the reasoning model."
)
def test_perplexity_reasoning_models_support_reasoning(self):
"""
Test that Perplexity Sonar reasoning models are correctly identified as supporting reasoning
"""
from litellm.utils import supports_reasoning
# Set up local model cost map
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
reasoning_models = [
"perplexity/sonar-reasoning",
"perplexity/sonar-reasoning-pro",
]
for model in reasoning_models:
assert supports_reasoning(model, None), f"{model} should support reasoning"
def test_perplexity_non_reasoning_models_dont_support_reasoning(self):
"""
Test that non-reasoning Perplexity models don't support reasoning
"""
from litellm.utils import supports_reasoning
# Set up local model cost map
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
non_reasoning_models = [
"perplexity/sonar",
"perplexity/sonar-pro",
"perplexity/llama-3.1-sonar-large-128k-chat",
"perplexity/mistral-7b-instruct",
]
for model in non_reasoning_models:
# These models should not support reasoning (should return False or raise exception)
try:
result = supports_reasoning(model, None)
# If it doesn't raise an exception, it should return False
assert result is False, f"{model} should not support reasoning"
except Exception:
# If it raises an exception, that's also acceptable behavior
pass
@pytest.mark.parametrize(
"model,expected_api_base",

27
tests/local_testing/test_completion_cost.py
View file

@ -6,8 +6,7 @@ import litellm.cost_calculator
import asyncio
import time
from typing import Optional
from unittest.mock import AsyncMock, MagicMock, patch
import base64
from unittest.mock import MagicMock, patch
import pytest
import litellm
@ -15,9 +14,7 @@ from litellm import (
TranscriptionResponse,
completion_cost,
cost_per_token,
get_max_tokens,
model_cost,
open_ai_chat_completion_models,
)
from litellm.llms.custom_httpx.http_handler import HTTPHandler
import json
@ -162,12 +159,6 @@ def test_custom_pricing_as_completion_cost_param():
# test_get_palm_tokens()
def test_zephyr_hf_tokens():
max_tokens = get_max_tokens("huggingface/HuggingFaceH4/zephyr-7b-beta")
print(max_tokens)
assert max_tokens == 32768
# test_zephyr_hf_tokens()
@ -426,10 +417,8 @@ def test_groq_response_cost_tracking(is_streaming):
from litellm.utils import (
CallTypes,
Choices,
Delta,
Message,
ModelResponse,
StreamingChoices,
Usage,
)
@ -548,12 +537,6 @@ def test_gemini_completion_cost(provider):
assert calculated_output_cost == output_cost
def _count_characters(text):
# Remove white spaces and count characters
filtered_text = "".join(char for char in text if not char.isspace())
return len(filtered_text)
def test_vertex_ai_completion_cost():
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
@ -817,10 +800,8 @@ def test_completion_cost_azure_common_deployment_name():
from litellm.utils import (
CallTypes,
Choices,
Delta,
Message,
ModelResponse,
StreamingChoices,
Usage,
)
@ -1252,7 +1233,7 @@ def test_cost_openai_prompt_caching():
],
)
def test_completion_cost_azure_ai_rerank(model):
from litellm import RerankResponse, rerank
from litellm import RerankResponse
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
@ -1283,7 +1264,7 @@ def test_completion_cost_azure_ai_rerank(model):
def test_together_ai_embedding_completion_cost():
from litellm.utils import Choices, EmbeddingResponse, Message, ModelResponse, Usage
from litellm.utils import EmbeddingResponse, Usage
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
@ -2222,7 +2203,6 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream):
ModelResponse,
Usage,
ChatCompletionAudioResponse,
PromptTokensDetails,
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
)
@ -2464,7 +2444,6 @@ def test_add_known_models():
@pytest.mark.skip(reason="flaky test")
def test_bedrock_cost_calc_with_region():
from litellm import completion
from litellm import ModelResponse

27
tests/local_testing/test_get_model_info.py
View file

@ -47,12 +47,6 @@ def test_get_model_info_custom_llm_with_same_name_vllm(monkeypatch):
assert model_info["input_cost_per_token"] == 0.0
def test_get_model_info_gemini_pro():
info = litellm.get_model_info("gemini-2.0-flash")
print("info", info)
assert info["key"] == "gemini-2.0-flash"
def test_get_model_info_ollama_chat():
from litellm.llms.ollama.completion.transformation import OllamaConfig
@ -354,27 +348,6 @@ def test_get_model_info_huggingface_models(monkeypatch):
)
@pytest.mark.parametrize(
"model, provider",
[
("bedrock/us-east-2/us.anthropic.claude-3-haiku-20240307-v1:0", None),
(
"bedrock/us-east-2/us.anthropic.claude-3-haiku-20240307-v1:0",
"bedrock",
),
],
)
def test_get_model_info_cost_calculator_bedrock_region_cris_stripped(model, provider):
"""
ensure cross region inferencing model is used correctly
Relevant Issue: https://github.com/BerriAI/litellm/issues/8115
"""
info = get_model_info(model=model, custom_llm_provider=provider)
print("info", info)
assert info["key"] == "us.anthropic.claude-3-haiku-20240307-v1:0"
assert info["litellm_provider"] == "bedrock"
def test_get_model_info_case_insensitive_lookup(monkeypatch):
"""
Test that model info lookup is case-insensitive.

43
tests/local_testing/test_prompt_caching.py
View file

@ -1,43 +0,0 @@
"""Asserts that prompt caching information is correctly returned for Anthropic, OpenAI, and Deepseek"""
import io
import litellm
import pytest
def _usage_format_tests(usage: litellm.Usage):
"""
OpenAI prompt caching
- prompt_tokens = sum of non-cache hit tokens + cache-hit tokens
- total_tokens = prompt_tokens + completion_tokens
Example
```
"usage": {
"prompt_tokens": 2006,
"completion_tokens": 300,
"total_tokens": 2306,
"prompt_tokens_details": {
"cached_tokens": 1920
},
"completion_tokens_details": {
"reasoning_tokens": 0
}
# ANTHROPIC_ONLY #
"cache_creation_input_tokens": 0
}
```
"""
assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens
assert usage.prompt_tokens > usage.prompt_tokens_details.cached_tokens
def test_supports_prompt_caching():
from litellm.utils import supports_prompt_caching
supports_pc = supports_prompt_caching(model="anthropic/claude-sonnet-4-5-20250929")
assert supports_pc

12
tests/local_testing/test_register_model.py
View file

@ -2,8 +2,6 @@
# This tests calling batch_completions by running 100 messages together
import ast
import sys, os
import traceback
from pathlib import Path
import pytest
@ -32,16 +30,6 @@ def test_update_model_cost():
# test_update_model_cost()
def test_update_model_cost_map_url():
try:
litellm.register_model(
model_cost="https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
)
assert litellm.model_cost["gpt-4"]["input_cost_per_token"] == 0.00003
except Exception as e:
pytest.fail(f"An error occurred: {e}")
# test_update_model_cost_map_url()

15
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_fixture_models.py
View file

@ -3,7 +3,6 @@ from __future__ import annotations
import base64
from collections.abc import Callable
from datetime import date
from pathlib import Path
from typing import Final, cast
from unittest.mock import patch
from urllib.parse import parse_qs, urlparse
@ -262,20 +261,6 @@ def _reducto_document() -> ReductoDocumentUrlDocument:
)
def test_fixture_catalogs_match_active_registered_ocr_models() -> None:
registry_path: Final = Path(__file__).resolve().parents[6] / "model_prices_and_context_window.json"
registry: Final = MODEL_REGISTRY.validate_json(registry_path.read_text(encoding="utf-8"))
active_registered: Final = frozenset(
model
for model, raw_metadata in registry.items()
if raw_metadata.get("mode") == "ocr" and raw_metadata.get("litellm_provider") in SUPPORTED_OCR_PROVIDERS
for metadata in (_ModelRegistryEntry.model_validate(raw_metadata),)
if metadata.deprecation_date is None or metadata.deprecation_date > date.today()
)
assert ACTIVE_OCR_MODELS == active_registered
@pytest.mark.parametrize(
("fixture_model", "provider_config", "model"),
(

21
tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
View file

@ -1,15 +1,12 @@
import copy
import datetime
import json
import os
import subprocess
import sys
import textwrap
import unittest
from typing import List, Optional, Tuple
from unittest.mock import ANY, MagicMock, Mock, patch
from unittest.mock import MagicMock, patch
import httpx
import pytest
import litellm
@ -19,7 +16,6 @@ from litellm.integrations.anthropic_cache_control_hook import (
)
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import StandardCallbackDynamicParams
@pytest.fixture(autouse=True)
@ -2984,18 +2980,6 @@ class TestPromptCacheBreakpointCapability:
yield
litellm.utils._cached_get_model_info_helper.cache_clear()
def test_public_helper_reads_the_model_map(self):
from litellm.utils import supports_prompt_cache_breakpoint
assert supports_prompt_cache_breakpoint("gpt-5.6") is True
assert supports_prompt_cache_breakpoint("openai/gpt-5.6-sol") is True
assert supports_prompt_cache_breakpoint("gpt-5.6", custom_llm_provider="openai") is True
assert supports_prompt_cache_breakpoint("gpt-4.1") is False
@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
def test_model_map_flags_every_openai_gpt_5_6_entry(self, model):
assert litellm.model_cost[model]["litellm_provider"] == "openai"
assert litellm.model_cost[model]["supports_prompt_cache_breakpoint"] is True
def test_listed_model_uses_the_model_map_flag(self, monkeypatch):
flagged = {**litellm.model_cost["gpt-4.1"], "supports_prompt_cache_breakpoint": True}
@ -3014,9 +2998,6 @@ class TestPromptCacheBreakpointCapability:
)
assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is False
def test_listed_gpt_model_without_the_flag_follows_the_version_rule(self):
assert "supports_prompt_cache_breakpoint" not in litellm.model_cost["gpt-4.1"]
assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is False
def test_published_map_without_the_flag_still_injects_on_gpt_5_6(self, monkeypatch):
unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"}

17
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py
View file

@ -1,4 +1,3 @@
import os
import pytest
@ -121,22 +120,6 @@ def test_billed_guardrail_cost_by_unit_treats_none_in_spend_as_billed():
assert billed_guardrail_cost_by_unit(entry) == {"contentPolicyUnits": 0.15}
def test_shipped_bedrock_guardrail_prices_match_aws_pricing_page(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
assert litellm.model_cost["bedrock/guardrails"]["guardrail_cost_per_unit"] == {
"automatedReasoningPolicyUnits": 0.00017,
"contentPolicyImageUnits": 0.00075,
"contentPolicyUnits": 0.00015,
"contextualGroundingPolicyUnits": 0.0001,
"sensitiveInformationPolicyFreeUnits": 0.0,
"sensitiveInformationPolicyUnits": 0.0001,
"topicPolicyUnits": 0.00015,
"wordPolicyUnits": 0.0,
}
assert "bedrock/guardrails" not in litellm.bedrock_models
def test_guardrail_information_cost_sums_entries():
entries = [
{"guardrail_name": "a", "guardrail_cost": 0.0003},

84
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -1575,59 +1575,6 @@ def test_generic_cost_per_token_tiered_pricing_bills_reasoning_at_tier_rate():
litellm.model_cost.pop(model, None)
def test_gpt_5_6_alias_prices_match_sol(local_model_cost_map):
"""Regression: the bare gpt-5.6 alias routes to GPT-5.6 Sol, so every cost field on
the two entries has to hold the same value. They drifted once before, when Sol took
its promotional cut and gpt-5.6 was left on the pre-cut rates, overbilling callers
who used the alias."""
alias = litellm.model_cost["gpt-5.6"]
sol = litellm.model_cost["gpt-5.6-sol"]
cost_fields = sorted(field for field in sol if "cost" in field)
assert len(cost_fields) == 27
for field in cost_fields:
assert alias.get(field) == sol.get(field), field
@pytest.mark.parametrize(
"model,expected_none,expected_xhigh,expected_minimal",
[
# Verified against OpenAI's live API on 2026-04-24:
# gpt-5.5 -> supports: none, low, medium, high, xhigh
# gpt-5.5-pro -> supports: medium, high, xhigh
# Neither supports "minimal"; gpt-5.5-pro additionally does not support "none".
# The JSON must reflect this so LiteLLM rejects unsupported values locally
# (or drops them with drop_params=True) instead of round-tripping to OpenAI
# for a 400.
("gpt-5.5", True, True, False),
("gpt-5.5-2026-04-23", True, True, False),
("gpt-5.5-pro", False, True, False),
("gpt-5.5-pro-2026-04-23", False, True, False),
],
)
def test_gpt55_reasoning_effort_flags_match_live_openai_api(
_local_model_cost_map, model, expected_none, expected_xhigh, expected_minimal
):
"""Pin reasoning_effort capability flags to OpenAI's actual API contract.
Observed via `POST /v1/chat/completions` with reasoning_effort=minimal:
``Unsupported value: 'reasoning_effort' does not support 'minimal' with
this model``. gpt-5.5-pro additionally rejects 'none' and 'low'.
"""
m = litellm.model_cost[model]
assert m.get("supports_none_reasoning_effort") is expected_none, (
f"{model}: supports_none_reasoning_effort expected {expected_none}"
)
assert m.get("supports_xhigh_reasoning_effort") is expected_xhigh, (
f"{model}: supports_xhigh_reasoning_effort expected {expected_xhigh}"
)
assert m.get("supports_minimal_reasoning_effort") is expected_minimal, (
f"{model}: supports_minimal_reasoning_effort expected {expected_minimal}"
)
@pytest.mark.parametrize(
"base_model,dated_model",
[
@ -1662,29 +1609,6 @@ def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities(_local_mo
)
@pytest.mark.parametrize(
"model,expected_none,expected_minimal,expected_xhigh",
[
# Mirror live OpenAI API contract (verified via openai/gpt-5.5* on
# 2026-04-24): chat accepts {none, low, medium, high, xhigh} but NOT
# minimal; pro accepts {medium, high, xhigh} only.
# NOTE: openai/gpt-5.5* entries currently set supports_minimal=true on
# main (pre #26456). Once that PR lands, OpenAI + Azure flags align.
("azure/gpt-5.5", True, False, True),
("azure/gpt-5.5-pro", False, False, True),
],
)
def test_azure_gpt55_reasoning_effort_flags_match_live_openai_api(
_local_model_cost_map, model, expected_none, expected_minimal, expected_xhigh
):
"""Azure entries pin reasoning_effort flags to OpenAI's actual API contract."""
m = litellm.model_cost[model]
assert m.get("supports_none_reasoning_effort") is expected_none
assert m.get("supports_minimal_reasoning_effort") is expected_minimal
assert m.get("supports_xhigh_reasoning_effort") is expected_xhigh
def test_string_cost_values():
"""Test that cost values defined as strings are properly converted to floats."""
from unittest.mock import patch
@ -3413,14 +3337,6 @@ GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH = (
)
@pytest.mark.parametrize("prefix", ["", "gemini/", "vertex_ai/"])
def test_gemini_38_flash_matches_37_flash_promotional_pricing(prefix, _local_model_cost_map):
new_model = litellm.model_cost[f"{prefix}gemini-3.8-flash"]
old_model = litellm.model_cost[f"{prefix}gemini-3.7-flash"]
for field in GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH:
assert new_model[field] == old_model[field], field
@pytest.mark.parametrize(
("response_quality", "requested_quality", "expected_cost"),
[

35
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py
View file

@ -1,4 +1,3 @@
from collections.abc import Mapping, Sequence
import pytest
@ -527,7 +526,6 @@ def _openai_responses_with_web_search_calls(model, num_calls):
ResponseFunctionWebSearch,
)
from litellm.types.llms.openai import ResponsesAPIResponse
output = [
ResponseFunctionWebSearch(
@ -585,7 +583,6 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map):
counter must read their "type" key like the detection gate does, instead of flooring
a multi-search response to a single billable search.
"""
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import Usage
model = "gpt-4o-search-preview"
@ -631,7 +628,6 @@ def test_response_includes_output_type_reads_dict_output_items():
items without an "action" field) stay plain dicts in the output union. The gate must
read their "type" key instead of returning False and skipping the web search fee.
"""
from litellm.types.llms.openai import ResponsesAPIResponse
response = ResponsesAPIResponse.model_validate(
{
@ -699,34 +695,3 @@ _BEDROCK_MANTLE_WEB_SEARCH_MODELS = (
_BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012
def _responses_with_web_search(
model: str, actions: Sequence[Mapping[str, str]], tool_usage: Mapping[str, object] | None = None
) -> ResponsesAPIResponse:
payload = {
"id": "resp_1",
"created_at": 1756900000,
"model": model.split("/", 1)[-1],
"object": "response",
"status": "completed",
"output": [
{"type": "web_search_call", "id": f"ws_{i}", "status": "completed", "action": action}
for i, action in enumerate(actions)
],
}
return ResponsesAPIResponse.model_validate(
payload if tool_usage is None else {**payload, "tool_usage": tool_usage}
)
def _web_search_cost(model: str, response: ResponsesAPIResponse, custom_llm_provider: str) -> float:
from litellm.types.utils import Usage
return StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=model,
response_object=response,
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=None,
)

74
tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py
View file

@ -14,7 +14,6 @@ rather than forwarded as a no-op the provider can reject. See BerriAI/litellm#33
import pytest
from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt
from litellm.llms.bedrock.common_utils import bedrock_converse_supports_strict_tools
_STRICT_TOOL = [
{
@ -163,76 +162,3 @@ def test_bedrock_tools_pt_strict_dropped_for_non_anthropic(model_id: str) -> Non
assert "strict" not in result[0]["toolSpec"]
def test_bedrock_converse_supports_strict_tools_helper() -> None:
"""Direct check for the gate helper used by factory.py."""
assert (
bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-7")
is False
)
assert (
bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-8")
is False
)
assert (
bedrock_converse_supports_strict_tools(
"anthropic.claude-sonnet-4-5-20250929-v1:0"
)
is True
)
assert (
bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-6")
is True
)
assert bedrock_converse_supports_strict_tools("us.amazon.nova-micro-v1:0") is False
assert bedrock_converse_supports_strict_tools("") is False
# Sonnet 4 also rejects strict on Bedrock Converse
assert (
bedrock_converse_supports_strict_tools(
"anthropic.claude-sonnet-4-20250514-v1:0"
)
is False
)
assert (
bedrock_converse_supports_strict_tools(
"bedrock/global.anthropic.claude-sonnet-4-20250514-v1:0"
)
is False
)
assert bedrock_converse_supports_strict_tools("anthropic.claude-sonnet-5") is False
assert (
bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-sonnet-5")
is False
)
assert (
bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0")
is True
)
@pytest.mark.parametrize(
"cost_map_key",
[
"anthropic.claude-opus-4-7",
"us.anthropic.claude-opus-4-7",
"anthropic.claude-opus-4-8",
"us.anthropic.claude-opus-4-8",
"anthropic.claude-sonnet-4-20250514-v1:0",
"global.anthropic.claude-sonnet-4-20250514-v1:0",
"us.anthropic.claude-sonnet-4-20250514-v1:0",
"eu.anthropic.claude-sonnet-4-20250514-v1:0",
"apac.anthropic.claude-sonnet-4-20250514-v1:0",
"anthropic.claude-sonnet-5",
"global.anthropic.claude-sonnet-5",
"us.anthropic.claude-sonnet-5",
"eu.anthropic.claude-sonnet-5",
"au.anthropic.claude-sonnet-5",
"jp.anthropic.claude-sonnet-5",
],
)
def test_strict_tools_flag_set_in_model_cost_map(cost_map_key: str) -> None:
"""The gate is driven by ``bedrock_converse_supports_strict_tools: false`` in
``model_prices_and_context_window.json``, not hardcoded model patterns."""
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
cost_map = GetModelCostMap.load_local_model_cost_map()
assert cost_map[cost_map_key]["bedrock_converse_supports_strict_tools"] is False

3
tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
View file

@ -1,5 +1,4 @@
import base64
import json
import logging
import os
import re
@ -10,7 +9,6 @@ import pytest
import litellm
from litellm.litellm_core_utils.prompt_templates.factory import (
BAD_MESSAGE_ERROR_STR,
BEDROCK_DOCUMENT_PLACEHOLDER_TEXT,
BedrockConverseMessagesProcessor,
BedrockImageProcessor,
@ -1243,7 +1241,6 @@ def test_bedrock_image_processor_content_type_document_formats():
"""
Test that _post_call_image_processing handles various document formats
"""
import base64
# Create mock response
mock_response = MagicMock()

57
tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
View file

@ -488,13 +488,6 @@ def test_shipped_gemini_chat_baseline_resolves_unmapped_ids(shipped_cost_map, mo
assert not info.get("output_cost_per_token")
def test_shipped_gemini_chat_baseline_loses_to_perplexity_exact_entries(shipped_cost_map):
info = litellm.get_model_info("google/gemini-2.5-pro", custom_llm_provider="perplexity")
entry = litellm.model_cost["perplexity/google/gemini-2.5-pro"]
assert info["mode"] == "responses"
assert entry["supports_reasoning"] is False
def test_shipped_gemini_chat_baseline_skips_non_chat_and_pre_2_5_ids(shipped_cost_map):
for model in (
"gemini/gemini-4-flash-image",
@ -809,20 +802,6 @@ def test_shipped_rules_flag_unmapped_wandb_ids_as_reasoning(shipped_cost_map):
assert litellm.supports_reasoning(model="zai-org/GLM-6-Turbo", custom_llm_provider="wandb") is True
def test_shipped_wandb_rule_loses_to_mapped_non_reasoning_entries(shipped_cost_map):
"""The whole point of a fallback is that it only fills gaps. A wandb model the map
describes as non-reasoning must stay non-reasoning, otherwise the rule silently
re-introduces the blanket supports_reasoning it exists to avoid."""
for model in (
"meta-llama/Llama-3.1-8B-Instruct",
"microsoft/Phi-4-mini-instruct",
"moonshotai/Kimi-K2-Instruct",
"Qwen/Qwen3-Coder-480B-A35B-Instruct",
):
assert f"wandb/{model}" in litellm.model_cost, model
assert litellm.supports_reasoning(model=model, custom_llm_provider="wandb") is False, model
def test_shipped_wandb_rule_does_not_fill_missing_mapped_entries(shipped_cost_map):
assert match_fill_missing_generalizations("wandb/meta-llama/Llama-3.1-8B-Instruct", "wandb") is None
@ -941,47 +920,11 @@ def test_shipped_openai_reasoning_rule_skips_non_reasoning_gpt_ids(shipped_cost_
assert match_capability_generalizations(model) is None, model
def test_shipped_openai_reasoning_rule_loses_to_mapped_entries(shipped_cost_map):
assert "gpt-5-search-api" in litellm.model_cost
assert litellm.supports_reasoning(model="gpt-5-search-api", custom_llm_provider="openai") is False
@pytest.mark.parametrize(
"model,provider,expected_supports_reasoning",
[
("azure/us/o1-2024-12-17", "azure", True),
("github_copilot/gpt-5", "github_copilot", None),
("perplexity/openai/gpt-5.4-mini", "perplexity", None),
],
)
def test_shipped_openai_reasoning_rule_backfills_only_approved_providers(
shipped_cost_map, model, provider, expected_supports_reasoning
):
assert model in litellm.model_cost
raw_entry = litellm.model_cost[model]
assert "supports_reasoning" not in raw_entry
model_without_provider = model.removeprefix(f"{provider}/")
info = litellm.get_model_info(model=model_without_provider, custom_llm_provider=provider)
assert info.get("supports_reasoning") is expected_supports_reasoning
assert info["input_cost_per_token"] == raw_entry.get("input_cost_per_token", 0)
def test_shipped_openai_reasoning_rule_matches_only_openai(shipped_cost_map):
assert match_fill_missing_generalizations("gpt-5.4", "openai") == {"supports_reasoning": True}
assert match_fill_missing_generalizations("gpt-5.4", "openrouter") is None
def test_shipped_openai_reasoning_rule_skips_non_text_modes(shipped_cost_map):
model = "gemini/deep-research-pro-preview-12-2025"
assert model in litellm.model_cost
raw_entry = litellm.model_cost[model]
assert "supports_reasoning" not in raw_entry
assert raw_entry["mode"] == "image_generation"
info = litellm.get_model_info("deep-research-pro-preview-12-2025", custom_llm_provider="gemini")
assert info.get("supports_reasoning") is None
def test_shipped_claude_thinking_rules_backfill_only_anthropic(shipped_cost_map):
model = "perplexity/anthropic/claude-sonnet-4-6"
assert model in litellm.model_cost

4
tests/test_litellm/litellm_core_utils/test_litellm_logging.py
View file

@ -2378,7 +2378,7 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path()
Test that _generate_cold_storage_object_key uses s3_path from custom logger instance.
"""
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import MagicMock, patch
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
@ -2425,7 +2425,7 @@ async def test_e2e_generate_cold_storage_object_key_with_logger_no_s3_path():
Test that _generate_cold_storage_object_key falls back to empty s3_path when logger has no s3_path.
"""
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import MagicMock, patch
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup

2
tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py
View file

@ -1,4 +1,3 @@
import json
from collections.abc import Mapping, Sequence
from typing import Final
@ -400,7 +399,6 @@ def test_streaming_preserves_anthropic_1hr_cache_creation_breakdown():
assert usage.cache_read_input_tokens == 8728
def test_streaming_keeps_cache_creation_breakdown_from_final_chunk():
"""When the final usage chunk itself carries the cache-creation breakdown,
aggregation must keep that breakdown instead of re-attaching a stale one

18
tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py
View file

@ -9,7 +9,7 @@ Covers:
import json
import os
from typing import Any, Dict, Optional
from typing import Any, Dict
import pytest
@ -42,22 +42,6 @@ class TestGetModelInfoReasoningEffortFields:
"""get_model_info should expose supports_minimal_reasoning_effort and
supports_max_reasoning_effort from the model registry."""
def test_opus_4_6_has_supports_minimal(self):
info = get_model_info("claude-opus-4-6")
assert "supports_minimal_reasoning_effort" in info
def test_opus_4_6_has_supports_max(self):
info = get_model_info("claude-opus-4-6")
assert "supports_max_reasoning_effort" in info
def test_opus_4_7_has_supports_minimal(self):
info = get_model_info("claude-opus-4-7")
assert "supports_minimal_reasoning_effort" in info
def test_opus_4_7_has_supports_max(self):
info = get_model_info("claude-opus-4-7")
assert "supports_max_reasoning_effort" in info
# ---------------------------------------------------------------------------
# Commit 2: JSON registry has correct reasoning effort fields

23
tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py
View file

@ -1974,20 +1974,6 @@ class TestClaudeOpus48AdaptiveThinking:
assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True
def test_resolver_reads_flag_through_bedrock_invoke_prefix(self, local_model_cost_map):
"""The resolver fix: ``bedrock/invoke/...`` resolves to the flagged
Bedrock entry. Pure ``_supports_factory`` without prefix-stripping
returns False here, which is why the data-only fix alone was not enough."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert (
AnthropicModelInfo._supports_model_capability(
"bedrock/invoke/us.anthropic.claude-opus-4-8",
"supports_adaptive_thinking",
"anthropic",
)
is True
)
@pytest.mark.parametrize(
"model",
@ -2172,15 +2158,6 @@ class TestCapabilityProbeUsesCallerProvider:
assert AnthropicModelInfo._is_adaptive_thinking_model(self.BEDROCK_MODEL, "bedrock") is False
def test_native_anthropic_probe_still_reads_anthropic_entry(self, local_model_cost_map, monkeypatch):
import litellm
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
monkeypatch.setitem(litellm.model_cost[self.BEDROCK_MODEL], "supports_adaptive_thinking", False)
litellm.get_model_info.cache_clear()
assert AnthropicModelInfo._is_adaptive_thinking_model("claude-opus-4-8", "anthropic") is True
def test_create_anthropic_model_list_response_shape():
from litellm.llms.anthropic.common_utils import (

11
tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py
View file

@ -1,6 +1,4 @@
import io
import json
from pathlib import Path
from unittest.mock import MagicMock
import httpx
@ -228,12 +226,3 @@ def test_azure_speech_transcription_routes_through_provider_config(monkeypatch):
assert audio_handler.call_args.kwargs["custom_llm_provider"] == "azure"
def test_azure_speech_stt_has_non_zero_input_pricing():
pricing_path = Path(__file__).parents[4] / "model_prices_and_context_window.json"
pricing = json.loads(pricing_path.read_text())
assert pricing["azure/speech/azure-stt"]["input_cost_per_second"] > 0
assert (
pricing["azure/speech/azure-stt"]["audio_transcription_config"]
== "azure_speech"
)

1
tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py
View file

@ -317,7 +317,6 @@ class TestProviderConfigManagerAzureAnthropicMessages:
assert config is None
def test_messages_thinking_shape_follows_exact_azure_entry_flag(local_model_cost_map, monkeypatch):
"""The Azure messages config must probe capabilities under ``azure_ai`` so an
operator setting ``supports_adaptive_thinking: false`` on the exact

117
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -1,15 +1,13 @@
import asyncio
import json
import os
import httpx
import pytest
from fastapi.testclient import TestClient
from unittest.mock import MagicMock, patch
import litellm
from litellm import ModelResponse, RateLimitError, completion
from litellm import ModelResponse
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
from litellm.types.llms.bedrock import ConverseTokenUsageBlock
@ -222,35 +220,6 @@ def test_bedrock_invoke_nova_cache_read_billed_at_discounted_rate(monkeypatch):
assert completion_cost == pytest.approx(3 * model_info["output_cost_per_token"])
@pytest.mark.parametrize(
"model",
[
"amazon.nova-micro-v1:0",
"amazon.nova-lite-v1:0",
"amazon.nova-pro-v1:0",
"us.amazon.nova-micro-v1:0",
"us.amazon.nova-lite-v1:0",
"us.amazon.nova-pro-v1:0",
"eu.amazon.nova-micro-v1:0",
"eu.amazon.nova-lite-v1:0",
"eu.amazon.nova-pro-v1:0",
"apac.amazon.nova-micro-v1:0",
"apac.amazon.nova-lite-v1:0",
"apac.amazon.nova-pro-v1:0",
"bedrock/us-gov-west-1/amazon.nova-micro-v1:0",
"bedrock/us-gov-west-1/amazon.nova-lite-v1:0",
"bedrock/us-gov-west-1/amazon.nova-pro-v1:0",
"bedrock/us-gov-east-1/amazon.nova-pro-v1:0",
],
)
def test_nova_prompt_caching_models_price_cache_reads_below_the_input_rate(model, monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
entry = litellm.model_cost[model]
assert entry["supports_prompt_caching"] is True
assert 0 < entry["cache_read_input_token_cost"] < entry["input_cost_per_token"]
def test_transform_usage_with_reasoning_content():
"""Test that completion_tokens_details correctly tracks reasoning vs text tokens."""
usage = ConverseTokenUsageBlock(
@ -1377,13 +1346,8 @@ def test_parallel_tool_calls_config_dropped_for_ttl_only_model(
def test_transform_response_with_computer_use_tool():
"""Test response transformation with computer use tool call."""
import httpx
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
from litellm.types.llms.bedrock import (
ConverseResponseBlock,
ConverseTokenUsageBlock,
)
from litellm.types.utils import ModelResponse
# Simulate a Bedrock Converse response with a computer-use tool call
@ -1472,13 +1436,8 @@ def test_transform_response_with_computer_use_tool():
def test_transform_response_with_bash_tool():
"""Test response transformation with bash tool call."""
import httpx
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
from litellm.types.llms.bedrock import (
ConverseResponseBlock,
ConverseTokenUsageBlock,
)
from litellm.types.utils import ModelResponse
# Simulate a Bedrock Converse response with a bash tool call
@ -4206,79 +4165,6 @@ def test_drop_thinking_param_when_thinking_blocks_missing():
litellm.modify_params = original_modify_params
def test_supports_native_structured_outputs(monkeypatch):
"""Test model detection for native structured outputs support.
Support is driven by the ``supports_native_structured_output`` flag in the
cost JSON (litellm.model_cost), not a hardcoded model set.
"""
old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
old_cost = litellm.model_cost
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
try:
config = AmazonConverseConfig()
# Supported models (have supports_native_structured_output=true in cost JSON)
assert config._supports_native_structured_outputs(
"anthropic.claude-sonnet-4-5-20250929-v1:0"
)
assert config._supports_native_structured_outputs(
"anthropic.claude-haiku-4-5-20251001-v1:0"
)
assert config._supports_native_structured_outputs(
"anthropic.claude-opus-4-6-v1"
)
# Regional prefix is stripped by get_bedrock_base_model
assert config._supports_native_structured_outputs(
"eu.anthropic.claude-opus-4-5-20251101-v1:0"
)
# Claude 4.6 Sonnet
assert config._supports_native_structured_outputs("anthropic.claude-sonnet-4-6")
assert config._supports_native_structured_outputs(
"us.anthropic.claude-sonnet-4-6"
)
# Non-Anthropic models
assert config._supports_native_structured_outputs(
"qwen.qwen3-235b-a22b-2507-v1:0"
)
assert config._supports_native_structured_outputs(
"mistral.mistral-large-3-675b-instruct"
)
assert config._supports_native_structured_outputs("minimax.minimax-m2")
assert config._supports_native_structured_outputs("moonshot.kimi-k2-thinking")
assert config._supports_native_structured_outputs("nvidia.nemotron-nano-3-30b")
# DeepSeek: old substring "deepseek-v3.1" didn't match real ID
assert config._supports_native_structured_outputs("deepseek.v3-v1:0")
assert config._supports_native_structured_outputs("deepseek.v3.2")
assert config._supports_native_structured_outputs("zai.glm-5")
# Unsupported models -- should fall back to tool-call approach
assert not config._supports_native_structured_outputs(
"anthropic.claude-sonnet-4-20250514-v1:0"
)
assert not config._supports_native_structured_outputs(
"meta.llama3-3-70b-instruct-v1:0"
)
assert not config._supports_native_structured_outputs("amazon.nova-pro-v1:0")
# Excluded: broken constrained decoding on Bedrock
assert not config._supports_native_structured_outputs("openai.gpt-oss-120b-1:0")
assert not config._supports_native_structured_outputs(
"mistral.magistral-small-2509"
)
# Excluded: ignores schema or broken on Bedrock
assert not config._supports_native_structured_outputs("google.gemma-3-27b-it")
assert not config._supports_native_structured_outputs(
"nvidia.nemotron-nano-12b-v2"
)
finally:
litellm.model_cost = old_cost
if old_env is None:
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
else:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env)
def test_create_output_config_for_response_format():
"""Test outputConfig dict creation from JSON schema."""
config = AmazonConverseConfig()
@ -7356,7 +7242,6 @@ def test_update_optional_params_with_thinking_tokens_bool_thinking_does_not_cras
assert "maxTokens" not in optional_params
@pytest.mark.parametrize(
"model, expected_dropped",
[

51
tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py
View file

@ -3,7 +3,7 @@
import base64
import io
from typing import cast
from unittest.mock import Mock, patch
from unittest.mock import Mock
import httpx
import pytest
@ -483,55 +483,6 @@ def test_transform_request_unknown_quality_reaches_image_generation_config():
assert body["imageGenerationConfig"]["quality"] == "auto"
def test_is_nova_canvas_image_edit_model_uses_model_cost_flag(monkeypatch):
"""Routing uses supports_nova_canvas_image_edit in model_cost, not a hardcoded name substring."""
fake_id = "amazon.custom-bedrock-image-edit-v99:0"
monkeypatch.setitem(
litellm.model_cost,
fake_id,
{
"litellm_provider": "bedrock",
"mode": "image_generation",
"supports_nova_canvas_image_edit": True,
},
)
assert (
BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(fake_id)
is True
)
monkeypatch.setitem(
litellm.model_cost,
"amazon.not-nova-canvas-v1:0",
{
"litellm_provider": "bedrock",
"mode": "image_generation",
},
)
assert (
BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(
"amazon.not-nova-canvas-v1:0"
)
is False
)
# Name-shaped ids do not route without supports_nova_canvas_image_edit (no substring heuristic).
monkeypatch.setitem(
litellm.model_cost,
"amazon.nova-canvas-v2:0",
{
"litellm_provider": "bedrock",
"mode": "image_generation",
},
)
assert (
BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(
"amazon.nova-canvas-v2:0"
)
is False
)
def test_transform_response_to_openai_format():
"""Response maps images[] to ImageResponse.data b64_json."""
config = BedrockAmazonNovaCanvasImageEditConfig()

18
tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py
View file

@ -32,7 +32,6 @@ from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_tran
)
@pytest.mark.asyncio
async def test_bedrock_sse_wrapper_encodes_dict_chunks():
"""Verify that `bedrock_sse_wrapper` converts dictionary chunks to properly formatted Server-Sent Events and forwards non-dict chunks unchanged."""
@ -1913,7 +1912,6 @@ async def test_unified_bedrock_messages_sse_usage_and_cost_claude_sonnet_46():
same logging reconstruction as Anthropic /messages. Ensures token counts and
completion_cost match model_prices for us.anthropic.claude-sonnet-4-6.
"""
from litellm import completion_cost
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
)
@ -2902,22 +2900,6 @@ def test_bedrock_messages_tool_search_follows_claude_tool_search_rule(local_mode
assert cfg._supports_tool_search_on_bedrock(model) is expected
def test_bedrock_messages_tool_search_rule_fills_mapped_entry_without_flag(local_model_cost_map, monkeypatch):
"""LIT-5851: a Bedrock entry that is in the map but carries no ``supports_tool_search``
key, the state Opus 4.8, Opus 5 and Sonnet 5 shipped in, is filled by the
``claude-tool-search`` rule instead of resolving to ``None`` and losing the beta."""
import litellm
model = "us.anthropic.claude-opus-5"
cfg = AmazonAnthropicClaudeMessagesConfig()
monkeypatch.delitem(litellm.model_cost[model], "supports_tool_search")
litellm.get_model_info.cache_clear()
assert litellm.get_model_info(model, custom_llm_provider="bedrock")["supports_tool_search"] is True
assert cfg._supports_tool_search_on_bedrock(model) is True
def test_bedrock_messages_thinking_shape_follows_exact_bedrock_entry_flag(
local_model_cost_map, monkeypatch
):

1
tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py
View file

@ -2,7 +2,6 @@
import pytest
from litellm.llms.bedrock.common_utils import BedrockModelInfo
# --------------------------------------------------------------------------- #

73
tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py
View file

@ -484,19 +484,6 @@ class TestBedrockMantleResponsesWebSearch:
)
assert body["tools"] == [self._WEB_SEARCH_TOOL]
@pytest.mark.parametrize(
"model",
[
"bedrock_mantle/openai.gpt-5.6-sol",
"bedrock_mantle/openai.gpt-5.6-terra",
"bedrock_mantle/openai.gpt-5.6-luna",
"bedrock_mantle/openai.gpt-5.5",
"bedrock_mantle/openai.gpt-5.4",
],
)
def test_cost_map_advertises_web_search_support(self, model):
assert litellm.supports_web_search(model=model) is True
def _codex_exec_tool():
return {
@ -1175,21 +1162,6 @@ class TestBedrockMantleResponsesRegistry:
assert isinstance(cfg, BedrockMantleResponsesAPIConfig)
assert cfg.use_openai_path is True
def test_gpt_5_5_price_map_declares_openai_responses_path(self, local_cost_map):
# The gpt-5.x entries must carry the data-driven flag so frontier routing
# does not rely on the name-string fallback alone.
assert (
litellm.model_cost["bedrock_mantle/openai.gpt-5.5"].get(
"use_openai_responses_path"
)
is True
)
assert (
litellm.model_cost["bedrock_mantle/openai.gpt-5.4"].get(
"use_openai_responses_path"
)
is True
)
@pytest.mark.parametrize(
"model",
@ -1361,51 +1333,6 @@ class TestMantleSupportsResponses:
model-name match: per-model, so gpt-oss-120b is supported but the safeguard
variant is not despite the shared substring."""
@pytest.mark.parametrize(
"model,model_cost,expected",
[
# supported_endpoints lists responses -> supported
(
"openai.gpt-oss-120b",
{
"bedrock_mantle/openai.gpt-oss-120b": {
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
}
},
True,
),
# chat-only supported_endpoints -> not supported (the discriminator)
(
"openai.gpt-oss-safeguard-120b",
{
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
"supported_endpoints": ["/v1/chat/completions"]
}
},
False,
),
# mode=responses (no supported_endpoints) -> supported
(
"somelab.future-model",
{"bedrock_mantle/somelab.future-model": {"mode": "responses"}},
True,
),
# mode=chat, no responses endpoint -> not supported
(
"google.gemma-3-27b-it",
{"bedrock_mantle/google.gemma-3-27b-it": {"mode": "chat"}},
False,
),
# absent from model_cost -> no signal -> not supported
("somelab.unmapped", {}, False),
(None, {}, False),
],
)
def test_supports_responses(self, model, model_cost, expected):
from litellm.llms.bedrock_mantle.common_utils import mantle_supports_responses
assert mantle_supports_responses(model, model_cost) is expected
class TestBedrockMantlePerModelResponsesURL:
"""End-to-end: the registry-selected config must build the correct wire URL

24
tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py
View file

@ -46,21 +46,6 @@ class TestBedrockMantleProviderRegistration:
def test_provider_in_provider_list(self):
assert "bedrock_mantle" in litellm.provider_list
def test_models_loaded(self, monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
litellm.add_known_models()
assert len(litellm.bedrock_mantle_models) > 0
assert "bedrock_mantle/openai.gpt-oss-120b" in litellm.bedrock_mantle_models
assert "bedrock_mantle/openai.gpt-oss-20b" in litellm.bedrock_mantle_models
assert (
"bedrock_mantle/openai.gpt-oss-safeguard-120b"
in litellm.bedrock_mantle_models
)
assert (
"bedrock_mantle/openai.gpt-oss-safeguard-20b"
in litellm.bedrock_mantle_models
)
class TestBedrockMantleConfig:
def test_custom_llm_provider(self):
@ -836,15 +821,6 @@ class TestBedrockMantleProviderResolution:
class TestBedrockMantlePricing:
"""Tests that verify Bedrock Mantle uses correct AWS Bedrock pricing, not OpenAI pricing."""
def test_safeguard_models_have_larger_output_tokens(self, monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
litellm.add_known_models()
info_120b = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-120b")
info_safeguard = litellm.get_model_info(
"bedrock_mantle/openai.gpt-oss-safeguard-120b"
)
assert info_safeguard["max_output_tokens"] > info_120b["max_output_tokens"]
@pytest.mark.parametrize(
"model_id",

28
tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py
View file

@ -1,28 +0,0 @@
from pathlib import Path
import pytest
import litellm
from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo
REPO_ROOT = Path(__file__).parents[5]
COST_MAPS = [
REPO_ROOT / "model_prices_and_context_window.json",
REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json",
]
MODELS = [("cohere/parse-v5.0", "cohere"), ("azure_ai/Cohere-parse-v5", "azure_ai")]
def _ocr_response(model: str, pages_processed: int) -> OCRResponse:
return OCRResponse(
pages=[OCRPage(index=i, markdown=f"page {i}") for i in range(pages_processed)],
model=model,
usage_info=OCRUsageInfo(pages_processed=pages_processed),
)
@pytest.mark.parametrize("model, provider", MODELS)
def test_model_info_resolves_ocr_mode_and_price(local_model_cost_map, model: str, provider: str) -> None:
info = litellm.get_model_info(model=model, custom_llm_provider=provider)
assert info["mode"] == "ocr"

28
tests/test_litellm/llms/crusoe/test_crusoe.py
View file

@ -105,31 +105,3 @@ def test_crusoe_provider_detection_by_prefix():
assert model == "meta-llama/Llama-3.3-70B-Instruct"
def test_crusoe_model_list_populated(monkeypatch):
"""Test Crusoe models are present in model_prices_and_context_window.json"""
import litellm
original_model_cost = litellm.model_cost
original_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
try:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
expected = [
"crusoe/meta-llama/Llama-3.3-70B-Instruct",
"crusoe/deepseek-ai/DeepSeek-R1-0528",
"crusoe/deepseek-ai/DeepSeek-V3-0324",
"crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507",
"crusoe/moonshotai/Kimi-K2-Thinking",
"crusoe/openai/gpt-oss-120b",
"crusoe/google/gemma-3-12b-it",
]
for model in expected:
assert model in litellm.model_cost, f"{model} not found in model_cost"
assert litellm.model_cost[model].get("litellm_provider") == "crusoe"
finally:
litellm.model_cost = original_model_cost
if original_env is None:
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
else:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", original_env)

47
tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py
View file

@ -4,7 +4,6 @@ from unittest.mock import MagicMock, patch
import pytest
import litellm
from litellm import supports_reasoning, supports_vision
from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY
from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id
@ -282,40 +281,6 @@ def test_handle_message_content_with_tool_calls():
)
def test_supports_reasoning_effort():
"""Test that reasoning_effort is only supported for specific Fireworks AI models."""
supported_models = [
"fireworks_ai/accounts/fireworks/models/qwen3-8b",
"fireworks_ai/accounts/fireworks/models/qwen3-32b",
"fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-a35b-instruct",
"fireworks_ai/accounts/fireworks/models/deepseek-v3p1",
"fireworks_ai/accounts/fireworks/models/deepseek-v3p2",
"fireworks_ai/accounts/fireworks/models/glm-4p5",
"fireworks_ai/accounts/fireworks/models/glm-4p5-air",
"fireworks_ai/accounts/fireworks/models/glm-4p6",
"fireworks_ai/accounts/fireworks/models/glm-4p7",
"fireworks_ai/accounts/fireworks/models/glm-5p1",
"fireworks_ai/accounts/fireworks/models/gpt-oss-120b",
"fireworks_ai/accounts/fireworks/models/gpt-oss-20b",
"fireworks_ai/glm-5p1",
]
unsupported_models = [
"fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct",
"fireworks_ai/accounts/fireworks/models/mixtral-8x7b-instruct",
]
for model in supported_models:
assert (
supports_reasoning(model=model, custom_llm_provider="fireworks_ai") is True
), f"{model} should support reasoning_effort"
for model in unsupported_models:
assert (
supports_reasoning(model=model, custom_llm_provider="fireworks_ai") is False
), f"{model} should not support reasoning_effort"
def test_get_supported_openai_params_reasoning_effort():
"""Test that reasoning_effort is only included in supported params for models that support it."""
config = FireworksAIConfig()
@ -973,18 +938,6 @@ def test_thinking_and_reasoning_effort_conflict_rejected():
)
def test_llama_vision_supports_vision_from_model_map():
config = FireworksAIConfig()
for model in [
"fireworks_ai/accounts/fireworks/models/llama-v3p2-11b-vision-instruct",
"fireworks_ai/accounts/fireworks/models/minimax-m3",
"fireworks_ai/minimax-m3",
]:
assert supports_vision(model=model, custom_llm_provider="fireworks_ai") is True
assert config.get_provider_info(model)["supports_vision"] is True
def test_transform_messages_helper_rejects_file_blocks():
config = FireworksAIConfig()
messages = [

13
tests/test_litellm/llms/inception/test_inception_chat_transformation.py
View file

@ -7,7 +7,6 @@ import os
from unittest import mock
import httpx
import pytest
import litellm
from litellm.llms.inception.chat.transformation import InceptionChatConfig
@ -232,18 +231,6 @@ def test_inception_in_provider_lists():
assert "https://api.inceptionlabs.ai/v1" in litellm.openai_compatible_endpoints
def test_inception_model_list_populated(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
litellm.inception_models = set()
litellm.add_known_models()
assert "inception/mercury-2" in litellm.inception_models
assert "inception/mercury-2.5" in litellm.inception_models
for model in litellm.inception_models:
assert model.startswith("inception/")
def test_inception_completion_targets_inception_endpoint():
"""
End-to-end: a completion routed through the inception provider must hit

4
tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py
View file

@ -730,10 +730,6 @@ class TestMoonshotResponseSchemaSupport:
def model_cost_map(self):
return GetModelCostMap.load_local_model_cost_map()
def test_supports_response_schema_utility_reports_true(self, model_cost_map, monkeypatch):
monkeypatch.setattr(litellm, "model_cost", model_cost_map)
assert litellm.utils.supports_response_schema(model="moonshot/kimi-k2.5") is True
class TestMoonshotReasoningEffort:
"""Moonshot documents reasoning_effort as a top-level chat completions field for its reasoning

70
tests/test_litellm/llms/oci/embed/test_oci_embedding.py
View file

@ -1,5 +1,3 @@
import json
import os
from unittest.mock import MagicMock, patch
import httpx
@ -308,72 +306,4 @@ class TestOCIEmbeddingConfig:
litellm_params={},
)
def test_model_prices_embedding_models(self):
"""test all 8 OCI embedding models exist in model_prices_and_context_window.json with mode=embedding."""
model_prices_path = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"..",
"..",
"..",
"model_prices_and_context_window.json",
)
with open(model_prices_path) as f:
model_prices = json.load(f)
expected_embedding_models = [
"oci/cohere.embed-english-v3.0",
"oci/cohere.embed-english-light-v3.0",
"oci/cohere.embed-multilingual-v3.0",
"oci/cohere.embed-multilingual-light-v3.0",
"oci/cohere.embed-english-image-v3.0",
"oci/cohere.embed-english-light-image-v3.0",
"oci/cohere.embed-multilingual-light-image-v3.0",
"oci/cohere.embed-v4.0",
]
for model_key in expected_embedding_models:
assert model_key in model_prices, f"Missing model: {model_key}"
assert (
model_prices[model_key].get("mode") == "embedding"
), f"Model {model_key} does not have mode='embedding'"
def test_model_prices_new_chat_models(self):
"""test the 16 new OCI chat models exist in model_prices_and_context_window.json with mode=chat."""
model_prices_path = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"..",
"..",
"..",
"model_prices_and_context_window.json",
)
with open(model_prices_path) as f:
model_prices = json.load(f)
expected_chat_models = [
"oci/xai.grok-3",
"oci/xai.grok-3-fast",
"oci/xai.grok-3-mini",
"oci/xai.grok-3-mini-fast",
"oci/xai.grok-4",
"oci/xai.grok-4-fast",
"oci/xai.grok-4.1-fast",
"oci/xai.grok-4.20",
"oci/xai.grok-4.20-multi-agent",
"oci/xai.grok-code-fast-1",
"oci/cohere.command-a-03-2025",
"oci/cohere.command-a-reasoning-08-2025",
"oci/cohere.command-a-vision-07-2025",
"oci/cohere.command-a-translate-08-2025",
"oci/google.gemini-2.5-pro",
"oci/google.gemini-2.5-flash",
]
for model_key in expected_chat_models:
assert model_key in model_prices, f"Missing model: {model_key}"
assert (
model_prices[model_key].get("mode") == "chat"
), f"Model {model_key} does not have mode='chat'"

4
tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py
View file

@ -1,9 +1,8 @@
import json
from types import SimpleNamespace
from typing import Final
from unittest.mock import AsyncMock, MagicMock, Mock, patch
from unittest.mock import MagicMock, Mock, patch
import httpx
import pytest
@ -15,7 +14,6 @@ from litellm.types.llms.openai import (
ImageGenerationPartialImageEvent,
OutputTextDeltaEvent,
ResponseCompletedEvent,
ResponsesAPIRequestParams,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
)

26
tests/test_litellm/llms/openai/test_gpt5_transformation.py
View file

@ -288,24 +288,6 @@ def test_gpt5_1_gpt5_2_gpt5_4_drop_minimal_reasoning_effort(config: OpenAIConfig
# GPT-5.1 temperature handling tests
def test_gpt5_1_model_detection(gpt5_config: OpenAIGPT5Config):
"""Test that models supporting reasoning_effort='none' are correctly detected via model map."""
# gpt-5.1 and gpt-5.2 chat variants support none
assert gpt5_config._supports_reasoning_effort_level("gpt-5.1", "none")
assert gpt5_config._supports_reasoning_effort_level("gpt-5.1-2025-11-13", "none")
assert gpt5_config._supports_reasoning_effort_level("gpt-5.1-chat-latest", "none")
assert gpt5_config._supports_reasoning_effort_level("gpt-5.2", "none")
assert gpt5_config._supports_reasoning_effort_level("gpt-5.2-2025-12-11", "none")
# codex/pro/chat variants do not support none
assert not gpt5_config._supports_reasoning_effort_level("gpt-5.1-codex", "none")
assert not gpt5_config._supports_reasoning_effort_level("gpt-5.1-codex-max", "none")
assert not gpt5_config._supports_reasoning_effort_level(
"gpt-5.2-chat-latest", "none"
)
assert not gpt5_config._supports_reasoning_effort_level("gpt-5.2-pro", "none")
assert not gpt5_config._supports_reasoning_effort_level("gpt-5", "none")
assert not gpt5_config._supports_reasoning_effort_level("gpt-5-mini", "none")
assert not gpt5_config._supports_reasoning_effort_level("gpt-5-codex", "none")
def test_gpt5_1_temperature_with_reasoning_effort_none(config: OpenAIConfig):
@ -491,14 +473,6 @@ def test_gpt5_minimal_dict_accepted_for_supported_model(config: OpenAIConfig):
assert params["reasoning_effort"] == "minimal"
def test_gpt5_supports_reasoning_effort_level_minimal(gpt5_config: OpenAIGPT5Config):
"""Test that _supports_reasoning_effort_level correctly identifies minimal support."""
assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4", "minimal")
assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-pro", "minimal")
assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-mini", "minimal")
assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-nano", "minimal")
def test_gpt5_minimal_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config):
"""_is_reasoning_effort_level_explicitly_disabled returns True only for explicit False entries.

32
tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py
View file

@ -6,7 +6,6 @@ import os
import sys
from unittest.mock import patch
import pytest
sys.path.insert(
0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))
@ -58,12 +57,6 @@ class TestSimpleProviderConfigSupportedEndpoints:
class TestJSONProviderRegistryResponsesAPI:
"""Test supports_responses_api on JSONProviderRegistry."""
def test_existing_provider_no_responses(self):
"""Existing providers without supported_endpoints don't support responses"""
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
# publicai has no supported_endpoints in JSON, defaults to []
assert JSONProviderRegistry.supports_responses_api("publicai") is False
def test_nonexistent_provider(self):
"""Non-existent provider returns False"""
@ -74,31 +67,6 @@ class TestJSONProviderRegistryResponsesAPI:
is False
)
def test_provider_with_responses_endpoint(self):
"""A provider with /v1/responses in supported_endpoints returns True"""
from litellm.llms.openai_like.json_loader import (
JSONProviderRegistry,
SimpleProviderConfig,
)
# Temporarily inject a test provider
test_config = SimpleProviderConfig(
"test_responses_provider",
{
"base_url": "https://test.example.com",
"api_key_env": "TEST_API_KEY",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
},
)
JSONProviderRegistry._providers["test_responses_provider"] = test_config
try:
assert (
JSONProviderRegistry.supports_responses_api("test_responses_provider")
is True
)
finally:
del JSONProviderRegistry._providers["test_responses_provider"]
class TestCreateResponsesConfigClass:
"""Test dynamic responses config class generation."""

8
tests/test_litellm/llms/openai_like/test_cognition_provider.py
View file

@ -112,13 +112,6 @@ class TestCognitionProviderIdentity:
class TestCognitionCostTracking:
def test_lightning_is_five_times_the_standard_tier(self):
standard = litellm.get_model_info(model="cognition/swe-1.7")
lightning = litellm.get_model_info(model="cognition/swe-1.7-lightning")
assert lightning["input_cost_per_token"] == pytest.approx(standard["input_cost_per_token"] * 5)
assert lightning["output_cost_per_token"] == pytest.approx(standard["output_cost_per_token"] * 5)
def test_supported_endpoints_matrix(self):
matrix = json.loads((Path(litellm.__file__).parent / "provider_endpoints_support_backup.json").read_text())
@ -129,4 +122,3 @@ class TestCognitionCostTracking:
assert endpoints["embeddings"] is False

5
tests/test_litellm/llms/openai_like/test_meta_provider.py
View file

@ -24,10 +24,6 @@ class TestMetaProviderConfig:
assert meta.api_key_env == "META_API_KEY"
assert meta.api_base_env == "META_API_BASE"
def test_meta_supports_responses_api(self):
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
assert JSONProviderRegistry.supports_responses_api("meta")
def test_meta_in_openai_compatible_providers(self):
from litellm.constants import openai_compatible_providers
@ -192,4 +188,3 @@ class TestMetaAnthropicMessages:
assert headers["anthropic-version"] == "2023-06-01"

20
tests/test_litellm/llms/openai_like/test_scx_ai_provider.py
View file

@ -154,26 +154,6 @@ class TestSCXAIModelMetadata:
with open(json_path) as f:
return json.load(f)
def test_scx_ai_models_registered_with_correct_metadata(self):
model_cost = self._load(("model_prices_and_context_window.json",))
for model in self.SCX_MODELS:
info = model_cost.get(model)
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
assert info["litellm_provider"] == "scx-ai"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] > 0
assert info["output_cost_per_token"] > 0
assert info["supports_function_calling"] is True
assert info["supports_tool_choice"] is True
assert info["supports_reasoning"] is True
assert info["supports_response_schema"] is True
assert info.get("supports_vision", False) is (model in self.VISION_MODELS)
assert info["supports_prompt_caching"] is True
assert 0 < info["cache_read_input_token_cost"] < info["input_cost_per_token"]
assert info["max_tokens"] == info["max_output_tokens"]
assert info["max_input_tokens"] >= 1_000_000
def test_scx_ai_models_synced_to_backup(self):
model_cost = self._load(("model_prices_and_context_window.json",))

9
tests/test_litellm/llms/openai_like/test_tensormesh_provider.py
View file

@ -129,15 +129,6 @@ class TestTensormeshCostMap:
litellm.model_cost = original_model_cost
litellm.get_model_info.cache_clear()
def test_models_registered_with_capabilities(self):
for model in TENSORMESH_MODELS:
info = litellm.get_model_info(model)
assert info["litellm_provider"] == "tensormesh"
assert info["mode"] == "chat"
assert litellm.supports_function_calling(model) is True, model
assert litellm.supports_response_schema(model) is True, model
assert litellm.model_cost[model]["supports_tool_choice"] is True, model
assert litellm.model_cost[model]["supports_prompt_caching"] is True, model
def test_reasoning_flag_matches_expected_set(self):
reasoning_models = {

31
tests/test_litellm/llms/reducto/test_model_info.py
View file

@ -1,9 +1,6 @@
import uuid
import litellm
from litellm.utils import _invalidate_model_cost_lowercase_map
def test_reducto_provider_registration():
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
@ -14,31 +11,3 @@ def test_reducto_provider_registration():
assert custom_llm_provider == "reducto"
def test_get_model_info_preserves_ocr_cost_per_credit():
test_model_name = f"reducto/test-cost-propagation-{uuid.uuid4().hex[:12]}"
previous_model_entry = litellm.model_cost.get(test_model_name)
_invalidate_model_cost_lowercase_map()
try:
litellm.register_model(
{
test_model_name: {
"litellm_provider": "reducto",
"mode": "ocr",
"ocr_cost_per_credit": 0.003,
}
}
)
model_info = litellm.get_model_info(
model=test_model_name,
custom_llm_provider="reducto",
)
assert model_info.get("ocr_cost_per_credit") == 0.003
finally:
if previous_model_entry is None:
litellm.model_cost.pop(test_model_name, None)
else:
litellm.model_cost[test_model_name] = previous_model_entry
_invalidate_model_cost_lowercase_map()

17
tests/test_litellm/llms/tencent/chat/test_tencent_chat_transformation.py
View file

@ -247,23 +247,6 @@ class TestAdaptiveThinkingCoercion:
assert config._is_adaptive_thinking_model("tencent/no-such-model") is False
def test_minimax_m3_cost_map_entry_marks_adaptive_thinking():
"""The capability flag driving the coercion must exist in the cost map
(and its backup, which is shipped with the package)."""
import json
from pathlib import Path
repo_root = Path(__file__).parents[5]
for filename in ("model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json"):
with open(repo_root / filename) as f:
entry = json.load(f).get("tencent/minimax-m3")
assert entry is not None, f"tencent/minimax-m3 not found in {filename}"
assert entry["litellm_provider"] == "tencent"
assert entry.get("supports_adaptive_thinking") is True
assert entry.get("supports_reasoning") is True
def test_get_complete_url_default():
config = TencentChatConfig()

20
tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
View file

@ -389,7 +389,6 @@ def test_build_vertex_schema_array_branch_missing_items_in_anyof():
def test_vertex_ai_complex_response_schema():
import json
from copy import deepcopy
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
@ -1192,7 +1191,7 @@ async def test_vertex_ai_token_counter_routes_partner_models():
Test that VertexAITokenCounter correctly routes partner models (Claude, Mistral, etc.)
to the partner models token counter instead of the Gemini token counter.
"""
from unittest.mock import AsyncMock, patch
from unittest.mock import patch
from litellm.llms.vertex_ai.common_utils import VertexAITokenCounter
from litellm.types.utils import TokenCountResponse
@ -1242,7 +1241,6 @@ async def test_vertex_ai_token_counter_uses_count_tokens_location():
from unittest.mock import patch
from litellm.llms.vertex_ai.common_utils import VertexAITokenCounter
from litellm.types.utils import TokenCountResponse
token_counter = VertexAITokenCounter()
@ -1283,7 +1281,7 @@ async def test_vertex_ai_token_counter_routes_gemini_models():
Test that VertexAITokenCounter correctly routes Gemini models
to the Gemini token counter (not partner models).
"""
from unittest.mock import AsyncMock, patch
from unittest.mock import patch
from litellm.llms.vertex_ai.common_utils import VertexAITokenCounter
from litellm.types.utils import TokenCountResponse
@ -1757,17 +1755,3 @@ def test_get_vertex_ai_lyria_model_info_is_none_for_non_lyria_speech_models(mode
assert get_vertex_ai_lyria_model_info(model=model) is None
def test_get_vertex_ai_lyria_model_info_falls_back_to_bundled_map(monkeypatch):
import litellm
from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info
stale_runtime_model_cost = {
key: value for key, value in litellm.model_cost.items() if not key.startswith("vertex_ai/lyria")
}
monkeypatch.setattr(litellm, "model_cost", stale_runtime_model_cost)
model_info = get_vertex_ai_lyria_model_info(model="lyria-3-pro-preview")
assert model_info is not None
assert model_info["vertex_ai_audio_api"] == "lyria_interactions"
assert model_info["supported_audio_formats"] == ("mp3", "wav")

4
tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py
View file

@ -34,7 +34,6 @@ def test_get_supported_params_thinking():
def test_vertex_ai_anthropic_web_search_header_in_completion():
"""Test that web search tool adds the required beta header for Vertex AI completion requests"""
from unittest.mock import MagicMock, patch
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
@ -463,9 +462,6 @@ def test_vertex_ai_partner_models_anthropic_remove_prompt_caching_scope_beta_hea
Test that remove_unsupported_beta correctly filters out prompt-caching-scope-2026-01-05
from the anthropic-beta headers.
"""
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
VertexAIPartnerModelsAnthropicMessagesConfig,
)
# This beta header should be removed
PROMPT_CACHING_BETA_HEADER = "prompt-caching-scope-2026-01-05"

22
tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py
View file

@ -180,28 +180,6 @@ class TestCreateVertexURLGemma:
# ---------------------------------------------------------------------------
def test_gemma_maas_supports_function_calling():
"""supports_function_calling=true in model_cost must be surfaced by the utility."""
with patch.dict(litellm.model_cost, _GEMMA_MODEL_COST_ENTRY, clear=False):
assert (
litellm.utils.supports_function_calling(
model="vertex_ai/google/gemma-4-26b-a4b-it-maas"
)
is True
)
def test_gemma_maas_supports_vision():
"""supports_vision=true in model_cost must be surfaced by the utility."""
with patch.dict(litellm.model_cost, _GEMMA_MODEL_COST_ENTRY, clear=False):
assert (
litellm.utils.supports_vision(
model="vertex_ai/google/gemma-4-26b-a4b-it-maas"
)
is True
)
# ---------------------------------------------------------------------------
# Integration tests: verify payloads reach the global OpenAI endpoint
#

1
tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py
View file

@ -14,7 +14,6 @@ import pytest
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.openai.cost_calculation import video_generation_cost
from litellm.llms.vertex_ai.videos.transformation import (
VertexAIVideoConfig,
_convert_image_to_vertex_format,

18
tests/test_litellm/llms/xai/test_xai_model_registry.py
View file

@ -29,24 +29,6 @@ def cost_map(request: pytest.FixtureRequest) -> dict:
return json.loads(path.read_text(encoding="utf-8"))
@pytest.mark.parametrize("model", RESPONSES_ONLY_MODELS)
def test_multi_agent_models_are_responses_only(cost_map: dict, model: str):
entry = cost_map[model]
assert entry["supported_endpoints"] == ["/v1/responses"]
assert entry["mode"] == "responses"
def test_surviving_xai_chat_models_still_serve_chat_completions(cost_map: dict):
"""Guard against the removal above over-reaching into live models."""
chat_models = [
key
for key, value in cost_map.items()
if isinstance(value, dict) and value.get("litellm_provider") == "xai" and value.get("mode") == "chat"
]
assert "xai/grok-4.3" in chat_models
assert "xai/grok-4.6" in chat_models
def test_both_cost_maps_agree_on_xai_entries():
prices = json.loads(PRICES_PATH.read_text(encoding="utf-8"))
backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8"))

5
tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py
View file

@ -85,11 +85,6 @@ def test_code_slug_bills_at_grok_build_rate(cost_map: dict, slug: str):
assert entry[field] == target[field], field
def test_a_live_xai_model_is_untouched(cost_map: dict):
"""Guard against the repricing leaking onto models xAI still serves directly."""
assert cost_map["xai/grok-4.6"]["input_cost_per_token"] != cost_map[REDIRECT_TARGET]["input_cost_per_token"]
@pytest.mark.parametrize("slug", REDIRECTED_SLUGS)
def test_redirected_slug_carries_the_target_tier_rates(cost_map: dict, slug: str):
"""The request executes as grok-4.3, so it is tiered at grok-4.3's 200k boundary."""

22
tests/test_litellm/proxy/auth/test_model_checks.py
View file

@ -1,10 +1,7 @@
from unittest.mock import AsyncMock, patch
from unittest.mock import patch
import pytest
from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable, Member
from litellm.proxy.auth.handle_jwt import JWTAuthManager
def test_get_team_models_for_all_models_and_team_only_models():
from litellm.proxy.auth.model_checks import get_team_models
@ -858,23 +855,6 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion():
assert fake_model not in litellm.models_by_provider["vertex_ai"]
def test_azure_ai_wildcard_lists_the_foundry_gpt_6_astra_entry(monkeypatch):
import litellm
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
foundry_key = "azure_ai/gpt-6-astra"
local_entry = litellm.get_model_cost_map(url="")[foundry_key]
registered_before = foundry_key in litellm.azure_ai_models
try:
litellm.add_known_models(model_cost_map={foundry_key: local_entry})
assert foundry_key in get_known_models_from_wildcard("azure_ai/*")
finally:
if not registered_before:
litellm.azure_ai_models.discard(foundry_key)
litellm.add_known_models(model_cost_map={})
def test_get_complete_model_list_drops_no_default_models_sentinel():
from litellm.proxy.auth.model_checks import get_complete_model_list

75
tests/test_litellm/test_azure_audio_price_aliases.py
View file

@ -1,75 +0,0 @@
"""Undated azure aliases for the audio models must exist and match their dated
variants. Azure deployments are commonly created under an admin-chosen name, so
the served model name means nothing to the cost lookup and `base_model:
azure/gpt-audio-mini` is what prices the call. That key resolved to nothing, the
lookup raised "This model isn't mapped yet", and the proxy logged the request at
$0. Issue #33170."""
import json
from pathlib import Path
import pytest
import litellm
pytestmark = pytest.mark.usefixtures("local_model_cost_map")
COST_FIELDS = (
"input_cost_per_token",
"output_cost_per_token",
"input_cost_per_audio_token",
"output_cost_per_audio_token",
)
ALIAS_PAIRS = (
("azure/gpt-audio-mini", "azure/gpt-audio-mini-2025-10-06"),
("azure/gpt-realtime-mini", "azure/gpt-realtime-mini-2025-10-06"),
)
def _load_root_cost_map() -> dict:
root_map_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
with open(root_map_path) as f:
return json.load(f)
@pytest.mark.parametrize("undated, dated", ALIAS_PAIRS)
def test_undated_azure_audio_alias_matches_dated_entry(undated, dated):
undated_info = litellm.get_model_info(undated)
dated_info = litellm.get_model_info(dated)
for field in COST_FIELDS:
assert undated_info.get(field) == dated_info.get(field), field
assert (undated_info.get(field) or 0) > 0, f"{undated}.{field} must be non-zero"
assert undated_info.get("litellm_provider") == "azure"
assert undated_info.get("mode") == dated_info.get("mode")
@pytest.mark.parametrize("undated, dated", ALIAS_PAIRS)
def test_undated_azure_audio_alias_is_exact_mirror(undated, dated):
"""The undated alias must be a byte-for-byte mirror of its dated entry, covering
every field (incl. realtime-specific cache/audio cost keys) so any future drift
between the pair is caught, not just the core COST_FIELDS."""
model_map = litellm.model_cost
assert undated in model_map, f"{undated} missing from model cost map"
assert model_map[undated] == model_map[dated], (
f"{undated} must exactly mirror {dated}; "
f"diff keys: {[k for k in set(model_map[undated]) | set(model_map[dated]) if model_map[undated].get(k) != model_map[dated].get(k)]}"
)
@pytest.mark.parametrize("undated, dated", ALIAS_PAIRS)
def test_undated_azure_audio_alias_is_in_the_root_cost_map(undated, dated):
"""`local_model_cost_map` pins `litellm.model_cost` to the packaged backup, but a
proxy left on its defaults fetches the root map instead, and that is the copy
that ships to the CDN. An alias added to only one of the two files still bills
$0 for every proxy reading the other, which is the very bug this file guards, so
assert the root map directly and assert the two files agree."""
root_map = _load_root_cost_map()
assert undated in root_map, f"{undated} missing from the root cost map"
assert root_map[undated] == root_map[dated], f"{undated} must exactly mirror {dated} in the root cost map"
assert root_map[undated] == litellm.model_cost[undated], (
f"{undated} differs between the root cost map and the packaged backup"
)

12
tests/test_litellm/test_baseten_glm_5_3_model_metadata.py
View file

@ -4,7 +4,6 @@ from pathlib import Path
import pytest
import litellm
from litellm.utils import supports_function_calling, supports_prompt_caching
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
@ -33,17 +32,6 @@ def local_model_cost_map(monkeypatch):
litellm.get_model_info.cache_clear()
def test_baseten_glm_5_3_capabilities_are_visible_to_callers(local_model_cost_map):
"""The entry advertises prompt caching and tool calling, so the helpers every
caller checks before sending a request must say so too."""
assert supports_prompt_caching(model=MODEL) is True
assert supports_function_calling(model=MODEL) is True
info = litellm.get_model_info(model="zai-org/GLM-5.3", custom_llm_provider="baseten")
assert info["max_input_tokens"] > 0
assert info["max_output_tokens"] > 0
def test_backup_matches_main():
"""Ensure the bundled (backup) cost map stays in sync with the canonical file.

8
tests/test_litellm/test_bedrock_marengo_embed_3_model_metadata.py
View file

@ -3,7 +3,6 @@ from pathlib import Path
import pytest
import litellm
from litellm.constants import bedrock_embedding_models
REPO_ROOT = Path(__file__).parents[2]
@ -31,13 +30,6 @@ def _load(path):
return json.load(f)
@pytest.mark.parametrize("model", ALL_MODELS)
def test_marengo_embed_3_is_visible_to_callers(model, local_model_cost_map):
info = litellm.get_model_info(model=model, custom_llm_provider="bedrock")
assert info["mode"] == "embedding"
assert info["output_vector_size"] == 512
def test_marengo_embed_3_is_a_known_bedrock_embedding_model():
assert BASE_MODEL in bedrock_embedding_models

77
tests/test_litellm/test_bedrock_usgov_pricing.py
View file

@ -1,77 +0,0 @@
"""
Validate AWS GovCloud (Bedrock us-gov-*) Anthropic pricing entries.
AWS Bedrock pricing in GovCloud carries a +20% premium over the global
Anthropic prices (not the +10% commercial-US premium). Until 2026-05-22
these entries silently mirrored commercial US, undercharging customers
by ~9%.
Source: https://aws.amazon.com/bedrock/pricing/
Sonnet 4.5 in us-gov-* (per million tokens):
input = $3.60
output = $18.00
cache write 5m = $4.50
cache write 1h = $7.20
cache read = $0.36
Reference: https://github.com/BerriAI/litellm/issues/27120
"""
import json
import os
import pytest
@pytest.fixture(scope="module")
def model_data():
json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
with open(json_path) as f:
return json.load(f)
def test_usgov_east_haiku_profile_mirrors_in_region_row(model_data):
"""us-gov-east-1 serves claude-3-haiku through the us-gov. inference profile
only, so the profile row must bill exactly like the in-region gov row.
"""
profile = model_data["us-gov.anthropic.claude-3-haiku-20240307-v1:0"]
in_region = model_data["bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0"]
assert profile["litellm_provider"] == "bedrock_converse"
assert {k: v for k, v in profile.items() if k != "litellm_provider"} == {
k: v for k, v in in_region.items() if k != "litellm_provider"
}
GOV_ROW_SOURCES = {
"us-gov.anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1",
"bedrock/us-gov-west-1/anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1",
"bedrock/us-gov-east-1/anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1",
"us-gov.nvidia.nemotron-nano-9b-v2": "nvidia.nemotron-nano-9b-v2",
"bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": "nvidia.nemotron-nano-9b-v2",
"bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": "nvidia.nemotron-nano-9b-v2",
"us-gov.xai.grok-4.6": "us.xai.grok-4.6",
"bedrock_mantle/us-gov-west-1/xai.grok-4.6": "bedrock_mantle/xai.grok-4.6",
"bedrock_mantle/us-gov-east-1/xai.grok-4.6": "bedrock_mantle/xai.grok-4.6",
"bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0": "amazon.nova-2-multimodal-embeddings-v1:0",
"bedrock/us-gov-west-1/amazon.nova-lite-v1:0": "amazon.nova-lite-v1:0",
"bedrock/us-gov-west-1/amazon.nova-micro-v1:0": "amazon.nova-micro-v1:0",
"bedrock_mantle/us-gov-west-1/google.gemma-4-e2b": "bedrock_mantle/google.gemma-4-e2b",
"bedrock_mantle/us-gov-west-1/google.gemma-4-26b-a4b": "bedrock_mantle/google.gemma-4-26b-a4b",
"bedrock_mantle/us-gov-west-1/google.gemma-4-31b": "bedrock_mantle/google.gemma-4-31b",
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": "bedrock_mantle/openai.gpt-oss-20b",
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": "bedrock_mantle/openai.gpt-oss-20b",
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": "bedrock_mantle/openai.gpt-oss-120b",
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": "bedrock_mantle/openai.gpt-oss-120b",
}
def _non_pricing_fields(info):
return {k: v for k, v in info.items() if "cost" not in k and k not in ("litellm_provider", "source")}
@pytest.mark.parametrize("gov_key", GOV_ROW_SOURCES)
def test_usgov_rows_keep_commercial_limits_and_capabilities(model_data, gov_key):
"""Gov rows preserve the commercial row's non-pricing fields."""
gov = model_data[gov_key]
assert _non_pricing_fields(gov) == _non_pricing_fields(model_data[GOV_ROW_SOURCES[gov_key]])

76
tests/test_litellm/test_claude_fable_5_config.py
View file

@ -26,67 +26,10 @@ def _load_root_cost_map() -> dict:
return json.load(f)
def test_fable_5_present_in_bundled_backup():
"""The bundled backup is the runtime fallback (and what tests load with
``LITELLM_LOCAL_MODEL_COST_MAP=True``) — it must carry the same entries as
the root cost map, otherwise the model resolves on one path but not the
other."""
backup = GetModelCostMap.load_local_model_cost_map()
root = _load_root_cost_map()
for model_name in (
"claude-fable-5",
"anthropic.claude-fable-5",
"global.anthropic.claude-fable-5",
"us.anthropic.claude-fable-5",
"eu.anthropic.claude-fable-5",
"vertex_ai/claude-fable-5",
"vertex_ai/claude-fable-5@default",
"azure_ai/claude-fable-5",
):
assert model_name in backup, f"Missing from backup cost map: {model_name}"
assert backup[model_name] == root[model_name], model_name
def test_fable_5_registered_for_bedrock_converse():
assert "anthropic.claude-fable-5" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_fable_5_all_variants_carry_adaptive_thinking_flag(cost_map):
"""Every Fable 5 entry must advertise ``supports_adaptive_thinking``.
Adaptive-thinking detection is cost-map driven, so a single variant missing
the flag silently sends the legacy ``thinking.type='enabled'`` shape and the
provider 400s (issue #29188 for the Opus 4.8 equivalent). Fable 5 is even
stricter than Opus 4.8: an explicit ``thinking.type='disabled'`` also 400s,
so adaptive is the only valid thinking shape LiteLLM can emit for it."""
variants = [k for k in cost_map if "claude-fable-5" in k]
assert variants, "no claude-fable-5 entries found in cost map"
missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True]
assert not missing, f"missing supports_adaptive_thinking: {missing}"
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_fable_5_all_variants_carry_thinking_always_on_flag(cost_map):
"""Every Fable 5 entry must advertise ``thinking_always_on``.
The flag drives the Anthropic transformations to omit an explicit
``thinking.type='disabled'``, which Fable 5 rejects with a 400; a variant
missing the flag forwards the param verbatim and the provider 400s."""
variants = [k for k in cost_map if "claude-fable-5" in k]
assert variants, "no claude-fable-5 entries found in cost map"
missing = [k for k in variants if cost_map[k].get("thinking_always_on") is not True]
assert not missing, f"missing thinking_always_on: {missing}"
@pytest.mark.parametrize(
"model",
[
@ -151,22 +94,3 @@ def test_adaptive_thinking_detected_for_fable_5_1(local_model_cost_map, model):
assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_sampling_params_flag_on_all_models_that_removed_them(cost_map):
"""Fable 5 and Opus 4.7/4.8 reject ``top_p``/``top_k``/``temperature != 1``;
the drop/raise gating is cost-map driven, so every variant must carry an
explicit ``supports_sampling_params: false``. The perplexity route is
exempt: it is OpenAI-compatible and maps sampling params upstream."""
variants = [
k
for k in cost_map
if any(v in k for v in ("claude-fable-5", "claude-opus-4-7", "claude-opus-4-8"))
and not k.startswith("perplexity/")
]
assert variants, "no matching entries found in cost map"
missing = [k for k in variants if cost_map[k].get("supports_sampling_params") is not False]
assert not missing, f"missing supports_sampling_params=false: {missing}"

46
tests/test_litellm/test_claude_haiku_4_5_config.py
View file

@ -1,46 +0,0 @@
"""
Test Claude Haiku 4.5 model configurations for Bedrock
https://github.com/BerriAI/litellm/issues/15818
"""
import json
import os
def test_bedrock_haiku_4_5_matches_sonnet_capabilities():
"""
Test that Haiku 4.5 has same capabilities as Sonnet 4.5
(including computer_use, vision, tools, etc.)
"""
# Load model configuration
json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
with open(json_path) as f:
model_data = json.load(f)
haiku_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
sonnet_model = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
haiku_info = model_data[haiku_model]
sonnet_info = model_data[sonnet_model]
# Both should use bedrock_converse
assert haiku_info["litellm_provider"] == "bedrock_converse"
assert sonnet_info["litellm_provider"] == "bedrock_converse"
# Shared capabilities that should match
shared_capabilities = [
"supports_vision",
"supports_computer_use",
"supports_function_calling",
"supports_tool_choice",
"supports_prompt_caching",
"supports_response_schema",
"supports_pdf_input",
"supports_assistant_prefill",
"supports_reasoning",
]
for capability in shared_capabilities:
assert haiku_info.get(capability) == sonnet_info.get(capability), (
f"Capability {capability} mismatch: Haiku={haiku_info.get(capability)}, Sonnet={sonnet_info.get(capability)}"
)

90
tests/test_litellm/test_claude_opus_4_6_config.py
View file

@ -2,100 +2,10 @@
Validate Claude Opus 4.6 model configuration entries.
"""
import json
import os
import litellm
def test_claude_4_6_australia_region_uses_au_prefix_not_apac():
"""
Test that Australia region Claude 4.6 models use 'au.' prefix instead of incorrect 'apac.' prefix.
AWS Bedrock cross-region inference uses specific regional prefixes:
- 'us.' for United States
- 'eu.' for Europe
- 'au.' for Australia (ap-southeast-2)
- 'apac.' for Asia-Pacific (Singapore, ap-southeast-1)
This test ensures the Claude 4.6 models correctly use 'au.' for Australia,
and that 'apac.' is NOT incorrectly used for Australia region.
Related: The 'apac.' prefix is valid for Asia-Pacific (Singapore) region models,
but should not be used for Australia which has its own 'au.' prefix.
"""
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
# Verify au.anthropic.claude-opus-4-6-v1 exists (correct)
assert (
"au.anthropic.claude-opus-4-6-v1" in model_data
), "Missing Australia region model: au.anthropic.claude-opus-4-6-v1"
# Verify apac.anthropic.claude-opus-4-6-v1 does NOT exist (incorrect)
assert (
"apac.anthropic.claude-opus-4-6-v1" not in model_data
), "Incorrect model entry exists: apac.anthropic.claude-opus-4-6-v1 should be au.anthropic.claude-opus-4-6-v1"
# Verify au.anthropic.claude-sonnet-4-6 exists (correct)
assert (
"au.anthropic.claude-sonnet-4-6" in model_data
), "Missing Australia region model: au.anthropic.claude-sonnet-4-6"
# Verify apac.anthropic.claude-sonnet-4-6 does NOT exist (incorrect)
assert (
"apac.anthropic.claude-sonnet-4-6" not in model_data
), "Incorrect model entry exists: apac.anthropic.claude-sonnet-4-6 should be au.anthropic.claude-sonnet-4-6"
# Verify the au. model is registered in bedrock_converse_models
assert (
"au.anthropic.claude-opus-4-6-v1" in litellm.bedrock_converse_models
), "au.anthropic.claude-opus-4-6-v1 not registered in bedrock_converse_models"
# Verify apac. is NOT registered for this model
assert (
"apac.anthropic.claude-opus-4-6-v1" not in litellm.bedrock_converse_models
), "apac.anthropic.claude-opus-4-6-v1 should not be in bedrock_converse_models"
# Verify the au. model is registered in bedrock_converse_models
assert (
"au.anthropic.claude-sonnet-4-6" in litellm.bedrock_converse_models
), "au.anthropic.claude-sonnet-4-6 not registered in bedrock_converse_models"
# Verify apac. is NOT registered for this model
assert (
"apac.anthropic.claude-sonnet-4-6" not in litellm.bedrock_converse_models
), "apac.anthropic.claude-sonnet-4-6 should not be in bedrock_converse_models"
def test_opus_4_6_alias_and_dated_metadata_match():
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
alias = model_data["claude-opus-4-6"]
dated = model_data["claude-opus-4-6-20260205"]
keys_to_match = [
"max_input_tokens",
"max_output_tokens",
"max_tokens",
"input_cost_per_token",
"output_cost_per_token",
"cache_creation_input_token_cost",
"cache_creation_input_token_cost_above_1hr",
"cache_read_input_token_cost",
"supports_assistant_prefill",
]
for key in keys_to_match:
assert alias[key] == dated[key], f"Mismatch for {key}"
def test_opus_4_6_bedrock_converse_registration():
assert "anthropic.claude-opus-4-6-v1" in litellm.BEDROCK_CONVERSE_MODELS
assert "global.anthropic.claude-opus-4-6-v1" in litellm.bedrock_converse_models

28
tests/test_litellm/test_claude_opus_4_8_config.py
View file

@ -11,43 +11,15 @@ for Anthropic, Bedrock, Vertex AI, and Azure AI; those entries are what populate
in ``get_llm_provider`` consumes.
"""
import json
import os
import pytest
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
def _load_root_cost_map() -> dict:
json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json")
with open(json_path) as f:
return json.load(f)
def test_opus_4_8_registered_for_bedrock_converse():
assert "anthropic.claude-opus-4-8" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_opus_4_8_all_variants_carry_adaptive_thinking_flag(cost_map):
"""Every Opus 4.8 entry must advertise ``supports_adaptive_thinking``.
Adaptive-thinking detection is cost-map driven, so a single variant missing
the flag silently sends the legacy ``thinking.type='enabled'`` shape and the
provider 400s (issue #29188, which the Bedrock/Vertex/Azure variants hit
because only the bare ``claude-opus-4-8`` entry carried the flag). This guards
against a future variant being added without it."""
variants = [k for k in cost_map if "claude-opus-4-8" in k]
assert variants, "no claude-opus-4-8 entries found in cost map"
missing = [
k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True
]
assert not missing, f"missing supports_adaptive_thinking: {missing}"

32
tests/test_litellm/test_claude_opus_5_config.py
View file

@ -12,13 +12,11 @@ validator accepts the full effort ladder, so the entries must not carry the
``anthropic/*`` wildcard deployment).
"""
import json
import os
import pytest
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
@ -45,12 +43,6 @@ BEDROCK_OPUS_5_VARIANTS = (
)
def _load_root_cost_map() -> dict:
json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json")
with open(json_path) as f:
return json.load(f)
@pytest.mark.parametrize("model_name", BEDROCK_OPUS_5_VARIANTS)
def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map):
"""Bedrock Converse routes Opus through a validator that rejects
@ -62,31 +54,7 @@ def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map):
assert bedrock_converse_supports_strict_tools(model_name) is False
def test_opus_5_present_in_bundled_backup():
"""The bundled backup is the runtime fallback (and what tests load with
``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the
root cost map, otherwise the model resolves on one path but not the other."""
backup = GetModelCostMap.load_local_model_cost_map()
for model_name in ALL_OPUS_5_VARIANTS:
assert model_name in backup, f"Missing from backup cost map: {model_name}"
def test_opus_5_registered_for_bedrock_converse():
assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_opus_5_all_variants_carry_adaptive_thinking_flag(cost_map):
"""Every Opus 5 entry must advertise ``supports_adaptive_thinking``.
Adaptive-thinking detection is cost-map driven, so a single variant missing
the flag silently sends the legacy ``thinking.type='enabled'`` shape, which
Opus 5 rejects with a 400."""
variants = [k for k in cost_map if "claude-opus-5" in k]
assert variants, "no claude-opus-5 entries found in cost map"
missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True]
assert not missing, f"missing supports_adaptive_thinking: {missing}"

38
tests/test_litellm/test_claude_sonnet_4_6_config.py
View file

@ -1,38 +0,0 @@
"""
Test Claude Sonnet 4.6 model configurations for Bedrock cross-region inference.
Pins the set of region-prefixed entries in model_prices_and_context_window.json
so future drops of a region (or pricing drift between regions) is caught.
https://github.com/BerriAI/litellm/issues/22972
"""
import json
import os
def test_bedrock_sonnet_4_6_jp_matches_other_regional_pricing():
"""The jp. cross-region inference profile shares pricing with the other
regional profiles (us./eu./au.), which carry a 10% premium over the
base/global entries.
"""
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
jp_info = model_data["jp.anthropic.claude-sonnet-4-6"]
au_info = model_data["au.anthropic.claude-sonnet-4-6"]
pricing_fields = [
"input_cost_per_token",
"output_cost_per_token",
"cache_creation_input_token_cost",
"cache_read_input_token_cost",
]
for field in pricing_fields:
assert jp_info[field] == au_info[field], (
f"{field} mismatch between jp. and au. variants: "
f"jp={jp_info[field]}, au={au_info[field]}"
)

33
tests/test_litellm/test_claude_sonnet_5_config.py
View file

@ -10,13 +10,10 @@ populate ``litellm.anthropic_models`` at import, which is what lets a bare
``anthropic/*`` wildcard deployment).
"""
import json
import os
import pytest
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
@ -34,37 +31,7 @@ ALL_SONNET_5_VARIANTS = (
)
def _load_root_cost_map() -> dict:
json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json")
with open(json_path) as f:
return json.load(f)
def test_sonnet_5_present_in_bundled_backup():
"""The bundled backup is the runtime fallback (and what tests load with
``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the
root cost map, otherwise the model resolves on one path but not the other."""
backup = GetModelCostMap.load_local_model_cost_map()
for model_name in ALL_SONNET_5_VARIANTS:
assert model_name in backup, f"Missing from backup cost map: {model_name}"
def test_sonnet_5_registered_for_bedrock_converse():
assert "anthropic.claude-sonnet-5" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_sonnet_5_all_variants_carry_adaptive_thinking_flag(cost_map):
"""Every Sonnet 5 entry must advertise ``supports_adaptive_thinking``.
Adaptive-thinking detection is cost-map driven, so a single variant missing
the flag silently sends the legacy ``thinking.type='enabled'`` shape and the
provider 400s. This guards against a future variant being added without it."""
variants = [k for k in cost_map if "claude-sonnet-5" in k]
assert variants, "no claude-sonnet-5 entries found in cost map"
missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True]
assert not missing, f"missing supports_adaptive_thinking: {missing}"

41
tests/test_litellm/test_cost_calculator.py
View file

@ -27,7 +27,6 @@ from litellm.types.utils import (
PromptTokensDetailsWrapper,
Usage,
)
from litellm.utils import TranscriptionResponse
@pytest.fixture
@ -2375,28 +2374,6 @@ def test_anthropic_geo_and_fast_multipliers_compose(_local_model_cost_map, monke
assert completion_cost == pytest.approx(500 * 25e-6 * 2.0 * 1.1)
@pytest.mark.parametrize(
"model,expected_fast",
[
("claude-opus-5", 2.0),
("claude-opus-4-8", 2.0),
("claude-opus-4-6", None),
("claude-opus-4-6-20260205", None),
("claude-opus-4-7", None),
("claude-opus-4-7-20260416", None),
],
)
def test_anthropic_fast_multiplier_only_on_models_with_fast_mode(_local_model_cost_map, model, expected_fast):
"""
Anthropic serves fast mode on Opus 5 and Opus 4.8 only, at 2x. Opus 4.6 and
4.7 accept the ``speed`` request param but are always served standard, so a
``fast`` multiplier on their map entries overbills every request that asked
for fast and was served standard.
"""
entry = litellm.model_cost[model]
assert entry["provider_specific_entry"].get("fast") == expected_fast
@pytest.mark.parametrize(
"model",
["claude-sonnet-4-6", "claude-mythos-5", "claude-mythos-preview"],
@ -3376,24 +3353,6 @@ def test_combine_usage_objects_sums_mirrored_cache_write_fields_once():
assert combined_pair.prompt_tokens_details.cache_creation_tokens == 100
def _together_chat_response(
model: str, prompt_tokens: int, completion_tokens: int, cached_tokens: int
) -> ModelResponse:
return ModelResponse(
id="chatcmpl-together-cache",
choices=[{"finish_reason": "stop", "index": 0, "message": {"content": "acknowledged", "role": "assistant"}}],
created=1756164000,
model=model,
object="chat.completion",
usage=Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens),
),
)
def test_select_model_name_strips_unregistered_alias_prefix(_local_model_cost_map):
"""A router-facing model_name alias containing "/" whose leading segment is NOT a
registered provider must not be double-prefixed into a non-existent cost key.

37
tests/test_litellm/test_dashscope_image_generation.py
View file

@ -5,7 +5,6 @@ qwen-image-3.0, qwen-image-3.0-pro).
Run in docker: pytest tests/test_litellm/test_dashscope_image_generation.py -v
"""
import json
from unittest.mock import MagicMock, patch
import httpx
@ -16,7 +15,7 @@ from litellm.llms.dashscope.image_generation.transformation import (
DashScopeImageGenerationConfig,
DEFAULT_API_BASE,
)
from litellm.types.utils import ImageObject, ImageResponse
from litellm.types.utils import ImageResponse
from litellm.utils import get_llm_provider
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -46,40 +45,6 @@ def test_get_llm_provider_returns_dashscope(model_string: str):
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"model_string, custom_provider",
[
("dashscope/qwen-image-2.0", "dashscope"),
("dashscope/qwen-image-2.0-pro", "dashscope"),
("dashscope/qwen-image-3.0", "dashscope"),
("dashscope/qwen-image-3.0-pro", "dashscope"),
],
)
def test_get_model_info_mode_is_image_generation(
model_string: str, custom_provider: str
):
import os
prev_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
prev_model_cost = litellm.model_cost
try:
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
info = litellm.get_model_info(
model=model_string, custom_llm_provider=custom_provider
)
assert (
info["mode"] == "image_generation"
), f"Expected mode='image_generation', got '{info['mode']}'"
finally:
if prev_env is None:
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
else:
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = prev_env
litellm.model_cost = prev_model_cost
# ---------------------------------------------------------------------------
# 3. Request transformation
# ---------------------------------------------------------------------------

13
tests/test_litellm/test_deepseek_model_metadata.py
View file

@ -15,7 +15,6 @@ import os
import litellm
from litellm.utils import (
_supports_factory,
supports_response_schema,
)
# ---------------------------------------------------------------------------
@ -59,18 +58,6 @@ class TestSupportsResponseSchemaDeepSeek:
"""All calling conventions for DeepSeek should return True for
``supports_response_schema``."""
def test_provider_slash_model(self):
assert supports_response_schema(model="deepseek/deepseek-chat") is True
def test_explicit_provider(self):
assert supports_response_schema(model="deepseek-chat", custom_llm_provider="deepseek") is True
def test_reasoner_provider_slash_model(self):
assert supports_response_schema(model="deepseek/deepseek-reasoner") is True
def test_reasoner_explicit_provider(self):
assert supports_response_schema(model="deepseek-reasoner", custom_llm_provider="deepseek") is True
# ---------------------------------------------------------------------------
# Fallback-logic test – bare model entry used when prefixed is incomplete

11
tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py
View file

@ -4,7 +4,6 @@ from pathlib import Path
import pytest
import litellm
from litellm.utils import supports_prompt_caching, supports_reasoning
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
@ -33,16 +32,6 @@ def local_model_cost_map(monkeypatch):
litellm.get_model_info.cache_clear()
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
def test_zai_glm_5_2_capabilities_are_visible_to_callers(local_model_cost_map, model):
"""Mistral advertises reasoning and prompt caching on this model, so the helpers
every caller checks before sending a request must say so too."""
assert supports_reasoning(model=model) is True
assert supports_prompt_caching(model=model) is True
assert litellm.get_model_info(model=model)
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
def test_backup_matches_main(model):
"""Ensure the bundled (backup) cost map stays in sync with the canonical file."""

537
tests/test_litellm/test_utils.py
View file

@ -162,15 +162,6 @@ def test_prompt_tokens_details_cache_write_creation_stay_in_sync_on_assignment()
assert details.cache_write_tokens == details.cache_creation_tokens == 375
def test_get_model_info_surfaces_supported_endpoints(local_model_cost_map):
"""supported_endpoints ships in the cost map and is declared on ModelInfoBase,
but the constructor never copied it, so get_model_info always returned None.
The realtime health check reads it to spot GA-only transcription models
(LIT-6240)."""
info = litellm.get_model_info(model="gpt-realtime-whisper", custom_llm_provider="azure")
assert info["supported_endpoints"] == ["/v1/realtime", "/v1/realtime/transcription_sessions"]
def test_potential_model_names_keeps_provider_prefixed_candidate():
"""A provider whose own model ids repeat the litellm provider name (Perplexity's
Agent API serves `perplexity/glm-5.2`, mapped as `perplexity/perplexity/glm-5.2`)
@ -236,23 +227,6 @@ def test_check_provider_match_github_allows_upstream_provider_metadata():
)
def test_supports_function_calling_github_openai_alias():
assert litellm.utils.supports_function_calling(model="github/gpt-4o-mini") is True
assert litellm.utils.supports_function_calling(model="gpt-4o-mini", custom_llm_provider="github") is True
def test_supports_function_calling_github_anthropic_alias():
assert litellm.utils.supports_function_calling(model="github/claude-3-7-sonnet-20250219") is True
def test_supports_function_calling_deepinfra_llama():
"""Test that deepinfra Llama models correctly report function calling support.
Regression test for https://github.com/BerriAI/litellm/issues/22619
"""
assert litellm.utils.supports_function_calling(model="deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo") is True
def test_supports_function_calling_unknown_github_alias_returns_false():
assert litellm.utils.supports_function_calling(model="github/non-existent-model-for-capability-check") is False
@ -565,25 +539,6 @@ def test_all_model_configs():
) == {"max_output_tokens": 10}
def test_anthropic_web_search_in_model_info(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
supported_models = [
"anthropic/claude-4-sonnet-20250514",
"anthropic/claude-sonnet-4-5-20250929",
]
for model in supported_models:
from litellm.utils import get_model_info
model_info = get_model_info(model)
assert model_info is not None
assert model_info["supports_web_search"] is True, f"Model {model} should support web search"
assert model_info["search_context_cost_per_query"] is not None, (
f"Model {model} should have a search context cost per query"
)
def test_cohere_embedding_optional_params():
from litellm import get_optional_params_embeddings
@ -1129,13 +1084,6 @@ def test_get_model_info_bedrock_regional_inference_profile_pricing(local_model_c
assert control["key"] == "au.anthropic.claude-opus-4-8"
def test_get_model_info_bedrock_double_provider_prefix_resolves(local_model_cost_map):
"""A doubled bedrock/ prefix routes at runtime via strip_bedrock_routing_prefix,
so model info must resolve it to the same entry the request actually bills as."""
info = litellm.get_model_info(model="bedrock/bedrock/us.anthropic.claude-sonnet-4-6")
assert info["key"] == "us.anthropic.claude-sonnet-4-6"
def test_openai_models_in_model_info(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
@ -1149,51 +1097,6 @@ def test_openai_models_in_model_info(monkeypatch):
assert len(violated_models) == 0, f"The following models should support pdf input: {violated_models}"
def test_supports_tool_choice_simple_tests():
"""
simple sanity checks
"""
assert litellm.utils.supports_tool_choice(model="gpt-4o") == True
assert litellm.utils.supports_tool_choice(model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0") == True
assert litellm.utils.supports_tool_choice(model="anthropic.claude-3-sonnet-20240229-v1:0") is True
assert (
litellm.utils.supports_tool_choice(
model="anthropic.claude-3-sonnet-20240229-v1:0",
custom_llm_provider="bedrock_converse",
)
is True
)
assert litellm.utils.supports_tool_choice(model="perplexity/sonar") is False
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize(
"model",
[
"amazon.nova-lite-v1:0",
"amazon.nova-micro-v1:0",
"amazon.nova-pro-v1:0",
"apac.amazon.nova-lite-v1:0",
"apac.amazon.nova-micro-v1:0",
"apac.amazon.nova-pro-v1:0",
"bedrock/us-gov-east-1/amazon.nova-pro-v1:0",
"bedrock/us-gov-west-1/amazon.nova-lite-v1:0",
"bedrock/us-gov-west-1/amazon.nova-micro-v1:0",
"bedrock/us-gov-west-1/amazon.nova-pro-v1:0",
"eu.amazon.nova-lite-v1:0",
"eu.amazon.nova-micro-v1:0",
"eu.amazon.nova-pro-v1:0",
"us.amazon.nova-lite-v1:0",
"us.amazon.nova-micro-v1:0",
"us.amazon.nova-pro-v1:0",
],
)
def test_amazon_nova_v1_understanding_models_support_tool_choice(model: str) -> None:
assert litellm.utils.supports_tool_choice(model=model) is True
def test_check_provider_match():
"""
Test the _check_provider_match function for various provider scenarios
@ -1303,42 +1206,6 @@ for commitment in BEDROCK_COMMITMENTS:
print("block_list", block_list)
def test_supports_computer_use_utility(monkeypatch):
"""
Tests the litellm.utils.supports_computer_use utility function.
"""
from litellm.utils import supports_computer_use
# Ensure LITELLM_LOCAL_MODEL_COST_MAP is set for consistent test behavior,
# as supports_computer_use relies on get_model_info.
# This also requires litellm.model_cost to be populated.
original_env_var = os.getenv("LITELLM_LOCAL_MODEL_COST_MAP")
original_model_cost = getattr(litellm, "model_cost", None)
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="") # Load with local/backup
try:
# Test a model known to support computer_use from backup JSON
supports_cu_anthropic = supports_computer_use(model="anthropic/claude-4-sonnet-20250514")
assert supports_cu_anthropic is True
# Test a model known not to have the flag or set to false (defaults to False via get_model_info)
supports_cu_gpt = supports_computer_use(model="gpt-3.5-turbo")
assert supports_cu_gpt is False
finally:
# Restore original environment and model_cost to avoid side effects
if original_env_var is None:
del os.environ["LITELLM_LOCAL_MODEL_COST_MAP"]
else:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", original_env_var)
if original_model_cost is not None:
litellm.model_cost = original_model_cost
elif hasattr(litellm, "model_cost"):
delattr(litellm, "model_cost")
@pytest.mark.parametrize(
"model, custom_llm_provider",
[
@ -1658,32 +1525,6 @@ class TestProxyFunctionCalling:
# For now, we expect False (current behavior), but document the limitation
assert proxy_result is False, f"Current limitation: {proxy_model_with_hints} returns False without inference"
@pytest.mark.parametrize(
"proxy_model,expected_result",
[
# Test specific proxy models that should support function calling
("litellm_proxy/gpt-3.5-turbo", True),
("litellm_proxy/gpt-4", True),
("litellm_proxy/gpt-4o", True),
("litellm_proxy/claude-sonnet-4-6", True),
("litellm_proxy/gemini/gemini-2.5-pro", True),
# Test proxy models that should not support function calling
("litellm_proxy/command-nightly", False),
("litellm_proxy/anthropic.claude-instant-v1", False),
],
)
def test_proxy_only_function_calling_support(self, proxy_model, expected_result):
"""
Test proxy models independently to ensure they report correct function calling support.
This test focuses on proxy models without comparing to direct models,
useful for cases where we only care about the proxy behavior.
"""
try:
result = supports_function_calling(model=proxy_model)
assert result == expected_result, f"Proxy model {proxy_model} returned {result}, expected {expected_result}"
except Exception as e:
pytest.fail(f"Error testing proxy model {proxy_model}: {e}")
def test_litellm_utils_supports_function_calling_import(self):
"""Test that supports_function_calling can be imported from litellm.utils."""
@ -1704,28 +1545,6 @@ class TestProxyFunctionCalling:
except Exception as e:
pytest.fail(f"Failed to access litellm.supports_function_calling: {e}")
@pytest.mark.parametrize(
"model_name",
[
"litellm_proxy/gpt-3.5-turbo",
"litellm_proxy/gpt-4",
"litellm_proxy/claude-sonnet-4-6",
"litellm_proxy/gemini/gemini-2.5-pro",
],
)
def test_proxy_model_with_custom_llm_provider_none(self, model_name):
"""
Test proxy models with custom_llm_provider=None parameter.
This tests the supports_function_calling function with the custom_llm_provider
parameter explicitly set to None, which is a common usage pattern.
"""
try:
result = supports_function_calling(model=model_name, custom_llm_provider=None)
# All the models in this test should support function calling
assert result is True, f"Model {model_name} should support function calling but returned {result}"
except Exception as e:
pytest.fail(f"Error testing {model_name} with custom_llm_provider=None: {e}")
def test_edge_cases_and_malformed_proxy_models(self):
"""Test edge cases and malformed proxy model names."""
@ -1963,84 +1782,6 @@ class TestProxyFunctionCalling:
f"(without config context). Description: {description}"
)
def test_real_world_proxy_config_documentation(self):
"""
Document how real-world proxy configurations would handle model mappings.
This test provides documentation on how the proxy server configuration
would typically map custom model names to underlying models.
"""
print("""
REAL-WORLD PROXY SERVER CONFIGURATION EXAMPLE:
===============================================
In a proxy_server_config.yaml file, you would define:
model_list:
- model_name: bedrock-claude-3-haiku
litellm_params:
model: bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
- model_name: bedrock-claude-3-sonnet
litellm_params:
model: bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
- model_name: prod-claude-haiku
litellm_params:
model: bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0
aws_access_key_id: os.environ/PROD_AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/PROD_AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
FUNCTION CALLING WITH PROXY SERVER:
===================================
When using the proxy server with this configuration:
1. Client calls: supports_function_calling("bedrock-claude-3-haiku")
2. Proxy server resolves to: bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0
3. LiteLLM evaluates the underlying model's capabilities
4. Returns: True (because Claude 3 Haiku supports function calling)
Without the proxy server configuration context, LiteLLM cannot resolve
the custom model name and returns False.
BEDROCK CONVERSE API BENEFITS:
==============================
The Bedrock Converse API provides:
- Standardized function calling interface across providers
- Better tool use capabilities compared to legacy APIs
- Consistent request/response format
- Enhanced streaming support for function calls
""")
# Verify that direct underlying models work as expected
bedrock_models = [
"bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0",
"bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0",
"bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
]
for model in bedrock_models:
try:
result = supports_function_calling(model)
print(f"Direct test - {model}: {result}")
# Claude 3 models should support function calling
assert result is True, f"Claude 3 model should support function calling: {model}"
except Exception as e:
print(f"Could not test {model}: {e}")
def test_register_model_with_scientific_notation():
"""
@ -3637,28 +3378,6 @@ _FIREWORKS_ROUTER_SHORT_FORMS = [
]
def _assert_fireworks_entry(
model_cost,
model_path,
expected_max_input,
expected_max_output,
expected_vision,
expected_reasoning,
):
info = model_cost.get(f"fireworks_ai/{model_path}")
assert info is not None, f"fireworks_ai/{model_path} missing from model cost map"
assert info["litellm_provider"] == "fireworks_ai"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] > 0
assert info["output_cost_per_token"] > 0
assert "cache_read_input_token_cost" in info
assert info["supports_function_calling"] is True
assert info["supports_tool_choice"] is True
assert info["supports_reasoning"] is expected_reasoning
assert info["supports_response_schema"] is True
assert info["supports_vision"] is expected_vision
@pytest.fixture
def fireworks_short_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
monkeypatch.setattr(
@ -3985,21 +3704,6 @@ def test_get_prompt_cache_min_tokens_resolves_per_model(
assert get_prompt_cache_min_tokens(model=model) == expected_min_tokens
def test_get_prompt_cache_min_tokens_uniform_for_fable_5_across_platforms(local_model_cost_map: None) -> None:
"""Anthropic removed the Amazon Bedrock override for Claude Fable 5, so its 512-token minimum
now applies on every platform. The Bedrock entries carried the old 1024 and the re-export
entries carried nothing, so the router judged 512-1023-token prefixes uncacheable and skipped
prompt-cache-affinity routing for prompts the provider demonstrably caches (issue #35011)."""
wrong: Final = {
model: get_prompt_cache_min_tokens(model=model)
for model, info in litellm.model_cost.items()
if "fable-5" in model
and info.get("supports_prompt_caching")
and get_prompt_cache_min_tokens(model=model) != 512
}
assert not wrong, f"every Claude Fable 5 entry must carry prompt_cache_min_tokens 512: {wrong}"
ANTHROPIC_REEXPORT_CACHE_MIN: Final = {
"azure_ai/claude-fable-5": 512,
"azure_ai/claude-haiku-4-5": 4096,
@ -4048,21 +3752,6 @@ ANTHROPIC_REEXPORT_CACHE_MIN: Final = {
}
def test_anthropic_reexport_entries_carry_explicit_prompt_cache_min_tokens(local_model_cost_map: None) -> None:
"""Regression for issue #35011: these re-export entries carried no prompt_cache_min_tokens, so
they silently inherited the 1024 default. That skipped cache-affinity routing for Fable 5's
512-1023-token prefixes and reported 1024-4095-token prompts as cacheable on the 2048/4096
models. The entry must be explicit so a default change can never re-break them, which is why
this asserts the cost-map value itself and not just the resolver's answer."""
wrong: Final = {
model: (litellm.model_cost[model].get("prompt_cache_min_tokens"), get_prompt_cache_min_tokens(model=model))
for model, expected in ANTHROPIC_REEXPORT_CACHE_MIN.items()
if litellm.model_cost[model].get("prompt_cache_min_tokens") != expected
or get_prompt_cache_min_tokens(model=model) != expected
}
assert not wrong, f"(cost-map value, resolved value) diverge from Anthropic's published minimums: {wrong}"
GEMINI_4096_CACHE_MIN_MODELS: Final = tuple(
prefix + base
for base in (
@ -5981,82 +5670,6 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th
assert snapshot["api_base"]
def test_fireworks_models_in_backup_cost_map():
import json
from pathlib import Path
json_path = Path(__file__).parents[2] / "litellm" / "model_prices_and_context_window_backup.json"
with open(json_path) as f:
model_cost = json.load(f)
for entry in _FIREWORKS_MODELS:
_assert_fireworks_entry(model_cost, *entry)
for short in _FIREWORKS_SHORT_FORMS:
long_key = f"fireworks_ai/accounts/fireworks/models/{short}"
short_key = f"fireworks_ai/{short}"
assert model_cost.get(short_key) == model_cost.get(long_key), (
f"short-form {short_key} does not match long-form {long_key}"
)
for short in _FIREWORKS_ROUTER_SHORT_FORMS:
long_key = f"fireworks_ai/accounts/fireworks/routers/{short}"
short_key = f"fireworks_ai/{short}"
assert model_cost.get(short_key) == model_cost.get(long_key), (
f"short-form {short_key} does not match long-form {long_key}"
)
def test_fireworks_models_in_cost_map():
import json
from pathlib import Path
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
with open(json_path) as f:
model_cost = json.load(f)
for entry in _FIREWORKS_MODELS:
_assert_fireworks_entry(model_cost, *entry)
for short in _FIREWORKS_SHORT_FORMS:
long_key = f"fireworks_ai/accounts/fireworks/models/{short}"
short_key = f"fireworks_ai/{short}"
assert model_cost.get(short_key) == model_cost.get(long_key), (
f"short-form {short_key} does not match long-form {long_key}"
)
for short in _FIREWORKS_ROUTER_SHORT_FORMS:
long_key = f"fireworks_ai/accounts/fireworks/routers/{short}"
short_key = f"fireworks_ai/{short}"
assert model_cost.get(short_key) == model_cost.get(long_key), (
f"short-form {short_key} does not match long-form {long_key}"
)
def test_fireworks_short_model_names_resolve_to_long_cost_map_keys(fireworks_short_model_cost_map: None) -> None:
model_info = litellm.get_model_info("fireworks_ai/glm-5p3")
assert model_info["key"] == "fireworks_ai/accounts/fireworks/models/glm-5p3"
model_info = litellm.get_model_info("glm-5p3", custom_llm_provider="fireworks_ai")
assert model_info["key"] == "fireworks_ai/accounts/fireworks/models/glm-5p3"
model_info = litellm.get_model_info("fireworks_ai/glm-5p3-fast")
assert model_info["key"] == "fireworks_ai/accounts/fireworks/routers/glm-5p3-fast"
model_info = litellm.get_model_info("fireworks_ai/nomic-ai/nomic-embed-text-v1.5")
assert model_info["key"] == "fireworks_ai/nomic-ai/nomic-embed-text-v1.5"
with pytest.raises(Exception, match="isn't mapped"):
litellm.get_model_info("fireworks_ai/does-not-exist")
def test_get_model_info_bedrock_regional_profile_without_entry_falls_back_to_base(local_model_cost_map):
"""A regional profile with no dedicated cost-map entry must still resolve to its
region-stripped base entry."""
info = litellm.get_model_info(model="bedrock/apac.anthropic.claude-opus-4-8")
assert info["key"] == "anthropic.claude-opus-4-8"
def test_get_model_info_gemini(monkeypatch):
"""
Tests if ALL gemini models have 'tpm' and 'rpm' in the model info
@ -6079,153 +5692,3 @@ def test_get_model_info_gemini(monkeypatch):
assert info.get("rpm") is not None, f"{model} does not have rpm"
def test_get_model_info_resolves_provider_prefixed_model_ids(local_model_cost_map):
"""Perplexity's Agent API third-party models are keyed `perplexity/perplexity/<id>`
because Perplexity's own id already starts with `perplexity/`. Callers run
`get_llm_provider` first, which hands `_get_potential_model_names` model
`perplexity/glm-5.2` with provider `perplexity`, and every candidate but the
provider-prefixed one strips that second `perplexity/` off. Regression: the
entries were unreachable from `supports_reasoning` and from the cost calculator's
per-token fallback, so a mapped model reported no reasoning support and raised
"This model isn't mapped yet" on the only path where its rates are ever used."""
for model, reasoning in (
("perplexity/perplexity/glm-5.2", True),
("perplexity/perplexity/kimi-k3", True),
("perplexity/perplexity/deepseek-v4-flash-0731", True),
("perplexity/perplexity/kimi-k2.7-code", False),
("perplexity/perplexity/nemotron-3.5-lightning-30b-a3b", True),
("perplexity/perplexity/nemotron-3-ultra-550b-a55b", True),
):
assert litellm.supports_reasoning(model=model) is reasoning, model
via_provider = litellm.get_model_info(model="perplexity/glm-5.2", custom_llm_provider="perplexity")
assert via_provider["key"] == "perplexity/perplexity/glm-5.2"
assert via_provider["mode"] == "responses"
lightning = litellm.get_model_info(
model="perplexity/nemotron-3.5-lightning-30b-a3b", custom_llm_provider="perplexity"
)
assert lightning["key"] == "perplexity/perplexity/nemotron-3.5-lightning-30b-a3b"
assert lightning["mode"] == "responses"
ultra = litellm.get_model_info(model="perplexity/perplexity/nemotron-3-ultra-550b-a55b")
assert ultra["key"] == "perplexity/perplexity/nemotron-3-ultra-550b-a55b"
def test_get_model_info_shows_supports_computer_use(monkeypatch):
"""
Tests if 'supports_computer_use' is correctly retrieved by get_model_info.
We'll use 'claude-4-sonnet-20250514' as it's configured
in the backup JSON to have supports_computer_use: True.
"""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
# Ensure litellm.model_cost is loaded, relying on the backup mechanism if primary fails
# as per previous debugging.
litellm.model_cost = litellm.get_model_cost_map(url="")
# This model should have 'supports_computer_use': True in the backup JSON
model_known_to_support_computer_use = "claude-4-sonnet-20250514"
info = litellm.get_model_info(model_known_to_support_computer_use)
# After the fix in utils.py, this should now be present and True
assert info.get("supports_computer_use") is True
def test_get_model_info_surfaces_supports_adaptive_thinking(local_model_cost_map):
"""supports_adaptive_thinking must flow through get_model_info like every other
capability flag: both from an explicit cost-map entry and from a
fallback-generalization rule for an unmapped model. Regression: the field shipped
in the JSON but was never declared on ModelInfo nor copied during construction, so
get_model_info (and _supports_factory) silently dropped it for any provider-prefixed
or unmapped name."""
explicit = litellm.get_model_info(model="claude-opus-4-8")
assert explicit["supports_adaptive_thinking"] is True
generalized = litellm.get_model_info(model="claude-opus-4-9", custom_llm_provider="anthropic")
assert generalized["supports_adaptive_thinking"] is True
def test_get_model_info_surfaces_supports_parallel_function_calling(local_model_cost_map):
"""A registry entry's supports_parallel_function_calling must read back through get_model_info
and litellm.supports_parallel_function_calling. Regression: the key was never copied into
ModelInfo, so provider-prefixed entries read None / False even when the map said True, and an
explicit False was indistinguishable from unset."""
declared_true = litellm.get_model_info(model="together_ai/zai-org/GLM-5.3-Flash")
assert declared_true["supports_parallel_function_calling"] is True
assert litellm.supports_parallel_function_calling(model="together_ai/zai-org/GLM-5.3-Flash") is True
def test_model_info_for_fireworks_short_form_models():
"""
Test that fireworks_ai short-form model entries (fireworks_ai/<model>)
are correctly configured in model_prices_and_context_window.json.
These entries enable cost attribution for models called via short-form
names (e.g., fireworks_ai/glm-4p7 instead of
fireworks_ai/accounts/fireworks/models/glm-4p7).
"""
import json
from pathlib import Path
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
with open(json_path) as f:
model_cost = json.load(f)
# glm-4p7: short-form and long-form
for key in [
"fireworks_ai/glm-4p7",
"fireworks_ai/accounts/fireworks/models/glm-4p7",
]:
info = model_cost.get(key)
assert info is not None, f"{key} not found in model_prices_and_context_window.json"
assert info["litellm_provider"] == "fireworks_ai"
assert info["mode"] == "chat"
assert info["supports_reasoning"] is True
# minimax-m2p1: short-form and long-form
for key in [
"fireworks_ai/minimax-m2p1",
"fireworks_ai/accounts/fireworks/models/minimax-m2p1",
]:
info = model_cost.get(key)
assert info is not None, f"{key} not found in model_prices_and_context_window.json"
assert info["litellm_provider"] == "fireworks_ai"
assert info["mode"] == "chat"
# kimi-k2p5: short-form only (long-form already existed)
info = model_cost.get("fireworks_ai/kimi-k2p5")
assert info is not None, "fireworks_ai/kimi-k2p5 not found in model_prices_and_context_window.json"
assert info["litellm_provider"] == "fireworks_ai"
assert info["mode"] == "chat"
def test_model_info_for_vertex_ai_deepseek_model():
model_info = litellm.get_model_info(model="vertex_ai/deepseek-ai/deepseek-r1-0528-maas")
assert model_info is not None
assert model_info["litellm_provider"] == "vertex_ai-deepseek_models"
assert model_info["mode"] == "chat"
assert model_info["input_cost_per_token"] is not None
assert model_info["output_cost_per_token"] is not None
def test_provider_prefixed_lookup_never_outranks_an_existing_row(local_model_cost_map):
"""The provider-prefixed candidate is tried last, after every candidate that
already existed, so no model that resolves today can change answer. `perplexity/sonar`
is the case that proves it: both `perplexity/sonar` and `perplexity/perplexity/sonar`
are cost-map keys, and the shorter one must keep winning."""
sonar = litellm.get_model_info(model="sonar", custom_llm_provider="perplexity")
assert sonar["key"] == "perplexity/sonar"
assert sonar["mode"] == "chat"
still_sonar = litellm.get_model_info(model="perplexity/sonar", custom_llm_provider="perplexity")
assert still_sonar["key"] == "perplexity/sonar"
assert still_sonar["mode"] == "chat"
for model, provider, expected_key in (
("claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"),
("anthropic/claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"),
("gemini/gemini-2.0-flash", "gemini", "gemini/gemini-2.0-flash"),
("openrouter/openai/gpt-4o", "openrouter", "openrouter/openai/gpt-4o"),
):
assert litellm.get_model_info(model=model, custom_llm_provider=provider)["key"] == expected_key