mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-23 00:41:40 +00:00
8ecbf3dbc1
The repo rule is that a test must only fail when litellm code changes, never when a vendor updates a price, renames a field, or drops a model. These tests asserted shipped catalog entries directly, comparing lookup results to literals copied from model_prices_and_context_window.json or requiring named entries to exist or be absent, so every cost map sync could break them without any litellm code changing Tests that exercise real litellm behavior with an injected local model_cost, invariants like backup parity, and assertions on non-lookup code paths are untouched Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
111 lines
4.5 KiB
Python
111 lines
4.5 KiB
Python
"""
|
||
Regression tests for #20885 – ``supports_response_schema`` (and related
|
||
capability flags) must be consistent between the bare model-name entry
|
||
(e.g. ``deepseek-chat``) and the provider-prefixed entry
|
||
(e.g. ``deepseek/deepseek-chat``) in the model-cost map.
|
||
|
||
The bug caused ``supports_response_schema("deepseek/deepseek-chat")`` to
|
||
return ``False`` even though the canonical ``deepseek-chat`` entry has the
|
||
field set to ``True``.
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
|
||
import litellm
|
||
from litellm.utils import (
|
||
_supports_factory,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Data-level tests – verify the JSON files are in sync
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _load_backup_json() -> dict:
|
||
"""Load the backup JSON directly from disk."""
|
||
backup_path = os.path.join(
|
||
os.path.dirname(litellm.__file__),
|
||
"model_prices_and_context_window_backup.json",
|
||
)
|
||
with open(backup_path, encoding="utf-8") as f:
|
||
return json.load(f)
|
||
|
||
|
||
class TestDeepSeekModelCostEntries:
|
||
"""Verify that provider-prefixed DeepSeek entries contain the same
|
||
capability flags as their bare-name counterparts in the JSON files."""
|
||
|
||
def test_deepseek_chat_max_input_tokens_matches_bare_in_backup(self):
|
||
data = _load_backup_json()
|
||
bare = data.get("deepseek-chat", {})
|
||
prefixed = data.get("deepseek/deepseek-chat", {})
|
||
assert prefixed.get("max_input_tokens") == bare.get("max_input_tokens")
|
||
|
||
def test_deepseek_reasoner_max_output_tokens_matches_bare_in_backup(self):
|
||
data = _load_backup_json()
|
||
bare = data.get("deepseek-reasoner", {})
|
||
prefixed = data.get("deepseek/deepseek-reasoner", {})
|
||
assert prefixed.get("max_output_tokens") == bare.get("max_output_tokens")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# API-level tests – verify supports_response_schema returns True
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestSupportsResponseSchemaDeepSeek:
|
||
"""All calling conventions for DeepSeek should return True for
|
||
``supports_response_schema``."""
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fallback-logic test – bare model entry used when prefixed is incomplete
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestBareModelFallback:
|
||
"""When a provider-prefixed entry is missing a capability flag, the
|
||
``_supports_factory`` fallback should consult the bare model-name
|
||
entry in ``litellm.model_cost``."""
|
||
|
||
def test_fallback_uses_bare_entry(self):
|
||
"""Temporarily remove ``supports_response_schema`` from the prefixed
|
||
entry and verify the fallback still returns True."""
|
||
key = "deepseek/deepseek-chat"
|
||
original = litellm.model_cost.get(key, {}).get("supports_response_schema")
|
||
try:
|
||
# Simulate the pre-fix state: field missing from prefixed entry
|
||
if key in litellm.model_cost:
|
||
litellm.model_cost[key].pop("supports_response_schema", None)
|
||
result = _supports_factory(
|
||
model="deepseek-chat",
|
||
custom_llm_provider="deepseek",
|
||
key="supports_response_schema",
|
||
)
|
||
assert result is True
|
||
finally:
|
||
# Restore
|
||
if key in litellm.model_cost and original is not None:
|
||
litellm.model_cost[key]["supports_response_schema"] = original
|
||
|
||
def test_no_fallback_when_explicitly_false(self):
|
||
"""If the prefixed entry explicitly sets a capability to ``False``,
|
||
the fallback must NOT override it."""
|
||
key = "deepseek/deepseek-reasoner"
|
||
# After the data fix, deepseek/deepseek-reasoner has
|
||
# supports_function_calling=false (matching the bare entry).
|
||
# Explicitly set it to False to test the guard.
|
||
original = litellm.model_cost.get(key, {}).get("supports_function_calling")
|
||
try:
|
||
if key in litellm.model_cost:
|
||
litellm.model_cost[key]["supports_function_calling"] = False
|
||
result = _supports_factory(
|
||
model="deepseek-reasoner",
|
||
custom_llm_provider="deepseek",
|
||
key="supports_function_calling",
|
||
)
|
||
assert result is False
|
||
finally:
|
||
if key in litellm.model_cost and original is not None:
|
||
litellm.model_cost[key]["supports_function_calling"] = original
|