mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-29 01:42:19 +00:00
f6882246d4
* ci: run the unit_selection.sh shard files on every event instead of only fork pull requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: rename fork-flag to unit-flag now that it applies on every event * test: move tests/test_litellm root and small trees into tests/unit Pure renames, no content changes. Follow-up commits in this PR fix references, merge the three files that already existed in tests/unit, keep live-provider tests in tests/test_litellm and wire CI. * test: carry tests/test_litellm conftest isolation into tests/unit Callback lists, routing fallbacks, cached HTTP clients, logger state, AWS, proxy-URL and keychain env, and session-end client cleanup now reset for unit tests too. The environment isolation owns its MonkeyPatch so a test's own monkeypatch is undone before the model-cost teardown runs. * test: merge, split and prune the moved root and small-tree tests Merge batches/test_batch_utils.py and the chat_completions and messages dispatch tests into the files that already existed in tests/unit. Keep the live Gemini interactions tests, the async image-fetch format test and the OpenAI embedding scorer test in tests/test_litellm since they need real network or keys. Put test_router.py under tests/unit/test_router so the existing package no longer shadows it. Delete eight tests the audit found superseded by stronger ones kept in this move. * ci: run the moved root and small-tree tests under their legacy flags Add the misc and responses-caching-types flags to unit_selection.sh and CircleCI, extend enterprise-routing and mcp-integration, and point the legacy GHA shards, Makefile, redis-compat workflow, merge smoke manifest and change classifier at the new paths. * test: make the new tests/unit directories packages tests/unit/test_package_layout.py requires every directory to carry an __init__.py, and without one the moved and retained test_litellm_responses_bridge.py modules collide on import. * test: scope the unit socket block to tests/unit in shared sessions The GHA shards collect the legacy test-path and the unit selection in one pytest session. The unit conftest's loopback-only block leaked into legacy modules that reach the network at import. The legacy conftest now lifts the restriction at collect and setup time, and the unit conftest re-applies it when collecting its own modules. * test: give the shard-script tests their own GITHUB_OUTPUT They only passed where the runner set it. The CircleCI unit job's env allowlist drops it, so the script's redirect failed there. * test: point the router and module-deletion checks at tests/unit router_code_coverage and code_qa_check_tests only searched tests/test_litellm, so the moved router tests no longer counted. The two silent-experiment tests the audit deleted were the only direct callers of those methods; they are replaced with tests that assert the forwarded shadow request and the recursion guard. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
108 lines
3.8 KiB
Python
108 lines
3.8 KiB
Python
"""
|
|
Validate Claude Opus 5 model configuration entries.
|
|
|
|
Opus 5.5 (``claude-opus-5-5``) is covered here too.
|
|
|
|
Opus 5 carries Opus 4.8's pricing ($5 / $25 per MTok) and the gen-5 adaptive
|
|
thinking profile, but differs from 4.8 in two ways that are behavior-bearing in
|
|
LiteLLM: the cacheable-prefix minimum drops to 512 tokens, and Bedrock's Opus 5
|
|
validator accepts the full effort ladder, so the entries must not carry the
|
|
``bedrock_output_config_effort_ceiling`` that silently clamps ``max`` to
|
|
``xhigh`` on 4.8. The cost-map entries are also what populate
|
|
``litellm.anthropic_models`` at import, which is what lets a bare
|
|
``claude-opus-5`` name resolve to the ``anthropic`` provider (and match an
|
|
``anthropic/*`` wildcard deployment).
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
|
|
import pytest
|
|
|
|
from litellm.constants import BEDROCK_CONVERSE_MODELS
|
|
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
|
|
|
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
|
|
|
|
|
|
def _load_root_cost_map() -> dict:
|
|
json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json")
|
|
with open(json_path) as f:
|
|
return json.load(f)
|
|
|
|
|
|
ALL_OPUS_5_VARIANTS = (
|
|
"claude-opus-5",
|
|
"anthropic.claude-opus-5",
|
|
"global.anthropic.claude-opus-5",
|
|
"us.anthropic.claude-opus-5",
|
|
"eu.anthropic.claude-opus-5",
|
|
"au.anthropic.claude-opus-5",
|
|
"jp.anthropic.claude-opus-5",
|
|
"vertex_ai/claude-opus-5",
|
|
"vertex_ai/claude-opus-5@default",
|
|
"vertex_ai/claude-opus-5-5",
|
|
"vertex_ai/claude-opus-5-5@default",
|
|
"azure_ai/claude-opus-5",
|
|
"azure_ai/claude-opus-5-5",
|
|
)
|
|
|
|
BEDROCK_OPUS_5_VARIANTS = (
|
|
"anthropic.claude-opus-5",
|
|
"global.anthropic.claude-opus-5",
|
|
"us.anthropic.claude-opus-5",
|
|
"eu.anthropic.claude-opus-5",
|
|
"au.anthropic.claude-opus-5",
|
|
"jp.anthropic.claude-opus-5",
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("model_name", BEDROCK_OPUS_5_VARIANTS)
|
|
def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map):
|
|
"""Bedrock Converse routes Opus through a validator that rejects
|
|
``toolSpec.strict`` (``tools.0.custom.strict: Extra inputs are not
|
|
permitted``), same as Opus 4.7/4.8; verified against Bedrock on 2026-07-24.
|
|
Without the flag LiteLLM forwards ``strict`` and every tool call 400s."""
|
|
from litellm.llms.bedrock.common_utils import bedrock_converse_supports_strict_tools
|
|
|
|
assert bedrock_converse_supports_strict_tools(model_name) is False
|
|
|
|
|
|
def test_opus_5_registered_for_bedrock_converse():
|
|
assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS
|
|
|
|
|
|
OPUS_5_5_VARIANTS = (
|
|
"claude-opus-5-5",
|
|
"vertex_ai/claude-opus-5-5",
|
|
"vertex_ai/claude-opus-5-5@default",
|
|
"azure_ai/claude-opus-5-5",
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("model_name", OPUS_5_5_VARIANTS)
|
|
def test_opus_5_5_present_in_bundled_backup(model_name):
|
|
backup = GetModelCostMap.load_local_model_cost_map()
|
|
root = _load_root_cost_map()
|
|
assert model_name in backup
|
|
assert model_name in root
|
|
assert backup[model_name] == root[model_name]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("model", "provider"),
|
|
[
|
|
("claude-opus-5-5", "anthropic"),
|
|
("anthropic/claude-opus-5-5", "anthropic"),
|
|
("vertex_ai/claude-opus-5-5", "vertex_ai"),
|
|
("azure_ai/claude-opus-5-5", "azure_ai"),
|
|
],
|
|
)
|
|
def test_opus_5_5_thinking_profile(local_model_cost_map, model, provider):
|
|
"""Opus 5.5 has thinking always on with the adaptive thinking surface, and
|
|
no forced tool use, same as Fable 5.1."""
|
|
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
|
|
|
assert AnthropicModelInfo._is_adaptive_thinking_model(model, provider) is True
|
|
assert AnthropicModelInfo._is_always_on_thinking_model(model, provider) is True
|
|
assert AnthropicModelInfo.forced_tool_use_unsupported(model.removeprefix("anthropic/")) is True
|