mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
fix(router): ignore the azure gpt5_series/ selector when resolving model info
An Azure deployment such as azure/gpt5_series/gpt-5.6-sol with no base_model looked up azure/gpt5_series/gpt-5.6-sol in the cost map, found only the zeroed stub the Router registers for deployment names, and so got no context window, no pricing and no pre-call context check. The fallback now strips the selector before the cost-map lookup, and the request itself still keeps it.
This commit is contained in:
parent
ff3983c1f6
commit
0d9872d9ac
2 changed files with 36 additions and 1 deletions
|
|
@ -223,6 +223,7 @@ from litellm.litellm_core_utils.token_counter import (
|
|||
)
|
||||
from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, anthropic_error_frame_exception
|
||||
from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
|
||||
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
|
||||
from litellm.llms.base_llm.vector_store.transformation import (
|
||||
RouterVectorStoreEmbeddingExecutor,
|
||||
|
|
@ -12885,13 +12886,17 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
# Router init auto-registers every deployment name into
|
||||
# litellm.model_cost as a zeroed stub, so membership alone can't
|
||||
# tell a resolvable name apart; require usable limits/costs.
|
||||
_azure_fallback_key = _model if _model.startswith("azure/") else f"azure/{_model}"
|
||||
_metadata_model: Final = _model.removeprefix(AzureOpenAIGPT5Config.GPT5_SERIES_ROUTE)
|
||||
_azure_fallback_key: Final = (
|
||||
_metadata_model if _metadata_model.startswith("azure/") else f"azure/{_metadata_model}"
|
||||
)
|
||||
_fallback_entry = self._host.dynamic().model_cost.get(_azure_fallback_key)
|
||||
_fallback_resolves = isinstance(_fallback_entry, Mapping) and (
|
||||
(_fallback_entry.get("max_input_tokens") or 0) > 0
|
||||
or (_fallback_entry.get("input_cost_per_token") or 0) > 0
|
||||
)
|
||||
if _fallback_resolves:
|
||||
model = _metadata_model
|
||||
verbose_router_logger.debug(
|
||||
"Azure deployment '%s' has no base_model set; using '%s' from the model cost map for max tokens, cost tracking, etc.",
|
||||
_model,
|
||||
|
|
|
|||
|
|
@ -17196,6 +17196,36 @@ class TestAzureBaseModelFallbackLogging:
|
|||
)
|
||||
assert model_info["max_input_tokens"] == litellm.model_cost["azure/gpt-4o-mini"]["max_input_tokens"]
|
||||
|
||||
def test_gpt5_series_selector_resolves_the_underlying_model(self):
|
||||
router = self._router_with_azure_deployment("azure/gpt5_series/gpt-5.6-sol")
|
||||
|
||||
model_info = router.get_router_model_info(
|
||||
deployment=None, received_model_name="my-group", id="azure-base-model-test-id"
|
||||
)
|
||||
|
||||
assert model_info["max_input_tokens"] == litellm.model_cost["azure/gpt-5.6-sol"]["max_input_tokens"]
|
||||
assert model_info["input_cost_per_token"] == litellm.model_cost["azure/gpt-5.6-sol"]["input_cost_per_token"]
|
||||
|
||||
def test_gpt5_series_selector_cannot_bypass_the_context_window_check(self):
|
||||
router = self._router_with_azure_deployment("azure/gpt5_series/gpt-5.6-sol")
|
||||
limit = litellm.model_cost["azure/gpt-5.6-sol"]["max_input_tokens"]
|
||||
messages = [{"role": "user", "content": "hi"}]
|
||||
|
||||
with patch.object(litellm, "token_counter", return_value=limit):
|
||||
router._pre_call_checks(model="my-group", healthy_deployments=router.model_list, messages=messages)
|
||||
with patch.object(litellm, "token_counter", return_value=limit + 1):
|
||||
with pytest.raises(litellm.ContextWindowExceededError):
|
||||
router._pre_call_checks(model="my-group", healthy_deployments=router.model_list, messages=messages)
|
||||
|
||||
def test_gpt5_series_selector_does_not_give_an_unknown_deployment_any_limits(self):
|
||||
router = self._router_with_azure_deployment("azure/gpt5_series/my-custom-deployment-name")
|
||||
|
||||
model_info = router.get_router_model_info(
|
||||
deployment=None, received_model_name="my-group", id="azure-base-model-test-id"
|
||||
)
|
||||
|
||||
assert model_info.get("max_input_tokens") is None
|
||||
|
||||
def test_model_group_info_intersects_supported_reasoning_efforts():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue