mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-31 06:56:59 +00:00
test(proxy): feed wildcard discovery real provider data in model info tests
This commit is contained in:
mateo-berri
parent
7a9e8abdbd
commit
6de7d49616
2 changed files with 56 additions and 56 deletions
|
|
@ -4865,7 +4865,7 @@ class TestModelInfoCostMapEchoFilter:
|
|||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
router = litellm.Router(
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "my-openai/*",
|
||||
|
|
@ -4874,21 +4874,21 @@ class TestModelInfoCostMapEchoFilter:
|
|||
}
|
||||
]
|
||||
)
|
||||
entry = litellm.get_model_info("openai/gpt-5.6")
|
||||
echo = {**entry, "id": "dep-echo-wildcard", "db_model": True, "access_groups": ["prod"]}
|
||||
db_model = Deployment(
|
||||
entry: Final = litellm.get_model_info("openai/gpt-5.6")
|
||||
echo: Final = {**entry, "id": "dep-echo-wildcard", "db_model": True, "access_groups": ["prod"]}
|
||||
db_model: Final = Deployment(
|
||||
model_name="my-openai/*",
|
||||
litellm_params=LiteLLM_Params(model="openai/*"),
|
||||
model_info=ModelInfo(id="dep-echo-wildcard"),
|
||||
)
|
||||
|
||||
result = update_db_model(
|
||||
result: Final = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(model_name="my-openai/gpt-5.6", model_info=ModelInfo(**echo)),
|
||||
llm_router=router,
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
info: Final = json.loads(result["model_info"])
|
||||
assert info["access_groups"] == ["prod"]
|
||||
assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard"
|
||||
|
||||
|
|
@ -4897,7 +4897,7 @@ class TestModelInfoCostMapEchoFilter:
|
|||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
router_without_the_wildcard = litellm.Router(
|
||||
router_without_the_wildcard: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai-mini",
|
||||
|
|
@ -4906,14 +4906,14 @@ class TestModelInfoCostMapEchoFilter:
|
|||
}
|
||||
]
|
||||
)
|
||||
entry = litellm.get_model_info("openai/gpt-5.6")
|
||||
db_model = Deployment(
|
||||
entry: Final = litellm.get_model_info("openai/gpt-5.6")
|
||||
db_model: Final = Deployment(
|
||||
model_name="my-openai/*",
|
||||
litellm_params=LiteLLM_Params(model="openai/*"),
|
||||
model_info=ModelInfo(id="dep-echo-wildcard"),
|
||||
)
|
||||
|
||||
result = update_db_model(
|
||||
result: Final = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(
|
||||
model_name="my-openai/gpt-5.6",
|
||||
|
|
@ -4922,7 +4922,7 @@ class TestModelInfoCostMapEchoFilter:
|
|||
llm_router=router_without_the_wildcard,
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
info: Final = json.loads(result["model_info"])
|
||||
assert info["access_groups"] == ["prod"]
|
||||
assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard"
|
||||
|
||||
|
|
@ -4934,7 +4934,7 @@ class TestModelInfoCostMapEchoFilter:
|
|||
)
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
router = litellm.Router(
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "my-openai/*",
|
||||
|
|
@ -4943,14 +4943,14 @@ class TestModelInfoCostMapEchoFilter:
|
|||
}
|
||||
]
|
||||
)
|
||||
entry = litellm.get_model_info("openai/gpt-5.6")
|
||||
db_model = Deployment(
|
||||
entry: Final = litellm.get_model_info("openai/gpt-5.6")
|
||||
db_model: Final = Deployment(
|
||||
model_name="my-openai/*",
|
||||
litellm_params=LiteLLM_Params(model="openai/*"),
|
||||
model_info=ModelInfo(id="dep-echo-wildcard"),
|
||||
)
|
||||
|
||||
changes = legacy_update_model_info_changes(
|
||||
changes: Final = legacy_update_model_info_changes(
|
||||
db_model,
|
||||
ModelInfo(**{**entry, "id": "dep-echo-wildcard", "access_groups": ["prod"]}),
|
||||
loaded_catalog=lambda: {},
|
||||
|
|
@ -4965,7 +4965,7 @@ class TestModelInfoCostMapEchoFilter:
|
|||
from litellm.proxy.management_endpoints.model_management_endpoints import _update_team_model_in_db
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
router = litellm.Router(
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "team-openai/*",
|
||||
|
|
@ -4974,9 +4974,9 @@ class TestModelInfoCostMapEchoFilter:
|
|||
}
|
||||
]
|
||||
)
|
||||
entry = litellm.get_model_info("openai/gpt-5.6")
|
||||
echo = {**entry, "id": "dep-team-wildcard", "team_id": "team-a", "access_groups": ["prod"]}
|
||||
db_model = Deployment(
|
||||
entry: Final = litellm.get_model_info("openai/gpt-5.6")
|
||||
echo: Final = {**entry, "id": "dep-team-wildcard", "team_id": "team-a", "access_groups": ["prod"]}
|
||||
db_model: Final = Deployment(
|
||||
model_name="team-openai/*",
|
||||
litellm_params=LiteLLM_Params(model="openai/*"),
|
||||
model_info=ModelInfo(id="dep-team-wildcard", team_id="team-a"),
|
||||
|
|
@ -4987,7 +4987,7 @@ class TestModelInfoCostMapEchoFilter:
|
|||
patch("litellm.proxy.management_endpoints.model_management_endpoints.team_model_add"),
|
||||
patch("litellm.proxy.management_endpoints.model_management_endpoints.team_model_delete"),
|
||||
):
|
||||
result = await _update_team_model_in_db(
|
||||
result: Final = await _update_team_model_in_db(
|
||||
db_model=db_model,
|
||||
patch_data=updateDeployment(model_name="team-openai/gpt-5.6", model_info=ModelInfo(**echo)),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
|
|
@ -4996,7 +4996,7 @@ class TestModelInfoCostMapEchoFilter:
|
|||
llm_router=router,
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
info: Final = json.loads(result["model_info"])
|
||||
assert info["access_groups"] == ["prod"]
|
||||
assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto a team's wildcard"
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
|
@ -539,6 +539,11 @@ def test_model_info_id_lookup_reports_the_same_cost_as_the_list(
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def _discoverable_models(monkeypatch: pytest.MonkeyPatch, models: Mapping[str, list[str]]) -> None:
|
||||
for provider, provider_models in models.items():
|
||||
monkeypatch.setitem(litellm.models_by_provider, provider, provider_models)
|
||||
|
||||
|
||||
async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monkeypatch):
|
||||
from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth
|
||||
from litellm.proxy.auth import model_checks
|
||||
|
|
@ -585,7 +590,6 @@ def test_model_info_prices_expanded_wildcard_rows_as_the_model_their_calls_route
|
|||
"""A catch-all over Fireworks lists ``acme-chat``, which also names a DeepSeek entry, and a
|
||||
prefixed OpenAI wildcard lists ``my-openai/gpt-acme``, which names no entry at all. Each row must
|
||||
show the provider and price its calls are billed at."""
|
||||
from litellm.proxy.auth import model_checks
|
||||
|
||||
catalog: Final = {
|
||||
"fireworks_ai/accounts/fireworks/models/acme-chat": {
|
||||
|
|
@ -609,14 +613,6 @@ def test_model_info_prices_expanded_wildcard_rows_as_the_model_their_calls_route
|
|||
}
|
||||
monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog})
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
monkeypatch.setattr(
|
||||
model_checks,
|
||||
"get_provider_models",
|
||||
lambda provider, litellm_params=None: {
|
||||
"fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-chat"],
|
||||
"openai": ["gpt-acme"],
|
||||
}.get(provider, []),
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -632,6 +628,13 @@ def test_model_info_prices_expanded_wildcard_rows_as_the_model_their_calls_route
|
|||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
_discoverable_models(
|
||||
monkeypatch,
|
||||
{
|
||||
"fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-chat"],
|
||||
"openai": ["gpt-acme"],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
|
||||
monkeypatch.setattr(proxy_server, "user_model", None)
|
||||
|
||||
|
|
@ -678,7 +681,6 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip(
|
|||
calls to those names never reach the catch-all, which still lists them. Such a row shows no price or
|
||||
provider, even where another provider's model goes by the same name, and every price shown is the one
|
||||
calls are billed at."""
|
||||
from litellm.proxy.auth import model_checks
|
||||
|
||||
catalog: Final = {
|
||||
"fireworks_ai/accounts/fireworks/models/acme-lite": {
|
||||
|
|
@ -720,17 +722,6 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip(
|
|||
}
|
||||
monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog})
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
monkeypatch.setattr(
|
||||
model_checks,
|
||||
"get_provider_models",
|
||||
lambda provider, litellm_params=None: {
|
||||
"fireworks_ai": [
|
||||
"fireworks_ai/accounts/fireworks/models/acme-lite",
|
||||
"fireworks_ai/accounts/fireworks/models/acme-pro",
|
||||
"fireworks_ai/accounts/fireworks/models/acme-max",
|
||||
],
|
||||
}.get(provider, []),
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -748,6 +739,16 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip(
|
|||
routing_groups=[{"group_name": "acme-max", "models": ["acme-lite"], "routing_strategy": "simple-shuffle"}],
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
_discoverable_models(
|
||||
monkeypatch,
|
||||
{
|
||||
"fireworks_ai": [
|
||||
"fireworks_ai/accounts/fireworks/models/acme-lite",
|
||||
"fireworks_ai/accounts/fireworks/models/acme-pro",
|
||||
"fireworks_ai/accounts/fireworks/models/acme-max",
|
||||
],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
|
||||
monkeypatch.setattr(proxy_server, "user_model", None)
|
||||
|
||||
|
|
@ -808,7 +809,6 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip(
|
|||
|
||||
|
||||
def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) -> litellm.Router:
|
||||
from litellm.proxy.auth import model_checks
|
||||
|
||||
catalog: Final = {
|
||||
"gpt-5.6-acme": {
|
||||
|
|
@ -844,18 +844,6 @@ def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) ->
|
|||
}
|
||||
monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog})
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
monkeypatch.setattr(
|
||||
model_checks,
|
||||
"get_provider_models",
|
||||
lambda provider, litellm_params=None: {
|
||||
"openai": ["gpt-5.6-acme"],
|
||||
"fireworks_ai": [
|
||||
"fireworks_ai/accounts/fireworks/models/acme-pro",
|
||||
"fireworks_ai/accounts/fireworks/models/acme-max",
|
||||
"fireworks_ai/accounts/fireworks/models/acme-mini",
|
||||
],
|
||||
}.get(provider, []),
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -877,6 +865,17 @@ def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) ->
|
|||
model_group_alias={"my-fw/acme-pro": "acme-lite", "my-fw/acme-max": "my-fw/acme-mini"},
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
_discoverable_models(
|
||||
monkeypatch,
|
||||
{
|
||||
"openai": ["gpt-5.6-acme"],
|
||||
"fireworks_ai": [
|
||||
"fireworks_ai/accounts/fireworks/models/acme-pro",
|
||||
"fireworks_ai/accounts/fireworks/models/acme-max",
|
||||
"fireworks_ai/accounts/fireworks/models/acme-mini",
|
||||
],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
|
||||
monkeypatch.setattr(proxy_server, "user_model", None)
|
||||
return router
|
||||
|
|
@ -1025,6 +1024,7 @@ def test_model_info_shows_no_wildcard_price_for_a_name_the_callers_team_publishe
|
|||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
_discoverable_models(monkeypatch, {"fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-mini"]})
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
|
||||
|
||||
with auth_as(team_id="team-a"):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue