mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
fix(proxy): drop cost-map metadata echoed back on model save (#41944)
* fix(proxy): drop cost-map metadata echoed back on model save Filter unchanged cost-map fields from model-info save echoes while preserving edited overrides. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): drop a stored override when an echoed save resets it to the cost-map value Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): compare model_info echo against the deployment's cost-map lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): decrypt the stored model before the cost-map lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): treat a reset to the bundled catalog value as an echo even after router registration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): compare the reset against the catalog as loaded, not only the bundled backup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): type the catalog snapshot and echo filter parameters as Mapping[str, object] Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): inject the loaded catalog into update_db_model instead of patching the class Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): use contextlib.suppress for cost-map lookup miss to stay under BLE001 budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: ryan <ryan@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
5a764205a5
commit
5d3d99eb9f
5 changed files with 396 additions and 7 deletions
|
|
@ -17,14 +17,16 @@ import random
|
|||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime, timezone
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -37,6 +39,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
|
|||
)
|
||||
|
||||
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
|
||||
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"})
|
||||
|
||||
|
||||
|
|
@ -88,6 +91,18 @@ class GetModelCostMap:
|
|||
"""Load the local backup model cost map bundled with the package."""
|
||||
return GetModelCostMap.load_local_model_cost_map_with_revision().model_cost_map
|
||||
|
||||
_loaded_catalog: Mapping[str, Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
@classmethod
|
||||
def loaded_model_cost_map(cls) -> Mapping[str, Mapping[str, object]]:
|
||||
"""The catalog as last loaded (bundled or remote), untouched by ``register_model`` or router registrations."""
|
||||
return cls._loaded_catalog
|
||||
|
||||
@classmethod
|
||||
def _snapshot_loaded_catalog(cls, model_cost: Mapping[str, object]) -> None:
|
||||
raw: Final = _CATALOG_ADAPTER.validate_python(model_cost)
|
||||
cls._loaded_catalog = MappingProxyType({key: MappingProxyType(entry) for key, entry in raw.items()})
|
||||
|
||||
@classmethod
|
||||
def _get_backup_model_count(cls) -> int:
|
||||
"""Return the number of models in the local backup (cached int)."""
|
||||
|
|
@ -533,7 +548,9 @@ def _finalize_model_cost_map(model_cost: dict) -> dict:
|
|||
def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded:
|
||||
_cost_map_source_info.source_revision = loaded.revision
|
||||
_cost_map_source_info.etag = loaded.etag
|
||||
return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map))
|
||||
finalized: Final = _finalize_model_cost_map(loaded.model_cost_map)
|
||||
GetModelCostMap._snapshot_loaded_catalog(finalized) # pyright: ignore[reportPrivateUsage] # same module
|
||||
return replace(loaded, model_cost_map=finalized)
|
||||
|
||||
|
||||
def adopt_model_cost_map(
|
||||
|
|
|
|||
|
|
@ -14,12 +14,12 @@ import asyncio
|
|||
import datetime
|
||||
import json
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
|
||||
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
||||
from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress
|
||||
from dataclasses import dataclass
|
||||
from fnmatch import fnmatchcase
|
||||
from json import JSONDecodeError
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator
|
||||
|
|
@ -29,6 +29,7 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm._uuid import uuid
|
||||
from litellm.constants import LITELLM_PROXY_ADMIN_NAME
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.ptu_pricing import (
|
||||
CUSTOM_PRICING_FIELDS,
|
||||
PTU_EMPTIED_PRICING_FIELDS,
|
||||
|
|
@ -139,7 +140,12 @@ from litellm.types.router import (
|
|||
updateDeployment,
|
||||
updateLiteLLMParams,
|
||||
)
|
||||
from litellm.types.utils import echoed_cost_map_pricing_fields, without_server_derived_pricing
|
||||
from litellm.types.utils import (
|
||||
COST_MAP_LOOKUP_KEY,
|
||||
echoed_cost_map_fields,
|
||||
echoed_cost_map_pricing_fields,
|
||||
without_server_derived_pricing,
|
||||
)
|
||||
from litellm.utils import get_utc_datetime
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -928,7 +934,33 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment:
|
|||
)
|
||||
|
||||
|
||||
def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> PrismaCompatibleUpdateDBModel:
|
||||
def _cost_map_entry(db_model: Deployment, incoming_model_info: Mapping[str, object]) -> Mapping[str, object]:
|
||||
base_model: Final = incoming_model_info.get("base_model")
|
||||
lookup: Final = base_model if isinstance(base_model, str) else _decrypted_model(db_model.litellm_params.model)
|
||||
if lookup is None:
|
||||
return MappingProxyType({})
|
||||
with suppress(Exception):
|
||||
return MappingProxyType(dict(litellm.get_model_info(model=lookup)))
|
||||
return MappingProxyType({})
|
||||
|
||||
|
||||
LoadedCatalog: TypeAlias = Callable[[], Mapping[str, Mapping[str, object]]] # mutable-ok: Callable parameter syntax
|
||||
|
||||
|
||||
def _loaded_catalog_entry(
|
||||
incoming_model_info: Mapping[str, object], loaded_catalog: LoadedCatalog
|
||||
) -> Mapping[str, object]:
|
||||
catalog_key: Final = incoming_model_info.get(COST_MAP_LOOKUP_KEY)
|
||||
if not isinstance(catalog_key, str):
|
||||
return MappingProxyType({})
|
||||
return loaded_catalog().get(catalog_key, MappingProxyType({}))
|
||||
|
||||
|
||||
def update_db_model(
|
||||
db_model: Deployment,
|
||||
updated_patch: updateDeployment,
|
||||
loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map,
|
||||
) -> PrismaCompatibleUpdateDBModel:
|
||||
if updated_patch.model_info is not None:
|
||||
_raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True))
|
||||
merged_model_name: Final = updated_patch.model_name or db_model.model_name
|
||||
|
|
@ -955,7 +987,24 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr
|
|||
|
||||
# update model info
|
||||
if updated_patch.model_info:
|
||||
merged_model_info.update(without_server_derived_pricing(updated_patch.model_info.model_dump(exclude_none=True)))
|
||||
incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True)
|
||||
echoed_fields: Final = echoed_cost_map_fields(
|
||||
incoming_model_info,
|
||||
_cost_map_entry(db_model, incoming_model_info),
|
||||
_loaded_catalog_entry(incoming_model_info, loaded_catalog),
|
||||
)
|
||||
merged_model_info.update(
|
||||
MappingProxyType(
|
||||
dict(
|
||||
(k, v)
|
||||
for k, v in without_server_derived_pricing(incoming_model_info).items()
|
||||
if k not in echoed_fields
|
||||
)
|
||||
)
|
||||
)
|
||||
for k in echoed_fields:
|
||||
if k in merged_model_info and merged_model_info[k] != incoming_model_info[k]:
|
||||
del merged_model_info[k]
|
||||
|
||||
# Honor explicit-null clears LAST, after both merges, so a model_info blob a client
|
||||
# passes through cannot silently undo a litellm_params clear via .update().
|
||||
|
|
|
|||
|
|
@ -3842,6 +3842,24 @@ def echoed_cost_map_pricing_fields(model_info: Mapping[str, Any]) -> tuple[str,
|
|||
return tuple(sorted(k for k in model_info if is_server_derived_pricing_key(k)))
|
||||
|
||||
|
||||
def echoed_cost_map_fields(
|
||||
model_info: Mapping[str, object], *cost_map_entries: Mapping[str, object]
|
||||
) -> tuple[str, ...]:
|
||||
"""Fields a ``/model/info`` echo copied from the cost map unchanged.
|
||||
|
||||
Only ``litellm.get_model_info`` emits ``key``, so a blob carrying it is an echo of that
|
||||
response. Anything in it that still equals a resolved cost-map entry is a display value
|
||||
nobody typed; a value the operator edited differs from every entry and stays a real override.
|
||||
Callers pass both the live entry, which the router rewrites with each deployment's own
|
||||
overrides, and the catalog entry as loaded, so a reset to the catalog value reads as an echo either way.
|
||||
"""
|
||||
if COST_MAP_LOOKUP_KEY not in model_info:
|
||||
return ()
|
||||
return tuple(
|
||||
sorted(k for k, v in model_info.items() if any(k in entry and entry[k] == v for entry in cost_map_entries))
|
||||
)
|
||||
|
||||
|
||||
def pricing_override_fields(*sources: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
|
|
|
|||
|
|
@ -448,6 +448,23 @@ async def test_refetch_records_the_blob_id_of_the_bytes_served_and_the_fetch_eta
|
|||
assert get_model_cost_map_provenance() == {"source_revision": git_blob_id(body), "etag": 'W/"abc123"'}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_loaded_catalog_snapshot_follows_the_fetched_map_and_ignores_later_registrations(monkeypatch):
|
||||
import litellm
|
||||
|
||||
edited = json.loads(_real_map_bytes())
|
||||
edited["gpt-5.4-mini"]["max_input_tokens"] = 777
|
||||
client, _ = _mock_client([httpx.Response(200, content=json.dumps(edited).encode())])
|
||||
|
||||
result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client)
|
||||
|
||||
assert isinstance(result, ModelCostMapReloaded)
|
||||
monkeypatch.setattr(litellm, "model_cost", result.model_cost_map)
|
||||
litellm.register_model({"gpt-5.4-mini": {"max_input_tokens": 2048}}, persist_across_reloads=False)
|
||||
assert litellm.model_cost["gpt-5.4-mini"]["max_input_tokens"] == 2048
|
||||
assert GetModelCostMap.loaded_model_cost_map()["gpt-5.4-mini"]["max_input_tokens"] == 777
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_revision_follows_the_bytes_not_the_url():
|
||||
edited = json.loads(_real_map_bytes())
|
||||
|
|
|
|||
|
|
@ -4063,6 +4063,294 @@ class TestModelInfoServerDerivedPricingFilter:
|
|||
assert written["access_groups"] == ["prod"]
|
||||
|
||||
|
||||
class TestModelInfoCostMapEchoFilter:
|
||||
"""LIT-5534. ``/model/info`` fills a deployment's ``model_info`` from the cost map (context
|
||||
limits, mode, provider, supported params, capability flags), and the Admin UI edit form sends
|
||||
that whole blob back on any save. Only values that still equal the cost-map entry are the
|
||||
echo; a value the operator changed is a real override and stays."""
|
||||
|
||||
def test_echoed_cost_map_metadata_is_not_persisted(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
entry = litellm.get_model_info("openai/gpt-5.6")
|
||||
echo = {**entry, "id": "dep-echo-0", "db_model": True, "access_groups": ["prod"]}
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.6"),
|
||||
model_info=ModelInfo(id="dep-echo-0"),
|
||||
)
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(model_info=ModelInfo(**echo)),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert info["access_groups"] == ["prod"]
|
||||
assert set(info).isdisjoint(entry)
|
||||
assert "max_input_tokens" not in info and "mode" not in info and "supports_vision" not in info, (
|
||||
"cost-map metadata must not be persisted from an unchanged /model/info echo"
|
||||
)
|
||||
|
||||
def test_an_edited_value_survives_the_echo_filter(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
entry = litellm.get_model_info("openai/gpt-5.6")
|
||||
echo = {
|
||||
**entry,
|
||||
"id": "dep-echo-1",
|
||||
"db_model": True,
|
||||
"access_groups": ["prod"],
|
||||
"max_input_tokens": entry["max_input_tokens"] + 1,
|
||||
"mode": "completion" if entry["mode"] != "completion" else "chat",
|
||||
}
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.6"),
|
||||
model_info=ModelInfo(id="dep-echo-1"),
|
||||
)
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(model_info=ModelInfo(**echo)),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert info["max_input_tokens"] == echo["max_input_tokens"]
|
||||
assert info["mode"] == echo["mode"]
|
||||
assert "litellm_provider" not in info
|
||||
assert "supported_openai_params" not in info
|
||||
|
||||
def test_metadata_without_a_cost_map_key_is_persisted(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
from litellm.types.utils import echoed_cost_map_fields
|
||||
|
||||
entry = litellm.get_model_info("openai/gpt-5.6")
|
||||
assert echoed_cost_map_fields({"max_input_tokens": entry["max_input_tokens"]}, entry) == ()
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.6"),
|
||||
model_info=ModelInfo(id="dep-echo-2"),
|
||||
)
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(
|
||||
model_info=ModelInfo(
|
||||
id="dep-echo-2",
|
||||
max_input_tokens=entry["max_input_tokens"],
|
||||
mode=entry["mode"],
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert info["max_input_tokens"] == entry["max_input_tokens"]
|
||||
assert info["mode"] == entry["mode"]
|
||||
|
||||
def test_a_stored_mode_survives_an_echoed_save(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
entry = litellm.get_model_info("openai/gpt-5.6")
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.6"),
|
||||
model_info=ModelInfo(id="dep-echo-3", mode=entry["mode"]),
|
||||
)
|
||||
echo = {**entry, "id": "dep-echo-3", "db_model": True, "access_groups": ["prod"]}
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(model_info=ModelInfo(**echo)),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert info["mode"] == entry["mode"]
|
||||
assert "max_input_tokens" not in info
|
||||
|
||||
def test_resetting_an_override_to_the_cost_map_value_removes_it(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
entry = litellm.get_model_info("openai/gpt-5.6")
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.6"),
|
||||
model_info=ModelInfo(id="dep-echo-4", mode="chat", max_input_tokens=2048),
|
||||
)
|
||||
echo = {**entry, "id": "dep-echo-4", "db_model": True, "access_groups": ["staging"]}
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(model_info=ModelInfo(**echo)),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert "max_input_tokens" not in info
|
||||
assert info["mode"] == "chat"
|
||||
assert info["access_groups"] == ["staging"]
|
||||
|
||||
def test_reset_is_recognised_after_the_router_registered_the_override(self, monkeypatch: pytest.MonkeyPatch):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
pristine = litellm.get_model_info("openai/gpt-5.6")
|
||||
polluted = {**pristine, "max_input_tokens": 2048}
|
||||
monkeypatch.setattr(litellm, "get_model_info", lambda model, **_: polluted)
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.6"),
|
||||
model_info=ModelInfo(id="dep-echo-8", max_input_tokens=2048),
|
||||
)
|
||||
echo = {**pristine, "id": "dep-echo-8", "db_model": True}
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(model_info=ModelInfo(**echo)),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert "max_input_tokens" not in info, info
|
||||
|
||||
def test_reset_to_a_remote_catalog_value_that_differs_from_the_bundled_one(self, monkeypatch: pytest.MonkeyPatch):
|
||||
from types import MappingProxyType
|
||||
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
bundled = litellm.get_model_info("openai/gpt-5.6")
|
||||
remote = {**bundled, "max_input_tokens": bundled["max_input_tokens"] + 1}
|
||||
remote_catalog = MappingProxyType({remote["key"]: MappingProxyType(remote)})
|
||||
monkeypatch.setattr(litellm, "get_model_info", lambda model, **_: {**remote, "max_input_tokens": 2048})
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.6"),
|
||||
model_info=ModelInfo(id="dep-echo-9", max_input_tokens=2048),
|
||||
)
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(model_info=ModelInfo(**{**remote, "id": "dep-echo-9", "db_model": True})),
|
||||
loaded_catalog=lambda: remote_catalog,
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert "max_input_tokens" not in info, info
|
||||
|
||||
def test_echo_is_compared_against_the_deployments_lookup_not_the_key(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
lookup_pairs: Final = (
|
||||
("openai/gpt-5.6", "gpt-5.6"),
|
||||
("openai/gpt-4.1-mini", "gpt-4.1-mini"),
|
||||
)
|
||||
lookup_data: Final = tuple(
|
||||
(deployment_model, deployment_entry, differing_fields)
|
||||
for deployment_model, key_model in lookup_pairs
|
||||
for deployment_entry in (litellm.get_model_info(deployment_model),)
|
||||
for key_entry in (litellm.get_model_info(key_model),)
|
||||
for differing_fields in (
|
||||
frozenset(
|
||||
k for k in deployment_entry if k in key_entry and deployment_entry[k] != key_entry[k]
|
||||
),
|
||||
)
|
||||
if differing_fields
|
||||
)
|
||||
if not lookup_data:
|
||||
pytest.skip("No deployment/key cost-map lookup differences are available")
|
||||
|
||||
deployment_model, entry, differing_fields = lookup_data[0]
|
||||
assert differing_fields
|
||||
db_model = Deployment(
|
||||
model_name=deployment_model,
|
||||
litellm_params=LiteLLM_Params(model=deployment_model),
|
||||
model_info=ModelInfo(id="dep-echo-5"),
|
||||
)
|
||||
echo = {**entry, "id": "dep-echo-5", "db_model": True, "access_groups": ["prod"]}
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(model_info=ModelInfo(**echo)),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",)))
|
||||
|
||||
def test_base_model_wins_over_litellm_params_model_for_the_lookup(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
entry = litellm.get_model_info("azure/gpt-5.6")
|
||||
db_model = Deployment(
|
||||
model_name="azure/my-deploy",
|
||||
litellm_params=LiteLLM_Params(model="azure/my-deploy"),
|
||||
model_info=ModelInfo(id="dep-echo-6", base_model="azure/gpt-5.6"),
|
||||
)
|
||||
echo = {
|
||||
**entry,
|
||||
"id": "dep-echo-6",
|
||||
"base_model": "azure/gpt-5.6",
|
||||
"db_model": True,
|
||||
}
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(model_info=ModelInfo(**echo)),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",)))
|
||||
assert info["base_model"] == "azure/gpt-5.6"
|
||||
|
||||
def test_encrypted_stored_model_is_decrypted_for_the_lookup(self, monkeypatch):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234")
|
||||
entry = litellm.get_model_info("openai/gpt-5.6")
|
||||
db_model = Deployment(
|
||||
model_name="gpt-5.6",
|
||||
litellm_params=LiteLLM_Params(model=encrypt_value_helper(value="openai/gpt-5.6")),
|
||||
model_info=ModelInfo(id="dep-echo-7", mode="chat"),
|
||||
)
|
||||
echo = {**entry, "id": "dep-echo-7", "db_model": True, "access_groups": ["prod"]}
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment(model_info=ModelInfo(**echo)),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",)))
|
||||
assert info["mode"] == "chat"
|
||||
assert info["access_groups"] == ["prod"]
|
||||
|
||||
|
||||
class TestUpdateDBModelClearCacheControlInjectionPoints:
|
||||
def test_explicit_null_removes_stored_injection_points(self):
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue