mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): satisfy strict typing gate
This commit is contained in:
parent
cebab8df0b
commit
8becb0e14e
3 changed files with 41 additions and 41 deletions
|
|
@ -11,7 +11,7 @@ from datetime import datetime, timedelta, timezone
|
|||
from itertools import chain, groupby
|
||||
from operator import attrgetter
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator
|
||||
|
|
@ -336,6 +336,9 @@ async def validate_complexity_router_config(
|
|||
return ComplexityRouterConfigValidationResponse(valid=error is None, error=error)
|
||||
|
||||
|
||||
_MODEL_REFERENCE_MAPPING: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _available_model_refs(
|
||||
llm_router: "Router", model_names: Sequence[str], team_id: str | None
|
||||
) -> Mapping[str, tuple[str, ...]]:
|
||||
|
|
@ -344,20 +347,15 @@ def _available_model_refs(
|
|||
deployments = llm_router.get_model_list(model_name=model_name, team_id=team_id) or ()
|
||||
resolved: list[str] = [model_name]
|
||||
for raw_deployment in deployments:
|
||||
deployment = cast(Mapping[str, object], raw_deployment)
|
||||
litellm_params = deployment.get("litellm_params")
|
||||
model_info = deployment.get("model_info")
|
||||
if isinstance(litellm_params, Mapping):
|
||||
typed_params = cast(Mapping[str, object], litellm_params)
|
||||
resolved.extend(
|
||||
value
|
||||
for key in ("model", "base_model")
|
||||
if isinstance(value := typed_params.get(key), str) and value
|
||||
)
|
||||
if isinstance(model_info, Mapping):
|
||||
base_model = cast(Mapping[str, object], model_info).get("base_model")
|
||||
if isinstance(base_model, str) and base_model:
|
||||
resolved.append(base_model)
|
||||
deployment = _MODEL_REFERENCE_MAPPING.validate_python(raw_deployment)
|
||||
litellm_params = _MODEL_REFERENCE_MAPPING.validate_python(deployment.get("litellm_params", {}))
|
||||
model_info = _MODEL_REFERENCE_MAPPING.validate_python(deployment.get("model_info", {}))
|
||||
resolved.extend(
|
||||
value for key in ("model", "base_model") if isinstance(value := litellm_params.get(key), str) and value
|
||||
)
|
||||
base_model = model_info.get("base_model")
|
||||
if isinstance(base_model, str) and base_model:
|
||||
resolved.append(base_model)
|
||||
refs[model_name] = tuple(dict.fromkeys(resolved))
|
||||
return MappingProxyType(refs)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@ from collections.abc import Mapping, Sequence
|
|||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from importlib.resources import files
|
||||
from typing import Final, Literal, cast
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator
|
||||
|
||||
from .config import (
|
||||
AutoSetupCandidate,
|
||||
|
|
@ -32,6 +32,7 @@ _TIER_COMPLEXITIES: Final[Mapping[str, SnapshotComplexity]] = {
|
|||
}
|
||||
_EASY_COMPLEXITIES: Final = frozenset(("trivial", "simple"))
|
||||
_VERSION_SEPARATOR: Final = re.compile(r"(?<=\d)\.(?=\d)")
|
||||
_SNAPSHOT_PAYLOAD_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class SnapshotQualityTier(BaseModel):
|
||||
|
|
@ -136,23 +137,14 @@ def _canonical_json(value: object) -> str:
|
|||
|
||||
@lru_cache(maxsize=1)
|
||||
def load_auto_router_snapshot() -> AutoRouterSnapshot:
|
||||
parsed = cast(
|
||||
object,
|
||||
json.loads(
|
||||
files("litellm.router_strategy.complexity_router")
|
||||
.joinpath("artifacts", _SNAPSHOT_FILENAME)
|
||||
.read_text(encoding="utf-8")
|
||||
),
|
||||
raw: Final = _SNAPSHOT_PAYLOAD_ADAPTER.validate_json(
|
||||
files("litellm.router_strategy.complexity_router")
|
||||
.joinpath("artifacts", _SNAPSHOT_FILENAME)
|
||||
.read_text(encoding="utf-8")
|
||||
)
|
||||
if not isinstance(parsed, dict):
|
||||
raise ValueError("Auto Router snapshot must be a JSON object")
|
||||
untyped = cast(dict[object, object], parsed)
|
||||
if not all(isinstance(key, str) for key in untyped):
|
||||
raise ValueError("Auto Router snapshot keys must be strings")
|
||||
raw = cast(dict[str, object], untyped)
|
||||
expected = raw.get("artifact_sha256")
|
||||
if not isinstance(expected, str):
|
||||
raise ValueError("Auto Router snapshot must carry an artifact_sha256")
|
||||
raise TypeError("Auto Router snapshot must carry an artifact_sha256")
|
||||
unsigned: Final = {key: value for key, value in raw.items() if key != "artifact_sha256"}
|
||||
actual: Final = hashlib.sha256(_canonical_json(unsigned).encode("utf-8")).hexdigest()
|
||||
if expected != actual:
|
||||
|
|
@ -271,19 +263,24 @@ def _hard_task_rank(
|
|||
),
|
||||
)
|
||||
), "cohort_scoped"
|
||||
balanced_comparable = tuple(item for item in comparable if item[1].cost_per_completed_task_usd is not None)
|
||||
balanced_comparable: list[tuple[_AvailableProfile, float, float]] = []
|
||||
for profile, speed in comparable:
|
||||
cost = speed.cost_per_completed_task_usd
|
||||
completion_ms = speed.retry_adjusted_completion_ms
|
||||
if cost is not None and completion_ms is not None:
|
||||
balanced_comparable.append((profile, cost, completion_ms))
|
||||
if not balanced_comparable:
|
||||
return cost_ranked, "fallback_no_comparable_cost_evidence"
|
||||
costs: Final = tuple(cast(float, speed.cost_per_completed_task_usd) for _, speed in balanced_comparable)
|
||||
times: Final = tuple(cast(float, speed.retry_adjusted_completion_ms) for _, speed in balanced_comparable)
|
||||
costs: Final = tuple(cost for _, cost, _ in balanced_comparable)
|
||||
times: Final = tuple(completion_ms for _, _, completion_ms in balanced_comparable)
|
||||
return tuple(
|
||||
profile
|
||||
for profile, _ in sorted(
|
||||
for profile, _, _ in sorted(
|
||||
balanced_comparable,
|
||||
key=lambda item: (
|
||||
math.hypot(
|
||||
_normalize_log(cast(float, item[1].cost_per_completed_task_usd), costs),
|
||||
_normalize_log(cast(float, item[1].retry_adjusted_completion_ms), times),
|
||||
_normalize_log(item[1], costs),
|
||||
_normalize_log(item[2], times),
|
||||
),
|
||||
-item[0].quality.quality_lower_bound,
|
||||
item[0].model_name,
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ import math
|
|||
import statistics
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Final, Protocol, cast
|
||||
from typing import Final, Protocol
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
|
||||
|
|
@ -18,6 +18,8 @@ _CACHE_TTL_SECONDS: Final = 7 * 24 * 60 * 60
|
|||
_MAX_SAMPLES_PER_MODEL: Final = 20
|
||||
_MAX_EXPECTED_OUTPUT_SAMPLES: Final = 100
|
||||
_MIN_SAMPLES_PER_MODEL: Final = 2
|
||||
_OBJECT_ADAPTER: Final = TypeAdapter(object)
|
||||
_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class ResponseLatencyCache(Protocol):
|
||||
|
|
@ -165,9 +167,12 @@ def _seconds(value: object) -> float | None:
|
|||
|
||||
def _field(value: object, key: str) -> object | None:
|
||||
if isinstance(value, Mapping):
|
||||
return cast(Mapping[object, object], value).get(key)
|
||||
try:
|
||||
return _OBJECT_MAPPING_ADAPTER.validate_python(value).get(key)
|
||||
except ValidationError:
|
||||
return None
|
||||
try:
|
||||
return cast(object, object.__getattribute__(value, key))
|
||||
return _OBJECT_ADAPTER.validate_python(object.__getattribute__(value, key))
|
||||
except AttributeError:
|
||||
return None
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue