From 8becb0e14e3c4bbae2e6b6c32554587c94b3085d Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 15:27:09 -0700 Subject: [PATCH] fix(router): satisfy strict typing gate --- .../auto_router_endpoints.py | 28 ++++++------- .../complexity_router/auto_setup.py | 41 +++++++++---------- .../complexity_router/response_latency.py | 13 ++++-- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 6b4ad77ec07..4b489e918a8 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -11,7 +11,7 @@ from datetime import datetime, timedelta, timezone from itertools import chain, groupby from operator import attrgetter from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast +from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol from uuid import uuid4 from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator @@ -336,6 +336,9 @@ async def validate_complexity_router_config( return ComplexityRouterConfigValidationResponse(valid=error is None, error=error) +_MODEL_REFERENCE_MAPPING: Final = TypeAdapter(dict[str, object]) + + def _available_model_refs( llm_router: "Router", model_names: Sequence[str], team_id: str | None ) -> Mapping[str, tuple[str, ...]]: @@ -344,20 +347,15 @@ def _available_model_refs( deployments = llm_router.get_model_list(model_name=model_name, team_id=team_id) or () resolved: list[str] = [model_name] for raw_deployment in deployments: - deployment = cast(Mapping[str, object], raw_deployment) - litellm_params = deployment.get("litellm_params") - model_info = deployment.get("model_info") - if isinstance(litellm_params, Mapping): - typed_params = cast(Mapping[str, object], litellm_params) - resolved.extend( - value - for key in ("model", "base_model") - if isinstance(value := typed_params.get(key), str) and value - ) - if isinstance(model_info, Mapping): - base_model = cast(Mapping[str, object], model_info).get("base_model") - if isinstance(base_model, str) and base_model: - resolved.append(base_model) + deployment = _MODEL_REFERENCE_MAPPING.validate_python(raw_deployment) + litellm_params = _MODEL_REFERENCE_MAPPING.validate_python(deployment.get("litellm_params", {})) + model_info = _MODEL_REFERENCE_MAPPING.validate_python(deployment.get("model_info", {})) + resolved.extend( + value for key in ("model", "base_model") if isinstance(value := litellm_params.get(key), str) and value + ) + base_model = model_info.get("base_model") + if isinstance(base_model, str) and base_model: + resolved.append(base_model) refs[model_name] = tuple(dict.fromkeys(resolved)) return MappingProxyType(refs) diff --git a/litellm/router_strategy/complexity_router/auto_setup.py b/litellm/router_strategy/complexity_router/auto_setup.py index 935bf723bb5..762fbea3854 100644 --- a/litellm/router_strategy/complexity_router/auto_setup.py +++ b/litellm/router_strategy/complexity_router/auto_setup.py @@ -8,9 +8,9 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from functools import lru_cache from importlib.resources import files -from typing import Final, Literal, cast +from typing import Final, Literal -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator from .config import ( AutoSetupCandidate, @@ -32,6 +32,7 @@ _TIER_COMPLEXITIES: Final[Mapping[str, SnapshotComplexity]] = { } _EASY_COMPLEXITIES: Final = frozenset(("trivial", "simple")) _VERSION_SEPARATOR: Final = re.compile(r"(?<=\d)\.(?=\d)") +_SNAPSHOT_PAYLOAD_ADAPTER: Final = TypeAdapter(dict[str, object]) class SnapshotQualityTier(BaseModel): @@ -136,23 +137,14 @@ def _canonical_json(value: object) -> str: @lru_cache(maxsize=1) def load_auto_router_snapshot() -> AutoRouterSnapshot: - parsed = cast( - object, - json.loads( - files("litellm.router_strategy.complexity_router") - .joinpath("artifacts", _SNAPSHOT_FILENAME) - .read_text(encoding="utf-8") - ), + raw: Final = _SNAPSHOT_PAYLOAD_ADAPTER.validate_json( + files("litellm.router_strategy.complexity_router") + .joinpath("artifacts", _SNAPSHOT_FILENAME) + .read_text(encoding="utf-8") ) - if not isinstance(parsed, dict): - raise ValueError("Auto Router snapshot must be a JSON object") - untyped = cast(dict[object, object], parsed) - if not all(isinstance(key, str) for key in untyped): - raise ValueError("Auto Router snapshot keys must be strings") - raw = cast(dict[str, object], untyped) expected = raw.get("artifact_sha256") if not isinstance(expected, str): - raise ValueError("Auto Router snapshot must carry an artifact_sha256") + raise TypeError("Auto Router snapshot must carry an artifact_sha256") unsigned: Final = {key: value for key, value in raw.items() if key != "artifact_sha256"} actual: Final = hashlib.sha256(_canonical_json(unsigned).encode("utf-8")).hexdigest() if expected != actual: @@ -271,19 +263,24 @@ def _hard_task_rank( ), ) ), "cohort_scoped" - balanced_comparable = tuple(item for item in comparable if item[1].cost_per_completed_task_usd is not None) + balanced_comparable: list[tuple[_AvailableProfile, float, float]] = [] + for profile, speed in comparable: + cost = speed.cost_per_completed_task_usd + completion_ms = speed.retry_adjusted_completion_ms + if cost is not None and completion_ms is not None: + balanced_comparable.append((profile, cost, completion_ms)) if not balanced_comparable: return cost_ranked, "fallback_no_comparable_cost_evidence" - costs: Final = tuple(cast(float, speed.cost_per_completed_task_usd) for _, speed in balanced_comparable) - times: Final = tuple(cast(float, speed.retry_adjusted_completion_ms) for _, speed in balanced_comparable) + costs: Final = tuple(cost for _, cost, _ in balanced_comparable) + times: Final = tuple(completion_ms for _, _, completion_ms in balanced_comparable) return tuple( profile - for profile, _ in sorted( + for profile, _, _ in sorted( balanced_comparable, key=lambda item: ( math.hypot( - _normalize_log(cast(float, item[1].cost_per_completed_task_usd), costs), - _normalize_log(cast(float, item[1].retry_adjusted_completion_ms), times), + _normalize_log(item[1], costs), + _normalize_log(item[2], times), ), -item[0].quality.quality_lower_bound, item[0].model_name, diff --git a/litellm/router_strategy/complexity_router/response_latency.py b/litellm/router_strategy/complexity_router/response_latency.py index eb33d37f73c..3b504cf53db 100644 --- a/litellm/router_strategy/complexity_router/response_latency.py +++ b/litellm/router_strategy/complexity_router/response_latency.py @@ -5,9 +5,9 @@ import math import statistics from collections.abc import Mapping, Sequence from datetime import datetime, timedelta -from typing import Final, Protocol, cast +from typing import Final, Protocol -from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from litellm._logging import verbose_router_logger @@ -18,6 +18,8 @@ _CACHE_TTL_SECONDS: Final = 7 * 24 * 60 * 60 _MAX_SAMPLES_PER_MODEL: Final = 20 _MAX_EXPECTED_OUTPUT_SAMPLES: Final = 100 _MIN_SAMPLES_PER_MODEL: Final = 2 +_OBJECT_ADAPTER: Final = TypeAdapter(object) +_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[str, object]) class ResponseLatencyCache(Protocol): @@ -165,9 +167,12 @@ def _seconds(value: object) -> float | None: def _field(value: object, key: str) -> object | None: if isinstance(value, Mapping): - return cast(Mapping[object, object], value).get(key) + try: + return _OBJECT_MAPPING_ADAPTER.validate_python(value).get(key) + except ValidationError: + return None try: - return cast(object, object.__getattribute__(value, key)) + return _OBJECT_ADAPTER.validate_python(object.__getattribute__(value, key)) except AttributeError: return None