mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): import cost tracking and provider helpers lazily in the Codex catalog
lite codex imports the catalog before the SDK is initialized, so importing cost tracking, the Bedrock helpers or provider resolution at module level hit a circular import and broke every lite command.
This commit is contained in:
parent
d5922c4bae
commit
eb5ccec4ab
2 changed files with 26 additions and 10 deletions
|
|
@ -35,10 +35,7 @@ from typing import TYPE_CHECKING, Final, Literal, TypeAlias
|
|||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider
|
||||
from litellm.llms.bedrock.common_utils import bedrock_model_name_without_provider
|
||||
from litellm.proxy.common_utils.codex_model_translation import (
|
||||
TIER_PRICE_KEYS,
|
||||
TOKEN_PRICE_KEY,
|
||||
CodexDeploymentCapabilities,
|
||||
CodexFallbackModel,
|
||||
|
|
@ -47,6 +44,7 @@ from litellm.proxy.common_utils.codex_model_translation import (
|
|||
codex_fallback_entry,
|
||||
combined_capabilities,
|
||||
no_priced_deployment_lacks_tier,
|
||||
tier_price_keys,
|
||||
)
|
||||
from litellm.types.llms.base_model import LiteLLMBaseModel
|
||||
|
||||
|
|
@ -246,19 +244,26 @@ def _codex_can_drive(model_id: str, mode: str | None) -> bool:
|
|||
return "*" not in model_id and (mode is None or mode in CODEX_CHAT_MODES)
|
||||
|
||||
|
||||
_PROVIDER_MODEL_NAMES: Final[Mapping[str, Callable[[str], str]]] = MappingProxyType(
|
||||
{"bedrock": bedrock_model_name_without_provider, "bedrock_mantle": bedrock_model_name_without_provider}
|
||||
)
|
||||
@cache
|
||||
def _provider_model_names() -> Mapping[str, Callable[[str], str]]:
|
||||
"""Imported on first use for the same reason as `tier_price_keys`."""
|
||||
from litellm.llms.bedrock.common_utils import bedrock_model_name_without_provider
|
||||
|
||||
return MappingProxyType(
|
||||
{"bedrock": bedrock_model_name_without_provider, "bedrock_mantle": bedrock_model_name_without_provider}
|
||||
)
|
||||
|
||||
|
||||
def _provider_model_name(upstream_model: str, upstream_provider: str | None) -> str:
|
||||
"""The upstream model without the prefixes its provider adds, the provider being the one the deployment
|
||||
declares in `custom_llm_provider`, else the model's `provider/` prefix."""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider
|
||||
|
||||
provider: Final = declared_provider(upstream_model, upstream_provider)
|
||||
if provider is None:
|
||||
return upstream_model
|
||||
model: Final = upstream_model.removeprefix(f"{provider}/")
|
||||
name_for: Final = _PROVIDER_MODEL_NAMES.get(provider)
|
||||
name_for: Final = _provider_model_names().get(provider)
|
||||
return model if name_for is None else name_for(model)
|
||||
|
||||
|
||||
|
|
@ -505,7 +510,7 @@ def deployment_capabilities(model_info: Mapping[str, object] | None, model_id: s
|
|||
if model_info is None:
|
||||
return CodexDeploymentCapabilities()
|
||||
prices_tokens: Final = model_info.get(TOKEN_PRICE_KEY) is not None
|
||||
priced: Final = frozenset(key for key in TIER_PRICE_KEYS if model_info.get(key) is not None)
|
||||
priced: Final = frozenset(key for key in tier_price_keys() if model_info.get(key) is not None)
|
||||
try:
|
||||
return CodexDeploymentCapabilities.model_validate(
|
||||
{**model_info, "prices_tokens": prices_tokens, "priced_service_tier_keys": priced}
|
||||
|
|
|
|||
|
|
@ -11,13 +11,13 @@ from __future__ import annotations
|
|||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from types import NoneType
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from packaging.version import Version
|
||||
from pydantic import ConfigDict, Field, StringConstraints
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES, get_service_tier_cost_key
|
||||
from litellm.litellm_core_utils.reasoning_effort_strength import nearest_declared_reasoning_effort
|
||||
from litellm.types.llms.base_model import LiteLLMBaseModel
|
||||
|
||||
|
|
@ -27,7 +27,6 @@ CodexInputModality: TypeAlias = Literal["text", "image"]
|
|||
_TEXT_ONLY: Final[tuple[CodexInputModality, ...]] = ("text",)
|
||||
_TEXT_AND_IMAGE: Final[tuple[CodexInputModality, ...]] = ("text", "image")
|
||||
TOKEN_PRICE_KEY: Final = "input_cost_per_token"
|
||||
TIER_PRICE_KEYS: Final = tuple(f"{TOKEN_PRICE_KEY}{suffix}" for suffix in SERVICE_TIER_COST_KEY_SUFFIXES)
|
||||
_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0")
|
||||
_EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"})
|
||||
_PREFERRED_DEFAULT_EFFORT: Final = "medium"
|
||||
|
|
@ -119,11 +118,23 @@ class CodexDeploymentCapabilities(LiteLLMBaseModel):
|
|||
priced_service_tier_keys: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
@cache
|
||||
def tier_price_keys() -> tuple[str, ...]:
|
||||
"""The input price key of every service tier cost tracking prices. Cost tracking is imported on first use
|
||||
because `lite codex` imports this module before the SDK is initialized, and cost tracking imports the rest
|
||||
of the SDK."""
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES
|
||||
|
||||
return tuple(f"{TOKEN_PRICE_KEY}{suffix}" for suffix in SERVICE_TIER_COST_KEY_SUFFIXES)
|
||||
|
||||
|
||||
def no_priced_deployment_lacks_tier(tier_id: str, deployments: Sequence[CodexDeploymentCapabilities]) -> bool:
|
||||
"""Whether every deployment that has a token price also has one for `tier_id`, the way cost tracking
|
||||
prices a request sent at that tier. A deployment with no token price at all (a subscription upstream,
|
||||
a deployment name the cost map does not know) says nothing about tiers, so it does not vote, and a tier
|
||||
cost tracking has no price key for is lacked by every deployment that prices tokens."""
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import get_service_tier_cost_key
|
||||
|
||||
price_key: Final = get_service_tier_cost_key(TOKEN_PRICE_KEY, tier_id)
|
||||
return all(
|
||||
price_key in deployment.priced_service_tier_keys for deployment in deployments if deployment.prices_tokens
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue