diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e05d3c4bfff..51bdcf9324e 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -147,7 +147,8 @@ from litellm.utils import ( ProviderConfigManager, TextCompletionResponse, TranscriptionResponse, - cached_get_model_info_helper, + get_model_info_for_pricing, + get_model_info_helper_for_pricing, model_info_with_pricing_overrides, runtime_model_cost_registrations, token_counter, @@ -362,7 +363,7 @@ def _per_second_pricing_cost( audio_seconds: float = 0.0, ) -> tuple[float, float] | None: try: - model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = get_model_info_helper_for_pricing(model=model, custom_llm_provider=custom_llm_provider) except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model return None if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): @@ -583,7 +584,9 @@ def cost_per_token( model = model_without_prefix if is_decisions_call_type(call_type): - decisions_model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + decisions_model_info: Final = get_model_info_helper_for_pricing( + model=model, custom_llm_provider=custom_llm_provider + ) decisions_overrides: Final = decisions_pricing_fields( litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded ) @@ -787,7 +790,7 @@ def cost_per_token( service_tier=service_tier, ) else: - model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = get_model_info_helper_for_pricing(model=model, custom_llm_provider=custom_llm_provider) if _has_token_or_tiered_pricing(model_info): return generic_cost_per_token( model=model, @@ -2353,7 +2356,7 @@ def _layered_ocr_pricing(*sources: Mapping[str, object] | None) -> OCRPricing: def _cost_map_model_info(model: str, custom_llm_provider: str | None) -> ModelInfo | None: try: - return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + return get_model_info_for_pricing(model=model, custom_llm_provider=custom_llm_provider) except Exception: return None diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index a67c1bb7df8..f1d01694b97 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -146,6 +146,10 @@ def handle_anthropic_text_model_custom_llm_provider( return model, custom_llm_provider +def declared_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None: + return custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None) + + def declared_authenticating_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None: """The authenticating provider this pair already names, or None. @@ -154,7 +158,7 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str and for a declared pair the resolver's answer is the declaration itself, so metadata callers adopt the declaration instead of resolving. """ - declared: Final = custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None) + declared: Final = declared_provider(model, custom_llm_provider) return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index eda1ed7e312..4e053425b8f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -181,7 +181,7 @@ from litellm.types.utils import ( Usage, ) from litellm.types.videos.main import VideoObject -from litellm.utils import get_base_model_from_metadata, print_verbose +from litellm.utils import get_base_model_from_metadata, get_model_info_for_pricing, print_verbose from ..integrations.argilla import ArgillaLogger from ..integrations.arize.arize_phoenix import ArizePhoenixLogger @@ -6324,7 +6324,7 @@ class StandardLoggingPayloadSetup: model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None) else: try: - _model_cost_information: Final = litellm.get_model_info( + _model_cost_information: Final = get_model_info_for_pricing( model=pricing_model_name or model_cost_name, custom_llm_provider=custom_llm_provider, api_base=api_base, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 6664d54a744..ed9a429fb2d 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -34,7 +34,7 @@ from litellm.types.utils import ( Usage, text_tokens_without_nested_reasoning, ) -from litellm.utils import get_model_info +from litellm.utils import get_model_info, get_model_info_for_pricing # Pre-resolved CallTypes enum values for fast membership checks _IMAGE_RESPONSE_CALL_TYPES: Final = frozenset( @@ -1385,7 +1385,9 @@ def generic_cost_per_token( # per-deployment override: those are registered under the deployment id and kept off # the shared model-name key, so resolving from the name here reads the public rate. resolved_model_info: Final = apply_provider_cache_read_default( - get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_info is None else model_info, + get_model_info_for_pricing(model=model, custom_llm_provider=custom_llm_provider) + if model_info is None + else model_info, custom_llm_provider, ) @@ -1708,7 +1710,7 @@ def get_billed_token_rates( resolved_model_info: Final = apply_provider_cache_read_default( model_info if model_info is not None - else get_model_info(model=model, custom_llm_provider=custom_llm_provider), + else get_model_info_for_pricing(model=model, custom_llm_provider=custom_llm_provider), custom_llm_provider, ) except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model: no rates diff --git a/litellm/utils.py b/litellm/utils.py index 5e3c2c886ec..b12b5557090 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6358,6 +6358,8 @@ def get_model_info_helper( custom_llm_provider: str | None = None, api_base: str | None = None, api_key: str | None = None, + *, + catalog_only: bool = False, ) -> ModelInfoBase: """ Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's @@ -6393,7 +6395,7 @@ def get_model_info_helper( model_cost_custom_llm_provider: Final = custom_llm_provider ######################### provider_config: BaseLLMModelInfo | None = None - if custom_llm_provider and custom_llm_provider in LlmProvidersSet: + if not catalog_only and custom_llm_provider and custom_llm_provider in LlmProvidersSet: provider_config = ProviderConfigManager.get_provider_model_info( model=model, provider=LlmProviders(custom_llm_provider) ) @@ -6417,7 +6419,7 @@ def get_model_info_helper( e, ) - if custom_llm_provider == "huggingface": + if not catalog_only and custom_llm_provider == "huggingface": max_tokens: Final = _get_max_position_embeddings(model_name=model) return ModelInfoBase( key=model, @@ -6894,6 +6896,39 @@ get_model_info.cache_clear = _cached_get_model_info.cache_clear get_model_info.cache_info = _cached_get_model_info.cache_info +def get_model_info_for_pricing( + model: str, custom_llm_provider: str | None = None, api_base: str | None = None +) -> ModelInfo: + from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider + + provider: Final = declared_provider(model, custom_llm_provider) + if provider not in ("ollama", "ollama_chat"): + return get_model_info(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base) + try: + catalog: Final = get_model_info_helper(model=model, custom_llm_provider=provider, catalog_only=True) + except ModelNotMappedError: + return ModelInfo( + key=model.removeprefix(f"{provider}/"), + litellm_provider="ollama", + mode="chat", + input_cost_per_token=0.0, + output_cost_per_token=0.0, + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + supported_openai_params=None, + ) + return ModelInfo(**catalog, supported_openai_params=None) + + +def get_model_info_helper_for_pricing(model: str, custom_llm_provider: str | None) -> ModelInfoBase: + from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider + + if declared_provider(model, custom_llm_provider) in ("ollama", "ollama_chat"): + return get_model_info_for_pricing(model=model, custom_llm_provider=custom_llm_provider) + return cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + + def json_schema_type(python_type_name: str): """Converts standard python types to json schema types diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py index 142d49bfff1..4dcb2b60c8b 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py @@ -9,6 +9,8 @@ import pytest import respx import litellm +from litellm.cost_calculator import cost_per_token, pricing_entry_for_cost_calc +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, get_billed_token_rates from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage @@ -237,3 +239,62 @@ async def test_router_prices_balanced_columns_by_the_same_rules_as_flex( assert balanced_cost == pytest.approx(flex_cost) if not overridden_columns: assert balanced_cost == pytest.approx(_cost_at(PARITY_ROW, "_balanced")) + + +@pytest.mark.parametrize("provider", ("ollama", "ollama_chat")) +@pytest.mark.parametrize("registered", (False, True)) +@pytest.mark.parametrize("qualified", (False, True)) +def test_ollama_pricing_uses_registered_rates_without_contacting_the_server( + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + provider: str, + registered: bool, + qualified: bool, +) -> None: + model: Final = "test-ollama-pricing-registered" if registered else "test-ollama-pricing-unmapped" + monkeypatch.setenv("OLLAMA_API_BASE", "https://ambient-pricing.example") + metadata: Final = respx_mock.post("https://ambient-pricing.example/api/show").respond( + json={"model_info": {"test.context_length": 32768}} + ) + if registered: + monkeypatch.setitem( + litellm.model_cost, + f"{provider}/{model}", + { + "litellm_provider": provider, + "mode": "chat", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 5e-06, + "input_cost_per_token_decisions": 3e-06, + "output_cost_per_token_decisions": 7e-06, + }, + ) + usage: Final = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120) + cost_model: Final = f"{provider}/{model}" if qualified else model + cost_provider: Final = None if qualified else provider + + assert cost_per_token( + model=cost_model, custom_llm_provider=cost_provider, usage_object=usage, call_type="systemone" + ) == pytest.approx((0.0003, 0.00014) if registered else (0.0, 0.0)) + assert generic_cost_per_token(model=cost_model, custom_llm_provider=cost_provider, usage=usage) == pytest.approx( + (0.0002, 0.0001) if registered else (0.0, 0.0) + ) + rates: Final = get_billed_token_rates(model=cost_model, custom_llm_provider=cost_provider, usage=usage) + assert rates is not None + assert (rates.input_cost_per_token, rates.output_cost_per_token) == ( + (2e-06, 5e-06) if registered else (0.0, 0.0) + ) + diagnostic: Final = pricing_entry_for_cost_calc( + model=cost_model, + completion_response=None, + custom_llm_provider=cost_provider, + custom_pricing=False, + base_model=None, + router_model_id=None, + region_name=None, + litellm_logging_obj=None, + ) + assert diagnostic is not None + assert diagnostic[1]["input_cost_per_token"] == (2e-06 if registered else 0.0) + assert not metadata.called diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index f4a39f17bed..957a241bc71 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -19,6 +19,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import respx import pytest_asyncio from mcp.types import AudioContent, CallToolResult, ImageContent, TextContent from openai import AsyncOpenAI @@ -12470,3 +12471,23 @@ def test_standard_logging_payload_reports_the_deployment_entry_the_request_was_b ) finally: litellm.model_cost.pop(deployment_id, None) + + +@pytest.mark.parametrize("provider", ("ollama", "ollama_chat")) +def test_ollama_cost_information_does_not_request_live_metadata( + local_model_cost_map: None, respx_mock: respx.MockRouter, provider: str +) -> None: + metadata: Final = respx_mock.post("https://cost-information.example/api/show").respond( + json={"model_info": {"test.context_length": 32768}} + ) + info: Final = StandardLoggingPayloadSetup.get_model_cost_information( + base_model="test-ollama-cost-information", + custom_pricing=False, + custom_llm_provider=provider, + init_response_obj=litellm.ModelResponse(model="test-ollama-cost-information"), + api_base="https://cost-information.example", + ) + rates: Final = info["model_map_value"] + assert rates is not None + assert (rates["input_cost_per_token"], rates["output_cost_per_token"]) == (0.0, 0.0) + assert not metadata.called diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7edfb22c121..d3ee643630d 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10009,3 +10009,25 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_ ) assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]} + + +@pytest.mark.parametrize("provider", ("ollama", "ollama_chat")) +def test_ollama_pricing_lookup_preserves_live_model_info_discovery( + local_model_cost_map: None, respx_mock: respx.MockRouter, provider: str +) -> None: + from litellm.utils import get_model_info_for_pricing + + metadata: Final = respx_mock.post("https://model-info.example/api/show").respond( + json={"model_info": {"test.context_length": 32768}} + ) + pricing: Final = get_model_info_for_pricing( + model="test-live-discovery", custom_llm_provider=provider, api_base="https://model-info.example" + ) + assert (pricing["input_cost_per_token"], pricing["output_cost_per_token"]) == (0.0, 0.0) + assert not metadata.called + + discovered: Final = litellm.get_model_info( + model="test-live-discovery", custom_llm_provider=provider, api_base="https://model-info.example" + ) + assert discovered["max_input_tokens"] == 32768 + assert metadata.call_count == 1