diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e66be233be4..5b39df887d8 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -155,6 +155,8 @@ from litellm.utils import ( TranscriptionResponse, cached_cost_map_key, cached_get_model_info_helper, + get_model_info_for_pricing, + get_model_info_helper_for_pricing, mapped_model_fill_missing, model_info_with_pricing_overrides, runtime_model_cost_registrations, @@ -379,7 +381,7 @@ def _per_second_pricing_cost( audio_seconds: float = 0.0, ) -> tuple[float, float] | None: try: - model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = get_model_info_helper_for_pricing(model=model, custom_llm_provider=custom_llm_provider) except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model return None if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): @@ -600,7 +602,9 @@ def cost_per_token( model = model_without_prefix if is_decisions_call_type(call_type): - decisions_model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + decisions_model_info: Final = get_model_info_helper_for_pricing( + model=model, custom_llm_provider=custom_llm_provider + ) decisions_overrides: Final = decisions_pricing_fields( litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded ) @@ -808,7 +812,7 @@ def cost_per_token( service_tier=service_tier, ) else: - model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = get_model_info_helper_for_pricing(model=model, custom_llm_provider=custom_llm_provider) if _has_token_or_tiered_pricing(model_info): return generic_cost_per_token( model=model, @@ -2604,7 +2608,7 @@ def _layered_ocr_pricing(*sources: Mapping[str, object] | None) -> OCRPricing: def _cost_map_model_info(model: str, custom_llm_provider: str | None) -> ModelInfo | None: try: - return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + return get_model_info_for_pricing(model=model, custom_llm_provider=custom_llm_provider) except Exception: return None diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 639003c9a6a..1a303901314 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -211,7 +211,7 @@ from litellm.types.utils import ( batch_chat_usage, ) from litellm.types.videos.main import VideoObject -from litellm.utils import get_base_model_from_metadata, print_verbose +from litellm.utils import get_base_model_from_metadata, get_model_info_for_pricing, print_verbose from ..integrations.argilla import ArgillaLogger from ..integrations.arize.arize_phoenix import ArizePhoenixLogger @@ -7113,7 +7113,7 @@ class StandardLoggingPayloadSetup: model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None) else: try: - _model_cost_information: Final = litellm.get_model_info( + _model_cost_information: Final = get_model_info_for_pricing( model=pricing_model_name or model_cost_name, custom_llm_provider=custom_llm_provider, api_base=api_base, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 50c26f4d4ca..f8a851ee880 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -35,7 +35,7 @@ from litellm.types.utils import ( Usage, text_tokens_without_nested_reasoning, ) -from litellm.utils import get_model_info +from litellm.utils import get_model_info, get_model_info_for_pricing # Pre-resolved CallTypes enum values for fast membership checks _IMAGE_RESPONSE_CALL_TYPES: Final = frozenset( @@ -1491,7 +1491,9 @@ def generic_cost_per_token( # per-deployment override: those are registered under the deployment id and kept off # the shared model-name key, so resolving from the name here reads the public rate. resolved_model_info: Final = apply_provider_cache_read_default( - get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_info is None else model_info, + get_model_info_for_pricing(model=model, custom_llm_provider=custom_llm_provider) + if model_info is None + else model_info, custom_llm_provider, ) @@ -1824,7 +1826,7 @@ def get_billed_token_rates( resolved_model_info: Final = apply_provider_cache_read_default( model_info if model_info is not None - else get_model_info(model=model, custom_llm_provider=custom_llm_provider), + else get_model_info_for_pricing(model=model, custom_llm_provider=custom_llm_provider), custom_llm_provider, ) except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model: no rates diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index c562c8d0f04..ed1b4011358 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -60,6 +60,7 @@ from litellm.types.decisions import ( systemone_choice_key, ) from litellm.types.utils import is_decisions_model_mode +from litellm.utils import get_model_info_helper _PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) _SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) @@ -500,7 +501,9 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR def catalog_mode(model: str, custom_llm_provider: str) -> str | None: try: - key: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)["key"] + key: Final = get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider, catalog_only=True)[ + "key" + ] except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which the catalog does not list return None entry: Final = GetModelCostMap.loaded_model_cost_map().get(key) @@ -510,7 +513,9 @@ def catalog_mode(model: str, custom_llm_provider: str) -> str | None: def _is_decisions_model(model: str, custom_llm_provider: str) -> bool: try: - model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode") + model_mode: Final = get_model_info_helper( + model=model, custom_llm_provider=custom_llm_provider, catalog_only=True + ).get("mode") except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which has no mode return False return is_decisions_model_mode(model_mode) diff --git a/litellm/utils.py b/litellm/utils.py index b0826f84006..61deb8baa18 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7542,6 +7542,39 @@ get_model_info.cache_clear = _cached_get_model_info.cache_clear get_model_info.cache_info = _cached_get_model_info.cache_info +def get_model_info_for_pricing( + model: str, custom_llm_provider: str | None = None, api_base: str | None = None +) -> ModelInfo: + from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider + + provider: Final = declared_provider(model, custom_llm_provider) + if provider not in ("ollama", "ollama_chat"): + return get_model_info(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base) + try: + catalog: Final = get_model_info_helper(model=model, custom_llm_provider=provider, catalog_only=True) + except ModelNotMappedError: + return ModelInfo( + key=model.removeprefix(f"{provider}/"), + litellm_provider="ollama", + mode="chat", + input_cost_per_token=0.0, + output_cost_per_token=0.0, + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + supported_openai_params=None, + ) + return ModelInfo(**catalog, supported_openai_params=None) + + +def get_model_info_helper_for_pricing(model: str, custom_llm_provider: str | None) -> ModelInfoBase: + from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider + + if declared_provider(model, custom_llm_provider) in ("ollama", "ollama_chat"): + return get_model_info_for_pricing(model=model, custom_llm_provider=custom_llm_provider) + return cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + + def json_schema_type(python_type_name: str): """Converts standard python types to json schema types diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py index b6c18e0da86..4025b7ce725 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py @@ -9,11 +9,73 @@ import pytest import respx import litellm +from litellm.cost_calculator import cost_per_token, pricing_entry_for_cost_calc from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, get_billed_token_rates from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage + +@pytest.mark.parametrize("provider", ("ollama", "ollama_chat")) +@pytest.mark.parametrize("registered", (False, True)) +@pytest.mark.parametrize("qualified", (False, True)) +def test_ollama_pricing_uses_registered_rates_without_contacting_the_server( + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + provider: str, + registered: bool, + qualified: bool, +) -> None: + model: Final = "test-ollama-pricing-registered" if registered else "test-ollama-pricing-unmapped" + monkeypatch.setenv("OLLAMA_API_BASE", "https://ambient-pricing.example") + metadata: Final = respx_mock.post("https://ambient-pricing.example/api/show").respond( + json={"model_info": {"test.context_length": 32768}} + ) + if registered: + monkeypatch.setitem( + litellm.model_cost, + f"{provider}/{model}", + { + "litellm_provider": provider, + "mode": "chat", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 5e-06, + "input_cost_per_token_decisions": 3e-06, + "output_cost_per_token_decisions": 7e-06, + }, + ) + usage: Final = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120) + cost_model: Final = f"{provider}/{model}" if qualified else model + cost_provider: Final = None if qualified else provider + + assert cost_per_token( + model=cost_model, custom_llm_provider=cost_provider, usage_object=usage, call_type="systemone" + ) == pytest.approx((0.0003, 0.00014) if registered else (0.0, 0.0)) + assert generic_cost_per_token(model=cost_model, custom_llm_provider=cost_provider, usage=usage) == pytest.approx( + (0.0002, 0.0001) if registered else (0.0, 0.0) + ) + rates: Final = get_billed_token_rates(model=cost_model, custom_llm_provider=cost_provider, usage=usage) + assert rates is not None + assert (rates.input_cost_per_token, rates.output_cost_per_token) == ( + (2e-06, 5e-06) if registered else (0.0, 0.0) + ) + diagnostic: Final = pricing_entry_for_cost_calc( + model=cost_model, + completion_response=None, + custom_llm_provider=cost_provider, + custom_pricing=False, + base_model=None, + router_model_id=None, + region_name=None, + litellm_logging_obj=None, + ) + assert diagnostic is not None + assert diagnostic[1]["input_cost_per_token"] == (2e-06 if registered else 0.0) + assert not metadata.called + + TIER_MODEL: Final = "tier-priced-test-model" TIER_ROW: Final[Mapping[str, float]] = MappingProxyType( { diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index f1854563cd5..272f8993588 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -20,6 +20,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import pytest_asyncio +import respx from mcp.types import AudioContent, CallToolResult, ImageContent, TextContent from openai import AsyncOpenAI from openai._legacy_response import HttpxBinaryResponseContent @@ -12589,6 +12590,26 @@ def test_get_model_cost_information(): assert all(field in result for field in StandardLoggingModelInformation.__annotations__) +@pytest.mark.parametrize("provider", ("ollama", "ollama_chat")) +def test_ollama_cost_information_does_not_request_live_metadata( + local_model_cost_map: None, respx_mock: respx.MockRouter, provider: str +) -> None: + metadata: Final = respx_mock.post("https://cost-information.example/api/show").respond( + json={"model_info": {"test.context_length": 32768}} + ) + info: Final = StandardLoggingPayloadSetup.get_model_cost_information( + base_model="test-ollama-cost-information", + custom_pricing=False, + custom_llm_provider=provider, + init_response_obj=litellm.ModelResponse(model="test-ollama-cost-information"), + api_base="https://cost-information.example", + ) + rates: Final = info["model_map_value"] + assert rates is not None + assert (rates["input_cost_per_token"], rates["output_cost_per_token"]) == (0.0, 0.0) + assert not metadata.called + + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_get_model_cost_information_custom_pricing_uses_base_model(): result = StandardLoggingPayloadSetup.get_model_cost_information( diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index 041c625eaef..fcac0b0d57d 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -4,6 +4,7 @@ from typing import Final import httpx import pytest +import respx from pydantic import TypeAdapter, ValidationError import litellm @@ -12,6 +13,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( BaseDecisionsConfig, DecisionsInputLimits, + catalog_mode, ir_to_systemone_request, ir_to_systemone_response, parse_systemone_response, @@ -53,6 +55,52 @@ _SYSTEMONE_REQUEST: Final[Mapping[str, object]] = { } +@pytest.mark.parametrize("provider", ("ollama", "ollama_chat")) +def test_decision_capability_checks_do_not_contact_an_ambient_ollama_server( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, provider: str +) -> None: + monkeypatch.setenv("OLLAMA_API_BASE", "https://ambient.example") + monkeypatch.setenv("OLLAMA_API_KEY", "ambient-test-key") + metadata: Final = respx_mock.post("https://ambient.example/api/show").respond( + json={"model_info": {"test.context_length": 32768}} + ) + config: Final = BaseDecisionsConfig() + + assert catalog_mode("unmapped-decisions-test", provider) is None + assert ( + config.supports_box_questions("unmapped-decisions-test", provider, {}, None), + config.supports_audio_input("unmapped-decisions-test", provider, None), + config.supports_video_input("unmapped-decisions-test", provider, None), + ) == (False, False, False) + assert not metadata.called + + +@pytest.mark.parametrize("mode", ("evaluation", "decisions", "chat")) +@pytest.mark.parametrize("supported", (False, True)) +def test_decision_capabilities_preserve_registered_modes_and_flags( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, mode: str, supported: bool +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "bespoke/test-decision-capabilities", + { + "litellm_provider": "bespoke", + "mode": mode, + "supports_box_questions": supported, + "supports_audio_input": supported, + "supports_video_input": supported, + }, + ) + config: Final = BaseDecisionsConfig() + expected: Final = supported and mode in ("evaluation", "decisions") + + assert ( + config.supports_box_questions("test-decision-capabilities", "bespoke", {}, None), + config.supports_audio_input("test-decision-capabilities", "bespoke", None), + config.supports_video_input("test-decision-capabilities", "bespoke", None), + ) == (expected, expected, expected) + + @pytest.mark.parametrize("declared", (4096, 0, -1, True, None)) def test_decision_input_limits_use_only_positive_catalog_limits( local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, declared: int | None diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 5cf872f4ea7..e0b378fc5e9 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -28,6 +28,8 @@ import respx from jsonschema import Draft7Validator, validate import litellm + + from litellm.exceptions import MidStreamFallbackError from litellm._internal_context import is_internal_call from litellm._logging import ( @@ -11415,3 +11417,25 @@ async def test_success_enqueue_is_tracked_without_starting_callbacks_synchronous assert GLOBAL_LOGGING_WORKER.pending_count > 0, "Tracked enqueue must finish before worker draining starts" await GLOBAL_LOGGING_WORKER.flush() assert logging_obj.model_call_details["has_logged_async_success"] is True + + +@pytest.mark.parametrize("provider", ("ollama", "ollama_chat")) +def test_ollama_pricing_lookup_preserves_live_model_info_discovery( + local_model_cost_map: None, respx_mock: respx.MockRouter, provider: str +) -> None: + from litellm.utils import get_model_info_for_pricing + + metadata: Final = respx_mock.post("https://model-info.example/api/show").respond( + json={"model_info": {"test.context_length": 32768}} + ) + pricing: Final = get_model_info_for_pricing( + model="test-live-discovery", custom_llm_provider=provider, api_base="https://model-info.example" + ) + assert (pricing["input_cost_per_token"], pricing["output_cost_per_token"]) == (0.0, 0.0) + assert not metadata.called + + discovered: Final = litellm.get_model_info( + model="test-live-discovery", custom_llm_provider=provider, api_base="https://model-info.example" + ) + assert discovered["max_input_tokens"] == 32768 + assert metadata.call_count == 1