From ce704fca73c75bbc48d98e7b782e7f7196376563 Mon Sep 17 00:00:00 2001 From: srinivassivaratri Date: Mon, 21 Sep 2026 16:30:42 +0000 Subject: [PATCH] fix(nebius): bill spend from the cost map for org/model ids Cost-map rows use litellm_provider=nebius but the provider set only ingested the old nebius-chat-models labels, so models_by_provider[nebius] stayed empty. Short response ids like Kimi-K3 also missed nebius/moonshotai/Kimi-K3. Streaming trusted usage.cost when present, which can be an order of magnitude below Token Factory list rates. Overwrite usage.cost on Nebius streams with the calculator/map cost so clients do not keep the provider's understated value. List models from live /v1/models via get_api_base and get_api_key, preserving path. Match cost-map keys case-insensitively; warn only when a suffix match is ambiguous. Fixes BerriAI/litellm#41782 --- litellm/__init__.py | 5 + litellm/llms/nebius/chat/transformation.py | 54 +++++ litellm/main.py | 8 +- litellm/utils.py | 2 + .../nebius/test_nebius_chat_transformation.py | 211 ++++++++++++++++++ 5 files changed, 278 insertions(+), 2 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index be8f59d210b..70ea707d48a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -903,6 +903,11 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None: sambanova_embedding_models.add(key) elif value.get("litellm_provider") == "novita": novita_models.add(key) + elif value.get("litellm_provider") == "nebius": + if value.get("mode") == "embedding": + nebius_embedding_models.add(key) + else: + nebius_models.add(key) elif value.get("litellm_provider") == "nebius-chat-models": nebius_models.add(key) elif value.get("litellm_provider") == "nebius-embedding-models": diff --git a/litellm/llms/nebius/chat/transformation.py b/litellm/llms/nebius/chat/transformation.py index cf65699d629..0dfa1083fe1 100644 --- a/litellm/llms/nebius/chat/transformation.py +++ b/litellm/llms/nebius/chat/transformation.py @@ -4,9 +4,12 @@ Nebius AI Studio Chat Completions API - Transformation This is OpenAI compatible - no translation needed / occurs """ +from types import MappingProxyType from typing import Final +from litellm._logging import verbose_logger from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.secret_managers.main import get_secret_str class NebiusConfig(OpenAIGPTConfig): @@ -27,3 +30,54 @@ class NebiusConfig(OpenAIGPTConfig): elif param in supported_openai_params: optional_params[param] = value return optional_params + + def get_model_cost_key(self, model: str) -> str | None: + import litellm + + stripped: Final = model.removeprefix("nebius/") + prefixed: Final = f"nebius/{stripped}" + catalog: Final = litellm.nebius_models | litellm.nebius_embedding_models + if prefixed in litellm.model_cost: + return prefixed + prefixed_folded: Final = prefixed.casefold() + prefixed_matches: Final = tuple(key for key in catalog if key.casefold() == prefixed_folded) + if len(prefixed_matches) == 1: + return prefixed_matches[0] + suffix: Final = f"/{stripped}".casefold() + matches: Final = tuple(key for key in catalog if key.casefold().endswith(suffix)) + if len(matches) == 1: + return matches[0] + if len(matches) > 1: + verbose_logger.warning( + "Nebius cost map has %s keys ending in %s; not using a suffix match. keys=%s", + len(matches), + suffix, + matches, + ) + return None + + def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + import litellm + + resolved_key: Final = self.get_api_key(api_key) + resolved_base: Final = self.get_api_base(api_base) + if resolved_key is None or resolved_base is None: + raise ValueError( + "NEBIUS_API_KEY is not set. Pass api_key or set NEBIUS_API_KEY to query Nebius /v1/models." + ) + response: Final = litellm.module_level_client.get( + url=f"{resolved_base.rstrip('/')}/models", + headers=MappingProxyType({"Authorization": f"Bearer {resolved_key}"}), + ) + if response.status_code != 200: + raise Exception(f"Failed to get models: {response.text}") + models: Final = response.json()["data"] + return [model["id"] for model in models] # mutable-ok: OpenAI-compatible get_models returns list[str] + + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or get_secret_str("NEBIUS_API_KEY") + + @staticmethod + def get_api_base(api_base: str | None = None) -> str | None: + return api_base or get_secret_str("NEBIUS_API_BASE") or "https://api.studio.nebius.ai/v1" diff --git a/litellm/main.py b/litellm/main.py index b1aaf5c5dab..eabc67b8d82 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8698,7 +8698,7 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) +_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value, LlmProviders.NEBIUS.value}) def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool: @@ -8752,7 +8752,11 @@ def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Opti def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - if isinstance(getattr(usage, "cost", None), (int, float)): + reported_cost: Final = getattr(usage, "cost", None) + if ( + isinstance(reported_cost, (int, float)) + and logging_obj.model_call_details.get("custom_llm_provider") != LlmProviders.NEBIUS.value + ): return computed_cost: Final = logging_obj._response_cost_calculator(result=response) if isinstance(computed_cost, (int, float)) and computed_cost > 0: diff --git a/litellm/utils.py b/litellm/utils.py index 252bc691301..d0661a69480 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9037,6 +9037,8 @@ class ProviderConfigManager: return VLLMModelInfo() elif LlmProviders.LEMONADE == provider: return litellm.LemonadeChatConfig() + elif LlmProviders.NEBIUS == provider: + return litellm.NebiusConfig() elif LlmProviders.CLARIFAI == provider: return litellm.ClarifaiConfig() elif LlmProviders.BEDROCK == provider: diff --git a/tests/test_litellm/llms/nebius/test_nebius_chat_transformation.py b/tests/test_litellm/llms/nebius/test_nebius_chat_transformation.py index 6d77e81b767..09ba5aafef9 100644 --- a/tests/test_litellm/llms/nebius/test_nebius_chat_transformation.py +++ b/tests/test_litellm/llms/nebius/test_nebius_chat_transformation.py @@ -7,11 +7,14 @@ Nebius AI Studio is an OpenAI-compatible provider with minor customizations. +from typing import Final + import pytest import litellm from litellm import completion from litellm.llms.nebius.chat.transformation import NebiusConfig +from litellm.types.utils import ModelResponse, Usage class TestNebiusConfig: @@ -106,3 +109,211 @@ class TestNebiusConfig: # Check for specific content in the response assert "```python" in response.choices[0].message.content assert "Hey from LiteLLM" in response.choices[0].message.content + + +_NEBIUS_KIMI_K3: Final = "nebius/moonshotai/Kimi-K3" + + +def _nebius_cost_map_keys() -> set[str]: + return { + key + for key, value in litellm.model_cost.items() + if isinstance(value, dict) and value.get("litellm_provider") == "nebius" + } + + +def _kimi_k3_row() -> dict: + row: Final = litellm.model_cost[_NEBIUS_KIMI_K3] + assert isinstance(row, dict) + return row + + +class TestNebiusCostTracking: + def test_cost_map_nebius_rows_are_registered_on_the_provider(self): + registered: Final = set(litellm.models_by_provider.get("nebius") or ()) + assert _nebius_cost_map_keys() <= registered + assert _NEBIUS_KIMI_K3 in registered + + def test_short_response_model_id_resolves_to_the_cost_map_row(self): + row: Final = _kimi_k3_row() + leaf: Final = _NEBIUS_KIMI_K3.rsplit("/", 1)[-1] + info: Final = litellm.get_model_info(model=leaf, custom_llm_provider="nebius") + assert info["input_cost_per_token"] == row["input_cost_per_token"] + assert info["output_cost_per_token"] == row["output_cost_per_token"] + + def test_short_response_model_id_resolution_is_case_insensitive(self): + row: Final = _kimi_k3_row() + leaf: Final = _NEBIUS_KIMI_K3.rsplit("/", 1)[-1] + swapped: Final = leaf.swapcase() + assert swapped != leaf + info: Final = litellm.get_model_info(model=swapped, custom_llm_provider="nebius") + assert info["input_cost_per_token"] == row["input_cost_per_token"] + assert info["output_cost_per_token"] == row["output_cost_per_token"] + assert NebiusConfig().get_model_cost_key(_NEBIUS_KIMI_K3.lower()) == _NEBIUS_KIMI_K3 + + def test_ambiguous_leaf_suffix_logs_and_returns_none( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ): + import logging + + leaf: Final = "shared-leaf" + monkeypatch.setattr( + litellm, + "nebius_models", + {f"nebius/org-a/{leaf}", f"nebius/org-b/{leaf}"}, + ) + monkeypatch.setattr(litellm, "nebius_embedding_models", set()) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + matched: Final = NebiusConfig().get_model_cost_key(leaf) + assert matched is None + assert "2 keys ending in /shared-leaf" in caplog.text + assert "nebius/org-a/shared-leaf" in caplog.text + assert "nebius/org-b/shared-leaf" in caplog.text + + def test_unmapped_leaf_returns_none_without_warning( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ): + import logging + + monkeypatch.setattr(litellm, "nebius_models", set()) + monkeypatch.setattr(litellm, "nebius_embedding_models", set()) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + matched: Final = NebiusConfig().get_model_cost_key("unmapped-leaf") + assert matched is None + assert "not using a suffix match" not in caplog.text + + def test_completion_cost_matches_cost_map_row_not_provider_usage_cost(self): + row: Final = _kimi_k3_row() + prompt_tokens: Final = 1_000_000 + completion_tokens: Final = 500_000 + map_cost: Final = ( + prompt_tokens * float(row["input_cost_per_token"]) + + completion_tokens * float(row["output_cost_per_token"]) + ) + response: Final = ModelResponse( + model="Kimi-K3", + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + cost=map_cost / 10, + ), + ) + response._hidden_params["custom_llm_provider"] = "nebius" + billed: Final = litellm.completion_cost( + completion_response=response, + model=_NEBIUS_KIMI_K3, + custom_llm_provider="nebius", + ) + assert billed == pytest.approx(map_cost) + assert billed != pytest.approx(map_cost / 10) + + def test_stream_builder_prices_nebius_from_the_cost_map(self): + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + row: Final = _kimi_k3_row() + prompt_tokens: Final = 1_000_000 + completion_tokens: Final = 500_000 + map_cost: Final = ( + prompt_tokens * float(row["input_cost_per_token"]) + + completion_tokens * float(row["output_cost_per_token"]) + ) + usage_chunk: Final = ModelResponseStream( + id="chatcmpl-nebius-cost", + created=1724900000, + model="Kimi-K3", + object="chat.completion.chunk", + choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="", role="assistant"))], + ) + usage_chunk.usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + cost=map_cost / 10, + ) + logging_obj: Final = LiteLLMLogging( + model=_NEBIUS_KIMI_K3, + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="test-nebius-cost", + function_id="test-function-id", + ) + logging_obj.update_environment_variables( + model=_NEBIUS_KIMI_K3, + user=None, + optional_params={}, + litellm_params={"custom_llm_provider": "nebius"}, + custom_llm_provider="nebius", + ) + response: Final = litellm.stream_chunk_builder( + chunks=[usage_chunk], + messages=[{"role": "user", "content": "hi"}], + logging_obj=logging_obj, + ) + assert response is not None + assert response._hidden_params.get("response_cost") is None + calculator_cost: Final = logging_obj._response_cost_calculator(result=response) + client_cost: Final = getattr(response.usage, "cost", None) + assert calculator_cost == pytest.approx(map_cost) + assert client_cost == pytest.approx(calculator_cost) + assert client_cost != pytest.approx(map_cost / 10) + + +class _FakeNebiusModelsResponse: + status_code: Final = 200 + text: Final = "" + + def json(self) -> dict[str, list[dict[str, str]]]: + return {"data": [{"id": "org/model-a"}, {"id": "org/model-b"}]} + + +class TestNebiusGetModels: + def test_get_models_lists_from_live_v1_models(self, monkeypatch: pytest.MonkeyPatch): + captured: dict[str, object] = {} + + def mock_get(*, url: str, headers: dict[str, str]) -> _FakeNebiusModelsResponse: + captured["url"] = url + captured["headers"] = headers + return _FakeNebiusModelsResponse() + + monkeypatch.setattr(litellm.module_level_client, "get", mock_get) + models: Final = NebiusConfig().get_models( + api_key="test-nebius-key", + api_base="https://gateway.example/nebius/v1", + ) + assert captured["url"] == "https://gateway.example/nebius/v1/models" + assert captured["headers"] == {"Authorization": "Bearer test-nebius-key"} + assert models == ["org/model-a", "org/model-b"] + + def test_get_models_uses_get_api_key_and_get_api_base(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("NEBIUS_API_KEY", "env-nebius-key") + monkeypatch.delenv("NEBIUS_API_BASE", raising=False) + captured: dict[str, object] = {} + + def mock_get(*, url: str, headers: dict[str, str]) -> _FakeNebiusModelsResponse: + captured["url"] = url + captured["headers"] = headers + return _FakeNebiusModelsResponse() + + monkeypatch.setattr(litellm.module_level_client, "get", mock_get) + models: Final = NebiusConfig().get_models() + assert captured["url"] == "https://api.studio.nebius.ai/v1/models" + assert captured["headers"] == {"Authorization": "Bearer env-nebius-key"} + assert models == ["org/model-a", "org/model-b"] + + def test_get_models_does_not_fall_back_to_openai_credentials(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("NEBIUS_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + + def mock_get(**_kwargs: object) -> _FakeNebiusModelsResponse: + raise AssertionError("must not call /v1/models without a Nebius key") + + monkeypatch.setattr(litellm.module_level_client, "get", mock_get) + with pytest.raises(ValueError, match="NEBIUS_API_KEY"): + NebiusConfig().get_models() +