fix(nebius): bill spend from the cost map for org/model ids

Cost-map rows use litellm_provider=nebius but the provider set only ingested the old nebius-chat-models labels, so models_by_provider[nebius] stayed empty. Short response ids like Kimi-K3 also missed nebius/moonshotai/Kimi-K3. Streaming trusted usage.cost when present, which can be an order of magnitude below Token Factory list rates.

Overwrite usage.cost on Nebius streams with the calculator/map cost so clients do not keep the provider's understated value. List models from live /v1/models via get_api_base and get_api_key, preserving path. Match cost-map keys case-insensitively; warn only when a suffix match is ambiguous.

Fixes BerriAI/litellm#41782
This commit is contained in:
srinivassivaratri 2026-09-21 16:30:42 +00:00
parent def37c6532
commit ce704fca73
No known key found for this signature in database
5 changed files with 278 additions and 2 deletions

View file

@ -903,6 +903,11 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None:
sambanova_embedding_models.add(key)
elif value.get("litellm_provider") == "novita":
novita_models.add(key)
elif value.get("litellm_provider") == "nebius":
if value.get("mode") == "embedding":
nebius_embedding_models.add(key)
else:
nebius_models.add(key)
elif value.get("litellm_provider") == "nebius-chat-models":
nebius_models.add(key)
elif value.get("litellm_provider") == "nebius-embedding-models":

View file

@ -4,9 +4,12 @@ Nebius AI Studio Chat Completions API - Transformation
This is OpenAI compatible - no translation needed / occurs
"""
from types import MappingProxyType
from typing import Final
from litellm._logging import verbose_logger
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.secret_managers.main import get_secret_str
class NebiusConfig(OpenAIGPTConfig):
@ -27,3 +30,54 @@ class NebiusConfig(OpenAIGPTConfig):
elif param in supported_openai_params:
optional_params[param] = value
return optional_params
def get_model_cost_key(self, model: str) -> str | None:
import litellm
stripped: Final = model.removeprefix("nebius/")
prefixed: Final = f"nebius/{stripped}"
catalog: Final = litellm.nebius_models | litellm.nebius_embedding_models
if prefixed in litellm.model_cost:
return prefixed
prefixed_folded: Final = prefixed.casefold()
prefixed_matches: Final = tuple(key for key in catalog if key.casefold() == prefixed_folded)
if len(prefixed_matches) == 1:
return prefixed_matches[0]
suffix: Final = f"/{stripped}".casefold()
matches: Final = tuple(key for key in catalog if key.casefold().endswith(suffix))
if len(matches) == 1:
return matches[0]
if len(matches) > 1:
verbose_logger.warning(
"Nebius cost map has %s keys ending in %s; not using a suffix match. keys=%s",
len(matches),
suffix,
matches,
)
return None
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
import litellm
resolved_key: Final = self.get_api_key(api_key)
resolved_base: Final = self.get_api_base(api_base)
if resolved_key is None or resolved_base is None:
raise ValueError(
"NEBIUS_API_KEY is not set. Pass api_key or set NEBIUS_API_KEY to query Nebius /v1/models."
)
response: Final = litellm.module_level_client.get(
url=f"{resolved_base.rstrip('/')}/models",
headers=MappingProxyType({"Authorization": f"Bearer {resolved_key}"}),
)
if response.status_code != 200:
raise Exception(f"Failed to get models: {response.text}")
models: Final = response.json()["data"]
return [model["id"] for model in models] # mutable-ok: OpenAI-compatible get_models returns list[str]
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return api_key or get_secret_str("NEBIUS_API_KEY")
@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:
return api_base or get_secret_str("NEBIUS_API_BASE") or "https://api.studio.nebius.ai/v1"

View file

@ -8698,7 +8698,7 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None
return TextCompletionResponse(**response)
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value})
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value, LlmProviders.NEBIUS.value})
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
@ -8752,7 +8752,11 @@ def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Opti
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if isinstance(getattr(usage, "cost", None), (int, float)):
reported_cost: Final = getattr(usage, "cost", None)
if (
isinstance(reported_cost, (int, float))
and logging_obj.model_call_details.get("custom_llm_provider") != LlmProviders.NEBIUS.value
):
return
computed_cost: Final = logging_obj._response_cost_calculator(result=response)
if isinstance(computed_cost, (int, float)) and computed_cost > 0:

View file

@ -9037,6 +9037,8 @@ class ProviderConfigManager:
return VLLMModelInfo()
elif LlmProviders.LEMONADE == provider:
return litellm.LemonadeChatConfig()
elif LlmProviders.NEBIUS == provider:
return litellm.NebiusConfig()
elif LlmProviders.CLARIFAI == provider:
return litellm.ClarifaiConfig()
elif LlmProviders.BEDROCK == provider:

View file

@ -7,11 +7,14 @@ Nebius AI Studio is an OpenAI-compatible provider with minor customizations.
from typing import Final
import pytest
import litellm
from litellm import completion
from litellm.llms.nebius.chat.transformation import NebiusConfig
from litellm.types.utils import ModelResponse, Usage
class TestNebiusConfig:
@ -106,3 +109,211 @@ class TestNebiusConfig:
# Check for specific content in the response
assert "```python" in response.choices[0].message.content
assert "Hey from LiteLLM" in response.choices[0].message.content
_NEBIUS_KIMI_K3: Final = "nebius/moonshotai/Kimi-K3"
def _nebius_cost_map_keys() -> set[str]:
return {
key
for key, value in litellm.model_cost.items()
if isinstance(value, dict) and value.get("litellm_provider") == "nebius"
}
def _kimi_k3_row() -> dict:
row: Final = litellm.model_cost[_NEBIUS_KIMI_K3]
assert isinstance(row, dict)
return row
class TestNebiusCostTracking:
def test_cost_map_nebius_rows_are_registered_on_the_provider(self):
registered: Final = set(litellm.models_by_provider.get("nebius") or ())
assert _nebius_cost_map_keys() <= registered
assert _NEBIUS_KIMI_K3 in registered
def test_short_response_model_id_resolves_to_the_cost_map_row(self):
row: Final = _kimi_k3_row()
leaf: Final = _NEBIUS_KIMI_K3.rsplit("/", 1)[-1]
info: Final = litellm.get_model_info(model=leaf, custom_llm_provider="nebius")
assert info["input_cost_per_token"] == row["input_cost_per_token"]
assert info["output_cost_per_token"] == row["output_cost_per_token"]
def test_short_response_model_id_resolution_is_case_insensitive(self):
row: Final = _kimi_k3_row()
leaf: Final = _NEBIUS_KIMI_K3.rsplit("/", 1)[-1]
swapped: Final = leaf.swapcase()
assert swapped != leaf
info: Final = litellm.get_model_info(model=swapped, custom_llm_provider="nebius")
assert info["input_cost_per_token"] == row["input_cost_per_token"]
assert info["output_cost_per_token"] == row["output_cost_per_token"]
assert NebiusConfig().get_model_cost_key(_NEBIUS_KIMI_K3.lower()) == _NEBIUS_KIMI_K3
def test_ambiguous_leaf_suffix_logs_and_returns_none(
self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
):
import logging
leaf: Final = "shared-leaf"
monkeypatch.setattr(
litellm,
"nebius_models",
{f"nebius/org-a/{leaf}", f"nebius/org-b/{leaf}"},
)
monkeypatch.setattr(litellm, "nebius_embedding_models", set())
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
matched: Final = NebiusConfig().get_model_cost_key(leaf)
assert matched is None
assert "2 keys ending in /shared-leaf" in caplog.text
assert "nebius/org-a/shared-leaf" in caplog.text
assert "nebius/org-b/shared-leaf" in caplog.text
def test_unmapped_leaf_returns_none_without_warning(
self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
):
import logging
monkeypatch.setattr(litellm, "nebius_models", set())
monkeypatch.setattr(litellm, "nebius_embedding_models", set())
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
matched: Final = NebiusConfig().get_model_cost_key("unmapped-leaf")
assert matched is None
assert "not using a suffix match" not in caplog.text
def test_completion_cost_matches_cost_map_row_not_provider_usage_cost(self):
row: Final = _kimi_k3_row()
prompt_tokens: Final = 1_000_000
completion_tokens: Final = 500_000
map_cost: Final = (
prompt_tokens * float(row["input_cost_per_token"])
+ completion_tokens * float(row["output_cost_per_token"])
)
response: Final = ModelResponse(
model="Kimi-K3",
usage=Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
cost=map_cost / 10,
),
)
response._hidden_params["custom_llm_provider"] = "nebius"
billed: Final = litellm.completion_cost(
completion_response=response,
model=_NEBIUS_KIMI_K3,
custom_llm_provider="nebius",
)
assert billed == pytest.approx(map_cost)
assert billed != pytest.approx(map_cost / 10)
def test_stream_builder_prices_nebius_from_the_cost_map(self):
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
row: Final = _kimi_k3_row()
prompt_tokens: Final = 1_000_000
completion_tokens: Final = 500_000
map_cost: Final = (
prompt_tokens * float(row["input_cost_per_token"])
+ completion_tokens * float(row["output_cost_per_token"])
)
usage_chunk: Final = ModelResponseStream(
id="chatcmpl-nebius-cost",
created=1724900000,
model="Kimi-K3",
object="chat.completion.chunk",
choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="", role="assistant"))],
)
usage_chunk.usage = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
cost=map_cost / 10,
)
logging_obj: Final = LiteLLMLogging(
model=_NEBIUS_KIMI_K3,
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="completion",
start_time=datetime.now(),
litellm_call_id="test-nebius-cost",
function_id="test-function-id",
)
logging_obj.update_environment_variables(
model=_NEBIUS_KIMI_K3,
user=None,
optional_params={},
litellm_params={"custom_llm_provider": "nebius"},
custom_llm_provider="nebius",
)
response: Final = litellm.stream_chunk_builder(
chunks=[usage_chunk],
messages=[{"role": "user", "content": "hi"}],
logging_obj=logging_obj,
)
assert response is not None
assert response._hidden_params.get("response_cost") is None
calculator_cost: Final = logging_obj._response_cost_calculator(result=response)
client_cost: Final = getattr(response.usage, "cost", None)
assert calculator_cost == pytest.approx(map_cost)
assert client_cost == pytest.approx(calculator_cost)
assert client_cost != pytest.approx(map_cost / 10)
class _FakeNebiusModelsResponse:
status_code: Final = 200
text: Final = ""
def json(self) -> dict[str, list[dict[str, str]]]:
return {"data": [{"id": "org/model-a"}, {"id": "org/model-b"}]}
class TestNebiusGetModels:
def test_get_models_lists_from_live_v1_models(self, monkeypatch: pytest.MonkeyPatch):
captured: dict[str, object] = {}
def mock_get(*, url: str, headers: dict[str, str]) -> _FakeNebiusModelsResponse:
captured["url"] = url
captured["headers"] = headers
return _FakeNebiusModelsResponse()
monkeypatch.setattr(litellm.module_level_client, "get", mock_get)
models: Final = NebiusConfig().get_models(
api_key="test-nebius-key",
api_base="https://gateway.example/nebius/v1",
)
assert captured["url"] == "https://gateway.example/nebius/v1/models"
assert captured["headers"] == {"Authorization": "Bearer test-nebius-key"}
assert models == ["org/model-a", "org/model-b"]
def test_get_models_uses_get_api_key_and_get_api_base(self, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("NEBIUS_API_KEY", "env-nebius-key")
monkeypatch.delenv("NEBIUS_API_BASE", raising=False)
captured: dict[str, object] = {}
def mock_get(*, url: str, headers: dict[str, str]) -> _FakeNebiusModelsResponse:
captured["url"] = url
captured["headers"] = headers
return _FakeNebiusModelsResponse()
monkeypatch.setattr(litellm.module_level_client, "get", mock_get)
models: Final = NebiusConfig().get_models()
assert captured["url"] == "https://api.studio.nebius.ai/v1/models"
assert captured["headers"] == {"Authorization": "Bearer env-nebius-key"}
assert models == ["org/model-a", "org/model-b"]
def test_get_models_does_not_fall_back_to_openai_credentials(self, monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("NEBIUS_API_KEY", raising=False)
monkeypatch.setenv("OPENAI_API_KEY", "openai-key")
def mock_get(**_kwargs: object) -> _FakeNebiusModelsResponse:
raise AssertionError("must not call /v1/models without a Nebius key")
monkeypatch.setattr(litellm.module_level_client, "get", mock_get)
with pytest.raises(ValueError, match="NEBIUS_API_KEY"):
NebiusConfig().get_models()