mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(nebius): bill spend from the cost map for org/model ids
Cost-map rows use litellm_provider=nebius but the provider set only ingested the old nebius-chat-models labels, so models_by_provider[nebius] stayed empty. Short response ids like Kimi-K3 also missed nebius/moonshotai/Kimi-K3. Streaming trusted usage.cost when present, which can be an order of magnitude below Token Factory list rates. Overwrite usage.cost on Nebius streams with the calculator/map cost so clients do not keep the provider's understated value. List models from live /v1/models via get_api_base and get_api_key, preserving path. Match cost-map keys case-insensitively; warn only when a suffix match is ambiguous. Fixes BerriAI/litellm#41782
This commit is contained in:
parent
def37c6532
commit
ce704fca73
5 changed files with 278 additions and 2 deletions
|
|
@ -903,6 +903,11 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None:
|
|||
sambanova_embedding_models.add(key)
|
||||
elif value.get("litellm_provider") == "novita":
|
||||
novita_models.add(key)
|
||||
elif value.get("litellm_provider") == "nebius":
|
||||
if value.get("mode") == "embedding":
|
||||
nebius_embedding_models.add(key)
|
||||
else:
|
||||
nebius_models.add(key)
|
||||
elif value.get("litellm_provider") == "nebius-chat-models":
|
||||
nebius_models.add(key)
|
||||
elif value.get("litellm_provider") == "nebius-embedding-models":
|
||||
|
|
|
|||
|
|
@ -4,9 +4,12 @@ Nebius AI Studio Chat Completions API - Transformation
|
|||
This is OpenAI compatible - no translation needed / occurs
|
||||
"""
|
||||
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
|
||||
class NebiusConfig(OpenAIGPTConfig):
|
||||
|
|
@ -27,3 +30,54 @@ class NebiusConfig(OpenAIGPTConfig):
|
|||
elif param in supported_openai_params:
|
||||
optional_params[param] = value
|
||||
return optional_params
|
||||
|
||||
def get_model_cost_key(self, model: str) -> str | None:
|
||||
import litellm
|
||||
|
||||
stripped: Final = model.removeprefix("nebius/")
|
||||
prefixed: Final = f"nebius/{stripped}"
|
||||
catalog: Final = litellm.nebius_models | litellm.nebius_embedding_models
|
||||
if prefixed in litellm.model_cost:
|
||||
return prefixed
|
||||
prefixed_folded: Final = prefixed.casefold()
|
||||
prefixed_matches: Final = tuple(key for key in catalog if key.casefold() == prefixed_folded)
|
||||
if len(prefixed_matches) == 1:
|
||||
return prefixed_matches[0]
|
||||
suffix: Final = f"/{stripped}".casefold()
|
||||
matches: Final = tuple(key for key in catalog if key.casefold().endswith(suffix))
|
||||
if len(matches) == 1:
|
||||
return matches[0]
|
||||
if len(matches) > 1:
|
||||
verbose_logger.warning(
|
||||
"Nebius cost map has %s keys ending in %s; not using a suffix match. keys=%s",
|
||||
len(matches),
|
||||
suffix,
|
||||
matches,
|
||||
)
|
||||
return None
|
||||
|
||||
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
|
||||
import litellm
|
||||
|
||||
resolved_key: Final = self.get_api_key(api_key)
|
||||
resolved_base: Final = self.get_api_base(api_base)
|
||||
if resolved_key is None or resolved_base is None:
|
||||
raise ValueError(
|
||||
"NEBIUS_API_KEY is not set. Pass api_key or set NEBIUS_API_KEY to query Nebius /v1/models."
|
||||
)
|
||||
response: Final = litellm.module_level_client.get(
|
||||
url=f"{resolved_base.rstrip('/')}/models",
|
||||
headers=MappingProxyType({"Authorization": f"Bearer {resolved_key}"}),
|
||||
)
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Failed to get models: {response.text}")
|
||||
models: Final = response.json()["data"]
|
||||
return [model["id"] for model in models] # mutable-ok: OpenAI-compatible get_models returns list[str]
|
||||
|
||||
@staticmethod
|
||||
def get_api_key(api_key: str | None = None) -> str | None:
|
||||
return api_key or get_secret_str("NEBIUS_API_KEY")
|
||||
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str | None:
|
||||
return api_base or get_secret_str("NEBIUS_API_BASE") or "https://api.studio.nebius.ai/v1"
|
||||
|
|
|
|||
|
|
@ -8698,7 +8698,7 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None
|
|||
return TextCompletionResponse(**response)
|
||||
|
||||
|
||||
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value})
|
||||
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value, LlmProviders.NEBIUS.value})
|
||||
|
||||
|
||||
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
|
||||
|
|
@ -8752,7 +8752,11 @@ def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Opti
|
|||
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
|
||||
if logging_obj is None:
|
||||
return
|
||||
if isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
reported_cost: Final = getattr(usage, "cost", None)
|
||||
if (
|
||||
isinstance(reported_cost, (int, float))
|
||||
and logging_obj.model_call_details.get("custom_llm_provider") != LlmProviders.NEBIUS.value
|
||||
):
|
||||
return
|
||||
computed_cost: Final = logging_obj._response_cost_calculator(result=response)
|
||||
if isinstance(computed_cost, (int, float)) and computed_cost > 0:
|
||||
|
|
|
|||
|
|
@ -9037,6 +9037,8 @@ class ProviderConfigManager:
|
|||
return VLLMModelInfo()
|
||||
elif LlmProviders.LEMONADE == provider:
|
||||
return litellm.LemonadeChatConfig()
|
||||
elif LlmProviders.NEBIUS == provider:
|
||||
return litellm.NebiusConfig()
|
||||
elif LlmProviders.CLARIFAI == provider:
|
||||
return litellm.ClarifaiConfig()
|
||||
elif LlmProviders.BEDROCK == provider:
|
||||
|
|
|
|||
|
|
@ -7,11 +7,14 @@ Nebius AI Studio is an OpenAI-compatible provider with minor customizations.
|
|||
|
||||
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import completion
|
||||
from litellm.llms.nebius.chat.transformation import NebiusConfig
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
|
||||
class TestNebiusConfig:
|
||||
|
|
@ -106,3 +109,211 @@ class TestNebiusConfig:
|
|||
# Check for specific content in the response
|
||||
assert "```python" in response.choices[0].message.content
|
||||
assert "Hey from LiteLLM" in response.choices[0].message.content
|
||||
|
||||
|
||||
_NEBIUS_KIMI_K3: Final = "nebius/moonshotai/Kimi-K3"
|
||||
|
||||
|
||||
def _nebius_cost_map_keys() -> set[str]:
|
||||
return {
|
||||
key
|
||||
for key, value in litellm.model_cost.items()
|
||||
if isinstance(value, dict) and value.get("litellm_provider") == "nebius"
|
||||
}
|
||||
|
||||
|
||||
def _kimi_k3_row() -> dict:
|
||||
row: Final = litellm.model_cost[_NEBIUS_KIMI_K3]
|
||||
assert isinstance(row, dict)
|
||||
return row
|
||||
|
||||
|
||||
class TestNebiusCostTracking:
|
||||
def test_cost_map_nebius_rows_are_registered_on_the_provider(self):
|
||||
registered: Final = set(litellm.models_by_provider.get("nebius") or ())
|
||||
assert _nebius_cost_map_keys() <= registered
|
||||
assert _NEBIUS_KIMI_K3 in registered
|
||||
|
||||
def test_short_response_model_id_resolves_to_the_cost_map_row(self):
|
||||
row: Final = _kimi_k3_row()
|
||||
leaf: Final = _NEBIUS_KIMI_K3.rsplit("/", 1)[-1]
|
||||
info: Final = litellm.get_model_info(model=leaf, custom_llm_provider="nebius")
|
||||
assert info["input_cost_per_token"] == row["input_cost_per_token"]
|
||||
assert info["output_cost_per_token"] == row["output_cost_per_token"]
|
||||
|
||||
def test_short_response_model_id_resolution_is_case_insensitive(self):
|
||||
row: Final = _kimi_k3_row()
|
||||
leaf: Final = _NEBIUS_KIMI_K3.rsplit("/", 1)[-1]
|
||||
swapped: Final = leaf.swapcase()
|
||||
assert swapped != leaf
|
||||
info: Final = litellm.get_model_info(model=swapped, custom_llm_provider="nebius")
|
||||
assert info["input_cost_per_token"] == row["input_cost_per_token"]
|
||||
assert info["output_cost_per_token"] == row["output_cost_per_token"]
|
||||
assert NebiusConfig().get_model_cost_key(_NEBIUS_KIMI_K3.lower()) == _NEBIUS_KIMI_K3
|
||||
|
||||
def test_ambiguous_leaf_suffix_logs_and_returns_none(
|
||||
self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
import logging
|
||||
|
||||
leaf: Final = "shared-leaf"
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"nebius_models",
|
||||
{f"nebius/org-a/{leaf}", f"nebius/org-b/{leaf}"},
|
||||
)
|
||||
monkeypatch.setattr(litellm, "nebius_embedding_models", set())
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
matched: Final = NebiusConfig().get_model_cost_key(leaf)
|
||||
assert matched is None
|
||||
assert "2 keys ending in /shared-leaf" in caplog.text
|
||||
assert "nebius/org-a/shared-leaf" in caplog.text
|
||||
assert "nebius/org-b/shared-leaf" in caplog.text
|
||||
|
||||
def test_unmapped_leaf_returns_none_without_warning(
|
||||
self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
import logging
|
||||
|
||||
monkeypatch.setattr(litellm, "nebius_models", set())
|
||||
monkeypatch.setattr(litellm, "nebius_embedding_models", set())
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
matched: Final = NebiusConfig().get_model_cost_key("unmapped-leaf")
|
||||
assert matched is None
|
||||
assert "not using a suffix match" not in caplog.text
|
||||
|
||||
def test_completion_cost_matches_cost_map_row_not_provider_usage_cost(self):
|
||||
row: Final = _kimi_k3_row()
|
||||
prompt_tokens: Final = 1_000_000
|
||||
completion_tokens: Final = 500_000
|
||||
map_cost: Final = (
|
||||
prompt_tokens * float(row["input_cost_per_token"])
|
||||
+ completion_tokens * float(row["output_cost_per_token"])
|
||||
)
|
||||
response: Final = ModelResponse(
|
||||
model="Kimi-K3",
|
||||
usage=Usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
cost=map_cost / 10,
|
||||
),
|
||||
)
|
||||
response._hidden_params["custom_llm_provider"] = "nebius"
|
||||
billed: Final = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=_NEBIUS_KIMI_K3,
|
||||
custom_llm_provider="nebius",
|
||||
)
|
||||
assert billed == pytest.approx(map_cost)
|
||||
assert billed != pytest.approx(map_cost / 10)
|
||||
|
||||
def test_stream_builder_prices_nebius_from_the_cost_map(self):
|
||||
from datetime import datetime
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
row: Final = _kimi_k3_row()
|
||||
prompt_tokens: Final = 1_000_000
|
||||
completion_tokens: Final = 500_000
|
||||
map_cost: Final = (
|
||||
prompt_tokens * float(row["input_cost_per_token"])
|
||||
+ completion_tokens * float(row["output_cost_per_token"])
|
||||
)
|
||||
usage_chunk: Final = ModelResponseStream(
|
||||
id="chatcmpl-nebius-cost",
|
||||
created=1724900000,
|
||||
model="Kimi-K3",
|
||||
object="chat.completion.chunk",
|
||||
choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="", role="assistant"))],
|
||||
)
|
||||
usage_chunk.usage = Usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
cost=map_cost / 10,
|
||||
)
|
||||
logging_obj: Final = LiteLLMLogging(
|
||||
model=_NEBIUS_KIMI_K3,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-nebius-cost",
|
||||
function_id="test-function-id",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model=_NEBIUS_KIMI_K3,
|
||||
user=None,
|
||||
optional_params={},
|
||||
litellm_params={"custom_llm_provider": "nebius"},
|
||||
custom_llm_provider="nebius",
|
||||
)
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[usage_chunk],
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
assert response is not None
|
||||
assert response._hidden_params.get("response_cost") is None
|
||||
calculator_cost: Final = logging_obj._response_cost_calculator(result=response)
|
||||
client_cost: Final = getattr(response.usage, "cost", None)
|
||||
assert calculator_cost == pytest.approx(map_cost)
|
||||
assert client_cost == pytest.approx(calculator_cost)
|
||||
assert client_cost != pytest.approx(map_cost / 10)
|
||||
|
||||
|
||||
class _FakeNebiusModelsResponse:
|
||||
status_code: Final = 200
|
||||
text: Final = ""
|
||||
|
||||
def json(self) -> dict[str, list[dict[str, str]]]:
|
||||
return {"data": [{"id": "org/model-a"}, {"id": "org/model-b"}]}
|
||||
|
||||
|
||||
class TestNebiusGetModels:
|
||||
def test_get_models_lists_from_live_v1_models(self, monkeypatch: pytest.MonkeyPatch):
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def mock_get(*, url: str, headers: dict[str, str]) -> _FakeNebiusModelsResponse:
|
||||
captured["url"] = url
|
||||
captured["headers"] = headers
|
||||
return _FakeNebiusModelsResponse()
|
||||
|
||||
monkeypatch.setattr(litellm.module_level_client, "get", mock_get)
|
||||
models: Final = NebiusConfig().get_models(
|
||||
api_key="test-nebius-key",
|
||||
api_base="https://gateway.example/nebius/v1",
|
||||
)
|
||||
assert captured["url"] == "https://gateway.example/nebius/v1/models"
|
||||
assert captured["headers"] == {"Authorization": "Bearer test-nebius-key"}
|
||||
assert models == ["org/model-a", "org/model-b"]
|
||||
|
||||
def test_get_models_uses_get_api_key_and_get_api_base(self, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("NEBIUS_API_KEY", "env-nebius-key")
|
||||
monkeypatch.delenv("NEBIUS_API_BASE", raising=False)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def mock_get(*, url: str, headers: dict[str, str]) -> _FakeNebiusModelsResponse:
|
||||
captured["url"] = url
|
||||
captured["headers"] = headers
|
||||
return _FakeNebiusModelsResponse()
|
||||
|
||||
monkeypatch.setattr(litellm.module_level_client, "get", mock_get)
|
||||
models: Final = NebiusConfig().get_models()
|
||||
assert captured["url"] == "https://api.studio.nebius.ai/v1/models"
|
||||
assert captured["headers"] == {"Authorization": "Bearer env-nebius-key"}
|
||||
assert models == ["org/model-a", "org/model-b"]
|
||||
|
||||
def test_get_models_does_not_fall_back_to_openai_credentials(self, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("NEBIUS_API_KEY", raising=False)
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "openai-key")
|
||||
|
||||
def mock_get(**_kwargs: object) -> _FakeNebiusModelsResponse:
|
||||
raise AssertionError("must not call /v1/models without a Nebius key")
|
||||
|
||||
monkeypatch.setattr(litellm.module_level_client, "get", mock_get)
|
||||
with pytest.raises(ValueError, match="NEBIUS_API_KEY"):
|
||||
NebiusConfig().get_models()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue