mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-19 09:53:33 +00:00
Merge fa6090b9d2 into e04e89c701
This commit is contained in:
commit
f6d81e021e
5 changed files with 243 additions and 63 deletions
|
|
@ -11,8 +11,12 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
handle_messages_with_content_list_to_str_conversion,
|
||||
)
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
DEFAULT_INTEGRATION_SOURCE: Final = "litellm"
|
||||
INTEGRATION_SOURCE_HEADER: Final = "X-Integration-Source"
|
||||
|
||||
|
||||
class SambanovaConfig(OpenAIGPTConfig):
|
||||
"""
|
||||
|
|
@ -59,9 +63,9 @@ class SambanovaConfig(OpenAIGPTConfig):
|
|||
Get the supported OpenAI params for the given model
|
||||
|
||||
"""
|
||||
from litellm.utils import supports_function_calling
|
||||
from litellm.utils import supports_function_calling, supports_reasoning
|
||||
|
||||
params: Final = [
|
||||
params: Final = (
|
||||
"max_completion_tokens",
|
||||
"max_tokens",
|
||||
"response_format",
|
||||
|
|
@ -71,14 +75,24 @@ class SambanovaConfig(OpenAIGPTConfig):
|
|||
"temperature",
|
||||
"top_p",
|
||||
"top_k",
|
||||
]
|
||||
"presence_penalty",
|
||||
"frequency_penalty",
|
||||
"logprobs",
|
||||
"top_logprobs",
|
||||
"n",
|
||||
"logit_bias",
|
||||
"seed",
|
||||
)
|
||||
tool_params: Final = (
|
||||
("tools", "tool_choice", "parallel_tool_calls")
|
||||
if supports_function_calling(model, custom_llm_provider="sambanova")
|
||||
else ()
|
||||
)
|
||||
reasoning_params: Final = (
|
||||
("reasoning_effort",) if supports_reasoning(model, custom_llm_provider="sambanova") else ()
|
||||
)
|
||||
|
||||
if supports_function_calling(model, custom_llm_provider="sambanova"):
|
||||
params.append("tools")
|
||||
params.append("tool_choice")
|
||||
params.append("parallel_tool_calls")
|
||||
|
||||
return params
|
||||
return [*params, *tool_params, *reasoning_params]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
|
|
@ -98,6 +112,43 @@ class SambanovaConfig(OpenAIGPTConfig):
|
|||
optional_params[param] = value
|
||||
return optional_params
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
"""Send X-Integration-Source: caller header, then extra_body, then env var, then default"""
|
||||
extra_headers: Final[dict[str, str]] = dict(optional_params.get("extra_headers") or {})
|
||||
extra_body: Final[dict[str, object]] = dict(optional_params.get("extra_body") or {})
|
||||
body_source: Final = extra_body.pop("integration_source", None)
|
||||
header_source: Final = next(
|
||||
(value for key, value in extra_headers.items() if key.lower() == INTEGRATION_SOURCE_HEADER.lower()),
|
||||
None,
|
||||
)
|
||||
integration_source: Final = (
|
||||
body_source or get_secret_str("SAMBANOVA_INTEGRATION_SOURCE") or DEFAULT_INTEGRATION_SOURCE
|
||||
)
|
||||
headers_with_source: Final = (
|
||||
extra_headers if header_source else {**extra_headers, INTEGRATION_SOURCE_HEADER: integration_source}
|
||||
)
|
||||
other_params: Final[dict[str, object]] = {k: v for k, v in optional_params.items() if k != "extra_body"}
|
||||
updated_params: Final[dict[str, object]] = {
|
||||
**other_params,
|
||||
"extra_headers": headers_with_source,
|
||||
**({"extra_body": extra_body} if extra_body else {}),
|
||||
}
|
||||
|
||||
return super().transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=updated_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
@overload
|
||||
def _transform_messages(
|
||||
self, messages: list[AllMessageValues], model: str, is_async: Literal[True]
|
||||
|
|
|
|||
|
|
@ -47622,39 +47622,26 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"sambanova/DeepSeek-R1": {
|
||||
"input_cost_per_token": 5e-06,
|
||||
"sambanova/MiniMax-M3": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "sambanova",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"max_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7e-06,
|
||||
"source": "https://cloud.sambanova.ai/plans/pricing"
|
||||
},
|
||||
"sambanova/Llama-4-Maverick-17B-128E-Instruct": {
|
||||
"input_cost_per_token": 6.3e-07,
|
||||
"litellm_provider": "sambanova",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"metadata": {
|
||||
"notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount"
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.8e-06,
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"source": "https://cloud.sambanova.ai/plans/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"sambanova/Meta-Llama-3.3-70B-Instruct": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "sambanova",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"max_output_tokens": 3072,
|
||||
"max_tokens": 3072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"source": "https://cloud.sambanova.ai/plans/pricing",
|
||||
|
|
@ -47663,9 +47650,9 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"sambanova/DeepSeek-V3.1": {
|
||||
"max_tokens": 131072,
|
||||
"max_tokens": 7168,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_output_tokens": 7168,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"output_cost_per_token": 4.5e-06,
|
||||
"litellm_provider": "sambanova",
|
||||
|
|
@ -47689,9 +47676,9 @@
|
|||
"source": "https://cloud.sambanova.ai/plans/pricing"
|
||||
},
|
||||
"sambanova/DeepSeek-V3.2": {
|
||||
"max_tokens": 32768,
|
||||
"max_tokens": 7168,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"max_output_tokens": 7168,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"output_cost_per_token": 4.5e-06,
|
||||
"litellm_provider": "sambanova",
|
||||
|
|
|
|||
|
|
@ -47622,39 +47622,26 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"sambanova/DeepSeek-R1": {
|
||||
"input_cost_per_token": 5e-06,
|
||||
"sambanova/MiniMax-M3": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "sambanova",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"max_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7e-06,
|
||||
"source": "https://cloud.sambanova.ai/plans/pricing"
|
||||
},
|
||||
"sambanova/Llama-4-Maverick-17B-128E-Instruct": {
|
||||
"input_cost_per_token": 6.3e-07,
|
||||
"litellm_provider": "sambanova",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"metadata": {
|
||||
"notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount"
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.8e-06,
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"source": "https://cloud.sambanova.ai/plans/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"sambanova/Meta-Llama-3.3-70B-Instruct": {
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "sambanova",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"max_output_tokens": 3072,
|
||||
"max_tokens": 3072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"source": "https://cloud.sambanova.ai/plans/pricing",
|
||||
|
|
@ -47663,9 +47650,9 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"sambanova/DeepSeek-V3.1": {
|
||||
"max_tokens": 131072,
|
||||
"max_tokens": 7168,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_output_tokens": 7168,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"output_cost_per_token": 4.5e-06,
|
||||
"litellm_provider": "sambanova",
|
||||
|
|
@ -47689,9 +47676,9 @@
|
|||
"source": "https://cloud.sambanova.ai/plans/pricing"
|
||||
},
|
||||
"sambanova/DeepSeek-V3.2": {
|
||||
"max_tokens": 32768,
|
||||
"max_tokens": 7168,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"max_output_tokens": 7168,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"output_cost_per_token": 4.5e-06,
|
||||
"litellm_provider": "sambanova",
|
||||
|
|
|
|||
127
tests/unit/llms/sambanova/test_sambanova_chat_transformation.py
Normal file
127
tests/unit/llms/sambanova/test_sambanova_chat_transformation.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
import pytest
|
||||
|
||||
from litellm.llms.sambanova.chat import SambanovaConfig
|
||||
from litellm.utils import get_optional_params
|
||||
|
||||
_ALWAYS_SUPPORTED = (
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"top_k",
|
||||
"stop",
|
||||
"stream",
|
||||
"stream_options",
|
||||
"response_format",
|
||||
"presence_penalty",
|
||||
"frequency_penalty",
|
||||
"logprobs",
|
||||
"top_logprobs",
|
||||
"n",
|
||||
"logit_bias",
|
||||
"seed",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param", _ALWAYS_SUPPORTED)
|
||||
def test_supported_params_include_spec_params(param):
|
||||
assert param in SambanovaConfig().get_supported_openai_params("MiniMax-M3")
|
||||
|
||||
|
||||
def test_reasoning_effort_only_for_reasoning_models():
|
||||
config = SambanovaConfig()
|
||||
assert "reasoning_effort" in config.get_supported_openai_params("gpt-oss-120b")
|
||||
assert "reasoning_effort" not in config.get_supported_openai_params("Meta-Llama-3.3-70B-Instruct")
|
||||
|
||||
|
||||
def test_tools_only_for_function_calling_models():
|
||||
config = SambanovaConfig()
|
||||
for param in ("tools", "tool_choice", "parallel_tool_calls"):
|
||||
assert param in config.get_supported_openai_params("MiniMax-M3")
|
||||
assert param not in config.get_supported_openai_params("gemma-4-31B-it")
|
||||
|
||||
|
||||
def test_max_completion_tokens_mapped_to_max_tokens():
|
||||
optional_params = SambanovaConfig().map_openai_params(
|
||||
non_default_params={"max_completion_tokens": 128},
|
||||
optional_params={},
|
||||
model="MiniMax-M3",
|
||||
drop_params=False,
|
||||
)
|
||||
assert optional_params == {"max_tokens": 128}
|
||||
|
||||
|
||||
def test_new_params_forwarded_via_get_optional_params():
|
||||
optional_params = get_optional_params(
|
||||
model="MiniMax-M3",
|
||||
custom_llm_provider="sambanova",
|
||||
seed=7,
|
||||
presence_penalty=0.5,
|
||||
frequency_penalty=0.25,
|
||||
reasoning_effort="low",
|
||||
)
|
||||
assert optional_params["seed"] == 7
|
||||
assert optional_params["presence_penalty"] == 0.5
|
||||
assert optional_params["frequency_penalty"] == 0.25
|
||||
assert optional_params["reasoning_effort"] == "low"
|
||||
|
||||
|
||||
def _transform(optional_params):
|
||||
return SambanovaConfig().transform_request(
|
||||
model="MiniMax-M3",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
|
||||
def test_integration_source_defaults_to_litellm(monkeypatch):
|
||||
monkeypatch.delenv("SAMBANOVA_INTEGRATION_SOURCE", raising=False)
|
||||
request = _transform({})
|
||||
assert request["extra_headers"]["X-Integration-Source"] == "litellm"
|
||||
|
||||
|
||||
def test_integration_source_env_overrides_default(monkeypatch):
|
||||
monkeypatch.setenv("SAMBANOVA_INTEGRATION_SOURCE", "langflow")
|
||||
request = _transform({})
|
||||
assert request["extra_headers"]["X-Integration-Source"] == "langflow"
|
||||
|
||||
|
||||
def test_integration_source_param_overrides_env(monkeypatch):
|
||||
monkeypatch.setenv("SAMBANOVA_INTEGRATION_SOURCE", "langflow")
|
||||
optional_params = {"extra_body": {"integration_source": "my-framework"}}
|
||||
request = _transform(optional_params)
|
||||
assert request["extra_headers"]["X-Integration-Source"] == "my-framework"
|
||||
assert "integration_source" not in request.get("extra_body", {})
|
||||
assert "integration_source" in optional_params["extra_body"], "caller's dict must not be mutated"
|
||||
|
||||
|
||||
def test_integration_source_explicit_header_wins(monkeypatch):
|
||||
monkeypatch.setenv("SAMBANOVA_INTEGRATION_SOURCE", "langflow")
|
||||
request = _transform({"extra_headers": {"X-Integration-Source": "custom", "X-Other": "1"}})
|
||||
assert request["extra_headers"] == {"X-Integration-Source": "custom", "X-Other": "1"}
|
||||
|
||||
|
||||
def test_integration_source_removed_from_body_when_header_also_set():
|
||||
request = _transform(
|
||||
{
|
||||
"extra_headers": {"X-Integration-Source": "custom"},
|
||||
"extra_body": {"integration_source": "ignored", "other": 1},
|
||||
}
|
||||
)
|
||||
assert request["extra_headers"] == {"X-Integration-Source": "custom"}
|
||||
assert request["extra_body"] == {"other": 1}
|
||||
|
||||
|
||||
def test_integration_source_header_matched_case_insensitively(monkeypatch):
|
||||
monkeypatch.setenv("SAMBANOVA_INTEGRATION_SOURCE", "langflow")
|
||||
request = _transform({"extra_headers": {"x-integration-source": "custom"}})
|
||||
assert request["extra_headers"] == {"x-integration-source": "custom"}
|
||||
|
||||
|
||||
def test_content_list_converted_to_string():
|
||||
messages = [{"role": "user", "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}]
|
||||
out = SambanovaConfig()._transform_messages(messages, model="MiniMax-M3")
|
||||
assert isinstance(out[0]["content"], str)
|
||||
assert "a" in out[0]["content"] and "b" in out[0]["content"]
|
||||
|
|
@ -23,3 +23,31 @@ def test_sambanova_minimax_m27_model_info():
|
|||
routed_model, provider, _, _ = get_llm_provider(model=model)
|
||||
assert routed_model == "MiniMax-M2.7"
|
||||
assert provider == "sambanova"
|
||||
|
||||
|
||||
def _sambanova_models() -> dict:
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
return {k: v for k, v in model_cost.items() if k.startswith("sambanova/")}
|
||||
|
||||
|
||||
def test_sambanova_chat_entries_are_consistent():
|
||||
chat_models = {k: v for k, v in _sambanova_models().items() if v["mode"] == "chat"}
|
||||
assert chat_models
|
||||
for name, info in chat_models.items():
|
||||
assert info["litellm_provider"] == "sambanova", name
|
||||
assert info["input_cost_per_token"] >= 0, name
|
||||
assert info["output_cost_per_token"] >= 0, name
|
||||
assert info["max_tokens"] == info["max_output_tokens"], name
|
||||
assert info["max_output_tokens"] <= info["max_input_tokens"], name
|
||||
if info.get("supports_tool_choice"):
|
||||
assert info.get("supports_function_calling"), name
|
||||
|
||||
|
||||
def test_sambanova_minimax_m3_is_routed_to_sambanova():
|
||||
assert "sambanova/MiniMax-M3" in _sambanova_models()
|
||||
|
||||
routed_model, provider, _, _ = get_llm_provider(model="sambanova/MiniMax-M3")
|
||||
assert routed_model == "MiniMax-M3"
|
||||
assert provider == "sambanova"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue