From fd40bc6ddd0740d738e27712626ac0ac50731fb4 Mon Sep 17 00:00:00 2001 From: Jorge Piedrahita Ortiz Date: Sun, 18 May 2025 22:45:20 -0500 Subject: [PATCH] update sambanova models and parameters (#10900) * add sambanova to completion input params table * update sambanova supported args * update sambanova supported models * minor changes * fix sambanova model list * update sambanova models * update sambanova models * update sambanova docs * minor chnage sambanova url * update type to match OpenAIGPTConfig * minor change --- docs/my-website/docs/completion/input.md | 1 + docs/my-website/docs/providers/sambanova.md | 190 +++++++++++++++++- litellm/__init__.py | 2 +- litellm/llms/sambanova/chat.py | 43 +++- ...odel_prices_and_context_window_backup.json | 184 ++++++++++++----- model_prices_and_context_window.json | 184 ++++++++++++----- 6 files changed, 485 insertions(+), 119 deletions(-) diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index d4ed0d2997d..f9751094249 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -55,6 +55,7 @@ Use `litellm.get_supported_openai_params()` for an updated list of params for ea |Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | ✅ (model dependent) | | |Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | |TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | | | ✅ | | ✅ | ✅ | | | | +|Sambanova| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | ✅ | | ✅ | ✅ | | | | |AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | |NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | |Petals| ✅ | ✅ | | ✅ | ✅ | | | | | | diff --git a/docs/my-website/docs/providers/sambanova.md b/docs/my-website/docs/providers/sambanova.md index 7dd837e1b0a..290b64a1f09 100644 --- a/docs/my-website/docs/providers/sambanova.md +++ b/docs/my-website/docs/providers/sambanova.md @@ -1,8 +1,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Sambanova -https://cloud.sambanova.ai/ +# SambaNova +[https://cloud.sambanova.ai/](http://cloud.sambanova.ai?utm_source=litellm&utm_medium=external&utm_campaign=cloud_signup) :::tip @@ -23,20 +23,17 @@ import os os.environ['SAMBANOVA_API_KEY'] = "" response = completion( - model="sambanova/Meta-Llama-3.1-8B-Instruct", + model="sambanova/Llama-4-Maverick-17B-128E-Instruct", messages=[ { "role": "user", - "content": "What do you know about sambanova.ai. Give your response in json format", + "content": "What do you know about SambaNova Systems", } ], max_tokens=10, - response_format={ "type": "json_object" }, - stop=["\n\n"], + stop=[], temperature=0.2, top_p=0.9, - tool_choice="auto", - tools=[], user="user", ) print(response) @@ -49,17 +46,17 @@ import os os.environ['SAMBANOVA_API_KEY'] = "" response = completion( - model="sambanova/Meta-Llama-3.1-8B-Instruct", + model="sambanova/Llama-4-Maverick-17B-128E-Instruct", messages=[ { "role": "user", - "content": "What do you know about sambanova.ai. Give your response in json format", + "content": "What do you know about SambaNova Systems", } ], stream=True, max_tokens=10, response_format={ "type": "json_object" }, - stop=["\n\n"], + stop=[], temperature=0.2, top_p=0.9, tool_choice="auto", @@ -139,3 +136,174 @@ Here's how to call a Sambanova model with the LiteLLM Proxy Server + +## SambaNova - Tool Calling + +```python +import litellm + +# Example dummy function + +def get_current_weather(location, unit="fahrenheit"): + if unit == "fahrenheit" + return{"location": location, "temperature": "72", "unit": "fahrenheit"} + else: + return{"location": location, "temperature": "22", "unit": "celsius"} + +messages = [{"role": "user", "content": "What's the weather like in San Francisco"}] + +tools = [ + { + "type": "function", + "function": { + "name": "import litellm", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } +] + +response = litellm.completion( + model="sambanova/Meta-Llama-3.3-70B-Instruct", + messages=messages, + tools=tools, + tool_choice="auto", # auto is default, but we'll be explicit +) + +print("\nFirst LLM Response:\n", response) +response_message = response.choices[0].message +tool_calls = response_message.tool_calls + +if tool_calls: + # Step 2: check if the model wanted to call a function +if tool_calls: + # Step 3: call the function + # Note: the JSON response may not always be valid; be sure to handle errors + available_functions = { + "get_current_weather": get_current_weather, + } + messages.append( + response_message + ) # extend conversation with assistant's reply + print("Response message\n", response_message) + # Step 4: send the info for each function call and function response to the model + for tool_call in tool_calls: + function_name = tool_call.function.name + function_to_call = available_functions[function_name] + function_args = json.loads(tool_call.function.arguments) + function_response = function_to_call( + location=function_args.get("location"), + unit=function_args.get("unit"), + ) + messages.append( + { + "tool_call_id": tool_call.id, + "role": "tool", + "name": function_name, + "content": function_response, + } + ) # extend conversation with function response + print(f"messages: {messages}") + second_response = litellm.completion( + model="sambanova/Meta-Llama-3.3-70B-Instruct", messages=messages + ) # get a new response from the model where it can see the function response + print("second response\n", second_response) +``` + +## SambaNova - Vision Example + +```python +import litellm + +# Auxiliary function to get b64 images +def data_url_from_image(file_path): + mime_type, _ = mimetypes.guess_type(file_path) + if mime_type is None: + raise ValueError("Could not determine MIME type of the file") + + with open(file_path, "rb") as image_file: + encoded_string = base64.b64encode(image_file.read()).decode("utf-8") + + data_url = f"data:{mime_type};base64,{encoded_string}" + return data_url + +response = litellm.completion( + model = "sambanova/Llama-4-Maverick-17B-128E-Instruct", + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What's in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": data_url_from_image("your_image_path"), + "format": "image/jpeg" + } + } + ] + } + ], + stream=False +) + +print(response.choices[0].message.content) +``` + + +## SambaNova - Structured Output + +```python +import litellm + +response = litellm.completion( + model="sambanova/Meta-Llama-3.3-70B-Instruct", + messages=[ + { + "role": "system", + "content": "You are an expert at structured data extraction. You will be given unstructured text should convert it into the given structure." + }, + { + "role": "user", + "content": "the section 24 has appliances, and videogames" + }, + ], + response_format={ + "type": "json_schema", + "json_schema": { + "title": "data", + "name": "data_extraction", + "schema": { + "type": "object", + "properties": { + "section": { + "type": "string" }, + "products": { + "type": "array", + "items": { "type": "string" } + } + }, + "required": ["section", "products"], + "additionalProperties": False + }, + "strict": False + } + }, + stream=False +) + +print(response.choices[0].message.content)) +``` diff --git a/litellm/__init__.py b/litellm/__init__.py index 2bba6a0233f..96c1552c36a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -599,7 +599,7 @@ def add_known_models(): cerebras_models.append(key) elif value.get("litellm_provider") == "galadriel": galadriel_models.append(key) - elif value.get("litellm_provider") == "sambanova_models": + elif value.get("litellm_provider") == "sambanova": sambanova_models.append(key) elif value.get("litellm_provider") == "novita": novita_models.append(key) diff --git a/litellm/llms/sambanova/chat.py b/litellm/llms/sambanova/chat.py index abf55d44fbb..3eda4d0bbfe 100644 --- a/litellm/llms/sambanova/chat.py +++ b/litellm/llms/sambanova/chat.py @@ -4,7 +4,7 @@ Sambanova Chat Completions API this is OpenAI compatible - no translation needed / occurs """ -from typing import Optional +from typing import Optional, Union from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig @@ -17,26 +17,28 @@ class SambanovaConfig(OpenAIGPTConfig): """ max_tokens: Optional[int] = None - response_format: Optional[dict] = None - seed: Optional[int] = None - stream: Optional[bool] = None + temperature: Optional[int] = None top_p: Optional[int] = None + top_k: Optional[int] = None + stop: Optional[Union[str, list]] = None + stream: Optional[bool] = None + stream_options: Optional[dict] = None tool_choice: Optional[str] = None + response_format: Optional[dict] = None tools: Optional[list] = None - user: Optional[str] = None - + def __init__( self, max_tokens: Optional[int] = None, response_format: Optional[dict] = None, - seed: Optional[int] = None, stop: Optional[str] = None, stream: Optional[bool] = None, + stream_options: Optional[dict] = None, temperature: Optional[float] = None, - top_p: Optional[int] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, tool_choice: Optional[str] = None, tools: Optional[list] = None, - user: Optional[str] = None, ) -> None: locals_ = locals().copy() for key, value in locals_.items(): @@ -56,12 +58,31 @@ class SambanovaConfig(OpenAIGPTConfig): return [ "max_tokens", "response_format", - "seed", "stop", "stream", + "stream_options", "temperature", "top_p", + "top_k", "tool_choice", "tools", - "user", + "parallel_tool_calls" ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + map max_completion_tokens param to max_tokens + """ + supported_openai_params = self.get_supported_openai_params(model=model) + for param, value in non_default_params.items(): + if param == "max_completion_tokens": + optional_params["max_tokens"] = value + elif param in supported_openai_params: + optional_params[param] = value + return optional_params \ No newline at end of file diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e7f63d18e13..649eda96e9f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12232,81 +12232,169 @@ "metadata": {"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."} }, "sambanova/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0000002, "litellm_provider": "sambanova", - "supports_function_calling": true, "mode": "chat", - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 0.0000006, - "output_cost_per_token": 0.0000012, - "litellm_provider": "sambanova", "supports_function_calling": true, - "mode": "chat", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true, + "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/Meta-Llama-3.1-405B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, "input_cost_per_token": 0.000005, "output_cost_per_token": 0.000010, "litellm_provider": "sambanova", - "supports_function_calling": true, "mode": "chat", - "supports_tool_choice": true + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/Meta-Llama-3.2-1B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000004, + "output_cost_per_token": 0.00000008, + "litellm_provider": "sambanova", + "mode": "chat", + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/Meta-Llama-3.2-3B-Instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000008, + "output_cost_per_token": 0.00000016, + "litellm_provider": "sambanova", + "mode": "chat", + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/Llama-4-Maverick-17B-128E-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 0.00000063, + "output_cost_per_token": 0.0000018, + "litellm_provider": "sambanova", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "source": "https://cloud.sambanova.ai/plans/pricing", + "metadata": {"notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount"} + }, + "sambanova/Llama-4-Scout-17B-16E-Instruct": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0000004, + "output_cost_per_token": 0.0000007, + "litellm_provider": "sambanova", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "source": "https://cloud.sambanova.ai/plans/pricing", + "metadata": {"notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount"} + }, + "sambanova/Meta-Llama-3.3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 0.0000006, + "output_cost_per_token": 0.0000012, + "litellm_provider": "sambanova", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/Meta-Llama-Guard-3-8B": { + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "input_cost_per_token": 0.0000003, + "output_cost_per_token": 0.0000003, + "litellm_provider": "sambanova", + "mode": "chat", + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/Qwen3-32B": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.0000004, "output_cost_per_token": 0.0000008, "litellm_provider": "sambanova", "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, "mode": "chat", - "supports_tool_choice": true + "source": "https://cloud.sambanova.ai/plans/pricing" }, - "sambanova/Meta-Llama-3.2-3B-Instruct": { - "max_tokens": 4000, - "max_input_tokens": 4000, - "max_output_tokens": 4000, - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000016, + "sambanova/QwQ-32B": { + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0000010, "litellm_provider": "sambanova", - "supports_function_calling": true, "mode": "chat", - "supports_tool_choice": true + "source": "https://cloud.sambanova.ai/plans/pricing" }, - "sambanova/Qwen2.5-Coder-32B-Instruct": { - "max_tokens": 8000, - "max_input_tokens": 8000, - "max_output_tokens": 8000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000003, + "sambanova/Qwen2-Audio-7B-Instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0001, "litellm_provider": "sambanova", - "supports_function_calling": true, "mode": "chat", - "supports_tool_choice": true + "supports_audio_input": true, + "source": "https://cloud.sambanova.ai/plans/pricing" }, - "sambanova/Qwen2.5-72B-Instruct": { - "max_tokens": 8000, - "max_input_tokens": 8000, - "max_output_tokens": 8000, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000004, + "sambanova/DeepSeek-R1-Distill-Llama-70B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 0.0000007, + "output_cost_per_token": 0.0000014, "litellm_provider": "sambanova", - "supports_function_calling": true, "mode": "chat", - "supports_tool_choice": true + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/DeepSeek-R1": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000007, + "litellm_provider": "sambanova", + "mode": "chat", + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/DeepSeek-V3-0324": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 0.0000030, + "output_cost_per_token": 0.0000045, + "litellm_provider": "sambanova", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "source": "https://cloud.sambanova.ai/plans/pricing" }, "assemblyai/nano": { "mode": "audio_transcription", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index aec5d2e314c..8ed64faceba 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12232,81 +12232,169 @@ "metadata": {"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."} }, "sambanova/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0000002, "litellm_provider": "sambanova", - "supports_function_calling": true, "mode": "chat", - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 0.0000006, - "output_cost_per_token": 0.0000012, - "litellm_provider": "sambanova", "supports_function_calling": true, - "mode": "chat", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_response_schema": true, + "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/Meta-Llama-3.1-405B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, "input_cost_per_token": 0.000005, "output_cost_per_token": 0.000010, "litellm_provider": "sambanova", - "supports_function_calling": true, "mode": "chat", - "supports_tool_choice": true + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/Meta-Llama-3.2-1B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000004, + "output_cost_per_token": 0.00000008, + "litellm_provider": "sambanova", + "mode": "chat", + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/Meta-Llama-3.2-3B-Instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000008, + "output_cost_per_token": 0.00000016, + "litellm_provider": "sambanova", + "mode": "chat", + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/Llama-4-Maverick-17B-128E-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 0.00000063, + "output_cost_per_token": 0.0000018, + "litellm_provider": "sambanova", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_vision": true, + "source": "https://cloud.sambanova.ai/plans/pricing", + "metadata": {"notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount"} + }, + "sambanova/Llama-4-Scout-17B-16E-Instruct": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0000004, + "output_cost_per_token": 0.0000007, + "litellm_provider": "sambanova", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "source": "https://cloud.sambanova.ai/plans/pricing", + "metadata": {"notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount"} + }, + "sambanova/Meta-Llama-3.3-70B-Instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 0.0000006, + "output_cost_per_token": 0.0000012, + "litellm_provider": "sambanova", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/Meta-Llama-Guard-3-8B": { + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "input_cost_per_token": 0.0000003, + "output_cost_per_token": 0.0000003, + "litellm_provider": "sambanova", + "mode": "chat", + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/Qwen3-32B": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, "input_cost_per_token": 0.0000004, "output_cost_per_token": 0.0000008, "litellm_provider": "sambanova", "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, "mode": "chat", - "supports_tool_choice": true + "source": "https://cloud.sambanova.ai/plans/pricing" }, - "sambanova/Meta-Llama-3.2-3B-Instruct": { - "max_tokens": 4000, - "max_input_tokens": 4000, - "max_output_tokens": 4000, - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000016, + "sambanova/QwQ-32B": { + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0000010, "litellm_provider": "sambanova", - "supports_function_calling": true, "mode": "chat", - "supports_tool_choice": true + "source": "https://cloud.sambanova.ai/plans/pricing" }, - "sambanova/Qwen2.5-Coder-32B-Instruct": { - "max_tokens": 8000, - "max_input_tokens": 8000, - "max_output_tokens": 8000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000003, + "sambanova/Qwen2-Audio-7B-Instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0001, "litellm_provider": "sambanova", - "supports_function_calling": true, "mode": "chat", - "supports_tool_choice": true + "supports_audio_input": true, + "source": "https://cloud.sambanova.ai/plans/pricing" }, - "sambanova/Qwen2.5-72B-Instruct": { - "max_tokens": 8000, - "max_input_tokens": 8000, - "max_output_tokens": 8000, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000004, + "sambanova/DeepSeek-R1-Distill-Llama-70B": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 0.0000007, + "output_cost_per_token": 0.0000014, "litellm_provider": "sambanova", - "supports_function_calling": true, "mode": "chat", - "supports_tool_choice": true + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/DeepSeek-R1": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000007, + "litellm_provider": "sambanova", + "mode": "chat", + "source": "https://cloud.sambanova.ai/plans/pricing" + }, + "sambanova/DeepSeek-V3-0324": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "input_cost_per_token": 0.0000030, + "output_cost_per_token": 0.0000045, + "litellm_provider": "sambanova", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "source": "https://cloud.sambanova.ai/plans/pricing" }, "assemblyai/nano": { "mode": "audio_transcription",