scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge pull request #38265 from BerriAI/litellm_together_tools_passthrough

Browse source 
fix(together_ai): pass tools through for models missing from the registry
This commit is contained in:
Mateo Wang 2026-08-25 15:12:08 -07:00 • committed by GitHub
commit e4ff44f623
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 226 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
litellm/llms/together_ai/chat/transformation.py
View file

@ -4,34 +4,74 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl
Docs: https://docs.together.ai/docs/chat-overview
"""
from collections.abc import Container
from types import MappingProxyType
from typing import Final
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.utils import supports_function_calling
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
FUNCTION_CALLING_ONLY_PARAMS: Final = ("tools", "tool_choice", "function_call", "response_format")
TOOL_CALLING_PARAMS: Final = ("tools", "tool_choice", "function_call")
PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
FUNCTION_CALLING_DOCS_URL: Final = "https://docs.together.ai/docs/function-calling"
def _function_calling_verdict(model: str) -> bool | None:
try:
if supports_function_calling(model, custom_llm_provider="together_ai"):
return True
except Exception as e:
verbose_logger.debug("Error checking together_ai function calling support for %s: %s", model, e)
registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
if isinstance(registry_entry, dict) and registry_entry.get("supports_function_calling") is False:
return False
return None
def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params: bool) -> tuple[str, ...]:
passed_tool_params: Final = tuple(param for param in TOOL_CALLING_PARAMS if param in passed_params)
if not passed_tool_params:
return ()
verdict: Final = _function_calling_verdict(model)
if verdict is True:
return ()
if verdict is None:
verbose_logger.warning(
"together_ai model %s has no function calling entry in the model registry; passing %s through for Together to validate. Docs - %s",
model,
", ".join(passed_tool_params),
FUNCTION_CALLING_DOCS_URL,
)
return ()
if drop_params or litellm.drop_params:
verbose_logger.warning(
"together_ai model %s does not support function calling per the model registry; dropping %s. Docs - %s",
model,
", ".join(passed_tool_params),
FUNCTION_CALLING_DOCS_URL,
)
return passed_tool_params
raise UnsupportedParamsError(
status_code=500,
message=f"together_ai does not support parameters: {', '.join(passed_tool_params)}, for model={model}. To drop it from the call, set `litellm.drop_params = True`.",
)
class TogetherAIChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list:
supports_fc: bool | None = None
try:
supports_fc = supports_function_calling(model, custom_llm_provider="together_ai")
except Exception as e:
verbose_logger.debug("Error getting supported openai params: %s", e)
supports_fc: Final = _function_calling_verdict(model)
supported_params: Final = super().get_supported_openai_params(model)
if supports_fc is True:
return supported_params
verbose_logger.debug(
"Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
"Only some together models support response_format. Docs - https://docs.together.ai/docs/function-calling"
)
return [ # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
param for param in supported_params if param not in FUNCTION_CALLING_ONLY_PARAMS
param for param in supported_params if param != "response_format"
]
def map_openai_params(
@ -42,7 +82,8 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
drop_params: bool,
) -> dict:
mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
for param in _tool_params_to_drop(mapped_openai_params, model, drop_params):
mapped_openai_params.pop(param)
if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT:
mapped_openai_params.pop("response_format")
return mapped_openai_params

195
tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
View file

@ -1,10 +1,12 @@
import json
import logging
from unittest.mock import MagicMock
import httpx
import pytest
import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.llms.openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
@ -14,10 +16,12 @@ from litellm.types.utils import LlmProviders, ModelResponse
TOOL_CALLING_MODEL = "openai/gpt-oss-20b"
REASONING_MODEL = "deepseek-ai/DeepSeek-V3.1"
PLAIN_MODEL = "Qwen/Qwen3-235B-A22B-fp8-tput"
UNMAPPED_MODEL = "MiniMaxAI/MiniMax-M3"
UNMAPPED_MODEL = "example-org/brand-new-model"
NO_TOOLS_MODEL = "example-org/no-tools-model"
FUNCTION_CALLING_PARAMS = ("tools", "tool_choice", "function_call", "response_format")
TOOL_PARAMS = ("tools", "tool_choice", "function_call")
WEATHER_TOOLS = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}]
@pytest.fixture(autouse=True)
@ -28,44 +32,103 @@ def force_local_model_cost(monkeypatch):
monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url))
@pytest.fixture
def registry_disables_function_calling(monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
f"together_ai/{NO_TOOLS_MODEL}",
{"litellm_provider": "together_ai", "mode": "chat", "supports_function_calling": False},
)
@pytest.fixture
def together_warning_log(caplog):
from litellm._logging import verbose_logger
verbose_logger.addHandler(caplog.handler)
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
yield caplog
verbose_logger.removeHandler(caplog.handler)
def test_supported_params_tool_calling_model():
supported = TogetherAIChatConfig().get_supported_openai_params(model=TOOL_CALLING_MODEL)
for param in FUNCTION_CALLING_PARAMS:
for param in (*TOOL_PARAMS, "response_format"):
assert param in supported
def test_supported_params_plain_model():
supported = TogetherAIChatConfig().get_supported_openai_params(model=PLAIN_MODEL)
for param in FUNCTION_CALLING_PARAMS:
assert param not in supported
assert "temperature" in supported
assert "max_tokens" in supported
def test_supported_params_unmapped_model_treated_as_plain():
def test_supported_params_unmapped_model_keeps_tool_params():
supported = TogetherAIChatConfig().get_supported_openai_params(model=UNMAPPED_MODEL)
for param in FUNCTION_CALLING_PARAMS:
assert param not in supported
for param in TOOL_PARAMS:
assert param in supported
assert "response_format" not in supported
assert "stream" in supported
assert "temperature" in supported
def test_supported_params_no_tools_model_keeps_tool_params(registry_disables_function_calling):
supported = TogetherAIChatConfig().get_supported_openai_params(model=NO_TOOLS_MODEL)
for param in TOOL_PARAMS:
assert param in supported
assert "response_format" not in supported
def test_map_openai_params_tool_calling_model_passes_tools():
tools = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}]
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"tools": tools, "tool_choice": "auto"},
non_default_params={"tools": WEATHER_TOOLS, "tool_choice": "auto"},
optional_params={},
model=TOOL_CALLING_MODEL,
drop_params=False,
)
assert mapped["tools"] == tools
assert mapped["tools"] == WEATHER_TOOLS
assert mapped["tool_choice"] == "auto"
@pytest.mark.parametrize("drop_params", [False, True])
def test_map_openai_params_unmapped_model_passes_tools_through(drop_params, together_warning_log):
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"tools": WEATHER_TOOLS, "tool_choice": "required"},
optional_params={},
model=UNMAPPED_MODEL,
drop_params=drop_params,
)
assert mapped["tools"] == WEATHER_TOOLS
assert mapped["tool_choice"] == "required"
assert UNMAPPED_MODEL in together_warning_log.text
assert "passing tools, tool_choice through" in together_warning_log.text
def test_map_openai_params_no_tools_model_drops_tools_with_warning(
registry_disables_function_calling, together_warning_log
):
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"tools": WEATHER_TOOLS, "temperature": 0.5},
optional_params={},
model=NO_TOOLS_MODEL,
drop_params=True,
)
assert "tools" not in mapped
assert mapped["temperature"] == 0.5
assert NO_TOOLS_MODEL in together_warning_log.text
assert "dropping tools" in together_warning_log.text
def test_map_openai_params_no_tools_model_raises_without_drop_params(registry_disables_function_calling):
with pytest.raises(UnsupportedParamsError, match="does not support parameters"):
TogetherAIChatConfig().map_openai_params(
non_default_params={"tools": WEATHER_TOOLS},
optional_params={},
model=NO_TOOLS_MODEL,
drop_params=False,
)
def test_map_openai_params_reasoning_model_passes_sampling_params():
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"temperature": 0.2, "max_tokens": 512},
@ -170,6 +233,41 @@ def test_streaming_chunk_maps_delta_reasoning_to_reasoning_content():
assert parsed.choices[0]["delta"]["reasoning_content"] == "thinking about 2+2"
def test_streaming_chunk_preserves_tool_call_index_and_id():
iterator = TogetherAIChatConfig().get_model_response_iterator(
streaming_response=iter(()), sync_stream=True
)
def parse_tool_call_chunk(tool_call: dict):
parsed = iterator.chunk_parser(
{
"id": "chunk-1",
"created": 1234567890,
"model": TOOL_CALLING_MODEL,
"choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [tool_call]}}],
}
)
return parsed.choices[0]["delta"]["tool_calls"][0]
opener = parse_tool_call_chunk(
{
"index": 1,
"id": "call_abc123",
"type": "function",
"function": {"name": "get_weather", "arguments": ""},
}
)
continuation = parse_tool_call_chunk(
{"index": 1, "id": "", "type": "function", "function": {"arguments": '{"city": "San'}}
)
assert opener["index"] == 1
assert opener["id"] == "call_abc123"
assert opener["function"]["name"] == "get_weather"
assert continuation["index"] == 1
assert continuation["function"]["arguments"] == '{"city": "San'
def test_together_ai_config_alias_points_at_chat_config():
assert litellm.TogetherAIConfig is litellm.TogetherAIChatConfig
config = litellm.TogetherAIConfig(max_tokens=10)
@ -230,3 +328,60 @@ def test_completion_routes_through_together_chat_config():
assert json.loads(request.content)["model"] == REASONING_MODEL
assert response.choices[0].message.content == "4"
assert response.choices[0].message.reasoning_content == "2+2 equals 4"
def test_completion_unmapped_model_sends_tools_to_together():
from litellm.llms.custom_httpx.http_handler import HTTPHandler
captured_requests = []
def respond(request: httpx.Request) -> httpx.Response:
captured_requests.append(request)
return httpx.Response(
200,
json={
"id": "chatcmpl-together-tools",
"object": "chat.completion",
"created": 1234567890,
"model": UNMAPPED_MODEL,
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_abc123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "San Francisco"}',
},
}
],
},
"finish_reason": "tool_calls",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
)
client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
response = litellm.completion(
model=f"together_ai/{UNMAPPED_MODEL}",
messages=[{"role": "user", "content": "What is the weather in San Francisco?"}],
tools=WEATHER_TOOLS,
tool_choice="auto",
api_key="fake-key",
client=client,
)
request_body = json.loads(captured_requests[0].content)
assert request_body["tools"] == WEATHER_TOOLS
assert request_body["tool_choice"] == "auto"
tool_call = response.choices[0].message.tool_calls[0]
assert tool_call.function.name == "get_weather"
assert json.loads(tool_call.function.arguments) == {"city": "San Francisco"}