diff --git a/litellm/llms/together_ai/chat/transformation.py b/litellm/llms/together_ai/chat/transformation.py index 88fd79f2366..3162a34f1b9 100644 --- a/litellm/llms/together_ai/chat/transformation.py +++ b/litellm/llms/together_ai/chat/transformation.py @@ -4,34 +4,74 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl Docs: https://docs.together.ai/docs/chat-overview """ +from collections.abc import Container from types import MappingProxyType from typing import Final +import litellm from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError from litellm.utils import supports_function_calling from ...openai.chat.gpt_transformation import OpenAIGPTConfig -FUNCTION_CALLING_ONLY_PARAMS: Final = ("tools", "tool_choice", "function_call", "response_format") +TOOL_CALLING_PARAMS: Final = ("tools", "tool_choice", "function_call") PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"}) +FUNCTION_CALLING_DOCS_URL: Final = "https://docs.together.ai/docs/function-calling" + + +def _function_calling_verdict(model: str) -> bool | None: + try: + if supports_function_calling(model, custom_llm_provider="together_ai"): + return True + except Exception as e: + verbose_logger.debug("Error checking together_ai function calling support for %s: %s", model, e) + registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}") + if isinstance(registry_entry, dict) and registry_entry.get("supports_function_calling") is False: + return False + return None + + +def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params: bool) -> tuple[str, ...]: + passed_tool_params: Final = tuple(param for param in TOOL_CALLING_PARAMS if param in passed_params) + if not passed_tool_params: + return () + verdict: Final = _function_calling_verdict(model) + if verdict is True: + return () + if verdict is None: + verbose_logger.warning( + "together_ai model %s has no function calling entry in the model registry; passing %s through for Together to validate. Docs - %s", + model, + ", ".join(passed_tool_params), + FUNCTION_CALLING_DOCS_URL, + ) + return () + if drop_params or litellm.drop_params: + verbose_logger.warning( + "together_ai model %s does not support function calling per the model registry; dropping %s. Docs - %s", + model, + ", ".join(passed_tool_params), + FUNCTION_CALLING_DOCS_URL, + ) + return passed_tool_params + raise UnsupportedParamsError( + status_code=500, + message=f"together_ai does not support parameters: {', '.join(passed_tool_params)}, for model={model}. To drop it from the call, set `litellm.drop_params = True`.", + ) class TogetherAIChatConfig(OpenAIGPTConfig): def get_supported_openai_params(self, model: str) -> list: - supports_fc: bool | None = None - try: - supports_fc = supports_function_calling(model, custom_llm_provider="together_ai") - except Exception as e: - verbose_logger.debug("Error getting supported openai params: %s", e) - + supports_fc: Final = _function_calling_verdict(model) supported_params: Final = super().get_supported_openai_params(model) if supports_fc is True: return supported_params verbose_logger.debug( - "Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling" + "Only some together models support response_format. Docs - https://docs.together.ai/docs/function-calling" ) return [ # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value - param for param in supported_params if param not in FUNCTION_CALLING_ONLY_PARAMS + param for param in supported_params if param != "response_format" ] def map_openai_params( @@ -42,7 +82,8 @@ class TogetherAIChatConfig(OpenAIGPTConfig): drop_params: bool, ) -> dict: mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params) - + for param in _tool_params_to_drop(mapped_openai_params, model, drop_params): + mapped_openai_params.pop(param) if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT: mapped_openai_params.pop("response_format") return mapped_openai_params diff --git a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py index 6216d3bf225..0b9fd5364f9 100644 --- a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py +++ b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py @@ -1,10 +1,12 @@ import json +import logging from unittest.mock import MagicMock import httpx import pytest import litellm +from litellm.exceptions import UnsupportedParamsError from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, @@ -14,10 +16,12 @@ from litellm.types.utils import LlmProviders, ModelResponse TOOL_CALLING_MODEL = "openai/gpt-oss-20b" REASONING_MODEL = "deepseek-ai/DeepSeek-V3.1" -PLAIN_MODEL = "Qwen/Qwen3-235B-A22B-fp8-tput" -UNMAPPED_MODEL = "MiniMaxAI/MiniMax-M3" +UNMAPPED_MODEL = "example-org/brand-new-model" +NO_TOOLS_MODEL = "example-org/no-tools-model" -FUNCTION_CALLING_PARAMS = ("tools", "tool_choice", "function_call", "response_format") +TOOL_PARAMS = ("tools", "tool_choice", "function_call") + +WEATHER_TOOLS = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}] @pytest.fixture(autouse=True) @@ -28,44 +32,103 @@ def force_local_model_cost(monkeypatch): monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url)) +@pytest.fixture +def registry_disables_function_calling(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + f"together_ai/{NO_TOOLS_MODEL}", + {"litellm_provider": "together_ai", "mode": "chat", "supports_function_calling": False}, + ) + + +@pytest.fixture +def together_warning_log(caplog): + from litellm._logging import verbose_logger + + verbose_logger.addHandler(caplog.handler) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + yield caplog + verbose_logger.removeHandler(caplog.handler) + + def test_supported_params_tool_calling_model(): supported = TogetherAIChatConfig().get_supported_openai_params(model=TOOL_CALLING_MODEL) - for param in FUNCTION_CALLING_PARAMS: + for param in (*TOOL_PARAMS, "response_format"): assert param in supported -def test_supported_params_plain_model(): - supported = TogetherAIChatConfig().get_supported_openai_params(model=PLAIN_MODEL) - - for param in FUNCTION_CALLING_PARAMS: - assert param not in supported - assert "temperature" in supported - assert "max_tokens" in supported - - -def test_supported_params_unmapped_model_treated_as_plain(): +def test_supported_params_unmapped_model_keeps_tool_params(): supported = TogetherAIChatConfig().get_supported_openai_params(model=UNMAPPED_MODEL) - for param in FUNCTION_CALLING_PARAMS: - assert param not in supported + for param in TOOL_PARAMS: + assert param in supported + assert "response_format" not in supported assert "stream" in supported + assert "temperature" in supported + + +def test_supported_params_no_tools_model_keeps_tool_params(registry_disables_function_calling): + supported = TogetherAIChatConfig().get_supported_openai_params(model=NO_TOOLS_MODEL) + + for param in TOOL_PARAMS: + assert param in supported + assert "response_format" not in supported def test_map_openai_params_tool_calling_model_passes_tools(): - tools = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}] - mapped = TogetherAIChatConfig().map_openai_params( - non_default_params={"tools": tools, "tool_choice": "auto"}, + non_default_params={"tools": WEATHER_TOOLS, "tool_choice": "auto"}, optional_params={}, model=TOOL_CALLING_MODEL, drop_params=False, ) - assert mapped["tools"] == tools + assert mapped["tools"] == WEATHER_TOOLS assert mapped["tool_choice"] == "auto" +@pytest.mark.parametrize("drop_params", [False, True]) +def test_map_openai_params_unmapped_model_passes_tools_through(drop_params, together_warning_log): + mapped = TogetherAIChatConfig().map_openai_params( + non_default_params={"tools": WEATHER_TOOLS, "tool_choice": "required"}, + optional_params={}, + model=UNMAPPED_MODEL, + drop_params=drop_params, + ) + + assert mapped["tools"] == WEATHER_TOOLS + assert mapped["tool_choice"] == "required" + assert UNMAPPED_MODEL in together_warning_log.text + assert "passing tools, tool_choice through" in together_warning_log.text + + +def test_map_openai_params_no_tools_model_drops_tools_with_warning( + registry_disables_function_calling, together_warning_log +): + mapped = TogetherAIChatConfig().map_openai_params( + non_default_params={"tools": WEATHER_TOOLS, "temperature": 0.5}, + optional_params={}, + model=NO_TOOLS_MODEL, + drop_params=True, + ) + + assert "tools" not in mapped + assert mapped["temperature"] == 0.5 + assert NO_TOOLS_MODEL in together_warning_log.text + assert "dropping tools" in together_warning_log.text + + +def test_map_openai_params_no_tools_model_raises_without_drop_params(registry_disables_function_calling): + with pytest.raises(UnsupportedParamsError, match="does not support parameters"): + TogetherAIChatConfig().map_openai_params( + non_default_params={"tools": WEATHER_TOOLS}, + optional_params={}, + model=NO_TOOLS_MODEL, + drop_params=False, + ) + + def test_map_openai_params_reasoning_model_passes_sampling_params(): mapped = TogetherAIChatConfig().map_openai_params( non_default_params={"temperature": 0.2, "max_tokens": 512}, @@ -170,6 +233,41 @@ def test_streaming_chunk_maps_delta_reasoning_to_reasoning_content(): assert parsed.choices[0]["delta"]["reasoning_content"] == "thinking about 2+2" +def test_streaming_chunk_preserves_tool_call_index_and_id(): + iterator = TogetherAIChatConfig().get_model_response_iterator( + streaming_response=iter(()), sync_stream=True + ) + + def parse_tool_call_chunk(tool_call: dict): + parsed = iterator.chunk_parser( + { + "id": "chunk-1", + "created": 1234567890, + "model": TOOL_CALLING_MODEL, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [tool_call]}}], + } + ) + return parsed.choices[0]["delta"]["tool_calls"][0] + + opener = parse_tool_call_chunk( + { + "index": 1, + "id": "call_abc123", + "type": "function", + "function": {"name": "get_weather", "arguments": ""}, + } + ) + continuation = parse_tool_call_chunk( + {"index": 1, "id": "", "type": "function", "function": {"arguments": '{"city": "San'}} + ) + + assert opener["index"] == 1 + assert opener["id"] == "call_abc123" + assert opener["function"]["name"] == "get_weather" + assert continuation["index"] == 1 + assert continuation["function"]["arguments"] == '{"city": "San' + + def test_together_ai_config_alias_points_at_chat_config(): assert litellm.TogetherAIConfig is litellm.TogetherAIChatConfig config = litellm.TogetherAIConfig(max_tokens=10) @@ -230,3 +328,60 @@ def test_completion_routes_through_together_chat_config(): assert json.loads(request.content)["model"] == REASONING_MODEL assert response.choices[0].message.content == "4" assert response.choices[0].message.reasoning_content == "2+2 equals 4" + + +def test_completion_unmapped_model_sends_tools_to_together(): + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + captured_requests = [] + + def respond(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-together-tools", + "object": "chat.completion", + "created": 1234567890, + "model": UNMAPPED_MODEL, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "San Francisco"}', + }, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, + ) + + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + + response = litellm.completion( + model=f"together_ai/{UNMAPPED_MODEL}", + messages=[{"role": "user", "content": "What is the weather in San Francisco?"}], + tools=WEATHER_TOOLS, + tool_choice="auto", + api_key="fake-key", + client=client, + ) + + request_body = json.loads(captured_requests[0].content) + assert request_body["tools"] == WEATHER_TOOLS + assert request_body["tool_choice"] == "auto" + tool_call = response.choices[0].message.tool_calls[0] + assert tool_call.function.name == "get_weather" + assert json.loads(tool_call.function.arguments) == {"city": "San Francisco"}