Merge pull request #38248 from BerriAI/litellm_together_chat_config

fix(together_ai): route chat completions through a dedicated TogetherAIChatConfig

litellm/__init__.py

@@ -1629,6 +1629,9 @@ if TYPE_CHECKING:
         AmazonMantleMessagesConfig as AmazonMantleMessagesConfig,
     )
     from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
+    from .llms.together_ai.chat.transformation import (
+        TogetherAIChatConfig as TogetherAIChatConfig,
+    )
     from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
     from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         VertexGeminiConfig as VertexGeminiConfig,

litellm/_lazy_imports_registry.py

@@ -177,6 +177,7 @@ LLM_CONFIG_NAMES: Final = (
     "AmazonAnthropicClaudeMessagesConfig",
     "AmazonMantleMessagesConfig",
     "TogetherAIConfig",
+    "TogetherAIChatConfig",
     "NLPCloudConfig",
     "VertexGeminiConfig",
     "GoogleAIStudioGeminiConfig",
@@ -741,6 +742,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "AmazonMantleMessagesConfig",
     ),
     "TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"),
+    "TogetherAIChatConfig": (
+        ".llms.together_ai.chat.transformation",
+        "TogetherAIChatConfig",
+    ),
     "NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"),
     "VertexGeminiConfig": (
         ".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini",

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -172,7 +172,7 @@ def get_supported_openai_params(
         if request_type == "embeddings":
             return litellm.JinaAIEmbeddingConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "together_ai":
-        return litellm.TogetherAIConfig().get_supported_openai_params(model=model)
+        return litellm.TogetherAIChatConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "databricks":
         if request_type == "chat_completion":
             return litellm.DatabricksConfig().get_supported_openai_params(model=model)

litellm/llms/together_ai/chat.py

@@ -1,58 +0,0 @@
-"""
-Support for OpenAI's `/v1/chat/completions` endpoint.
-
-Calls done in OpenAI/openai.py as TogetherAI is openai-compatible.
-
-Docs: https://docs.together.ai/reference/completions-1
-"""
-
-from typing import Final
-
-from litellm._logging import verbose_logger
-from litellm.utils import supports_function_calling
-
-from ..openai.chat.gpt_transformation import OpenAIGPTConfig
-
-
-class TogetherAIConfig(OpenAIGPTConfig):
-    def get_supported_openai_params(self, model: str) -> list:
-        """
-        Only some together models support response_format / tool calling
-
-        Docs: https://docs.together.ai/docs/json-mode
-        """
-        # Use supports_function_calling() — which reads _get_model_info_helper
-        # directly — instead of get_model_info(). get_model_info() calls
-        # get_supported_openai_params() as its first step, which routes back
-        # into this method for together_ai models, creating a recursion that
-        # only terminates when Python's recursion limit or the "not mapped"
-        # exception in _get_model_info_helper is hit (~332 deep calls).
-        supports_fc: bool | None = None
-        try:
-            supports_fc = supports_function_calling(model, custom_llm_provider="together_ai")
-        except Exception as e:
-            verbose_logger.debug("Error getting supported openai params: %s", e)
-
-        optional_params: Final = super().get_supported_openai_params(model)
-        if supports_fc is not True:
-            verbose_logger.debug(
-                "Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
-            )
-            optional_params.remove("tools")
-            optional_params.remove("tool_choice")
-            optional_params.remove("function_call")
-            optional_params.remove("response_format")
-        return optional_params
-
-    def map_openai_params(
-        self,
-        non_default_params: dict,
-        optional_params: dict,
-        model: str,
-        drop_params: bool,
-    ) -> dict:
-        mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
-
-        if "response_format" in mapped_openai_params and mapped_openai_params["response_format"] == {"type": "text"}:
-            mapped_openai_params.pop("response_format")
-        return mapped_openai_params

litellm/llms/together_ai/chat/__init__.py

@@ -0,0 +1,3 @@
+from .transformation import TogetherAIChatConfig as TogetherAIChatConfig
+
+TogetherAIConfig = TogetherAIChatConfig

litellm/llms/together_ai/chat/transformation.py

@@ -0,0 +1,48 @@
+"""
+Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/completions`.
+
+Docs: https://docs.together.ai/docs/chat-overview
+"""
+
+from types import MappingProxyType
+from typing import Final
+
+from litellm._logging import verbose_logger
+from litellm.utils import supports_function_calling
+
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+
+FUNCTION_CALLING_ONLY_PARAMS: Final = ("tools", "tool_choice", "function_call", "response_format")
+PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
+
+
+class TogetherAIChatConfig(OpenAIGPTConfig):
+    def get_supported_openai_params(self, model: str) -> list:
+        supports_fc: bool | None = None
+        try:
+            supports_fc = supports_function_calling(model, custom_llm_provider="together_ai")
+        except Exception as e:
+            verbose_logger.debug("Error getting supported openai params: %s", e)
+
+        supported_params: Final = super().get_supported_openai_params(model)
+        if supports_fc is True:
+            return supported_params
+        verbose_logger.debug(
+            "Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
+        )
+        return [  # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
+            param for param in supported_params if param not in FUNCTION_CALLING_ONLY_PARAMS
+        ]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
+
+        if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT:
+            mapped_openai_params.pop("response_format")
+        return mapped_openai_params

litellm/main.py

@@ -24,6 +24,7 @@ from concurrent import futures
 from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from copy import deepcopy
 from functools import partial
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
 
 from litellm._logging import _redact_string
@@ -1811,6 +1812,56 @@ def _complete_fireworks_ai(
     return response
 
 
+def _complete_together_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+    acompletion: Final = ctx.acompletion
+    api_base: Final = ctx.api_base
+    api_key: Final = ctx.api_key
+    client: Final = _dispatch_client_http(ctx)
+    custom_llm_provider: Final = ctx.custom_llm_provider
+    headers: Final = ctx.headers
+    litellm_params: Final = ctx.litellm_params
+    logging: Final = ctx.logging
+    messages: Final = ctx.messages
+    model: Final = ctx.model
+    model_response: Final = ctx.model_response
+    optional_params: Final = ctx.optional_params
+    provider_config: Final = ctx.provider_config
+    shared_session: Final = ctx.shared_session
+    stream: Final = ctx.stream
+    timeout: Final = ctx.timeout
+
+    try:
+        response: Final = base_llm_http_handler.completion(
+            model=model,
+            messages=messages,
+            headers=headers,
+            model_response=model_response,
+            api_key=api_key,
+            api_base=api_base,
+            acompletion=acompletion,
+            logging_obj=logging,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            shared_session=shared_session,
+            timeout=timeout,
+            client=client,
+            custom_llm_provider=custom_llm_provider,
+            encoding=_get_encoding(),
+            stream=stream,
+            provider_config=provider_config,
+        )
+    except Exception as e:
+        logging.post_call(
+            input=messages,
+            api_key=api_key,
+            original_response=str(e),
+            additional_args=MappingProxyType({"headers": headers}),
+        )
+        raise
+
+    return response
+
+
 def _complete_heroku(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
     acompletion: Final = ctx.acompletion
     api_base: Final = ctx.api_base
@@ -5600,6 +5651,8 @@ def completion(
         elif custom_llm_provider == "fireworks_ai":
             ## COMPLETION CALL
             response = _complete_fireworks_ai(_dispatch_ctx)
+        elif custom_llm_provider == "together_ai":
+            response = _complete_together_ai(_dispatch_ctx)
         elif custom_llm_provider == "heroku":
             response = _complete_heroku(_dispatch_ctx)
 
@@ -5649,7 +5702,6 @@ def completion(
             or custom_llm_provider == "volcengine"
             or custom_llm_provider == "anyscale"
             or custom_llm_provider == "openai"
-            or custom_llm_provider == "together_ai"
             or custom_llm_provider == "nebius"
             or custom_llm_provider == "wandb"
             or custom_llm_provider == "clarifai"
@@ -5699,14 +5751,6 @@ def completion(
             response = _complete_openrouter(_dispatch_ctx)
         elif custom_llm_provider == "vercel_ai_gateway":
             response = _complete_vercel_ai_gateway(_dispatch_ctx)
-        elif (
-            custom_llm_provider == "together_ai"
-            or ("togethercomputer" in model)
-            or (model in litellm.together_ai_models)
-        ):
-            """
-            Deprecated. We now do together ai calls via the openai client - https://docs.together.ai/docs/openai-api-compatibility
-            """
         elif custom_llm_provider == "palm":
             raise ValueError(
                 "Palm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en"

litellm/utils.py

@@ -4130,7 +4130,7 @@ def get_optional_params(
             drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False),
         )
     elif custom_llm_provider == "together_ai":
-        optional_params = litellm.TogetherAIConfig().map_openai_params(
+        optional_params = litellm.TogetherAIChatConfig().map_openai_params(
             non_default_params=non_default_params,
             optional_params=optional_params,
             model=model,
@@ -7898,7 +7898,7 @@ class ProviderConfigManager:
             LlmProviders.GALADRIEL: (lambda: litellm.GaladrielChatConfig(), False),
             LlmProviders.REPLICATE: (lambda: litellm.ReplicateConfig(), False),
             LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False),
-            LlmProviders.TOGETHER_AI: (lambda: litellm.TogetherAIConfig(), False),
+            LlmProviders.TOGETHER_AI: (lambda: litellm.TogetherAIChatConfig(), False),
             LlmProviders.OPENROUTER: (lambda: litellm.OpenrouterConfig(), False),
             LlmProviders.VERCEL_AI_GATEWAY: (
                 lambda: litellm.VercelAIGatewayConfig(),

tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py

@@ -0,0 +1,232 @@
+import json
+from unittest.mock import MagicMock
+
+import httpx
+import pytest
+
+import litellm
+from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
+from litellm.llms.openai.chat.gpt_transformation import (
+    OpenAIChatCompletionStreamingHandler,
+)
+from litellm.llms.together_ai.chat.transformation import TogetherAIChatConfig
+from litellm.types.utils import LlmProviders, ModelResponse
+
+TOOL_CALLING_MODEL = "openai/gpt-oss-20b"
+REASONING_MODEL = "deepseek-ai/DeepSeek-V3.1"
+PLAIN_MODEL = "Qwen/Qwen3-235B-A22B-fp8-tput"
+UNMAPPED_MODEL = "MiniMaxAI/MiniMax-M3"
+
+FUNCTION_CALLING_PARAMS = ("tools", "tool_choice", "function_call", "response_format")
+
+
+@pytest.fixture(autouse=True)
+def force_local_model_cost(monkeypatch):
+    monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+    from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map
+
+    monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url))
+
+
+def test_supported_params_tool_calling_model():
+    supported = TogetherAIChatConfig().get_supported_openai_params(model=TOOL_CALLING_MODEL)
+
+    for param in FUNCTION_CALLING_PARAMS:
+        assert param in supported
+
+
+def test_supported_params_plain_model():
+    supported = TogetherAIChatConfig().get_supported_openai_params(model=PLAIN_MODEL)
+
+    for param in FUNCTION_CALLING_PARAMS:
+        assert param not in supported
+    assert "temperature" in supported
+    assert "max_tokens" in supported
+
+
+def test_supported_params_unmapped_model_treated_as_plain():
+    supported = TogetherAIChatConfig().get_supported_openai_params(model=UNMAPPED_MODEL)
+
+    for param in FUNCTION_CALLING_PARAMS:
+        assert param not in supported
+    assert "stream" in supported
+
+
+def test_map_openai_params_tool_calling_model_passes_tools():
+    tools = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}]
+
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"tools": tools, "tool_choice": "auto"},
+        optional_params={},
+        model=TOOL_CALLING_MODEL,
+        drop_params=False,
+    )
+
+    assert mapped["tools"] == tools
+    assert mapped["tool_choice"] == "auto"
+
+
+def test_map_openai_params_reasoning_model_passes_sampling_params():
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"temperature": 0.2, "max_tokens": 512},
+        optional_params={},
+        model=REASONING_MODEL,
+        drop_params=False,
+    )
+
+    assert mapped["temperature"] == 0.2
+    assert mapped["max_tokens"] == 512
+
+
+def test_map_openai_params_drops_text_response_format():
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"response_format": {"type": "text"}, "temperature": 0.5},
+        optional_params={},
+        model=REASONING_MODEL,
+        drop_params=False,
+    )
+
+    assert "response_format" not in mapped
+    assert mapped["temperature"] == 0.5
+
+
+def test_map_openai_params_keeps_json_response_format():
+    response_format = {"type": "json_object"}
+
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"response_format": response_format},
+        optional_params={},
+        model=TOOL_CALLING_MODEL,
+        drop_params=False,
+    )
+
+    assert mapped["response_format"] == response_format
+
+
+def _transform_response(message: dict) -> ModelResponse:
+    raw_response_json = {
+        "id": "chatcmpl-test",
+        "object": "chat.completion",
+        "created": 1234567890,
+        "model": REASONING_MODEL,
+        "choices": [{"index": 0, "message": message, "finish_reason": "stop"}],
+        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+    }
+    mock_response = MagicMock(spec=httpx.Response)
+    mock_response.json.return_value = raw_response_json
+    mock_response.text = json.dumps(raw_response_json)
+    mock_response.headers = {}
+    logging_obj = MagicMock(spec=LiteLLMLoggingObj)
+    logging_obj.post_call = MagicMock()
+    logging_obj.model_call_details = {}
+
+    return TogetherAIChatConfig().transform_response(
+        model=REASONING_MODEL,
+        raw_response=mock_response,
+        model_response=ModelResponse(),
+        logging_obj=logging_obj,
+        request_data={},
+        messages=[{"role": "user", "content": "What is 2+2?"}],
+        optional_params={},
+        litellm_params={},
+        encoding=None,
+        api_key="test-key",
+        json_mode=False,
+    )
+
+
+def test_transform_response_maps_reasoning_to_reasoning_content():
+    result = _transform_response(
+        {"role": "assistant", "content": "4", "reasoning": "2+2 equals 4"}
+    )
+
+    assert result.choices[0].message.content == "4"
+    assert result.choices[0].message.reasoning_content == "2+2 equals 4"
+
+
+def test_transform_response_preserves_reasoning_content_field():
+    result = _transform_response(
+        {"role": "assistant", "content": "4", "reasoning_content": "adding 2 and 2"}
+    )
+
+    assert result.choices[0].message.reasoning_content == "adding 2 and 2"
+
+
+def test_streaming_chunk_maps_delta_reasoning_to_reasoning_content():
+    iterator = TogetherAIChatConfig().get_model_response_iterator(
+        streaming_response=iter(()), sync_stream=True
+    )
+    assert isinstance(iterator, OpenAIChatCompletionStreamingHandler)
+
+    parsed = iterator.chunk_parser(
+        {
+            "id": "chunk-1",
+            "created": 1234567890,
+            "model": REASONING_MODEL,
+            "choices": [{"index": 0, "delta": {"reasoning": "thinking about 2+2"}}],
+        }
+    )
+
+    assert parsed.choices[0]["delta"]["reasoning_content"] == "thinking about 2+2"
+
+
+def test_together_ai_config_alias_points_at_chat_config():
+    assert litellm.TogetherAIConfig is litellm.TogetherAIChatConfig
+    config = litellm.TogetherAIConfig(max_tokens=10)
+    assert isinstance(config, TogetherAIChatConfig)
+
+
+def test_provider_config_manager_returns_together_chat_config():
+    from litellm.utils import ProviderConfigManager
+
+    config = ProviderConfigManager.get_provider_chat_config(
+        model=REASONING_MODEL, provider=LlmProviders.TOGETHER_AI
+    )
+
+    assert isinstance(config, TogetherAIChatConfig)
+
+
+def test_completion_routes_through_together_chat_config():
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
+
+    captured_requests = []
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        captured_requests.append(request)
+        return httpx.Response(
+            200,
+            json={
+                "id": "chatcmpl-together",
+                "object": "chat.completion",
+                "created": 1234567890,
+                "model": REASONING_MODEL,
+                "choices": [
+                    {
+                        "index": 0,
+                        "message": {
+                            "role": "assistant",
+                            "content": "4",
+                            "reasoning": "2+2 equals 4",
+                        },
+                        "finish_reason": "stop",
+                    }
+                ],
+                "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+            },
+        )
+
+    client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
+
+    response = litellm.completion(
+        model=f"together_ai/{REASONING_MODEL}",
+        messages=[{"role": "user", "content": "What is 2+2?"}],
+        api_key="fake-key",
+        client=client,
+    )
+
+    request = captured_requests[0]
+    assert str(request.url) == "https://api.together.ai/v1/chat/completions"
+    assert request.headers["authorization"] == "Bearer fake-key"
+    assert json.loads(request.content)["model"] == REASONING_MODEL
+    assert response.choices[0].message.content == "4"
+    assert response.choices[0].message.reasoning_content == "2+2 equals 4"