From 5ab9e63628088f13024f5a949259bec69855481e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:24:58 -0700 Subject: [PATCH] fix(together_ai): fail open on response_format instead of dropping it for unregistered models --- .../llms/together_ai/chat/transformation.py | 79 ++++++--- .../test_together_ai_chat_transformation.py | 158 ++++++++++++++++-- 2 files changed, 195 insertions(+), 42 deletions(-) diff --git a/litellm/llms/together_ai/chat/transformation.py b/litellm/llms/together_ai/chat/transformation.py index 3162a34f1b9..5f0ab5e56af 100644 --- a/litellm/llms/together_ai/chat/transformation.py +++ b/litellm/llms/together_ai/chat/transformation.py @@ -4,32 +4,47 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl Docs: https://docs.together.ai/docs/chat-overview """ -from collections.abc import Container -from types import MappingProxyType +from collections.abc import Callable, Container from typing import Final import litellm from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError -from litellm.utils import supports_function_calling +from litellm.utils import supports_function_calling, supports_response_schema from ...openai.chat.gpt_transformation import OpenAIGPTConfig TOOL_CALLING_PARAMS: Final = ("tools", "tool_choice", "function_call") -PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"}) FUNCTION_CALLING_DOCS_URL: Final = "https://docs.together.ai/docs/function-calling" +STRUCTURED_OUTPUTS_DOCS_URL: Final = "https://docs.together.ai/docs/inference/chat/structured-outputs" + + +def _registry_verdict(model: str, flag: str, check: Callable[[str], bool]) -> bool | None: + try: + if check(model): + return True + except Exception as e: + verbose_logger.debug("Error checking together_ai %s for %s: %s", flag, model, e) + registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}") + if isinstance(registry_entry, dict) and registry_entry.get(flag) is False: + return False + return None def _function_calling_verdict(model: str) -> bool | None: - try: - if supports_function_calling(model, custom_llm_provider="together_ai"): - return True - except Exception as e: - verbose_logger.debug("Error checking together_ai function calling support for %s: %s", model, e) - registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}") - if isinstance(registry_entry, dict) and registry_entry.get("supports_function_calling") is False: - return False - return None + return _registry_verdict( + model, + "supports_function_calling", + lambda checked_model: supports_function_calling(checked_model, custom_llm_provider="together_ai"), + ) + + +def _response_schema_verdict(model: str) -> bool | None: + return _registry_verdict( + model, + "supports_response_schema", + lambda checked_model: supports_response_schema(checked_model, custom_llm_provider="together_ai"), + ) def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params: bool) -> tuple[str, ...]: @@ -61,19 +76,33 @@ def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params: ) -class TogetherAIChatConfig(OpenAIGPTConfig): - def get_supported_openai_params(self, model: str) -> list: - supports_fc: Final = _function_calling_verdict(model) - supported_params: Final = super().get_supported_openai_params(model) - if supports_fc is True: - return supported_params - verbose_logger.debug( - "Only some together models support response_format. Docs - https://docs.together.ai/docs/function-calling" +def _drop_response_format(passed_params: Container[str], model: str, drop_params: bool) -> bool: + if "response_format" not in passed_params: + return False + verdict: Final = _response_schema_verdict(model) + if verdict is True: + return False + if verdict is None: + verbose_logger.warning( + "together_ai model %s has no structured outputs entry in the model registry; passing response_format through for Together to validate. Docs - %s", + model, + STRUCTURED_OUTPUTS_DOCS_URL, ) - return [ # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value - param for param in supported_params if param != "response_format" - ] + return False + if drop_params or litellm.drop_params: + verbose_logger.warning( + "together_ai model %s does not support structured outputs per the model registry; dropping response_format. Docs - %s", + model, + STRUCTURED_OUTPUTS_DOCS_URL, + ) + return True + raise UnsupportedParamsError( + status_code=500, + message=f"together_ai does not support parameters: response_format, for model={model}. To drop it from the call, set `litellm.drop_params = True`.", + ) + +class TogetherAIChatConfig(OpenAIGPTConfig): def map_openai_params( self, non_default_params: dict, @@ -84,6 +113,6 @@ class TogetherAIChatConfig(OpenAIGPTConfig): mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params) for param in _tool_params_to_drop(mapped_openai_params, model, drop_params): mapped_openai_params.pop(param) - if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT: + if _drop_response_format(mapped_openai_params, model, drop_params): mapped_openai_params.pop("response_format") return mapped_openai_params diff --git a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py index 0b9fd5364f9..3848a9c7e6c 100644 --- a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py +++ b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py @@ -18,11 +18,24 @@ TOOL_CALLING_MODEL = "openai/gpt-oss-20b" REASONING_MODEL = "deepseek-ai/DeepSeek-V3.1" UNMAPPED_MODEL = "example-org/brand-new-model" NO_TOOLS_MODEL = "example-org/no-tools-model" +NO_SCHEMA_MODEL = "example-org/no-schema-model" TOOL_PARAMS = ("tools", "tool_choice", "function_call") WEATHER_TOOLS = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}] +VOICE_NOTE_SCHEMA = { + "type": "object", + "properties": {"title": {"type": "string"}, "summary": {"type": "string"}}, + "required": ["title", "summary"], + "additionalProperties": False, +} +JSON_SCHEMA_RESPONSE_FORMAT = { + "type": "json_schema", + "json_schema": {"name": "voice_note", "schema": VOICE_NOTE_SCHEMA, "strict": True}, +} +REGEX_RESPONSE_FORMAT = {"type": "regex", "pattern": "(positive|neutral|negative)"} + @pytest.fixture(autouse=True) def force_local_model_cost(monkeypatch): @@ -41,6 +54,15 @@ def registry_disables_function_calling(monkeypatch): ) +@pytest.fixture +def registry_disables_response_schema(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + f"together_ai/{NO_SCHEMA_MODEL}", + {"litellm_provider": "together_ai", "mode": "chat", "supports_response_schema": False}, + ) + + @pytest.fixture def together_warning_log(caplog): from litellm._logging import verbose_logger @@ -63,7 +85,7 @@ def test_supported_params_unmapped_model_keeps_tool_params(): for param in TOOL_PARAMS: assert param in supported - assert "response_format" not in supported + assert "response_format" in supported assert "stream" in supported assert "temperature" in supported @@ -73,7 +95,7 @@ def test_supported_params_no_tools_model_keeps_tool_params(registry_disables_fun for param in TOOL_PARAMS: assert param in supported - assert "response_format" not in supported + assert "response_format" in supported def test_map_openai_params_tool_calling_model_passes_tools(): @@ -141,21 +163,17 @@ def test_map_openai_params_reasoning_model_passes_sampling_params(): assert mapped["max_tokens"] == 512 -def test_map_openai_params_drops_text_response_format(): - mapped = TogetherAIChatConfig().map_openai_params( - non_default_params={"response_format": {"type": "text"}, "temperature": 0.5}, - optional_params={}, - model=REASONING_MODEL, - drop_params=False, - ) - - assert "response_format" not in mapped - assert mapped["temperature"] == 0.5 - - -def test_map_openai_params_keeps_json_response_format(): - response_format = {"type": "json_object"} - +@pytest.mark.parametrize( + "response_format", + [ + {"type": "text"}, + {"type": "json_object"}, + {"type": "json_object", "schema": VOICE_NOTE_SCHEMA}, + JSON_SCHEMA_RESPONSE_FORMAT, + REGEX_RESPONSE_FORMAT, + ], +) +def test_map_openai_params_schema_model_passes_response_format_through(response_format): mapped = TogetherAIChatConfig().map_openai_params( non_default_params={"response_format": response_format}, optional_params={}, @@ -166,6 +184,46 @@ def test_map_openai_params_keeps_json_response_format(): assert mapped["response_format"] == response_format +@pytest.mark.parametrize("drop_params", [False, True]) +def test_map_openai_params_unmapped_model_passes_response_format_through(drop_params, together_warning_log): + mapped = TogetherAIChatConfig().map_openai_params( + non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT}, + optional_params={}, + model=UNMAPPED_MODEL, + drop_params=drop_params, + ) + + assert mapped["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT + assert UNMAPPED_MODEL in together_warning_log.text + assert "passing response_format through" in together_warning_log.text + + +def test_map_openai_params_no_schema_model_drops_response_format_with_warning( + registry_disables_response_schema, together_warning_log +): + mapped = TogetherAIChatConfig().map_openai_params( + non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT, "temperature": 0.5}, + optional_params={}, + model=NO_SCHEMA_MODEL, + drop_params=True, + ) + + assert "response_format" not in mapped + assert mapped["temperature"] == 0.5 + assert NO_SCHEMA_MODEL in together_warning_log.text + assert "dropping response_format" in together_warning_log.text + + +def test_map_openai_params_no_schema_model_raises_without_drop_params(registry_disables_response_schema): + with pytest.raises(UnsupportedParamsError, match="response_format"): + TogetherAIChatConfig().map_openai_params( + non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT}, + optional_params={}, + model=NO_SCHEMA_MODEL, + drop_params=False, + ) + + def _transform_response(message: dict) -> ModelResponse: raw_response_json = { "id": "chatcmpl-test", @@ -385,3 +443,69 @@ def test_completion_unmapped_model_sends_tools_to_together(): tool_call = response.choices[0].message.tool_calls[0] assert tool_call.function.name == "get_weather" assert json.loads(tool_call.function.arguments) == {"city": "San Francisco"} + + +def _capture_completion_request(model: str, **completion_kwargs) -> dict: + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + captured_requests = [] + + def respond(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-together-structured", + "object": "chat.completion", + "created": 1234567890, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": '{"title": "t", "summary": "s"}'}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, + ) + + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + litellm.completion( + model=f"together_ai/{model}", + messages=[{"role": "user", "content": "Summarize with a title and summary."}], + api_key="fake-key", + client=client, + **completion_kwargs, + ) + return json.loads(captured_requests[0].content) + + +def test_completion_unmapped_model_sends_json_schema_to_together(): + request_body = _capture_completion_request( + UNMAPPED_MODEL, response_format=JSON_SCHEMA_RESPONSE_FORMAT, drop_params=True + ) + + assert request_body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT + + +def test_completion_pydantic_response_format_sends_json_schema_to_together(): + from pydantic import BaseModel + + class VoiceNote(BaseModel): + title: str + summary: str + + request_body = _capture_completion_request(TOOL_CALLING_MODEL, response_format=VoiceNote) + + sent = request_body["response_format"] + assert sent["type"] == "json_schema" + assert sent["json_schema"]["name"] == "VoiceNote" + assert sent["json_schema"]["strict"] is True + assert sent["json_schema"]["schema"]["required"] == ["title", "summary"] + + +def test_completion_regex_response_format_sends_pattern_to_together(): + request_body = _capture_completion_request(TOOL_CALLING_MODEL, response_format=REGEX_RESPONSE_FORMAT) + + assert request_body["response_format"] == REGEX_RESPONSE_FORMAT