scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

fix(together_ai): fail open on response_format instead of dropping it for unregistered models

Browse source
This commit is contained in:
mateo-berri 2026-08-25 15:24:58 -07:00
parent e4ff44f623
commit 5ab9e63628
2 changed files with 195 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
litellm/llms/together_ai/chat/transformation.py
View file

@ -4,32 +4,47 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl
Docs: https://docs.together.ai/docs/chat-overview
"""
from collections.abc import Container
from types import MappingProxyType
from collections.abc import Callable, Container
from typing import Final
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.utils import supports_function_calling
from litellm.utils import supports_function_calling, supports_response_schema
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
TOOL_CALLING_PARAMS: Final = ("tools", "tool_choice", "function_call")
PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
FUNCTION_CALLING_DOCS_URL: Final = "https://docs.together.ai/docs/function-calling"
STRUCTURED_OUTPUTS_DOCS_URL: Final = "https://docs.together.ai/docs/inference/chat/structured-outputs"
def _registry_verdict(model: str, flag: str, check: Callable[[str], bool]) -> bool | None:
try:
if check(model):
return True
except Exception as e:
verbose_logger.debug("Error checking together_ai %s for %s: %s", flag, model, e)
registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
if isinstance(registry_entry, dict) and registry_entry.get(flag) is False:
return False
return None
def _function_calling_verdict(model: str) -> bool | None:
try:
if supports_function_calling(model, custom_llm_provider="together_ai"):
return True
except Exception as e:
verbose_logger.debug("Error checking together_ai function calling support for %s: %s", model, e)
registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
if isinstance(registry_entry, dict) and registry_entry.get("supports_function_calling") is False:
return False
return None
return _registry_verdict(
model,
"supports_function_calling",
lambda checked_model: supports_function_calling(checked_model, custom_llm_provider="together_ai"),
)
def _response_schema_verdict(model: str) -> bool | None:
return _registry_verdict(
model,
"supports_response_schema",
lambda checked_model: supports_response_schema(checked_model, custom_llm_provider="together_ai"),
)
def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params: bool) -> tuple[str, ...]:
@ -61,19 +76,33 @@ def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params:
)
class TogetherAIChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list:
supports_fc: Final = _function_calling_verdict(model)
supported_params: Final = super().get_supported_openai_params(model)
if supports_fc is True:
return supported_params
verbose_logger.debug(
"Only some together models support response_format. Docs - https://docs.together.ai/docs/function-calling"
def _drop_response_format(passed_params: Container[str], model: str, drop_params: bool) -> bool:
if "response_format" not in passed_params:
return False
verdict: Final = _response_schema_verdict(model)
if verdict is True:
return False
if verdict is None:
verbose_logger.warning(
"together_ai model %s has no structured outputs entry in the model registry; passing response_format through for Together to validate. Docs - %s",
model,
STRUCTURED_OUTPUTS_DOCS_URL,
)
return [ # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
param for param in supported_params if param != "response_format"
]
return False
if drop_params or litellm.drop_params:
verbose_logger.warning(
"together_ai model %s does not support structured outputs per the model registry; dropping response_format. Docs - %s",
model,
STRUCTURED_OUTPUTS_DOCS_URL,
)
return True
raise UnsupportedParamsError(
status_code=500,
message=f"together_ai does not support parameters: response_format, for model={model}. To drop it from the call, set `litellm.drop_params = True`.",
)
class TogetherAIChatConfig(OpenAIGPTConfig):
def map_openai_params(
self,
non_default_params: dict,
@ -84,6 +113,6 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
for param in _tool_params_to_drop(mapped_openai_params, model, drop_params):
mapped_openai_params.pop(param)
if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT:
if _drop_response_format(mapped_openai_params, model, drop_params):
mapped_openai_params.pop("response_format")
return mapped_openai_params

158
tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
View file

@ -18,11 +18,24 @@ TOOL_CALLING_MODEL = "openai/gpt-oss-20b"
REASONING_MODEL = "deepseek-ai/DeepSeek-V3.1"
UNMAPPED_MODEL = "example-org/brand-new-model"
NO_TOOLS_MODEL = "example-org/no-tools-model"
NO_SCHEMA_MODEL = "example-org/no-schema-model"
TOOL_PARAMS = ("tools", "tool_choice", "function_call")
WEATHER_TOOLS = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}]
VOICE_NOTE_SCHEMA = {
"type": "object",
"properties": {"title": {"type": "string"}, "summary": {"type": "string"}},
"required": ["title", "summary"],
"additionalProperties": False,
}
JSON_SCHEMA_RESPONSE_FORMAT = {
"type": "json_schema",
"json_schema": {"name": "voice_note", "schema": VOICE_NOTE_SCHEMA, "strict": True},
}
REGEX_RESPONSE_FORMAT = {"type": "regex", "pattern": "(positive|neutral|negative)"}
@pytest.fixture(autouse=True)
def force_local_model_cost(monkeypatch):
@ -41,6 +54,15 @@ def registry_disables_function_calling(monkeypatch):
)
@pytest.fixture
def registry_disables_response_schema(monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
f"together_ai/{NO_SCHEMA_MODEL}",
{"litellm_provider": "together_ai", "mode": "chat", "supports_response_schema": False},
)
@pytest.fixture
def together_warning_log(caplog):
from litellm._logging import verbose_logger
@ -63,7 +85,7 @@ def test_supported_params_unmapped_model_keeps_tool_params():
for param in TOOL_PARAMS:
assert param in supported
assert "response_format" not in supported
assert "response_format" in supported
assert "stream" in supported
assert "temperature" in supported
@ -73,7 +95,7 @@ def test_supported_params_no_tools_model_keeps_tool_params(registry_disables_fun
for param in TOOL_PARAMS:
assert param in supported
assert "response_format" not in supported
assert "response_format" in supported
def test_map_openai_params_tool_calling_model_passes_tools():
@ -141,21 +163,17 @@ def test_map_openai_params_reasoning_model_passes_sampling_params():
assert mapped["max_tokens"] == 512
def test_map_openai_params_drops_text_response_format():
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"response_format": {"type": "text"}, "temperature": 0.5},
optional_params={},
model=REASONING_MODEL,
drop_params=False,
)
assert "response_format" not in mapped
assert mapped["temperature"] == 0.5
def test_map_openai_params_keeps_json_response_format():
response_format = {"type": "json_object"}
@pytest.mark.parametrize(
"response_format",
[
{"type": "text"},
{"type": "json_object"},
{"type": "json_object", "schema": VOICE_NOTE_SCHEMA},
JSON_SCHEMA_RESPONSE_FORMAT,
REGEX_RESPONSE_FORMAT,
],
)
def test_map_openai_params_schema_model_passes_response_format_through(response_format):
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"response_format": response_format},
optional_params={},
@ -166,6 +184,46 @@ def test_map_openai_params_keeps_json_response_format():
assert mapped["response_format"] == response_format
@pytest.mark.parametrize("drop_params", [False, True])
def test_map_openai_params_unmapped_model_passes_response_format_through(drop_params, together_warning_log):
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT},
optional_params={},
model=UNMAPPED_MODEL,
drop_params=drop_params,
)
assert mapped["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT
assert UNMAPPED_MODEL in together_warning_log.text
assert "passing response_format through" in together_warning_log.text
def test_map_openai_params_no_schema_model_drops_response_format_with_warning(
registry_disables_response_schema, together_warning_log
):
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT, "temperature": 0.5},
optional_params={},
model=NO_SCHEMA_MODEL,
drop_params=True,
)
assert "response_format" not in mapped
assert mapped["temperature"] == 0.5
assert NO_SCHEMA_MODEL in together_warning_log.text
assert "dropping response_format" in together_warning_log.text
def test_map_openai_params_no_schema_model_raises_without_drop_params(registry_disables_response_schema):
with pytest.raises(UnsupportedParamsError, match="response_format"):
TogetherAIChatConfig().map_openai_params(
non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT},
optional_params={},
model=NO_SCHEMA_MODEL,
drop_params=False,
)
def _transform_response(message: dict) -> ModelResponse:
raw_response_json = {
"id": "chatcmpl-test",
@ -385,3 +443,69 @@ def test_completion_unmapped_model_sends_tools_to_together():
tool_call = response.choices[0].message.tool_calls[0]
assert tool_call.function.name == "get_weather"
assert json.loads(tool_call.function.arguments) == {"city": "San Francisco"}
def _capture_completion_request(model: str, **completion_kwargs) -> dict:
from litellm.llms.custom_httpx.http_handler import HTTPHandler
captured_requests = []
def respond(request: httpx.Request) -> httpx.Response:
captured_requests.append(request)
return httpx.Response(
200,
json={
"id": "chatcmpl-together-structured",
"object": "chat.completion",
"created": 1234567890,
"model": model,
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": '{"title": "t", "summary": "s"}'},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
)
client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
litellm.completion(
model=f"together_ai/{model}",
messages=[{"role": "user", "content": "Summarize with a title and summary."}],
api_key="fake-key",
client=client,
**completion_kwargs,
)
return json.loads(captured_requests[0].content)
def test_completion_unmapped_model_sends_json_schema_to_together():
request_body = _capture_completion_request(
UNMAPPED_MODEL, response_format=JSON_SCHEMA_RESPONSE_FORMAT, drop_params=True
)
assert request_body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT
def test_completion_pydantic_response_format_sends_json_schema_to_together():
from pydantic import BaseModel
class VoiceNote(BaseModel):
title: str
summary: str
request_body = _capture_completion_request(TOOL_CALLING_MODEL, response_format=VoiceNote)
sent = request_body["response_format"]
assert sent["type"] == "json_schema"
assert sent["json_schema"]["name"] == "VoiceNote"
assert sent["json_schema"]["strict"] is True
assert sent["json_schema"]["schema"]["required"] == ["title", "summary"]
def test_completion_regex_response_format_sends_pattern_to_together():
request_body = _capture_completion_request(TOOL_CALLING_MODEL, response_format=REGEX_RESPONSE_FORMAT)
assert request_body["response_format"] == REGEX_RESPONSE_FORMAT