mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
fix(databricks): skip dropped assistant turns and blank parts when joining for json_object
The json_object join picked the last message before the request transform drops bare assistant turns, so a trailing thinking-only or reasoning-only turn (a Responses input ending in a reasoning item included) left the real last user message as a list and Databricks still returned its 400. The pick now skips the turns that transform drops. The join also ran when only one part had text, which sent whitespace-only parts the existing empty block filter used to drop on requests that already worked. It now joins only two or more non-blank parts, so a request with one non-blank part goes out exactly as before. Without drop_params, a message with a message-level cache_control now goes out as sent instead of being rewritten into one cached block.
This commit is contained in:
parent
9823467b01
commit
1ecfd61dfa
3 changed files with 132 additions and 7 deletions
|
|
@ -144,16 +144,28 @@ def _is_system(message: Mapping[str, object]) -> bool:
|
|||
|
||||
def _json_mode_message(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
|
||||
candidate: Final = _without_cache_control(message) if drop_params else message
|
||||
match _plain_texts(candidate):
|
||||
case ():
|
||||
return candidate if _has_string_content(candidate) else message
|
||||
case texts:
|
||||
return cast(AllMessageValues, {**candidate, "content": "".join(texts)}) # cast-ok: spread keeps the shape
|
||||
texts: Final = _plain_texts(candidate)
|
||||
if "cache_control" in candidate or not (texts or _has_string_content(candidate)):
|
||||
return message
|
||||
if sum(1 for text in texts if text.strip()) < 2:
|
||||
return candidate
|
||||
return cast(AllMessageValues, {**candidate, "content": "".join(texts)}) # cast-ok: spread keeps the shape
|
||||
|
||||
|
||||
def _is_dropped_on_send(message: Mapping[str, object]) -> bool:
|
||||
sanitized: Final = {**message}
|
||||
_sanitize_empty_content(sanitized)
|
||||
return _is_bare_assistant_message(sanitized)
|
||||
|
||||
|
||||
def _is_system_or_dropped(message: Mapping[str, object]) -> bool:
|
||||
return _is_system(message) or _is_dropped_on_send(message)
|
||||
|
||||
|
||||
def _json_checked_start(messages: Sequence[AllMessageValues]) -> int:
|
||||
trailing_system: Final = sum(1 for _ in takewhile(_is_system, reversed(messages)))
|
||||
return max(len(messages) - max(trailing_system, 1), 0)
|
||||
trailing: Final = tuple(takewhile(_is_system_or_dropped, reversed(messages)))
|
||||
last_sent_offset: Final = 0 if any(_is_system(message) for message in trailing) else 1
|
||||
return max(len(messages) - len(trailing) - last_sent_offset, 0)
|
||||
|
||||
|
||||
def _json_object_ready_messages(
|
||||
|
|
|
|||
|
|
@ -444,6 +444,26 @@ def _shape_reply(identity: str, stream: bool) -> Reply:
|
|||
"/chat/completions",
|
||||
id="A8c-chat-json-object-drop-params-drops-cache-control",
|
||||
),
|
||||
pytest.param(
|
||||
"json-object-trailing-thinking-turn",
|
||||
"/v1/chat/completions",
|
||||
"databricks-meta-llama-3-3-70b-instruct",
|
||||
{
|
||||
"messages": [
|
||||
{"role": "user", "content": [{"type": "text", "text": "Reply"}, {"type": "text", "text": " in JSON"}]},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"thinking_blocks": [{"type": "thinking", "thinking": "Plan.", "signature": "sig"}],
|
||||
},
|
||||
],
|
||||
"response_format": {"type": "json_object"},
|
||||
},
|
||||
[{"role": "user", "content": "Reply in JSON"}],
|
||||
False,
|
||||
"/chat/completions",
|
||||
id="A8d-chat-json-object-trailing-thinking-turn-dropped",
|
||||
),
|
||||
pytest.param(
|
||||
"json-object-responses-two-parts",
|
||||
"/v1/responses",
|
||||
|
|
@ -462,6 +482,25 @@ def _shape_reply(identity: str, stream: bool) -> Reply:
|
|||
"/chat/completions",
|
||||
id="A9-responses-json-object-two-parts-joined",
|
||||
),
|
||||
pytest.param(
|
||||
"json-object-responses-trailing-reasoning",
|
||||
"/v1/responses",
|
||||
"databricks-qwen35-122b-a10b",
|
||||
{
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "Reply"}, {"type": "input_text", "text": " in JSON"}],
|
||||
},
|
||||
{"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "Plan."}]},
|
||||
],
|
||||
"text": {"format": {"type": "json_object"}},
|
||||
},
|
||||
[{"role": "user", "content": "Reply in JSON"}],
|
||||
False,
|
||||
"/chat/completions",
|
||||
id="A9b-responses-json-object-trailing-reasoning-dropped",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_databricks_non_claude_plain_text_content_reaches_upstream_as_string(
|
||||
|
|
|
|||
|
|
@ -745,6 +745,80 @@ def test_transform_request_json_object_drops_last_message_cache_control_under_dr
|
|||
]
|
||||
|
||||
|
||||
_THINKING_BLOCK: Final = {"type": "thinking", "thinking": "Plan the reply.", "signature": "sig"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dropped_turn",
|
||||
[
|
||||
pytest.param({"role": "assistant", "content": "", "thinking_blocks": [_THINKING_BLOCK]}, id="thinking-only"),
|
||||
pytest.param({"role": "assistant", "content": None, "reasoning_content": "Plan."}, id="reasoning-only"),
|
||||
pytest.param({"role": "assistant", "content": [{"type": "text", "text": " "}]}, id="blank-text-part"),
|
||||
],
|
||||
)
|
||||
def test_transform_request_json_object_joins_the_message_before_a_dropped_assistant_turn(
|
||||
dropped_turn: AllMessageValues,
|
||||
) -> None:
|
||||
assert _outgoing_messages(
|
||||
"databricks-meta-llama-3-3-70b-instruct",
|
||||
[{"role": "user", "content": _TWO_TEXT_BLOCKS}, dropped_turn],
|
||||
_JSON_OBJECT,
|
||||
) == [{"role": "user", "content": "Reply in JSON"}]
|
||||
|
||||
|
||||
def test_transform_request_json_object_joins_a_system_run_split_by_a_dropped_assistant_turn() -> None:
|
||||
earlier_turn: Final[AllMessageValues] = {
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "Say"}, {"type": "text", "text": " hi"}],
|
||||
}
|
||||
messages: Final[list[AllMessageValues]] = [
|
||||
earlier_turn,
|
||||
{"role": "system", "content": [{"type": "text", "text": "Alpha"}, {"type": "text", "text": " rule."}]},
|
||||
{"role": "assistant", "content": "", "thinking_blocks": [_THINKING_BLOCK]},
|
||||
{"role": "system", "content": _TWO_TEXT_BLOCKS},
|
||||
]
|
||||
|
||||
assert _outgoing_messages("databricks-meta-llama-3-3-70b-instruct", messages, _JSON_OBJECT) == [
|
||||
earlier_turn,
|
||||
{"role": "system", "content": "Alpha rule.\n\nReply in JSON"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content,expected_content",
|
||||
[
|
||||
pytest.param(
|
||||
[{"type": "text", "text": "Reply in JSON"}, {"type": "text", "text": "\n\n"}],
|
||||
"Reply in JSON",
|
||||
id="one-non-blank-part-goes-out-as-before",
|
||||
),
|
||||
pytest.param(
|
||||
[{"type": "text", "text": "Reply"}, {"type": "text", "text": " "}, {"type": "text", "text": "in JSON"}],
|
||||
"Reply in JSON",
|
||||
id="blank-separator-between-parts-kept",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_transform_request_json_object_joins_only_two_or_more_non_blank_parts(
|
||||
content: list[object], expected_content: str
|
||||
) -> None:
|
||||
assert _outgoing_messages(
|
||||
"databricks-meta-llama-3-3-70b-instruct", [{"role": "user", "content": content}], _JSON_OBJECT
|
||||
) == [{"role": "user", "content": expected_content}]
|
||||
|
||||
|
||||
def test_transform_request_json_object_keeps_message_cache_control_parts_as_sent_without_drop_params() -> None:
|
||||
message: Final[AllMessageValues] = {
|
||||
"role": "user",
|
||||
"content": _TWO_TEXT_BLOCKS,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
|
||||
assert _outgoing_messages("databricks-meta-llama-3-3-70b-instruct", [message], {}) == _outgoing_messages(
|
||||
"databricks-meta-llama-3-3-70b-instruct", [message], _JSON_OBJECT
|
||||
)
|
||||
|
||||
|
||||
def test_transform_request_json_object_reads_the_global_drop_params(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue