diff --git a/litellm/llms/snowflake/chat/transformation.py b/litellm/llms/snowflake/chat/transformation.py index 0968185b084..c64fc583edc 100644 --- a/litellm/llms/snowflake/chat/transformation.py +++ b/litellm/llms/snowflake/chat/transformation.py @@ -8,23 +8,31 @@ Routes to native Cortex REST API endpoints based on model: Ref: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api """ +import copy import json +import re from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict import httpx from typing_extensions import ReadOnly -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.litellm_core_utils.prompt_templates.factory import ( + anthropic_process_openai_file_message, + convert_to_anthropic_tool_result, + create_anthropic_image_param, + select_anthropic_content_block_type_for_file, +) +from litellm.llms.anthropic.chat.handler import ModelResponseIterator as AnthropicStreamParser +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolMessage from litellm.types.utils import ( - ChatCompletionMessageToolCall, - ChatCompletionUsageBlock, Choices, - Function, GenericStreamingChunk, Message, ModelResponse, - Usage, + ModelResponseStream, ) from ...base_llm.base_model_iterator import BaseModelResponseIterator @@ -93,6 +101,103 @@ def _is_claude_model(model: str) -> bool: return any(name.startswith(p) for p in _CLAUDE_MODEL_PREFIXES) +def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object: + """One OpenAI ``image_url`` block in the native shape Cortex accepts. + + Cortex documents base64 sources only, so remote URLs are inlined the way every + other base64-only Anthropic dialect (Bedrock invoke, Vertex) inlines them, and + pdf/text data URIs become document blocks rather than malformed image blocks. + """ + image_url: Final = block.get("image_url") + url: Final = image_url if isinstance(image_url, str) else _image_url_field(image_url, "url") + if not url: + return block + + converted: Final = ( + anthropic_process_openai_file_message({"type": "file", "file": {"file_data": url}}) + if select_anthropic_content_block_type_for_file(_data_uri_media_type(url)) == "document" + else create_anthropic_image_param( + image_url if isinstance(image_url, dict) else url, # mutable-ok: caller's JSON block + format=_image_url_field(image_url, "format"), + is_bedrock_invoke=True, + ) + ) + cache_control: Final = block.get("cache_control") + if cache_control is None: + return converted + return {**converted, "cache_control": cache_control} # mutable-ok: JSON wire block + + +def _image_url_field(image_url: object, key: str) -> str | None: + value: Final = image_url.get(key) if isinstance(image_url, dict) else None + return value if isinstance(value, str) else None + + +def _data_uri_media_type(url: str) -> str: + match: Final = re.match(r"data:([^;,]+)", url) + return match.group(1) if match else "" + + +def _convert_image_url_blocks_to_anthropic(content: object) -> object: + if not isinstance(content, list): + return content + return [ # mutable-ok: JSON wire blocks + _convert_image_url_to_anthropic(block) + if isinstance(block, Mapping) and block.get("type") == "image_url" + else block + for block in content + ] + + +def _convert_tool_result_to_anthropic( + content: object, tool_call_id: str, cache_control: object +) -> Mapping[str, object]: + """The Anthropic ``tool_result`` block for one OpenAI tool message. + + Delegating to the shared converter keeps image, document and per-block cache + breakpoints identical to every other Anthropic dialect; only the plain-string + and non-list shapes it does not model are handled here. + """ + if not isinstance(content, list): + plain: Final[dict[str, object]] = { # mutable-ok: JSON wire block + "type": "tool_result", + "tool_use_id": tool_call_id, + "content": content if isinstance(content, str) else json.dumps(content), + } + return {**plain, "cache_control": cache_control} if cache_control is not None else plain + converted: Final = convert_to_anthropic_tool_result( + ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content), + force_base64=True, + ) + if cache_control is None: + return converted + return {**converted, "cache_control": cache_control} # mutable-ok: JSON wire block + + +def _signed_thinking_blocks(msg: object) -> list[dict[str, object]]: # mutable-ok: JSON wire blocks + """The assistant turn's thinking blocks that can legally be echoed back. + + Only signed blocks round-trip: Cortex rejects a thinking block whose signature is + missing, which is what an unsigned block from a non-thinking turn would produce. + """ + blocks: Final = msg.get("thinking_blocks") if isinstance(msg, dict) else getattr(msg, "thinking_blocks", None) + if not isinstance(blocks, list): + return [] # mutable-ok: JSON wire blocks + return [ # mutable-ok: JSON wire blocks + dict(block) + for block in blocks + if isinstance(block, Mapping) and (block.get("signature") or block.get("type") == "redacted_thinking") + ] + + +def _clean_input_schema(schema: object) -> object: # mutable-ok: JSON schema copy + return ( + {key: value for key, value in schema.items() if key != "$schema"} + if isinstance(schema, Mapping) + else schema # mutable-ok: JSON schema copy + ) # mutable-ok: JSON schema copy + + class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): """ Snowflake Cortex REST API — unified provider. @@ -178,7 +283,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): if "description" in func: anthropic_tool["description"] = func["description"] if "parameters" in func: - anthropic_tool["input_schema"] = func["parameters"] + anthropic_tool["input_schema"] = _clean_input_schema(func["parameters"]) else: anthropic_tool["input_schema"] = { "type": "object", @@ -186,10 +291,16 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): } anthropic_tools.append(anthropic_tool) else: - anthropic_tools.append(tool) + anthropic_tools.append( + {**tool, "input_schema": _clean_input_schema(tool["input_schema"])} # mutable-ok: JSON wire tool + if "input_schema" in tool + else tool + ) return anthropic_tools - def _extract_system_and_messages(self, messages: list[AllMessageValues]) -> tuple[str | None, list[dict]]: + def _extract_system_and_messages( # mutable-ok: JSON wire messages + self, messages: list[AllMessageValues] + ) -> tuple[list[dict] | None, list[dict]]: """ Split messages into system prompt and conversation turns for Anthropic format. @@ -197,26 +308,39 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): - assistant messages with tool_calls → tool_use content blocks - tool role messages → user role with tool_result content blocks """ - system_parts: Final[list[str]] = [] - conversation: Final[list[dict]] = [] + system_parts: Final[list[dict]] = [] # mutable-ok: JSON wire messages + conversation: Final[list[dict]] = [] # mutable-ok: JSON wire messages for msg in messages: if isinstance(msg, dict): role = msg.get("role", "") content: Any = msg.get("content", "") + msg_cache_control: object = msg.get("cache_control") else: role = getattr(msg, "role", "") content = getattr(msg, "content", "") + msg_cache_control = getattr(msg, "cache_control", None) if role == "system": if isinstance(content, str) and content: - system_parts.append(content) + system_parts.append({"type": "text", "text": content}) # mutable-ok: JSON wire system block elif isinstance(content, list): - system_parts.append("\n".join(b.get("text", "") for b in content if b.get("type") == "text")) + system_parts.extend( + { # mutable-ok: JSON wire system block + "type": "text", + "text": block.get("text", ""), + **( + {"cache_control": block["cache_control"]} if "cache_control" in block else {} + ), # mutable-ok: JSON wire block + } + for block in content + if isinstance(block, Mapping) and block.get("type") == "text" + ) elif role == "assistant": tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None) + thinking_blocks = _signed_thinking_blocks(msg) if tool_calls: - content_blocks: list[dict[str, object]] = [] + content_blocks: list[dict[str, object]] = list(thinking_blocks) # mutable-ok: JSON wire blocks if content: content_blocks.append({"type": "text", "text": content}) for tc in tool_calls: @@ -239,18 +363,26 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): } ) conversation.append({"role": "assistant", "content": content_blocks}) + elif thinking_blocks: + thinking_content = ( + [ + *thinking_blocks, + *copy.deepcopy(content), + ] + if isinstance(content, list) + else [*thinking_blocks, *([{"type": "text", "text": content}] if content else [])] + ) # rebind-ok: loop-local normalized content + conversation.append({"role": "assistant", "content": thinking_content}) else: conversation.append({"role": "assistant", "content": content}) elif role == "tool": - tool_call_id = ( + tool_call_id_value = ( msg.get("tool_call_id", "") if isinstance(msg, dict) else getattr(msg, "tool_call_id", "") ) - tool_content = content if isinstance(content, str) else json.dumps(content) - tool_result_block = { - "type": "tool_result", - "tool_use_id": tool_call_id, - "content": tool_content, - } + tool_call_id = ( + tool_call_id_value if isinstance(tool_call_id_value, str) else "" + ) # rebind-ok: normalized loop value + tool_result_block = _convert_tool_result_to_anthropic(content, tool_call_id, msg_cache_control) if ( conversation and conversation[-1]["role"] == "user" @@ -260,11 +392,18 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): ): conversation[-1]["content"].append(tool_result_block) else: - conversation.append({"role": "user", "content": [tool_result_block]}) + conversation.append( + {"role": "user", "content": [tool_result_block]} # mutable-ok: JSON wire message + ) # mutable-ok: JSON wire message else: - conversation.append({"role": role, "content": content}) + conversation.append( # mutable-ok: JSON wire message + { # mutable-ok: JSON wire message + "role": role, + "content": _convert_image_url_blocks_to_anthropic(content), + } # mutable-ok: JSON wire message + ) - system: Final[str | None] = "\n\n".join(system_parts) if system_parts else None + system: Final[list[dict] | None] = system_parts if system_parts else None # mutable-ok: JSON wire messages return system, conversation def transform_request( @@ -339,7 +478,9 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): extra_body: dict, ) -> dict: """Anthropic Messages format for /messages endpoint.""" - system, conversation = self._extract_system_and_messages(messages) + passthrough_system: Final = optional_params.pop("system", None) + extracted_system, conversation = self._extract_system_and_messages(messages) + system: Final = passthrough_system if passthrough_system is not None else extracted_system if "tools" in optional_params: optional_params["tools"] = self._transform_tools_to_anthropic(optional_params["tools"]) @@ -353,16 +494,19 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): model_name: Final = model.removeprefix("snowflake/") - body: Final[dict[str, object]] = { - "model": model_name, - "messages": conversation, - "stream": stream, - **optional_params, - **extra_body, - } - + body: Final[dict[str, object]] = normalize_cache_control_in_anthropic_payload( # mutable-ok: JSON wire body + { # mutable-ok: JSON wire body + "model": model_name, + "messages": conversation, + "stream": stream, + **optional_params, + **extra_body, # mutable-ok: JSON wire body + } + ) if system is not None: - body["system"] = system + body["system"] = normalize_cache_control_in_anthropic_payload( # mutable-ok: JSON wire payload + {"system": system} # mutable-ok: JSON wire payload + )["system"] if "max_tokens" not in body: body["max_tokens"] = 4096 # reasonable default; Anthropic API max varies by model @@ -435,23 +579,10 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): additional_args={"complete_input_dict": request_data}, ) - text_content = "" - tool_calls: Final = [] - - for block in response_json.get("content", []): - if block.get("type") == "text": - text_content += block.get("text", "") - elif block.get("type") == "tool_use": - tool_calls.append( - ChatCompletionMessageToolCall( - id=block.get("id", ""), - type="function", - function=Function( - name=block.get("name", ""), - arguments=json.dumps(block.get("input", {})), - ), - ) - ) + anthropic_config: Final = AnthropicConfig() + text_content, _, thinking_blocks, reasoning_content, tool_calls, _, _, _ = ( + anthropic_config.extract_response_content(completion_response=dict(response_json)) + ) _stop_reason_map: Final = { "end_turn": "stop", @@ -461,9 +592,13 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): } finish_reason: Final = _stop_reason_map.get(response_json.get("stop_reason", "end_turn"), "stop") - message: Final = Message(content=text_content or None, role="assistant") - if tool_calls: - message.tool_calls = tool_calls + message: Final = Message( + content=text_content or None, + role="assistant", + tool_calls=tool_calls or None, + thinking_blocks=thinking_blocks, + reasoning_content=reasoning_content, + ) choice: Final = Choices( finish_reason=finish_reason, @@ -471,11 +606,13 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): message=message, ) - usage_data: Final = response_json.get("usage", {}) - usage: Final = Usage( - prompt_tokens=usage_data.get("input_tokens", 0), - completion_tokens=usage_data.get("output_tokens", 0), - total_tokens=usage_data.get("input_tokens", 0) + usage_data.get("output_tokens", 0), + # Cortex reports prompt-cache creation/read counts alongside input_tokens; the + # shared calculator folds them into prompt_tokens_details so cached input is + # visible and billed at its own rate. + usage: Final = anthropic_config.calculate_usage( + usage_object=response_json.get("usage", {}), + reasoning_content=reasoning_content, + completion_response=dict(response_json), ) model_response.choices = [choice] @@ -516,15 +653,19 @@ class SnowflakeStreamingHandler(BaseModelResponseIterator): json_mode: bool | None = False, ): super().__init__(streaming_response=streaming_response, sync_stream=sync_stream) - self._tool_index = 0 - self._tool_id = "" - self._tool_name = "" - self._input_tokens = 0 + # Cortex streams the Anthropic SSE dialect on /messages, so its events are parsed + # by Anthropic's own parser: thinking deltas, signatures and prompt-cache usage + # all arrive the way they do on every other Anthropic-dialect provider. + self._anthropic_parser: Final = AnthropicStreamParser( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) - def chunk_parser(self, chunk: dict) -> GenericStreamingChunk: + def chunk_parser(self, chunk: dict) -> GenericStreamingChunk | ModelResponseStream: if "choices" in chunk: return self._parse_openai_chunk(chunk) - return self._parse_anthropic_chunk(chunk) + return self._anthropic_parser.chunk_parser(chunk) def _parse_openai_chunk(self, chunk: dict) -> GenericStreamingChunk: choices: Final = chunk.get("choices", []) @@ -566,117 +707,3 @@ class SnowflakeStreamingHandler(BaseModelResponseIterator): index=choice.get("index", 0), tool_use=tool_use, ) - - def _parse_anthropic_chunk(self, chunk: dict) -> GenericStreamingChunk: - event_type: Final = chunk.get("type", "") - - if event_type == "message_start": - message: Final = chunk.get("message", {}) - usage_data = message.get("usage", {}) - self._input_tokens = usage_data.get("input_tokens", 0) - return GenericStreamingChunk( - text="", - is_finished=False, - finish_reason="", - usage=None, - index=0, - tool_use=None, - ) - - elif event_type == "content_block_delta": - delta = chunk.get("delta", {}) - delta_type: Final = delta.get("type", "") - - if delta_type == "text_delta": - return GenericStreamingChunk( - text=delta.get("text", ""), - is_finished=False, - finish_reason="", - usage=None, - index=chunk.get("index", 0), - tool_use=None, - ) - elif delta_type == "input_json_delta": - return GenericStreamingChunk( - text="", - is_finished=False, - finish_reason="", - usage=None, - index=chunk.get("index", 0), - tool_use=ChatCompletionToolCallChunk( - id=self._tool_id, - type="function", - function={ - "name": self._tool_name, - "arguments": delta.get("partial_json", ""), - }, - index=self._tool_index, - ), - ) - - elif event_type == "content_block_start": - content_block: Final = chunk.get("content_block", {}) - if content_block.get("type") == "tool_use": - self._tool_id = content_block.get("id", "") - self._tool_name = content_block.get("name", "") - self._tool_index = chunk.get("index", 0) - return GenericStreamingChunk( - text="", - is_finished=False, - finish_reason="", - usage=None, - index=chunk.get("index", 0), - tool_use=ChatCompletionToolCallChunk( - id=self._tool_id, - type="function", - function={"name": self._tool_name, "arguments": ""}, - index=self._tool_index, - ), - ) - - elif event_type == "message_delta": - delta = chunk.get("delta", {}) - stop_reason: Final = delta.get("stop_reason", "") - usage_data = chunk.get("usage", {}) - _stop_map: Final = { - "end_turn": "stop", - "max_tokens": "length", - "tool_use": "tool_calls", - "stop_sequence": "stop", - } - usage = None - if usage_data or self._input_tokens: - output_t: Final = usage_data.get("output_tokens", 0) - input_t: Final = self._input_tokens or usage_data.get("input_tokens", 0) - usage = ChatCompletionUsageBlock( - prompt_tokens=input_t, - completion_tokens=output_t, - total_tokens=input_t + output_t, - ) - return GenericStreamingChunk( - text="", - is_finished=True, - finish_reason=_stop_map.get(stop_reason, "stop"), - usage=usage, - index=0, - tool_use=None, - ) - - elif event_type == "message_stop": - return GenericStreamingChunk( - text="", - is_finished=True, - finish_reason="stop", - usage=None, - index=0, - tool_use=None, - ) - - return GenericStreamingChunk( - text="", - is_finished=False, - finish_reason="", - usage=None, - index=0, - tool_use=None, - ) diff --git a/tests/test_litellm/llms/snowflake/chat/test_snowflake_chat_transformation.py b/tests/test_litellm/llms/snowflake/chat/test_snowflake_chat_transformation.py index a182656e4a8..25a961c3413 100644 --- a/tests/test_litellm/llms/snowflake/chat/test_snowflake_chat_transformation.py +++ b/tests/test_litellm/llms/snowflake/chat/test_snowflake_chat_transformation.py @@ -17,7 +17,7 @@ import pytest import litellm from litellm import completion, acompletion from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.llms.snowflake.chat.transformation import SnowflakeConfig +from litellm.llms.snowflake.chat.transformation import SnowflakeConfig, SnowflakeStreamingHandler from litellm.types.utils import ModelResponse @@ -114,8 +114,7 @@ class TestSnowflakeToolTransformation: ) assert transformed_request["tool_choice"] == value, ( - f"tool_choice='{value}' should pass through unchanged, " - f"got {transformed_request['tool_choice']}" + f"tool_choice='{value}' should pass through unchanged, got {transformed_request['tool_choice']}" ) def test_transform_response_with_tool_calls(self): @@ -159,9 +158,7 @@ class TestSnowflakeToolTransformation: headers={"Content-Type": "application/json"}, ) - model_response = ModelResponse( - choices=[litellm.Choices(index=0, message=litellm.Message())] - ) + model_response = ModelResponse(choices=[litellm.Choices(index=0, message=litellm.Message())]) logging_obj = MagicMock() @@ -232,9 +229,7 @@ class TestSnowflakeToolTransformation: headers={"Content-Type": "application/json"}, ) - model_response = ModelResponse( - choices=[litellm.Choices(index=0, message=litellm.Message())] - ) + model_response = ModelResponse(choices=[litellm.Choices(index=0, message=litellm.Message())]) logging_obj = MagicMock() @@ -280,9 +275,7 @@ class TestSnowflakeToolTransformation: headers={"Content-Type": "application/json"}, ) - model_response = ModelResponse( - choices=[litellm.Choices(index=0, message=litellm.Message())] - ) + model_response = ModelResponse(choices=[litellm.Choices(index=0, message=litellm.Message())]) logging_obj = MagicMock() @@ -300,10 +293,7 @@ class TestSnowflakeToolTransformation: # Verify standard response works assert isinstance(result, ModelResponse) - assert ( - result.choices[0].message.content - == "Hello! I'm doing well, thank you for asking." - ) + assert result.choices[0].message.content == "Hello! I'm doing well, thank you for asking." def test_get_supported_openai_params_includes_tools(self): """ @@ -318,6 +308,385 @@ class TestSnowflakeToolTransformation: assert "max_tokens" in supported_params +class TestSnowflakeCortexClaudeFixes: + def setup_method(self): + self.config = SnowflakeConfig() + + @staticmethod + def _transform(messages, optional_params=None): + return SnowflakeConfig().transform_request( + model="snowflake/claude-sonnet-4-6", + messages=messages, + optional_params=optional_params or {}, + litellm_params={}, + headers={}, + ) + + def test_thinking_is_offered_on_every_claude_model(self): + """Cortex documents extended thinking (budget_tokens) for Claude generally, so a + 4.6-only gate would silently drop it on the models that do support it.""" + for model in ( + "snowflake/claude-sonnet-4-6", + "snowflake/claude-sonnet-4-5", + "snowflake/claude-3-7-sonnet", + "snowflake/claude-4-opus", + ): + assert "thinking" in self.config.get_supported_openai_params(model), model + assert "thinking" not in self.config.get_supported_openai_params("snowflake/llama3.1-70b") + + def test_system_blocks_preserve_cache_control_and_strip_ttl(self): + body = self._transform( + [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are helpful", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + }, + {"role": "user", "content": "hi"}, + ] + ) + assert body["system"] == [{"type": "text", "text": "You are helpful", "cache_control": {"type": "ephemeral"}}] + + def test_direct_system_param_is_normalized(self): + body = self._transform( + [{"role": "user", "content": "hi"}], + {"system": [{"type": "text", "text": "direct", "cache_control": {"type": "ephemeral", "ttl": "1h"}}]}, + ) + assert body["system"] == [{"type": "text", "text": "direct", "cache_control": {"type": "ephemeral"}}] + + def test_message_and_tool_cache_control_are_normalized(self): + body = self._transform( + [ + { + "role": "user", + "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + } + ], + { + "tools": [ + { + "name": "f", + "input_schema": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ] + }, + ) + assert body["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + assert body["tools"][0]["cache_control"] == {"type": "ephemeral"} + + def test_extra_body_message_override_is_normalized(self): + body = self._transform( + [{"role": "user", "content": "original"}], + { + "extra_body": { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "override", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + } + ] + } + }, + ) + assert body["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + + def test_image_blocks_are_converted_to_anthropic_source(self): + body = self._transform( + [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,ZmFrZQ==", "format": "image/jpeg"}, + } + ], + } + ] + ) + assert body["messages"][0]["content"] == [ + {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": "ZmFrZQ=="}} + ] + + def test_tool_result_image_list_is_converted(self): + body = self._transform( + [ + {"role": "user", "content": "look"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,ZmFrZQ=="}}], + }, + ] + ) + assert body["messages"][2]["content"][0]["content"] == [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "ZmFrZQ=="}} + ] + + def test_tool_result_preserves_cache_control(self): + """A cache breakpoint the bridge puts on a tool message must survive onto the tool_result.""" + for tool_content in ("done", [{"type": "text", "text": "done"}]): + body = self._transform( + [ + {"role": "user", "content": "look"}, + { + "role": "tool", + "tool_call_id": "call_1", + "content": tool_content, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + }, + ] + ) + tool_result = body["messages"][1]["content"][0] + assert tool_result["cache_control"] == {"type": "ephemeral"}, tool_content + + def test_pdf_data_uri_becomes_a_document_block(self): + """A bridged pdf data URI is a document block; forwarding it as an image is malformed.""" + body = self._transform( + [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "data:application/pdf;base64,ZmFrZQ=="}}, + ], + } + ] + ) + assert body["messages"][0]["content"] == [ + { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": "ZmFrZQ=="}, + } + ] + + def test_multipart_tool_result_preserves_text_and_converts_image(self): + body = self._transform( + [ + {"role": "user", "content": "look"}, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [ + {"type": "text", "text": "first"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,ZmFrZQ=="}}, + {"type": "text", "text": "last"}, + ], + }, + ] + ) + assert body["messages"][1]["content"][0]["content"] == [ + {"type": "text", "text": "first"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "ZmFrZQ=="}}, + {"type": "text", "text": "last"}, + ] + + def test_plain_text_tool_result_remains_string(self): + body = self._transform( + [{"role": "user", "content": "look"}, {"role": "tool", "tool_call_id": "call_1", "content": "done"}] + ) + assert body["messages"][1]["content"][0]["content"] == "done" + + def test_anthropic_tool_schema_strips_only_top_level_schema_key(self): + tools = [ + { + "name": "f", + "input_schema": {"$schema": "schema", "type": "object", "properties": {"$schema": {"type": "string"}}}, + } + ] + body = self._transform([{"role": "user", "content": "hi"}], {"tools": tools}) + schema = body["tools"][0]["input_schema"] + assert "$schema" not in schema + assert "$schema" in schema["properties"] + + def test_tool_schema_strips_only_top_level_schema_key(self): + tools = [ + { + "type": "function", + "function": { + "name": "f", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"$schema": {"type": "string"}}, + }, + }, + } + ] + body = self._transform([{"role": "user", "content": "hi"}], {"tools": tools}) + schema = body["tools"][0]["input_schema"] + assert "$schema" not in schema + assert "$schema" in schema["properties"] + + def test_streaming_tool_identity_is_emitted_only_on_start(self): + handler = SnowflakeStreamingHandler(streaming_response=[], sync_stream=True) + start = handler.chunk_parser( + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "tool_use", "id": "tool_1", "name": "read"}, + } + ) + first_delta = handler.chunk_parser( + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"path":'}, + } + ) + second_delta = handler.chunk_parser( + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '"/tmp"}'}, + } + ) + + def _tool_call(chunk): + return chunk.choices[0].delta.tool_calls[0] + + assert _tool_call(start).id == "tool_1" + assert _tool_call(start).function.name == "read" + assert _tool_call(first_delta).id is None + assert _tool_call(first_delta).function.name is None + assert _tool_call(second_delta).id is None + assert _tool_call(second_delta).function.name is None + assert _tool_call(first_delta).function.arguments == '{"path":' + assert _tool_call(second_delta).function.arguments == '"/tmp"}' + + def test_signed_thinking_blocks_lead_the_assistant_turn(self): + """Multi-turn tool use with thinking only works if the signed block is echoed back first.""" + body = self._transform( + [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "thinking_blocks": [ + {"type": "thinking", "thinking": "391", "signature": "Eto"}, + {"type": "thinking", "thinking": "unsigned"}, + ], + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} + ], + }, + ] + ) + blocks = body["messages"][1]["content"] + assert blocks[0] == {"type": "thinking", "thinking": "391", "signature": "Eto"} + assert [b["type"] for b in blocks] == ["thinking", "tool_use"] + + def test_signed_thinking_blocks_lead_a_plain_text_assistant_turn(self): + """A thinking response without a tool call must also round-trip on the next request.""" + body = self._transform( + [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": "391", + "thinking_blocks": [{"type": "thinking", "thinking": "391", "signature": "Eto"}], + }, + {"role": "user", "content": "continue"}, + ] + ) + assert body["messages"][1] == { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "391", "signature": "Eto"}, + {"type": "text", "text": "391"}, + ], + } + + def test_signed_thinking_blocks_preserve_list_content(self): + """Cached assistant text reaches this transform as a content list, not a string.""" + body = self._transform( + [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [{"type": "text", "text": "391", "cache_control": {"type": "ephemeral"}}], + "thinking_blocks": [{"type": "thinking", "thinking": "391", "signature": "Eto"}], + }, + {"role": "user", "content": "continue"}, + ] + ) + assert body["messages"][1] == { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "391", "signature": "Eto"}, + {"type": "text", "text": "391", "cache_control": {"type": "ephemeral"}}, + ], + } + + def test_thinking_only_assistant_turn_sends_no_empty_text_block(self): + """Anthropic-shaped APIs reject empty text blocks, so a content-less thinking turn is thinking only.""" + body = self._transform( + [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "thinking_blocks": [{"type": "thinking", "thinking": "391", "signature": "Eto"}], + }, + {"role": "user", "content": "continue"}, + ] + ) + assert body["messages"][1]["content"] == [{"type": "thinking", "thinking": "391", "signature": "Eto"}] + + def test_streaming_surfaces_thinking_and_prompt_cache_usage(self): + """Cortex streams thinking deltas, signatures and cache counts; all must reach the caller.""" + handler = SnowflakeStreamingHandler(streaming_response=[], sync_stream=True) + handler.chunk_parser( + { + "type": "message_start", + "message": {"usage": {"input_tokens": 18, "cache_creation_input_tokens": 1323}}, + } + ) + thinking = handler.chunk_parser( + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": "391"}, + } + ) + signature = handler.chunk_parser( + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "Eto"}, + } + ) + final = handler.chunk_parser( + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 8, "cache_read_input_tokens": 1323}, + } + ) + + assert thinking.choices[0].delta.reasoning_content == "391" + assert signature.choices[0].delta.thinking_blocks[0]["signature"] == "Eto" + assert final.usage.prompt_tokens_details.cached_tokens == 1323 + + class TestSnowFlakeCompletion: model_name = "mistral" @@ -380,10 +749,7 @@ class TestSnowFlakeCompletion: # PAT key was used post_kwargs = mock_post.call_args_list[-1][1] assert "xxxxx" in post_kwargs["headers"]["Authorization"] - assert ( - post_kwargs["headers"]["X-Snowflake-Authorization-Token-Type"] - == "PROGRAMMATIC_ACCESS_TOKEN" - ) + assert post_kwargs["headers"]["X-Snowflake-Authorization-Token-Type"] == "PROGRAMMATIC_ACCESS_TOKEN" # account id was used assert "AAAA-BBBB" in post_kwargs["url"] @@ -495,9 +861,7 @@ class TestSnowflakeChatCompletion: ) mock_post.assert_called_once() else: - with patch.object( - AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_resp - ) as mock_post: + with patch.object(AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_resp) as mock_post: response = asyncio.run( acompletion( model="snowflake/mistral-7b", @@ -580,8 +944,4 @@ class TestSnowflakeChatCompletion: chunks_received = asyncio.run(_run()) assert len(chunks_received) > 0 - content = "".join( - c.choices[0].delta.content - for c in chunks_received - if c.choices[0].delta.content - ) + content = "".join(c.choices[0].delta.content for c in chunks_received if c.choices[0].delta.content) diff --git a/tests/test_litellm/llms/snowflake/test_snowflake_native_endpoints.py b/tests/test_litellm/llms/snowflake/test_snowflake_native_endpoints.py index fb21e2e6f6b..7970f7771fc 100644 --- a/tests/test_litellm/llms/snowflake/test_snowflake_native_endpoints.py +++ b/tests/test_litellm/llms/snowflake/test_snowflake_native_endpoints.py @@ -338,7 +338,7 @@ class TestAnthropicConfigRequest: litellm_params={}, headers={}, ) - assert body["system"] == "You are helpful." + assert body["system"] == [{"type": "text", "text": "You are helpful."}] assert all(m["role"] != "system" for m in body["messages"]) assert body["messages"][0] == {"role": "user", "content": "Hello"} @@ -422,6 +422,64 @@ class TestAnthropicConfigResponse: assert result.usage.completion_tokens == 5 assert result.usage.total_tokens == 15 + def test_prompt_cache_usage_is_surfaced(self): + """Cortex reports cache creation/read counts; dropping them hides caching and bills cached input at full price.""" + raw = httpx.Response( + 200, + json={ + "id": "msg_1", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 18, "cache_creation_input_tokens": 1323, "cache_read_input_tokens": 0}, + }, + ) + result = self.cfg.transform_response( + model="snowflake/claude-sonnet-4-6", + raw_response=raw, + model_response=ModelResponse(), + logging_obj=_mock_logging(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert result.usage.prompt_tokens == 1341 + assert result.usage.prompt_tokens_details.cache_creation_tokens == 1323 + assert result.usage.prompt_tokens_details.cached_tokens == 0 + + def test_thinking_block_and_signature_are_preserved(self): + """The signature must survive so a client can echo the thinking block on the next turn.""" + raw = httpx.Response( + 200, + json={ + "id": "msg_1", + "model": "claude-sonnet-4-6", + "content": [ + {"type": "thinking", "thinking": "391", "signature": "Eto"}, + {"type": "text", "text": "391"}, + ], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) + result = self.cfg.transform_response( + model="snowflake/claude-sonnet-4-6", + raw_response=raw, + model_response=ModelResponse(), + logging_obj=_mock_logging(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + message = result.choices[0].message + assert message.content == "391" + assert message.reasoning_content == "391" + assert message.thinking_blocks[0]["signature"] == "Eto" + def test_stop_reason_end_turn_maps_to_stop(self): raw = _make_anthropic_response() result = self.cfg.transform_response(