diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 8dc3061460a..2c5d930d58e 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -9,6 +9,10 @@ from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union import litellm from litellm._logging import verbose_logger from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _extract_reasoning_content, + _parse_content_for_reasoning, +) from litellm.types.llms.databricks import DatabricksTool from litellm.types.llms.openai import ( ChatCompletionThinkingBlock, @@ -274,49 +278,6 @@ def _handle_invalid_parallel_tool_calls( return tool_calls -def _parse_content_for_reasoning( - message_text: Optional[str], -) -> Tuple[Optional[str], Optional[str]]: - """ - Parse the content for reasoning - - Returns: - - reasoning_content: The content of the reasoning - - content: The content of the message - """ - if not message_text: - return None, message_text - - reasoning_match = re.match( - r"<(?:think|thinking)>(.*?)(.*)", message_text, re.DOTALL - ) - - if reasoning_match: - return reasoning_match.group(1), reasoning_match.group(2) - - return None, message_text - - -def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]: - """ - Extract reasoning content and main content from a message. - - Args: - message (dict): The message dictionary that may contain reasoning_content - - Returns: - tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content) - """ - message_content = message.get("content") - if "reasoning_content" in message: - return message["reasoning_content"], message["content"] - elif "reasoning" in message: - return message["reasoning"], message["content"] - elif isinstance(message_content, str): - return _parse_content_for_reasoning(message_content) - return None, message_content - - class LiteLLMResponseObjectHandler: @staticmethod def convert_to_image_response( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index a99883ef7b6..c5aa230427e 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -14,6 +14,7 @@ from typing import ( Literal, Mapping, Optional, + Tuple, Union, cast, ) @@ -869,3 +870,46 @@ def convert_prefix_message_to_non_prefix_messages( else: new_messages.append(message) return new_messages + + +def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]: + """ + Extract reasoning content and main content from a message. + + Args: + message (dict): The message dictionary that may contain reasoning_content + + Returns: + tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content) + """ + message_content = message.get("content") + if "reasoning_content" in message: + return message["reasoning_content"], message["content"] + elif "reasoning" in message: + return message["reasoning"], message["content"] + elif isinstance(message_content, str): + return _parse_content_for_reasoning(message_content) + return None, message_content + + +def _parse_content_for_reasoning( + message_text: Optional[str], +) -> Tuple[Optional[str], Optional[str]]: + """ + Parse the content for reasoning + + Returns: + - reasoning_content: The content of the reasoning + - content: The content of the message + """ + if not message_text: + return None, message_text + + reasoning_match = re.match( + r"<(?:think|thinking)>(.*?)(.*)", message_text, re.DOTALL + ) + + if reasoning_match: + return reasoning_match.group(1), reasoning_match.group(2) + + return None, message_text diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index fda9220ff7d..e3d65be8bbf 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -14,7 +14,7 @@ from litellm._logging import verbose_logger from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( +from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -397,7 +397,11 @@ class AmazonConverseConfig(BaseConfig): for param, value in non_default_params.items(): if param == "response_format" and isinstance(value, dict): optional_params = self._translate_response_format_param( - value=value, model=model, optional_params=optional_params, non_default_params=non_default_params, is_thinking_enabled=is_thinking_enabled + value=value, + model=model, + optional_params=optional_params, + non_default_params=non_default_params, + is_thinking_enabled=is_thinking_enabled, ) if param == "max_tokens" or param == "max_completion_tokens": optional_params["maxTokens"] = value @@ -446,11 +450,11 @@ class AmazonConverseConfig(BaseConfig): ) return optional_params - + def _translate_response_format_param( - self, - value: dict, - model: str, + self, + value: dict, + model: str, optional_params: dict, non_default_params: dict, is_thinking_enabled: bool, @@ -504,7 +508,7 @@ class AmazonConverseConfig(BaseConfig): optional_params["json_mode"] = True if non_default_params.get("stream", False) is True: optional_params["fake_stream"] = True - + return optional_params def update_optional_params_with_thinking_tokens( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py index d7ceec1f1c1..0fe84b0ce0c 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -3,7 +3,7 @@ from typing import Any, List, Optional, cast from httpx import Response from litellm import verbose_logger -from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( +from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 5dc0d2bb95d..f8717706157 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -118,7 +118,6 @@ class BaseLLMHTTPHandler: response: Optional[httpx.Response] = None for i in range(max(max_retry_on_unprocessable_entity_error, 1)): try: - response = await async_httpx_client.post( url=api_base, headers=headers, diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index ee0d3acef70..de61ae6e3bb 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -16,9 +16,17 @@ from httpx._models import Headers, Response from pydantic import BaseModel import litellm +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _extract_reasoning_content, + convert_content_list_to_str, +) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException -from litellm.types.llms.ollama import OllamaToolCall, OllamaToolCallFunction +from litellm.types.llms.ollama import ( + OllamaChatCompletionMessage, + OllamaToolCall, + OllamaToolCallFunction, +) from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantToolCall, @@ -299,7 +307,20 @@ class OllamaChatConfig(BaseConfig): ) new_tools.append(ollama_tool_call) cast(dict, m)["tool_calls"] = new_tools - new_messages.append(m) + reasoning_content, parsed_content = _extract_reasoning_content( + cast(dict, m) + ) + content_str = convert_content_list_to_str(cast(AllMessageValues, m)) + + ollama_message = OllamaChatCompletionMessage( + role=cast(str, m.get("role")), + ) + if reasoning_content is not None: + ollama_message["thinking"] = reasoning_content + if content_str is not None: + ollama_message["content"] = content_str + + new_messages.append(ollama_message) # Load Config config = self.get_config() @@ -361,7 +382,7 @@ class OllamaChatConfig(BaseConfig): del response_json_message["thinking"] elif response_json_message.get("content") is not None: # parse reasoning content from content - from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 71bcf0bb3f7..bfb0b7f1877 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -229,7 +229,7 @@ class OllamaConfig(BaseConfig): model = model.split("/", 1)[1] api_base = get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434" api_key = self.get_api_key() - headers = { "Authorization": f"Bearer {api_key}" } if api_key else {} + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} try: response = litellm.module_level_client.post( @@ -279,7 +279,7 @@ class OllamaConfig(BaseConfig): api_key: Optional[str] = None, json_mode: Optional[bool] = None, ) -> ModelResponse: - from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index e93902039b4..c30ac5f44aa 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -12,3 +12,6 @@ model_list: model: hosted_vllm/* api_base: https://webhook.site/6fbe498e-88b5-4a5f-8f07-edb9806c1937 api_key: fake-key + - model_name: deepseek-r1-5b + litellm_params: + model: ollama_chat/deepseek-r1:1.5b diff --git a/litellm/types/llms/ollama.py b/litellm/types/llms/ollama.py index 9d71904caa1..d8f20de514d 100644 --- a/litellm/types/llms/ollama.py +++ b/litellm/types/llms/ollama.py @@ -27,3 +27,12 @@ class OllamaToolCall(TypedDict): class OllamaVisionModelObject(TypedDict): prompt: str images: List[str] + + +class OllamaChatCompletionMessage(TypedDict, total=False): + role: Required[str] + content: str + thinking: str + images: List[str] + tool_calls: List[OllamaToolCall] + tool_name: str