From a8901f322f7e653575f35aabe6895aa32960f5c9 Mon Sep 17 00:00:00 2001 From: Bruno Felthes Date: Fri, 14 Aug 2026 09:29:04 -0300 Subject: [PATCH] feat(azure_ai): shape FW-Kimi / Kimi for Azure Foundry + Claude Code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make Azure AI Foundry FW-Kimi-K3 work with Claude Code multi-turn tool flows: drop fixed sampling, sanitize reasoning_effort, strip thinking_blocks into reasoning_content on chat, and shape the Anthropic→Responses bridge so tool calls keep reasoning/message ordering Foundry accepts. Co-authored-by: Cursor --- .../responses_adapters/handler.py | 22 +++ .../responses_adapters/transformation.py | 13 ++ litellm/llms/azure_ai/chat/transformation.py | 171 +++++++++++++++++- .../llms/openai/responses/transformation.py | 83 +++++++++ ...odel_prices_and_context_window_backup.json | 23 +++ model_prices_and_context_window.json | 23 +++ .../azure_ai/test_azure_ai_kimi_shaping.py | 120 ++++++++++++ 7 files changed, 453 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_shaping.py diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py index 9210719dd59..fb8f013f7d0 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -98,6 +98,28 @@ def _build_responses_kwargs( if stream: responses_kwargs["stream"] = True + # Foundry Kimi: drop fixed sampling and unsupported reasoning.effort values. + _model_id = (model or "").split("/")[-1].lower() + _is_kimi = ( + _model_id.startswith("fw-kimi") + or _model_id.startswith("kimi-k2.") + or "kimi-k3" in _model_id + ) + if _is_kimi: + for _param in ( + "temperature", + "top_p", + "n", + "presence_penalty", + "frequency_penalty", + ): + responses_kwargs.pop(_param, None) + _reasoning = responses_kwargs.get("reasoning") + if isinstance(_reasoning, dict): + _effort = _reasoning.get("effort") + if _effort not in ("low", "high", "max"): + responses_kwargs.pop("reasoning", None) + # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) excluded: Final = {"anthropic_messages"} for key, value in (extra_kwargs or {}).items(): diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index bf3f6153e7c..1461c348a8e 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -193,6 +193,19 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if btype == "text": asst_parts.append({"type": "output_text", "text": block.get("text", "")}) elif btype == "tool_use": + # Flush text/thinking before the tool call so Responses + # input keeps message/reasoning → function_call order. + # Without this, Claude Code turns (thinking + tool_use) + # emit function_call first and the thinking message after. + if asst_parts: + input_items.append( + { + "type": "message", + "role": "assistant", + "content": asst_parts, + } + ) + asst_parts = [] # tool_use becomes a top-level function_call item input_items.append( { diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 8545d646035..db0d772a671 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -21,7 +21,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelResponse, ProviderField -from litellm.utils import _add_path_to_api_base, supports_tool_choice +from litellm.utils import _add_path_to_api_base, supports_reasoning, supports_tool_choice class AzureFoundryErrorStrings(str, enum.Enum): @@ -57,6 +57,146 @@ class AzureAIStudioConfig(OpenAIConfig): return xai_config._supports_stop_reason(model) return True + # Kimi on Azure AI Foundry (native + Fireworks FW-* catalog) follows the same + # Moonshot chat contract: https://platform.kimi.ai/docs/guide/kimi-k3-quickstart + _KIMI_K3_FIXED_SAMPLING_PARAMS: Final = ( + "temperature", + "top_p", + "n", + "presence_penalty", + "frequency_penalty", + ) + _KIMI_K3_REASONING_EFFORT_VALUES: Final = frozenset({"low", "high", "max"}) + + def _normalize_azure_ai_model_id(self, model: str) -> str: + return model.split("/")[-1].lower() + + def _is_kimi_reasoning_model(self, model: str) -> bool: + """ + True for Kimi reasoning models on Azure AI Foundry / Fireworks catalog. + + Name matching covers FW-Kimi-* and native kimi-k2.5+ / kimi-k3 ids even when + the local cost map has not been refreshed yet. Cost-map supports_reasoning is + a fallback for future azure_ai Kimi entries. + """ + model_id = self._normalize_azure_ai_model_id(model) + if model_id.startswith("fw-kimi") or model_id.startswith("kimi-k2.") or model_id.startswith("kimi-k3"): + return True + if "kimi-k3" in model_id: + return True + try: + return supports_reasoning(model=model, custom_llm_provider="azure_ai") + except Exception: + return False + + def _is_kimi_k3_model(self, model: str) -> bool: + model_id = self._normalize_azure_ai_model_id(model) + return model_id == "fw-kimi-k3" or "kimi-k3" in model_id + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + optional_params = super().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ) + if not self._is_kimi_reasoning_model(model): + return optional_params + + # Reasoning Kimi rejects non-default temperature (Moonshot + Foundry Fireworks) + optional_params.pop("temperature", None) + + if self._is_kimi_k3_model(model): + # K3 fixes sampling server-side; omit or the Foundry gateway returns 400 + for param in self._KIMI_K3_FIXED_SAMPLING_PARAMS: + optional_params.pop(param, None) + # Parent OpenAI mapping may omit reasoning_effort; K3 accepts low/high/max + if "reasoning_effort" in non_default_params: + effort = non_default_params["reasoning_effort"] + if effort in self._KIMI_K3_REASONING_EFFORT_VALUES: + optional_params["reasoning_effort"] = effort + else: + # Claude Code / Anthropic adapters often emit "medium" + optional_params.pop("reasoning_effort", None) + + return optional_params + + @staticmethod + def _thinking_blocks_to_text(thinking_blocks: object) -> str: + if not isinstance(thinking_blocks, list): + return "" + parts: list[str] = [] + for block in thinking_blocks: + if not isinstance(block, dict): + continue + text = block.get("thinking") or block.get("text") or "" + if isinstance(text, str) and text.strip(): + parts.append(text) + return "\n".join(parts) + + def fill_reasoning_content(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: + """ + Kimi on Azure Foundry: + 1) Strip LiteLLM/Anthropic `thinking_blocks` (Foundry: Extra inputs not permitted). + 2) Require `reasoning_content` on assistant messages with tool_calls. + """ + result: Final[list[AllMessageValues]] = [] + for msg in messages: + has_tb = "thinking_blocks" in msg and msg.get("thinking_blocks") is not None + needs_rc = ( + msg.get("role") == "assistant" + and msg.get("tool_calls") + and not msg.get("reasoning_content") + ) + if not has_tb and not needs_rc: + result.append(msg) + continue + + patched = dict(cast(dict, msg)) + thinking_text = self._thinking_blocks_to_text(patched.pop("thinking_blocks", None)) + provider_fields = patched.get("provider_specific_fields") + if isinstance(provider_fields, dict): + provider_fields = dict(provider_fields) + provider_fields.pop("thinking_blocks", None) + stored = provider_fields.pop("reasoning_content", None) + patched["provider_specific_fields"] = provider_fields + else: + stored = None + + if ( + patched.get("role") == "assistant" + and patched.get("tool_calls") + and not patched.get("reasoning_content") + ): + if stored: + patched["reasoning_content"] = stored + elif thinking_text: + patched["reasoning_content"] = thinking_text + else: + litellm.verbose_logger.warning( + "Azure AI Kimi reasoning model: assistant tool-call message is missing " + "`reasoning_content`. Injecting a placeholder to satisfy API validation. " + "For best results, preserve `reasoning_content` from the original " + "assistant response when building multi-turn conversation history." + ) + patched["reasoning_content"] = " " + elif ( + patched.get("role") == "assistant" + and thinking_text + and not patched.get("reasoning_content") + ): + patched["reasoning_content"] = thinking_text + + result.append(cast(AllMessageValues, patched)) + return result + + def validate_environment( self, headers: dict, @@ -222,7 +362,20 @@ class AzureAIStudioConfig(OpenAIConfig): if extra_body and isinstance(extra_body, dict): optional_params.update(extra_body) optional_params.pop("max_retries", None) - return super().transform_request(model, messages, optional_params, litellm_params, headers) + if self._is_kimi_reasoning_model(model): + messages = self.fill_reasoning_content(messages) + payload: Final = super().transform_request(model, messages, optional_params, litellm_params, headers) + if self._is_kimi_reasoning_model(model): + payload_messages = payload.get("messages") + if isinstance(payload_messages, list): + cleaned: list = [] + for msg in payload_messages: + if isinstance(msg, dict) and "thinking_blocks" in msg: + msg = dict(msg) + msg.pop("thinking_blocks", None) + cleaned.append(msg) + payload["messages"] = cleaned + return payload def transform_response( self, @@ -286,9 +439,23 @@ class AzureAIStudioConfig(OpenAIConfig): request_data = self._drop_extra_params_from_request_data(request_data, error_text) if "Extra inputs are not permitted" in error_text and self._error_has_tool_level_extra_fields(error_text): request_data = self._drop_tool_level_extra_fields(request_data, error_text) + if "Extra inputs are not permitted" in error_text: + request_data = self._drop_message_level_extra_fields(request_data, error_text) data: Final = drop_params_from_unprocessable_entity_error(e=e, data=request_data) return data + def _drop_message_level_extra_fields(self, request_data: dict, error_text: str) -> dict: + """Strip fields Azure reports as messages[N].field (e.g. thinking_blocks).""" + fields: Final = set(re.findall(r"messages\[\d+\]\.([\w-]+)", error_text)) + messages = request_data.get("messages") + if not fields or not isinstance(messages, list): + return request_data + for msg in messages: + if isinstance(msg, dict): + for field in fields: + msg.pop(field, None) + return request_data + def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict: fields_to_drop: Final = set(re.findall(r"tools\[\d+\]\.([\w-]+)", error_text)) tools: Final = request_data.get("tools") diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index b2a69564908..b28ec76eab5 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -159,8 +159,91 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ResponsesAPIRequestParams(model=model, input=input, **response_api_optional_request_params) ) + if self._is_kimi_reasoning_model_responses(model): + final_request_params = self._shape_kimi_responses_request(final_request_params) + return final_request_params + @staticmethod + def _normalize_responses_model_id(model: str) -> str: + return (model or "").split("/")[-1].lower() + + def _is_kimi_reasoning_model_responses(self, model: str) -> bool: + model_id = self._normalize_responses_model_id(model) + return ( + model_id.startswith("fw-kimi") + or model_id.startswith("kimi-k2.") + or "kimi-k3" in model_id + ) + + def _shape_kimi_responses_request(self, request: dict) -> dict: + """Drop fixed sampling / bad reasoning.effort; ensure reasoning before tool calls.""" + shaped = dict(request) + for param in ( + "temperature", + "top_p", + "n", + "presence_penalty", + "frequency_penalty", + ): + shaped.pop(param, None) + reasoning = shaped.get("reasoning") + if isinstance(reasoning, dict): + effort = reasoning.get("effort") + if effort not in ("low", "high", "max"): + shaped.pop("reasoning", None) + inp = shaped.get("input") + if isinstance(inp, list): + shaped["input"] = self._shape_kimi_responses_input(inp) + return shaped + + def _shape_kimi_responses_input(self, items: list) -> list: + """ + Claude Code history often has thinking + tool_use. After adapter flush, + that is message(assistant/output_text) then function_call. Foundry/Kimi + expects a reasoning item before tool calls. + """ + shaped: list = [] + i = 0 + n = len(items) + while i < n: + item = items[i] + nxt = items[i + 1] if i + 1 < n else None + if ( + isinstance(item, dict) + and item.get("type") == "message" + and item.get("role") == "assistant" + and isinstance(nxt, dict) + and nxt.get("type") == "function_call" + ): + texts = [] + for part in item.get("content") or []: + if isinstance(part, dict) and part.get("type") == "output_text": + texts.append(part.get("text") or "") + text = "\n".join(t for t in texts if t).strip() or " " + shaped.append( + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": text[:2000]}], + } + ) + i += 1 + continue + if isinstance(item, dict) and item.get("type") == "function_call": + if not shaped or shaped[-1].get("type") != "reasoning": + shaped.append( + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": " "}], + } + ) + shaped.append(item) + i += 1 + continue + shaped.append(item) + i += 1 + return shaped + def remove_cache_control_flag_from_input_and_tools( self, model: str, # allows overrides to selectively run this diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f63a61f3c6e..6ba3e17199c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9419,6 +9419,29 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "azure_ai/FW-Kimi-K3": { + "input_cost_per_token": 3.3e-06, + "output_cost_per_token": 1.65e-05, + "cache_read_input_token_cost": 3.3e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 1048576, + "mode": "chat", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k3-through-fireworks-ai-on-microsoft-foundry/4540187", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure_ai/kimi-k2.5": { "input_cost_per_token": 6e-07, "litellm_provider": "azure_ai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f63a61f3c6e..6ba3e17199c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9419,6 +9419,29 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "azure_ai/FW-Kimi-K3": { + "input_cost_per_token": 3.3e-06, + "output_cost_per_token": 1.65e-05, + "cache_read_input_token_cost": 3.3e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 1048576, + "mode": "chat", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k3-through-fireworks-ai-on-microsoft-foundry/4540187", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure_ai/kimi-k2.5": { "input_cost_per_token": 6e-07, "litellm_provider": "azure_ai", diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_shaping.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_shaping.py new file mode 100644 index 00000000000..8eb4114e85d --- /dev/null +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_shaping.py @@ -0,0 +1,120 @@ +"""Tests for Azure AI Foundry / FW-Kimi K3 request shaping.""" + +from litellm.llms.azure_ai.chat.transformation import AzureAIStudioConfig +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig + + +config = AzureAIStudioConfig() +responses_config = OpenAIResponsesAPIConfig() + + +def test_is_kimi_reasoning_model_matches_fw_and_native_ids(): + assert config._is_kimi_reasoning_model("FW-Kimi-K3") is True + assert config._is_kimi_reasoning_model("azure_ai/FW-Kimi-K3") is True + assert config._is_kimi_k3_model("FW-Kimi-K3") is True + assert config._is_kimi_reasoning_model("kimi-k2.5") is True + assert config._is_kimi_reasoning_model("gpt-4o") is False + + +def test_map_openai_params_drops_fixed_sampling_and_medium_effort(): + optional = config.map_openai_params( + non_default_params={ + "temperature": 0.7, + "top_p": 0.9, + "n": 2, + "presence_penalty": 0.1, + "frequency_penalty": 0.2, + "reasoning_effort": "medium", + }, + optional_params={ + "temperature": 0.7, + "top_p": 0.9, + "n": 2, + "presence_penalty": 0.1, + "frequency_penalty": 0.2, + "reasoning_effort": "medium", + }, + model="FW-Kimi-K3", + drop_params=True, + ) + for key in ( + "temperature", + "top_p", + "n", + "presence_penalty", + "frequency_penalty", + "reasoning_effort", + ): + assert key not in optional + + +def test_map_openai_params_keeps_valid_reasoning_effort(): + optional = config.map_openai_params( + non_default_params={"reasoning_effort": "high"}, + optional_params={}, + model="FW-Kimi-K3", + drop_params=True, + ) + assert optional.get("reasoning_effort") == "high" + + +def test_fill_reasoning_content_placeholder_for_tool_calls(): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": "{}"}, + } + ], + } + ] + out = config.fill_reasoning_content(messages) + assert out[0]["reasoning_content"] == " " + + +def test_fill_reasoning_content_strips_thinking_blocks_into_reasoning_content(): + messages = [ + { + "role": "assistant", + "content": None, + "thinking_blocks": [ + {"type": "thinking", "thinking": "plan the tool call", "signature": ""} + ], + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": "{}"}, + } + ], + } + ] + out = config.fill_reasoning_content(messages) + assert "thinking_blocks" not in out[0] + assert out[0]["reasoning_content"] == "plan the tool call" + + +def test_shape_kimi_responses_inserts_reasoning_before_function_call(): + req = { + "model": "FW-Kimi-K3", + "temperature": 0.5, + "reasoning": {"effort": "medium"}, + "input": [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "thinking..."}], + }, + {"type": "function_call", "call_id": "c1", "name": "t", "arguments": "{}"}, + ], + } + shaped = responses_config._shape_kimi_responses_request(req) + assert "temperature" not in shaped + assert "reasoning" not in shaped + types = [x["type"] for x in shaped["input"]] + assert types == ["message", "reasoning", "function_call"]