From 62cb3a7c7d630cb95d9b54d37dc579b6444ed490 Mon Sep 17 00:00:00 2001 From: sharziki Date: Sat, 18 Apr 2026 16:27:13 -0400 Subject: [PATCH] fix(cloudflare): handle OpenAI-compatible response schema Newer Cloudflare Workers AI models (e.g. @cf/moonshotai/kimi-k2.5 and the OpenAI-compatible endpoints) return content under `result.choices[*].message.content` instead of the legacy `result.response` string. The old transform blindly indexed `result["response"]` and raised `KeyError: 'response'` against these models. Introduce `_extract_response_content` that tries the legacy shape first, falls back to the OpenAI-compatible shape, and raises a descriptive CloudflareError listing both expected paths if neither matches. Adds regression coverage for the OpenAI-compatible payload plus an error-path test for unknown shapes. Closes #25999. --- .../llms/cloudflare/chat/transformation.py | 39 ++++++++++- tests/llm_translation/test_cloudflare.py | 66 ++++++++++++++++++- 2 files changed, 101 insertions(+), 4 deletions(-) diff --git a/litellm/llms/cloudflare/chat/transformation.py b/litellm/llms/cloudflare/chat/transformation.py index 9e59782bf73..4b681a8fb5f 100644 --- a/litellm/llms/cloudflare/chat/transformation.py +++ b/litellm/llms/cloudflare/chat/transformation.py @@ -131,6 +131,39 @@ class CloudflareChatConfig(BaseConfig): } return data + @staticmethod + def _extract_response_content(completion_response: dict, status_code: int) -> str: + # Cloudflare Workers AI has two response shapes: + # legacy: {"result": {"response": "..."}} + # OpenAI-compatible: {"result": {"choices": [{"message": {"content": "..."}}]}} + # Newer models (e.g. kimi-k2, openai-compatible endpoints) use the second form. + result = completion_response.get("result") + if not isinstance(result, dict): + raise CloudflareError( + status_code=status_code, + message=f"Cloudflare response missing 'result' object: {completion_response}", + ) + + if isinstance(result.get("response"), str): + return result["response"] + + choices = result.get("choices") + if isinstance(choices, list) and choices: + message = ( + choices[0].get("message") if isinstance(choices[0], dict) else None + ) + if isinstance(message, dict) and isinstance(message.get("content"), str): + return message["content"] + + raise CloudflareError( + status_code=status_code, + message=( + "Cloudflare response did not contain a recognized content field " + "(expected 'result.response' or 'result.choices[0].message.content'): " + f"{completion_response}" + ), + ) + def transform_response( self, model: str, @@ -147,9 +180,9 @@ class CloudflareChatConfig(BaseConfig): ) -> ModelResponse: completion_response = raw_response.json() - model_response.choices[0].message.content = completion_response["result"][ # type: ignore - "response" - ] + model_response.choices[0].message.content = self._extract_response_content( # type: ignore + completion_response, raw_response.status_code + ) prompt_tokens = litellm.utils.get_token_count(messages=messages, model=model) completion_tokens = len( diff --git a/tests/llm_translation/test_cloudflare.py b/tests/llm_translation/test_cloudflare.py index 5d8e3e5990e..e4e5c684246 100644 --- a/tests/llm_translation/test_cloudflare.py +++ b/tests/llm_translation/test_cloudflare.py @@ -9,7 +9,9 @@ import pytest from litellm import acompletion, completion from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -FAKE_API_BASE = "https://fake-cloudflare.example.com/client/v4/accounts/fake-acct/ai/run/" +FAKE_API_BASE = ( + "https://fake-cloudflare.example.com/client/v4/accounts/fake-acct/ai/run/" +) FAKE_API_KEY = "fake-cf-api-key" @@ -33,6 +35,29 @@ def _chat_response() -> Dict[str, Any]: } +def _chat_response_openai_compatible() -> Dict[str, Any]: + # Newer Cloudflare Workers AI models (e.g. @cf/moonshotai/kimi-k2.5, + # OpenAI-compatible endpoints) return choices[*].message.content instead + # of the legacy top-level "response" field. Regression coverage for #25999. + return { + "result": { + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "I am a large language model created to assist you.", + }, + "finish_reason": "stop", + } + ], + }, + "success": True, + "errors": [], + "messages": [], + } + + def _streaming_chunks() -> list[str]: return [ json.dumps({"response": "I am"}), @@ -76,6 +101,45 @@ def test_completion_cloudflare(sync_mode): assert "language model" in response.choices[0].message.content.lower() +def test_completion_cloudflare_openai_compatible_response(): + """Regression test for issue #25999: newer Cloudflare Workers AI models + return an OpenAI-compatible `result.choices[*].message.content` payload + rather than the legacy `result.response` string.""" + messages = [{"role": "user", "content": "what llm are you"}] + mock_resp = _make_mock_response(_chat_response_openai_compatible()) + + with patch.object(HTTPHandler, "post", return_value=mock_resp) as mock_post: + response = completion( + model="cloudflare/@cf/moonshotai/kimi-k2.5", + messages=messages, + max_tokens=15, + api_base=FAKE_API_BASE, + api_key=FAKE_API_KEY, + ) + mock_post.assert_called_once() + + assert response is not None + assert response.choices[0].message.content is not None + assert "language model" in response.choices[0].message.content.lower() + + +def test_completion_cloudflare_unknown_response_shape_raises(): + messages = [{"role": "user", "content": "hello"}] + # Neither legacy `result.response` nor OpenAI-compatible `result.choices`. + mock_resp = _make_mock_response({"result": {"unexpected": "shape"}}) + + with patch.object(HTTPHandler, "post", return_value=mock_resp): + with pytest.raises(Exception) as exc_info: + completion( + model="cloudflare/@cf/meta/llama-2-7b-chat-int8", + messages=messages, + max_tokens=15, + api_base=FAKE_API_BASE, + api_key=FAKE_API_KEY, + ) + assert "recognized content field" in str(exc_info.value) + + @pytest.mark.parametrize("sync_mode", [True, False]) def test_completion_cloudflare_stream(sync_mode): messages = [{"role": "user", "content": "what llm are you"}]