mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat(sap): reasoning, gemini normalization, timeout/max_retries, model version
This commit is contained in:
parent
ec3f8183c3
commit
f2df9cae68
3 changed files with 458 additions and 105 deletions
|
|
@ -191,7 +191,8 @@ class LLMModelDetails(BaseModel):
|
|||
name: str
|
||||
version: str = "latest"
|
||||
params: dict | None = None
|
||||
|
||||
timeout: int | None = None
|
||||
max_retries: int | None = Field(default=None, ge=0, le=5)
|
||||
|
||||
class PromptTemplatingModuleConfig(BaseModel):
|
||||
prompt: Template
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Translate from OpenAI's `/v1/chat/completions` to SAP Generative AI Hub's Orchestration Service`v2/completion`
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any, Final, Union
|
||||
|
|
@ -12,8 +13,6 @@ import litellm
|
|||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
|
|
@ -23,6 +22,7 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from ..credentials import get_token_creator
|
||||
from .handler import (
|
||||
AsyncSAPStreamIterator,
|
||||
|
|
@ -44,19 +44,48 @@ from .models import (
|
|||
_SAP_MODEL_PARAMS_EXCLUDED_KEYS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"stream_options",
|
||||
"fallback_sap_modules",
|
||||
"placeholder_values",
|
||||
"model_version",
|
||||
"timeout",
|
||||
"max_retries",
|
||||
}
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SAP capability registry
|
||||
# ---------------------------------------------------------------------------
|
||||
# Models that accept reasoning_effort / thinking parameters on SAP GenAI Hub.
|
||||
_REASONING_MODELS: re.Pattern[str] = re.compile( # rebind-ok: module-level pattern constant; assignment is a one-time initialisation
|
||||
r"^(?:anthropic--claude-(?:4(?:\.[5-9])?|3-7)|o\d|gpt-5(?:[.\-]|$)|cohere--\S*reasoning\S*)"
|
||||
)
|
||||
|
||||
|
||||
def validate_dict(data: dict, model) -> dict:
|
||||
return model(**data).model_dump(by_alias=True, exclude_unset=True)
|
||||
|
||||
|
||||
def _build_model_details(
|
||||
model_name: str,
|
||||
model_version: str,
|
||||
params: dict, # mutable-ok: model params dict forwarded directly to wire payload
|
||||
timeout: int | None,
|
||||
max_retries: int | None,
|
||||
) -> dict: # mutable-ok: wire serialization helper; dict is the required output shape for JSON encoding
|
||||
"""Build the model dict for the orchestration request, adding optional fields only when set."""
|
||||
model_details: dict = { # mutable-ok: ephemeral wire dict built in one shot before JSON encoding
|
||||
"name": model_name,
|
||||
"params": params,
|
||||
"version": model_version,
|
||||
}
|
||||
if timeout is not None:
|
||||
model_details["timeout"] = timeout
|
||||
if max_retries is not None:
|
||||
model_details["max_retries"] = max_retries
|
||||
return model_details
|
||||
|
||||
|
||||
def _messages_to_sap_template(messages: list[dict[str, str]]) -> list:
|
||||
template: Final = []
|
||||
for message in messages:
|
||||
|
|
@ -207,13 +236,15 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
"temperature",
|
||||
"top_p",
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"function_call",
|
||||
"functions",
|
||||
"extra_headers",
|
||||
"parallel_tool_calls",
|
||||
"response_format",
|
||||
"timeout",
|
||||
"max_retries",
|
||||
"model_version",
|
||||
"user",
|
||||
]
|
||||
# Remove response_format for providers that don't support it on SAP GenAI Hub
|
||||
if (
|
||||
|
|
@ -223,8 +254,8 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
or model == "gpt-4"
|
||||
):
|
||||
params.remove("response_format")
|
||||
if model.startswith("gemini") or model.startswith("amazon"):
|
||||
params.remove("tool_choice")
|
||||
if self._sap_supports_reasoning(model):
|
||||
params.extend(["reasoning_effort", "thinking"]) # mutable-ok: params list is local to this call; extend is the standard list append
|
||||
return params
|
||||
|
||||
def validate_environment(
|
||||
|
|
@ -266,12 +297,14 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
params.pop("strict")
|
||||
|
||||
model_version: Final = params.pop("model_version", "latest")
|
||||
timeout = params.pop("timeout", None) # rebind-ok: popped from params for conditional inclusion below
|
||||
max_retries = params.pop("max_retries", None) # rebind-ok: popped from params for conditional inclusion below
|
||||
|
||||
tools_ = params.pop("tools", [])
|
||||
tools_ = [validate_dict(tool, ChatCompletionTool) for tool in tools_]
|
||||
tools_ = params.pop("tools", []) # mutable-ok: intermediate list; immediately validated on the next line
|
||||
tools_ = [validate_dict(tool, ChatCompletionTool) for tool in tools_] # mutable-ok: validated tool dicts; folded into wire payload below # rebind-ok: rebound to validated list
|
||||
tools: Final = {"tools": tools_} if tools_ else {}
|
||||
|
||||
response_format = params.pop("response_format", {})
|
||||
response_format = params.pop("response_format", {}) # mutable-ok: ephemeral wire dict consumed immediately # rebind-ok: popped for type inspection below
|
||||
resp_type: Final = response_format.get("type", None)
|
||||
if resp_type:
|
||||
if resp_type == "json_schema":
|
||||
|
|
@ -299,11 +332,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
**tools,
|
||||
**response_format,
|
||||
},
|
||||
"model": {
|
||||
"name": model_name,
|
||||
"params": params,
|
||||
"version": model_version,
|
||||
},
|
||||
"model": _build_model_details(model_name, model_version, params, timeout, max_retries),
|
||||
},
|
||||
**optional_modules,
|
||||
}
|
||||
|
|
@ -333,8 +362,6 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
if "delimiters" in stream_options:
|
||||
stream_config["delimiters"] = stream_options["delimiters"]
|
||||
|
||||
optional_params.pop("tool_choice", None)
|
||||
|
||||
modules: Final = [
|
||||
self._build_prompt_module(
|
||||
model_name=model,
|
||||
|
|
@ -387,13 +414,16 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=raw_response.text,
|
||||
additional_args={"complete_input_dict": request_data},
|
||||
)
|
||||
response = ModelResponse.model_validate(raw_response.json()["final_result"])
|
||||
final_result = raw_response.json()["final_result"] # rebind-ok: extracted from raw response for normalisation then model_validate
|
||||
|
||||
# SAP's orchestration service returns Gemini thinking as a list of
|
||||
# {"content": str, "signature": str} objects under reasoning_content.
|
||||
# Normalize to litellm's standard fields before model_validate, which
|
||||
# crashes on a list-shaped reasoning_content.
|
||||
if model.startswith("gemini"):
|
||||
self._normalize_gemini_reasoning(final_result)
|
||||
|
||||
response = ModelResponse.model_validate(final_result) # rebind-ok: constructed from final_result; referenced below for hidden params
|
||||
|
||||
# Strip markdown code blocks if JSON response_format was used with Anthropic models
|
||||
# SAP GenAI Hub with Anthropic models sometimes wraps JSON in ```json ... ```
|
||||
|
|
@ -406,6 +436,46 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def _sap_supports_reasoning(model: str) -> bool:
|
||||
"""Return True if *model* accepts reasoning_effort / thinking on SAP GenAI Hub."""
|
||||
return bool(_REASONING_MODELS.match(model))
|
||||
|
||||
@staticmethod
|
||||
def _normalize_gemini_reasoning(final_result: dict) -> None: # mutable-ok: final_result mutated in place to prepare for model_validate
|
||||
"""Normalize SAP Gemini reasoning_content from list to litellm's standard fields.
|
||||
|
||||
SAP's orchestration service returns Gemini thinking as a list of
|
||||
{"content": str, "signature": str} objects under reasoning_content.
|
||||
litellm uses two parallel fields for this:
|
||||
- thinking_blocks: list of ChatCompletionThinkingBlock dicts, preserving
|
||||
signature tokens that Gemini requires to replay thinking in multi-turn
|
||||
requests and that the Responses API serializes for round-trip preservation.
|
||||
- reasoning_content: plain str derived from the same blocks, exposed to
|
||||
callers and used by streaming accumulators and the Responses API.
|
||||
Both must be populated; populating only the string loses the signatures.
|
||||
"""
|
||||
for choice in final_result.get("choices") or []:
|
||||
msg = (choice.get("message") or {}) if isinstance(choice, dict) else {}
|
||||
rc = msg.get("reasoning_content")
|
||||
if not isinstance(rc, list):
|
||||
continue
|
||||
blocks: list[dict] = [] # mutable-ok: local accumulator; assigned once to msg field below
|
||||
texts: list[str] = [] # mutable-ok: local accumulator; joined to string below
|
||||
for item in rc:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
thought = item.get("content", "")
|
||||
block: dict = {"type": "thinking", "thinking": thought} # mutable-ok: local block dict; signature optionally appended then collected
|
||||
if item.get("signature") is not None:
|
||||
block["signature"] = item["signature"] # mutable-ok: block is a local dict
|
||||
blocks.append(block)
|
||||
texts.append(thought)
|
||||
msg["thinking_blocks"] = (
|
||||
blocks if blocks else None
|
||||
) # rebind-ok: normalizing SAP Gemini list to ChatCompletionThinkingBlock list
|
||||
msg["reasoning_content"] = "\n\n".join(texts) or None # rebind-ok: normalizing SAP Gemini list to plain str
|
||||
|
||||
def _strip_markdown_json(self, response: ModelResponse) -> ModelResponse:
|
||||
"""Strip markdown code block wrapper from JSON content if present.
|
||||
|
||||
|
|
|
|||
|
|
@ -34,9 +34,7 @@ class TestSAPTransformationIntegration:
|
|||
|
||||
result = mock_config.transform_request(model, messages, optional_params, {}, {})
|
||||
|
||||
model_params = result["config"]["modules"]["prompt_templating"]["model"][
|
||||
"params"
|
||||
]
|
||||
model_params = result["config"]["modules"]["prompt_templating"]["model"]["params"]
|
||||
|
||||
assert "temperature" in model_params
|
||||
assert "frequency_penalty" in model_params
|
||||
|
|
@ -44,18 +42,16 @@ class TestSAPTransformationIntegration:
|
|||
assert "model_version" not in model_params
|
||||
assert "tools" not in model_params
|
||||
|
||||
model_version = result["config"]["modules"]["prompt_templating"]["model"][
|
||||
"version"
|
||||
]
|
||||
model_version = result["config"]["modules"]["prompt_templating"]["model"]["version"]
|
||||
assert model_version == "v1.5"
|
||||
|
||||
prompt = result["config"]["modules"]["prompt_templating"]["prompt"]
|
||||
if "tools" in prompt:
|
||||
assert isinstance(prompt["tools"], list)
|
||||
for tool in prompt["tools"]:
|
||||
assert (
|
||||
tool["function"]["parameters"]["type"] == "object"
|
||||
), "SAP API requires parameters.type == 'object'"
|
||||
assert tool["function"]["parameters"]["type"] == "object", (
|
||||
"SAP API requires parameters.type == 'object'"
|
||||
)
|
||||
assert "properties" in tool["function"]["parameters"]
|
||||
|
||||
def test_transform_request_parameter_handling_robustness(self, mock_config):
|
||||
|
|
@ -96,33 +92,25 @@ class TestSAPTransformationIntegration:
|
|||
|
||||
for i, test_case in enumerate(test_cases):
|
||||
filtered_params = {
|
||||
k: v
|
||||
for k, v in test_case["params"].items()
|
||||
if k not in {"tools", "model_version", "deployment_url"}
|
||||
k: v for k, v in test_case["params"].items() if k not in {"tools", "model_version", "deployment_url"}
|
||||
}
|
||||
|
||||
for expected_param in test_case["expected_in_model"]:
|
||||
assert (
|
||||
expected_param in filtered_params
|
||||
), f"Case {i + 1}: {expected_param} should be in model params"
|
||||
assert expected_param in filtered_params, f"Case {i + 1}: {expected_param} should be in model params"
|
||||
|
||||
for excluded_param in test_case["expected_excluded"]:
|
||||
assert (
|
||||
excluded_param not in filtered_params
|
||||
), f"Case {i + 1}: {excluded_param} should be excluded from model params"
|
||||
assert excluded_param not in filtered_params, (
|
||||
f"Case {i + 1}: {excluded_param} should be excluded from model params"
|
||||
)
|
||||
|
||||
result = mock_config.transform_request(
|
||||
model, messages, test_case["params"], {}, {}
|
||||
)
|
||||
result = mock_config.transform_request(model, messages, test_case["params"], {}, {})
|
||||
if result and "config" in result:
|
||||
model_params = result["config"]["modules"]["prompt_templating"][
|
||||
"model"
|
||||
]["params"]
|
||||
model_params = result["config"]["modules"]["prompt_templating"]["model"]["params"]
|
||||
|
||||
for excluded_param in test_case["expected_excluded"]:
|
||||
assert (
|
||||
excluded_param not in model_params
|
||||
), f"Case {i + 1}: {excluded_param} should not be in actual model params"
|
||||
assert excluded_param not in model_params, (
|
||||
f"Case {i + 1}: {excluded_param} should not be in actual model params"
|
||||
)
|
||||
|
||||
def test_config_transform_with_response_format_json_object(self, mock_config):
|
||||
expected_dict = {
|
||||
|
|
@ -145,9 +133,7 @@ class TestSAPTransformationIntegration:
|
|||
}
|
||||
config = mock_config.transform_request(
|
||||
model="gpt-4o",
|
||||
messages=[
|
||||
{"role": "user", "content": "First man on the moon, answer in json"}
|
||||
],
|
||||
messages=[{"role": "user", "content": "First man on the moon, answer in json"}],
|
||||
optional_params={
|
||||
"response_format": {"type": "json_object"},
|
||||
"deployment_url": "shouldn't be in results",
|
||||
|
|
@ -189,9 +175,7 @@ class TestSAPTransformationIntegration:
|
|||
|
||||
config = mock_config.transform_request(
|
||||
model="gpt-4o",
|
||||
messages=[
|
||||
{"role": "user", "content": "First man on the moon, answer in json"}
|
||||
],
|
||||
messages=[{"role": "user", "content": "First man on the moon, answer in json"}],
|
||||
optional_params={
|
||||
"response_format": expected_response_format,
|
||||
"deployment_url": "shouldn't be in results",
|
||||
|
|
@ -199,27 +183,15 @@ class TestSAPTransformationIntegration:
|
|||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert (
|
||||
config["config"]["modules"]["prompt_templating"]["prompt"][
|
||||
"response_format"
|
||||
]
|
||||
== expected_response_format
|
||||
)
|
||||
assert (
|
||||
len(config["config"]["modules"]["prompt_templating"]["model"]["params"])
|
||||
== 0
|
||||
)
|
||||
assert config["config"]["modules"]["prompt_templating"]["prompt"]["response_format"] == expected_response_format
|
||||
assert len(config["config"]["modules"]["prompt_templating"]["model"]["params"]) == 0
|
||||
|
||||
def test_config_transform_with_stream(self, mock_config):
|
||||
expected_dict = {
|
||||
"config": {
|
||||
"modules": {
|
||||
"prompt_templating": {
|
||||
"prompt": {
|
||||
"template": [
|
||||
{"role": "user", "content": "Hello, how are you?"}
|
||||
]
|
||||
},
|
||||
"prompt": {"template": [{"role": "user", "content": "Hello, how are you?"}]},
|
||||
"model": {
|
||||
"name": "anthropic--claude-4-sonnet",
|
||||
"params": {},
|
||||
|
|
@ -257,9 +229,7 @@ class TestSAPTransformationIntegration:
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert config["config"]["modules"]["prompt_templating"]["prompt"][
|
||||
"defaults"
|
||||
] == {"user_query": "default value"}
|
||||
assert config["config"]["modules"]["prompt_templating"]["prompt"]["defaults"] == {"user_query": "default value"}
|
||||
assert config["config"]["modules"]["prompt_templating"]["model"]["params"] == {}
|
||||
|
||||
def test_sap_placeholder_values(self, mock_config):
|
||||
|
|
@ -323,14 +293,8 @@ class TestSAPTransformationIntegration:
|
|||
assert config["placeholder_values"] == placeholder_values
|
||||
modules = config["config"]["modules"]
|
||||
assert modules["grounding"]["type"] == "document_grounding_service"
|
||||
assert (
|
||||
modules["grounding"]["config"]["placeholders"]["output"]
|
||||
== "grounding_response"
|
||||
)
|
||||
assert (
|
||||
modules["grounding"]["config"]["filters"][0]["data_repository_type"]
|
||||
== "vector"
|
||||
)
|
||||
assert modules["grounding"]["config"]["placeholders"]["output"] == "grounding_response"
|
||||
assert modules["grounding"]["config"]["filters"][0]["data_repository_type"] == "vector"
|
||||
assert modules["prompt_templating"]["model"]["params"] == {}
|
||||
|
||||
def test_grounding_search_config_rejects_both_count_fields(self, mock_config):
|
||||
|
|
@ -442,10 +406,7 @@ class TestSAPTransformationIntegration:
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert (
|
||||
"For using SAP Filtering Module you must provide at least one property"
|
||||
in str(exc_info.value)
|
||||
)
|
||||
assert "For using SAP Filtering Module you must provide at least one property" in str(exc_info.value)
|
||||
|
||||
def test_sap_masking(self, mock_config):
|
||||
masking_config = {
|
||||
|
|
@ -509,9 +470,7 @@ class TestSAPTransformationIntegration:
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert "must set exactly one of: 'providers' or 'masking_providers'" in str(
|
||||
exc_info.value
|
||||
)
|
||||
assert "must set exactly one of: 'providers' or 'masking_providers'" in str(exc_info.value)
|
||||
|
||||
def test_masking_providers_deprecated_emits_warning(self, mock_config):
|
||||
masking_config = {
|
||||
|
|
@ -533,8 +492,7 @@ class TestSAPTransformationIntegration:
|
|||
headers={},
|
||||
)
|
||||
assert any(
|
||||
issubclass(warning.category, DeprecationWarning)
|
||||
and "masking_providers" in str(warning.message)
|
||||
issubclass(warning.category, DeprecationWarning) and "masking_providers" in str(warning.message)
|
||||
for warning in w
|
||||
), "Expected DeprecationWarning for 'masking_providers'"
|
||||
|
||||
|
|
@ -573,10 +531,7 @@ class TestSAPTransformationIntegration:
|
|||
headers={},
|
||||
)
|
||||
|
||||
assert (
|
||||
"TranslationModuleConfig requires at least one of 'input' or 'output'"
|
||||
in str(exc_info.value)
|
||||
)
|
||||
assert "TranslationModuleConfig requires at least one of 'input' or 'output'" in str(exc_info.value)
|
||||
|
||||
def test_sap_multiple_modules(self, mock_config):
|
||||
translation_config = {
|
||||
|
|
@ -629,13 +584,340 @@ class TestSAPTransformationIntegration:
|
|||
][0]["content"]
|
||||
== "Hello world!"
|
||||
)
|
||||
assert (
|
||||
config["config"]["modules"][0]["prompt_templating"]["prompt"][
|
||||
"template"
|
||||
][0]["content"]
|
||||
== "Hello."
|
||||
)
|
||||
assert (
|
||||
config["config"]["modules"][1]["translation"]["input"]["type"]
|
||||
== "sap_document_translation"
|
||||
assert config["config"]["modules"][0]["prompt_templating"]["prompt"]["template"][0]["content"] == "Hello."
|
||||
assert config["config"]["modules"][1]["translation"]["input"]["type"] == "sap_document_translation"
|
||||
|
||||
|
||||
class TestGeminiReasoningNormalization:
|
||||
"""Unit tests for _normalize_gemini_reasoning.
|
||||
|
||||
Verifies that list-shaped reasoning_content from Gemini is coerced to str
|
||||
before model_validate is called, without touching other shapes.
|
||||
"""
|
||||
|
||||
def _make_final_result(self, reasoning_content):
|
||||
"""Helper: build a minimal final_result dict with given reasoning_content."""
|
||||
result = {
|
||||
"id": "test-id",
|
||||
"object": "chat.completion",
|
||||
"model": "gemini-2.0-flash",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Hello",
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
if reasoning_content is not None:
|
||||
result["choices"][0]["message"]["reasoning_content"] = reasoning_content
|
||||
return result
|
||||
|
||||
def test_list_shaped_is_joined_to_string(self):
|
||||
"""Gemini list of content dicts is joined into a newline-separated string."""
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
final_result = self._make_final_result(
|
||||
[
|
||||
{"content": "First thought.", "signature": "sig1"},
|
||||
{"content": "Second thought.", "signature": "sig2"},
|
||||
]
|
||||
)
|
||||
GenAIHubOrchestrationConfig._normalize_gemini_reasoning(final_result)
|
||||
assert final_result["choices"][0]["message"]["reasoning_content"] == "First thought.\n\nSecond thought."
|
||||
|
||||
def test_string_reasoning_content_is_untouched(self):
|
||||
"""A reasoning_content that is already a str is left as-is."""
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
final_result = self._make_final_result("already a string")
|
||||
GenAIHubOrchestrationConfig._normalize_gemini_reasoning(final_result)
|
||||
assert final_result["choices"][0]["message"]["reasoning_content"] == "already a string"
|
||||
|
||||
def test_missing_reasoning_content_is_untouched(self):
|
||||
"""A message without reasoning_content is left unchanged."""
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
final_result = self._make_final_result(None)
|
||||
# reasoning_content key is absent — _make_final_result(None) does not add it
|
||||
assert "reasoning_content" not in final_result["choices"][0]["message"]
|
||||
GenAIHubOrchestrationConfig._normalize_gemini_reasoning(final_result)
|
||||
assert "reasoning_content" not in final_result["choices"][0]["message"]
|
||||
|
||||
def test_empty_list_becomes_none(self):
|
||||
"""An empty list collapses to None so model_validate sees no reasoning."""
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
final_result = self._make_final_result([])
|
||||
GenAIHubOrchestrationConfig._normalize_gemini_reasoning(final_result)
|
||||
assert final_result["choices"][0]["message"]["reasoning_content"] is None
|
||||
|
||||
def test_model_validate_succeeds_after_normalization(self):
|
||||
"""model_validate no longer raises after normalisation."""
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
final_result = self._make_final_result([{"content": "Thinking hard.", "signature": "abc"}])
|
||||
GenAIHubOrchestrationConfig._normalize_gemini_reasoning(final_result)
|
||||
response = ModelResponse.model_validate(final_result)
|
||||
assert response.choices[0].message.reasoning_content == "Thinking hard."
|
||||
|
||||
|
||||
class TestReasoningCapability:
|
||||
"""Unit tests for reasoning_effort / thinking parameter routing.
|
||||
|
||||
Verifies that capable models expose the params and that they land in
|
||||
model.params, while non-capable models have them silently dropped.
|
||||
"""
|
||||
|
||||
def _transform(self, model: str, **kwargs) -> dict:
|
||||
"""Run transform_request and return the parsed body."""
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
cfg = GenAIHubOrchestrationConfig()
|
||||
return cfg.transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
optional_params=dict(kwargs),
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
def _model_params(self, body: dict) -> dict:
|
||||
return body["config"]["modules"]["prompt_templating"]["model"]["params"]
|
||||
|
||||
# --- get_supported_openai_params ---
|
||||
|
||||
def test_reasoning_params_exposed_for_claude_3_7(self):
|
||||
"""reasoning_effort and thinking appear for claude-3-7 models."""
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
cfg = GenAIHubOrchestrationConfig()
|
||||
params = cfg.get_supported_openai_params("anthropic--claude-3-7-sonnet")
|
||||
assert "reasoning_effort" in params
|
||||
assert "thinking" in params
|
||||
|
||||
def test_reasoning_params_exposed_for_claude_4(self):
|
||||
"""reasoning_effort and thinking appear for claude-4 models."""
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
cfg = GenAIHubOrchestrationConfig()
|
||||
params = cfg.get_supported_openai_params("anthropic--claude-4-opus")
|
||||
assert "reasoning_effort" in params
|
||||
assert "thinking" in params
|
||||
|
||||
def test_reasoning_params_absent_for_gpt4o(self):
|
||||
"""reasoning_effort and thinking are not exposed for gpt-4o."""
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
cfg = GenAIHubOrchestrationConfig()
|
||||
params = cfg.get_supported_openai_params("gpt-4o")
|
||||
assert "reasoning_effort" not in params
|
||||
assert "thinking" not in params
|
||||
|
||||
# --- transform_request model.params ---
|
||||
|
||||
def test_reasoning_effort_lands_in_model_params_for_o3(self):
|
||||
"""reasoning_effort is forwarded into model.params for o-series models."""
|
||||
body = self._transform("o3", reasoning_effort="high")
|
||||
assert self._model_params(body).get("reasoning_effort") == "high"
|
||||
|
||||
def test_thinking_lands_in_model_params_for_claude_3_7(self):
|
||||
"""thinking dict is forwarded into model.params for claude-3-7."""
|
||||
thinking = {"type": "enabled", "budget_tokens": 8000}
|
||||
body = self._transform("anthropic--claude-3-7-sonnet", thinking=thinking)
|
||||
assert self._model_params(body).get("thinking") == thinking
|
||||
|
||||
|
||||
|
||||
|
||||
class TestModelVersionAdvertisement:
|
||||
"""model_version is advertised in get_supported_openai_params and lands in
|
||||
model.version (not model.params) in the serialised request body.
|
||||
"""
|
||||
|
||||
def _cfg(self):
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
cfg = GenAIHubOrchestrationConfig()
|
||||
return cfg
|
||||
|
||||
def _transform(self, model: str, **kwargs) -> dict:
|
||||
cfg = self._cfg()
|
||||
return cfg.transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
optional_params=dict(kwargs),
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
def test_model_version_advertised_for_all_models(self):
|
||||
for model in ("gpt-4o", "anthropic--claude-4-sonnet", "gemini-1.5-pro"):
|
||||
params = self._cfg().get_supported_openai_params(model)
|
||||
assert "model_version" in params, f"model_version missing for {model}"
|
||||
|
||||
def test_model_version_lands_in_model_version_field(self):
|
||||
body = self._transform("gpt-4o", model_version="1.2.3")
|
||||
pt = body["config"]["modules"]["prompt_templating"]
|
||||
assert pt["model"]["version"] == "1.2.3"
|
||||
|
||||
def test_model_version_absent_from_model_params(self):
|
||||
body = self._transform("gpt-4o", model_version="1.2.3")
|
||||
pt = body["config"]["modules"]["prompt_templating"]
|
||||
assert "model_version" not in pt["model"]["params"]
|
||||
|
||||
def test_model_version_defaults_to_latest(self):
|
||||
body = self._transform("gpt-4o")
|
||||
pt = body["config"]["modules"]["prompt_templating"]
|
||||
assert pt["model"]["version"] == "latest"
|
||||
|
||||
|
||||
class TestToolChoiceDropped:
|
||||
"""SAP orchestration v2 does not support tool_choice.
|
||||
|
||||
It is not advertised in get_supported_openai_params, so callers receive
|
||||
UnsupportedParamsError immediately via the generic litellm layer.
|
||||
tools themselves are still forwarded normally.
|
||||
"""
|
||||
|
||||
_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Return weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def _transform(self, model: str, **kwargs) -> dict:
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
cfg = GenAIHubOrchestrationConfig()
|
||||
return cfg.transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is the weather?"}],
|
||||
optional_params=dict(kwargs),
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
def _prompt(self, body: dict) -> dict:
|
||||
return body["config"]["modules"]["prompt_templating"]["prompt"]
|
||||
|
||||
def test_tool_choice_not_advertised(self):
|
||||
"""tool_choice must not appear in get_supported_openai_params for any model."""
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
cfg = GenAIHubOrchestrationConfig()
|
||||
for model in ("gpt-4o", "anthropic--claude-4-sonnet", "gemini-1.5-pro", "amazon--titan"):
|
||||
assert "tool_choice" not in cfg.get_supported_openai_params(model), (
|
||||
f"tool_choice must not be advertised for {model}"
|
||||
)
|
||||
|
||||
def test_tools_still_advertised(self):
|
||||
"""tools must still be advertised — only tool_choice is unsupported."""
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
cfg = GenAIHubOrchestrationConfig()
|
||||
for model in ("gpt-4o", "anthropic--claude-4-sonnet"):
|
||||
assert "tools" in cfg.get_supported_openai_params(model)
|
||||
|
||||
def test_tools_still_forwarded(self):
|
||||
"""Dropping tool_choice must not also suppress the tools list."""
|
||||
body = self._transform("gpt-4o", tools=[self._TOOL])
|
||||
assert "tools" in self._prompt(body)
|
||||
assert self._prompt(body)["tools"][0]["function"]["name"] == "get_weather"
|
||||
|
||||
|
||||
class TestTimeoutAndMaxRetries:
|
||||
"""timeout and max_retries land in model-level sibling fields,
|
||||
not inside model.params.
|
||||
"""
|
||||
|
||||
def _transform(self, model: str, **kwargs) -> dict:
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
cfg = GenAIHubOrchestrationConfig()
|
||||
return cfg.transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
optional_params=dict(kwargs),
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
def _model(self, body: dict) -> dict:
|
||||
return body["config"]["modules"]["prompt_templating"]["model"]
|
||||
|
||||
def test_timeout_and_max_retries_advertised(self):
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
cfg = GenAIHubOrchestrationConfig()
|
||||
for model in ("gpt-4o", "anthropic--claude-4-sonnet"):
|
||||
params = cfg.get_supported_openai_params(model)
|
||||
assert "timeout" in params, f"timeout missing for {model}"
|
||||
assert "max_retries" in params, f"max_retries missing for {model}"
|
||||
|
||||
def test_timeout_lands_at_model_level(self):
|
||||
body = self._transform("gpt-4o", timeout=120)
|
||||
model = self._model(body)
|
||||
assert model.get("timeout") == 120
|
||||
assert "timeout" not in model.get("params", {})
|
||||
|
||||
def test_max_retries_lands_at_model_level(self):
|
||||
body = self._transform("gpt-4o", max_retries=3)
|
||||
model = self._model(body)
|
||||
assert model.get("max_retries") == 3
|
||||
assert "max_retries" not in model.get("params", {})
|
||||
|
||||
def test_timeout_and_max_retries_together(self):
|
||||
body = self._transform("gpt-4o", timeout=60, max_retries=2)
|
||||
model = self._model(body)
|
||||
assert model.get("timeout") == 60
|
||||
assert model.get("max_retries") == 2
|
||||
assert "timeout" not in model.get("params", {})
|
||||
assert "max_retries" not in model.get("params", {})
|
||||
|
||||
def test_absent_when_not_passed(self):
|
||||
"""Neither key appears in the serialised body when not supplied."""
|
||||
body = self._transform("gpt-4o", temperature=0.7)
|
||||
model = self._model(body)
|
||||
assert "timeout" not in model
|
||||
assert "max_retries" not in model
|
||||
|
||||
|
||||
class TestUserForwarding:
|
||||
"""user param is advertised and lands in model.params (correct per SDK v2)."""
|
||||
|
||||
def _cfg(self):
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
return GenAIHubOrchestrationConfig()
|
||||
|
||||
def _transform(self, **kwargs) -> dict:
|
||||
cfg = self._cfg()
|
||||
return cfg.transform_request(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
optional_params=dict(kwargs),
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
def test_user_advertised(self):
|
||||
params = self._cfg().get_supported_openai_params("gpt-4o")
|
||||
assert "user" in params
|
||||
|
||||
def test_user_lands_in_model_params(self):
|
||||
body = self._transform(user="uid-abc123")
|
||||
pt = body["config"]["modules"]["prompt_templating"]
|
||||
assert pt["model"]["params"].get("user") == "uid-abc123"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue