mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
Merge ab47916a0c into f445e466b4
This commit is contained in:
Yamac Eren Ay
GitHub
commit
efcac41b5d
5 changed files with 1053 additions and 44 deletions
|
|
@ -32,6 +32,40 @@ def _now_ts() -> int:
|
|||
return int(time.time())
|
||||
|
||||
|
||||
def normalize_reasoning_content(raw: dict[str, object]) -> dict[str, object]: # mutable-ok: generic types
|
||||
choices: list[object] = list( # mutable-ok: immediately consumed, never escapes
|
||||
raw.get("choices") or [] # mutable-ok: transient default, consumed inline
|
||||
)
|
||||
return { # mutable-ok: one-shot dict passed directly to model_validate, never mutated
|
||||
**raw,
|
||||
"choices": [ # mutable-ok: built once, passed as value
|
||||
normalize_choice(c) for c in choices if isinstance(c, dict)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def normalize_choice(choice: dict[str, object]) -> dict[str, object]:
|
||||
for key in ("message", "delta"):
|
||||
carrier = choice.get(key)
|
||||
if not isinstance(carrier, dict):
|
||||
continue
|
||||
rc = carrier.get("reasoning_content")
|
||||
if not isinstance(rc, list):
|
||||
continue
|
||||
thinking_blocks = [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": item.get("content") or "",
|
||||
"signature": item.get("signature"),
|
||||
}
|
||||
for item in rc
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
carrier["thinking_blocks"] = thinking_blocks
|
||||
carrier["reasoning_content"] = "\n".join(b["thinking"] for b in thinking_blocks if b["thinking"]) or None
|
||||
return choice
|
||||
|
||||
|
||||
def _is_terminal_chunk(chunk: OpenAIChatCompletionChunk) -> bool:
|
||||
"""OpenAI-shaped chunk is terminal if any choice has a non-None finish_reason."""
|
||||
try:
|
||||
|
|
@ -55,22 +89,22 @@ class _StreamParser:
|
|||
if not orc:
|
||||
return None
|
||||
|
||||
return OpenAIChatCompletionChunk.model_validate(
|
||||
{
|
||||
"id": orc.get("id") or evt.get("request_id") or "stream-chunk",
|
||||
"object": orc.get("object") or "chat.completion.chunk",
|
||||
"created": orc.get("created") or evt.get("created") or _now_ts(),
|
||||
"model": orc.get("model") or "unknown",
|
||||
"choices": [
|
||||
{
|
||||
"index": c.get("index", 0),
|
||||
"delta": c.get("delta") or {},
|
||||
"finish_reason": c.get("finish_reason"),
|
||||
}
|
||||
for c in (orc.get("choices") or [])
|
||||
],
|
||||
}
|
||||
)
|
||||
chunk: Final[dict] = { # mutable-ok: local dict built once and passed to model_validate
|
||||
"id": orc.get("id") or evt.get("request_id") or "stream-chunk",
|
||||
"object": orc.get("object") or "chat.completion.chunk",
|
||||
"created": orc.get("created") or evt.get("created") or _now_ts(),
|
||||
"model": orc.get("model") or "unknown",
|
||||
"choices": [ # mutable-ok: list built inline as part of the outer Final dict literal
|
||||
{ # mutable-ok: each choice dict is a nested one-shot literal
|
||||
"index": c.get("index", 0),
|
||||
"delta": c.get("delta") or {}, # mutable-ok: transient default, consumed by model_validate
|
||||
"finish_reason": c.get("finish_reason"),
|
||||
**({"provider_specific_fields": c["provider_specific_fields"]} if "provider_specific_fields" in c else {}),
|
||||
}
|
||||
for c in (orc.get("choices") or []) # mutable-ok: transient default for iteration
|
||||
],
|
||||
}
|
||||
return OpenAIChatCompletionChunk.model_validate(normalize_reasoning_content(chunk))
|
||||
|
||||
@staticmethod
|
||||
def to_openai_chunk(event_obj: dict) -> OpenAIChatCompletionChunk | None:
|
||||
|
|
@ -90,10 +124,9 @@ class _StreamParser:
|
|||
# FINAL RESULT IS *NOT* TERMINAL: treat it as the next chunk
|
||||
if "final_result" in event_obj:
|
||||
fr: Final = event_obj["final_result"] or {}
|
||||
# ensure it looks like an OpenAI chunk
|
||||
if "object" not in fr:
|
||||
fr["object"] = "chat.completion.chunk"
|
||||
return OpenAIChatCompletionChunk.model_validate(fr)
|
||||
return OpenAIChatCompletionChunk.model_validate(normalize_reasoning_content(fr))
|
||||
|
||||
# Orchestration incremental delta
|
||||
if "orchestration_result" in event_obj:
|
||||
|
|
@ -101,7 +134,7 @@ class _StreamParser:
|
|||
|
||||
# Already an OpenAI-like chunk
|
||||
if "choices" in event_obj and "object" in event_obj:
|
||||
return OpenAIChatCompletionChunk.model_validate(event_obj)
|
||||
return OpenAIChatCompletionChunk.model_validate(normalize_reasoning_content(event_obj))
|
||||
|
||||
# Unknown / heartbeat / metrics
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -39,6 +39,12 @@ class ImageContent(BaseModel):
|
|||
image_url: ImageURLContent
|
||||
|
||||
|
||||
class FileContent(BaseModel):
|
||||
type_: Literal["file"] = Field(default="file", alias="type")
|
||||
file_data: str
|
||||
filename: str = ""
|
||||
|
||||
|
||||
class FunctionObj(BaseModel):
|
||||
name: str
|
||||
arguments: str
|
||||
|
|
@ -95,7 +101,12 @@ class SAPMessage(BaseModel):
|
|||
|
||||
class SAPUserMessage(BaseModel):
|
||||
role: Literal["user"] = "user"
|
||||
content: str | TextContent | ImageContent | list[TextContent | ImageContent]
|
||||
content: str | TextContent | ImageContent | FileContent | list[TextContent | ImageContent | FileContent]
|
||||
|
||||
|
||||
class ReasoningBlock(BaseModel):
|
||||
content: str = ""
|
||||
signature: str = ""
|
||||
|
||||
|
||||
class SAPAssistantMessage(BaseModel):
|
||||
|
|
@ -103,6 +114,7 @@ class SAPAssistantMessage(BaseModel):
|
|||
content: str = ""
|
||||
refusal: str = ""
|
||||
tool_calls: list[MessageToolCall] = []
|
||||
reasoning_content: list[ReasoningBlock] | None = None
|
||||
|
||||
_content_validator = field_validator("content", mode="before")(validate_different_content)
|
||||
|
||||
|
|
@ -194,7 +206,7 @@ class LLMModelDetails(BaseModel):
|
|||
|
||||
|
||||
class PromptTemplatingModuleConfig(BaseModel):
|
||||
prompt: Template
|
||||
prompt: Template | None = None
|
||||
model: LLMModelDetails
|
||||
|
||||
|
||||
|
|
@ -393,6 +405,43 @@ class MaskingModuleConfig(BaseModel):
|
|||
return self
|
||||
|
||||
|
||||
class AfterLastRoleTargetSelector(BaseModel):
|
||||
"""
|
||||
Scopes input filtering to all messages after the last message with a given role.
|
||||
|
||||
If no messages remain after applying this filter, filtering is skipped entirely
|
||||
rather than raising an error.
|
||||
|
||||
Args:
|
||||
after_last_role: The role used as the anchor. All messages that come after
|
||||
the last occurrence of this role in the combined message list
|
||||
(messages_history prepended to template) will be filtered.
|
||||
One of: 'system', 'user', 'assistant', 'developer', 'tool'.
|
||||
"""
|
||||
|
||||
after_last_role: Literal["system", "user", "assistant", "developer", "tool"]
|
||||
|
||||
|
||||
class LastMessagesTargetSelector(BaseModel):
|
||||
"""
|
||||
Scopes input filtering to the last N messages of the combined message list.
|
||||
|
||||
Args:
|
||||
last_messages: Number of messages from the end of the combined message list
|
||||
(messages_history prepended to template) to include in filtering.
|
||||
Must be >= 1 (0 is not allowed and returns 400 Bad Request).
|
||||
If the value exceeds the total number of messages, all messages are filtered.
|
||||
"""
|
||||
|
||||
last_messages: int = Field(ge=1)
|
||||
|
||||
|
||||
# InputFilterTargetSelector is a discriminated union: exactly one of the two selector
|
||||
# shapes must be present. The oneOf contract from the spec is enforced at instantiation
|
||||
# time because each shape carries a field the other does not.
|
||||
InputFilterTargetSelector = AfterLastRoleTargetSelector | LastMessagesTargetSelector
|
||||
|
||||
|
||||
class AzureThreshold(int, Enum):
|
||||
"""
|
||||
Enumerates the threshold levels for the Azure Content Safety service.
|
||||
|
|
@ -486,33 +535,25 @@ class LlamaGuard38bFilter(BaseModel):
|
|||
Specific implementation of ContentFilter for Llama Guard 3. Llama Guard 3 is a
|
||||
Llama-3.1-8B pretrained model, fine-tuned for content safety classification.
|
||||
|
||||
At least one category must be set to True (spec: minProperties: 1).
|
||||
|
||||
.. deprecated::
|
||||
Will be removed 2027-09-20. Use :class:`GraniteGuardian41` instead.
|
||||
|
||||
Args:
|
||||
violent_crimes: Responses that enable, encourage, or endorse the commission of violent crimes.
|
||||
|
||||
non_violent_crimes: Responses that enable, encourage, or endorse the commission of non-violent crimes.
|
||||
|
||||
sex_crimes: Responses that enable, encourage, or endorse the commission of sex-related crimes.
|
||||
|
||||
child_exploitation: Responses that contain, describe, enable or endorse the sexual abuse of children.
|
||||
|
||||
defamation: Responses that are both verifiably false and likely to injure a living person's reputation.
|
||||
|
||||
specialized_advice: Responses that contain specialized financial, medical or legal advice.
|
||||
|
||||
privacy: Responses that contain sensitive or nonpublic personal information.
|
||||
|
||||
intellectual_property: Responses that may violate the intellectual property rights of any third party.
|
||||
|
||||
indiscriminate_weapons: Responses that enable, encourage, or endorse the creation of indiscriminate weapons.
|
||||
|
||||
hate: Responses that demean or dehumanize people on the basis of their sensitive, personal characteristics.
|
||||
|
||||
self_harm: Responses that enable, encourage, or endorse acts of intentional self-harm.
|
||||
|
||||
sexual_content: Responses that contain erotica.
|
||||
|
||||
elections: Responses that contain factually incorrect information about electoral systems and processes.
|
||||
|
||||
code_interpreter_abuse: Responses that seek to abuse code interpreters.
|
||||
"""
|
||||
|
||||
|
|
@ -531,15 +572,111 @@ class LlamaGuard38bFilter(BaseModel):
|
|||
elections: bool = Field(default=False)
|
||||
code_interpreter_abuse: bool = Field(default=False)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_min_one_category(self) -> "LlamaGuard38bFilter":
|
||||
"""At least one category must be enabled (spec: minProperties: 1)."""
|
||||
if not any(
|
||||
[
|
||||
self.violent_crimes,
|
||||
self.non_violent_crimes,
|
||||
self.sex_crimes,
|
||||
self.child_exploitation,
|
||||
self.defamation,
|
||||
self.specialized_advice,
|
||||
self.privacy,
|
||||
self.intellectual_property,
|
||||
self.indiscriminate_weapons,
|
||||
self.hate,
|
||||
self.self_harm,
|
||||
self.sexual_content,
|
||||
self.elections,
|
||||
self.code_interpreter_abuse,
|
||||
]
|
||||
):
|
||||
raise ValueError(
|
||||
"LlamaGuard38bFilter requires at least one category set to True."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class LlamaGuard38bFilterConfig(BaseModel):
|
||||
type_: Literal["llama_guard_3_8b"] = Field(default="llama_guard_3_8b", alias="type")
|
||||
config: LlamaGuard38bFilter
|
||||
target_selector: InputFilterTargetSelector | None = None
|
||||
|
||||
|
||||
class GraniteGuardian41Categories(BaseModel):
|
||||
"""
|
||||
Content categories evaluated by IBM Granite Guardian 4.1.
|
||||
|
||||
At least one category must be set to True (minProperties: 1 in the spec).
|
||||
Granite Guardian issues a separate inference call per enabled category;
|
||||
for most use cases enabling only ``harm`` is recommended as a catch-all.
|
||||
|
||||
Args:
|
||||
harm: Catch-all criterion for generally harmful content.
|
||||
social_bias: Detect prejudice or discrimination based on identity or
|
||||
protected characteristics.
|
||||
jailbreak: Detect attempts to manipulate the model into producing harmful
|
||||
or otherwise undesired outputs.
|
||||
violence: Detect content promoting or depicting physical, mental, or
|
||||
sexual harm.
|
||||
profanity: Detect offensive language or insults.
|
||||
sexual_content: Detect explicit or suggestive material of a sexual nature.
|
||||
unethical_behavior: Detect content describing actions that violate moral
|
||||
or legal standards.
|
||||
"""
|
||||
|
||||
harm: bool = Field(default=False)
|
||||
social_bias: bool = Field(default=False)
|
||||
jailbreak: bool = Field(default=False)
|
||||
violence: bool = Field(default=False)
|
||||
profanity: bool = Field(default=False)
|
||||
sexual_content: bool = Field(default=False)
|
||||
unethical_behavior: bool = Field(default=False)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def enforce_min_one_category(self) -> "GraniteGuardian41Categories":
|
||||
"""At least one category must be enabled (spec: minProperties: 1)."""
|
||||
if not any(
|
||||
[
|
||||
self.harm,
|
||||
self.social_bias,
|
||||
self.jailbreak,
|
||||
self.violence,
|
||||
self.profanity,
|
||||
self.sexual_content,
|
||||
self.unethical_behavior,
|
||||
]
|
||||
):
|
||||
raise ValueError(
|
||||
"GraniteGuardian41Categories requires at least one category set to True."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class GraniteGuardian41(BaseModel):
|
||||
"""
|
||||
Configuration for IBM Granite Guardian 4.1 filter provider.
|
||||
|
||||
Args:
|
||||
enable_reasoning: Enable reasoning (think) mode. When True, the model returns
|
||||
an explanation alongside each verdict, e.g.
|
||||
``{'verdict': True, 'reasoning': '...'}``. Applies to every configured
|
||||
category. Defaults to False.
|
||||
categories: Content criteria to evaluate. At least one category must be
|
||||
enabled. Granite Guardian issues a separate inference call per category;
|
||||
using only ``harm`` is recommended as a catch-all to minimise latency.
|
||||
"""
|
||||
|
||||
enable_reasoning: bool = Field(default=False)
|
||||
categories: GraniteGuardian41Categories
|
||||
|
||||
|
||||
class AzureContentSafetyInputFilterConfig(BaseModel):
|
||||
type_: Literal["azure_content_safety"] = Field(default="azure_content_safety", alias="type")
|
||||
config: AzureContentSafetyInput | None = None
|
||||
target_selector: InputFilterTargetSelector | None = None
|
||||
|
||||
|
||||
class AzureContentSafetyOutputFilterConfig(BaseModel):
|
||||
|
|
@ -547,6 +684,22 @@ class AzureContentSafetyOutputFilterConfig(BaseModel):
|
|||
config: AzureContentSafetyOutput | None = None
|
||||
|
||||
|
||||
class GraniteGuardianFilterConfig(BaseModel):
|
||||
"""
|
||||
Filter configuration for the IBM Granite Guardian 4.1 provider.
|
||||
|
||||
Args:
|
||||
type_: Provider discriminator — always ``'granite_guardian_4_1'``.
|
||||
config: Category and reasoning settings for Granite Guardian.
|
||||
target_selector: Optional selector to scope filtering to a subset of the
|
||||
combined message list. When absent, all input content is filtered.
|
||||
"""
|
||||
|
||||
type_: Literal["granite_guardian_4_1"] = Field(default="granite_guardian_4_1", alias="type")
|
||||
config: GraniteGuardian41
|
||||
target_selector: InputFilterTargetSelector | None = None
|
||||
|
||||
|
||||
class FilteringStreamOptions(BaseModel):
|
||||
"""
|
||||
overlap: Number of characters that should be additionally sent to content filtering services
|
||||
|
|
@ -560,22 +713,26 @@ class InputFiltering(BaseModel):
|
|||
"""Module for managing and applying input content filters.
|
||||
|
||||
Args:
|
||||
filters: List of ContentFilter objects to be applied to input content.
|
||||
filters: List of filter provider configurations to be applied to input content.
|
||||
Supported providers: Azure Content Safety, Llama Guard 3 8B (deprecated),
|
||||
and IBM Granite Guardian 4.1.
|
||||
"""
|
||||
|
||||
filters: list[AzureContentSafetyInputFilterConfig | LlamaGuard38bFilterConfig] = Field(min_length=1)
|
||||
filters: list[AzureContentSafetyInputFilterConfig | LlamaGuard38bFilterConfig | GraniteGuardianFilterConfig] = Field(min_length=1)
|
||||
|
||||
|
||||
class OutputFiltering(BaseModel):
|
||||
"""Module for managing and applying output content filters.
|
||||
|
||||
Args:
|
||||
filters: List of ContentFilter objects to be applied to output content.
|
||||
|
||||
stream_options: Module-specific streaming options.
|
||||
filters: List of filter provider configurations to be applied to output content.
|
||||
Supported providers: Azure Content Safety, Llama Guard 3 8B (deprecated),
|
||||
and IBM Granite Guardian 4.1.
|
||||
stream_options: Module-specific streaming options. Ignored when streaming is
|
||||
disabled.
|
||||
"""
|
||||
|
||||
filters: list[AzureContentSafetyOutputFilterConfig | LlamaGuard38bFilterConfig] = Field(min_length=1)
|
||||
filters: list[AzureContentSafetyOutputFilterConfig | LlamaGuard38bFilterConfig | GraniteGuardianFilterConfig] = Field(min_length=1)
|
||||
stream_options: FilteringStreamOptions | None = None
|
||||
|
||||
|
||||
|
|
@ -712,3 +869,92 @@ class OrchestrationConfig(BaseModel):
|
|||
class OrchestrationRequest(BaseModel):
|
||||
config: OrchestrationConfig
|
||||
placeholder_values: dict[str, str] | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Partial config models — used by the config_ref request variants.
|
||||
# All fields are optional so callers only supply what they want to override.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PartialPromptTemplatingModuleConfig(BaseModel):
|
||||
"""Partial prompt-templating override for config_ref requests.
|
||||
|
||||
Both fields are optional: omit ``prompt`` to keep the referenced template,
|
||||
omit ``model`` to keep the referenced model.
|
||||
"""
|
||||
|
||||
prompt: Template | None = None
|
||||
model: LLMModelDetails | None = None
|
||||
|
||||
|
||||
class PartialModuleConfigs(BaseModel):
|
||||
"""Partial module configuration for config_ref overrides.
|
||||
|
||||
Only specify the modules you want to override; the remaining configuration
|
||||
is taken from the referenced orchestration config.
|
||||
"""
|
||||
|
||||
prompt_templating: PartialPromptTemplatingModuleConfig | None = None
|
||||
filtering: FilteringModuleConfig | None = None
|
||||
masking: MaskingModuleConfig | None = None
|
||||
grounding: GroundingModuleConfig | None = None
|
||||
translation: TranslationModuleConfig | None = None
|
||||
|
||||
|
||||
class PartialOrchestrationConfig(BaseModel):
|
||||
"""Partial orchestration configuration for config_ref overrides.
|
||||
|
||||
All fields are optional. Supply only the parts that should be overridden;
|
||||
the rest is taken from the referenced configuration stored in SAP AI Core.
|
||||
"""
|
||||
|
||||
modules: PartialModuleConfigs | None = None
|
||||
stream: GlobalStreamOptions | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# config_ref discriminated shapes (spec: CompletionPostRequest oneOf variants)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CompletionRequestConfigurationReferenceByIdConfigRef(BaseModel):
|
||||
"""Reference an SAP AI Core orchestration configuration by its UUID."""
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
class CompletionRequestConfigurationReferenceById(BaseModel):
|
||||
"""POST /v2/completion body variant: reference a saved config by ID.
|
||||
|
||||
The optional ``config`` field carries a partial override that is merged
|
||||
on top of the referenced configuration. ``placeholder_values`` and
|
||||
``messages_history`` work the same as in the full-config variant.
|
||||
"""
|
||||
|
||||
config_ref: CompletionRequestConfigurationReferenceByIdConfigRef
|
||||
config: PartialOrchestrationConfig | None = None
|
||||
placeholder_values: dict[str, str] | None = None
|
||||
messages_history: list[ChatMessage] | None = None
|
||||
|
||||
|
||||
class CompletionRequestConfigurationReferenceByNameScenarioVersionConfigRef(BaseModel):
|
||||
"""Reference an SAP AI Core orchestration configuration by name + scenario + version."""
|
||||
|
||||
scenario: str
|
||||
name: str
|
||||
version: str
|
||||
|
||||
|
||||
class CompletionRequestConfigurationReferenceByNameScenarioVersion(BaseModel):
|
||||
"""POST /v2/completion body variant: reference a saved config by name/scenario/version.
|
||||
|
||||
The optional ``config`` field carries a partial override that is merged
|
||||
on top of the referenced configuration. ``placeholder_values`` and
|
||||
``messages_history`` work the same as in the full-config variant.
|
||||
"""
|
||||
|
||||
config_ref: CompletionRequestConfigurationReferenceByNameScenarioVersionConfigRef
|
||||
config: PartialOrchestrationConfig | None = None
|
||||
placeholder_values: dict[str, str] | None = None
|
||||
messages_history: list[ChatMessage] | None = None
|
||||
|
|
|
|||
|
|
@ -27,10 +27,19 @@ from .handler import (
|
|||
AsyncSAPStreamIterator,
|
||||
GenAIHubOrchestrationError,
|
||||
SAPStreamIterator,
|
||||
normalize_reasoning_content,
|
||||
)
|
||||
from .handler import (
|
||||
normalize_choice as _normalize_choice_fn,
|
||||
)
|
||||
from .models import (
|
||||
ChatCompletionTool,
|
||||
CompletionRequestConfigurationReferenceById,
|
||||
CompletionRequestConfigurationReferenceByIdConfigRef,
|
||||
CompletionRequestConfigurationReferenceByNameScenarioVersion,
|
||||
CompletionRequestConfigurationReferenceByNameScenarioVersionConfigRef,
|
||||
OrchestrationRequest,
|
||||
PartialOrchestrationConfig,
|
||||
ResponseFormat,
|
||||
ResponseFormatJSONSchema,
|
||||
SAPAssistantMessage,
|
||||
|
|
@ -212,7 +221,8 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
"extra_headers",
|
||||
"parallel_tool_calls",
|
||||
"response_format",
|
||||
"timeout",
|
||||
"reasoning_effort",
|
||||
"thinking",
|
||||
]
|
||||
# Remove response_format for providers that don't support it on SAP GenAI Hub
|
||||
if (
|
||||
|
|
@ -307,6 +317,76 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
**optional_modules,
|
||||
}
|
||||
|
||||
def _build_config_ref_body(
|
||||
self,
|
||||
config_ref: dict,
|
||||
optional_params: dict,
|
||||
messages: list[AllMessageValues],
|
||||
) -> dict:
|
||||
"""Build a config_ref request body (ById or ByNameScenarioVersion variant).
|
||||
|
||||
The discriminator is duck-typed from the keys present in ``config_ref``:
|
||||
|
||||
* ``{"id": "<uuid>"}`` → ById variant
|
||||
* ``{"scenario": ..., "name": ..., "version": ...}`` → ByNameScenarioVersion variant
|
||||
|
||||
``optional_params`` may carry:
|
||||
* ``placeholder_values`` (dict[str, str]) — forwarded as-is.
|
||||
* ``messages_history`` (list of message dicts) — forwarded as-is.
|
||||
* ``config`` (dict matching PartialOrchestrationConfig) — partial override.
|
||||
|
||||
Any keys not consumed here are silently ignored so that LiteLLM's standard
|
||||
optional-params machinery does not break.
|
||||
"""
|
||||
# --- discriminate on config_ref shape -----------------------------------
|
||||
if "id" in config_ref:
|
||||
validated_ref = CompletionRequestConfigurationReferenceByIdConfigRef(
|
||||
**config_ref
|
||||
)
|
||||
model_cls = CompletionRequestConfigurationReferenceById
|
||||
elif "scenario" in config_ref or "name" in config_ref or "version" in config_ref:
|
||||
validated_ref = CompletionRequestConfigurationReferenceByNameScenarioVersionConfigRef(
|
||||
**config_ref
|
||||
)
|
||||
model_cls = CompletionRequestConfigurationReferenceByNameScenarioVersion
|
||||
else:
|
||||
raise ValueError(
|
||||
"config_ref must contain either 'id' (ById) or 'scenario'/'name'/'version' "
|
||||
"(ByNameScenarioVersion)."
|
||||
)
|
||||
|
||||
# --- optional fields ----------------------------------------------------
|
||||
placeholder_values: Final = optional_params.pop("placeholder_values", None)
|
||||
messages_history_raw: Final = optional_params.pop("messages_history", None)
|
||||
partial_config_raw: Final = optional_params.pop("config", None)
|
||||
|
||||
partial_config: PartialOrchestrationConfig | None = None
|
||||
if partial_config_raw is not None:
|
||||
partial_config = PartialOrchestrationConfig(**partial_config_raw)
|
||||
|
||||
messages_history = None
|
||||
if messages_history_raw is not None:
|
||||
messages_history = _messages_to_sap_template(messages_history_raw)
|
||||
elif messages:
|
||||
# When messages are provided through the standard LiteLLM path but
|
||||
# no explicit messages_history override was given, treat them as the
|
||||
# history so that the caller does not have to duplicate the payload.
|
||||
messages_history = _messages_to_sap_template(messages)
|
||||
|
||||
# --- assemble and validate the full body --------------------------------
|
||||
body_kwargs: dict = {
|
||||
"config_ref": validated_ref,
|
||||
}
|
||||
if partial_config is not None:
|
||||
body_kwargs["config"] = partial_config
|
||||
if placeholder_values is not None:
|
||||
body_kwargs["placeholder_values"] = placeholder_values
|
||||
if messages_history is not None:
|
||||
body_kwargs["messages_history"] = messages_history
|
||||
|
||||
validated = model_cls(**body_kwargs)
|
||||
return validated.model_dump(by_alias=True, exclude_unset=True)
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -318,6 +398,19 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
optional_params = dict(optional_params)
|
||||
optional_params.pop("deployment_url", None)
|
||||
|
||||
# --- config_ref routing -------------------------------------------------
|
||||
# When the caller supplies a `config_ref` key the request targets a
|
||||
# pre-saved SAP AI Core orchestration configuration. We build the
|
||||
# alternative body shape and return early, bypassing the full-config path.
|
||||
config_ref: Final = optional_params.pop("config_ref", None)
|
||||
if config_ref is not None:
|
||||
return self._build_config_ref_body(
|
||||
config_ref=dict(config_ref),
|
||||
optional_params=optional_params,
|
||||
messages=messages,
|
||||
)
|
||||
# ------------------------------------------------------------------------
|
||||
|
||||
template: Final = _messages_to_sap_template(messages)
|
||||
|
||||
placeholder_values: Final = optional_params.pop("placeholder_values", None)
|
||||
|
|
@ -386,13 +479,29 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
dropped_params = raw_response.headers.get("X-Orchestration-Dropped-Model-Params")
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
original_response=raw_response.text,
|
||||
additional_args={"complete_input_dict": request_data},
|
||||
additional_args={
|
||||
"complete_input_dict": request_data,
|
||||
**({
|
||||
"x_orchestration_dropped_model_params": dropped_params
|
||||
} if dropped_params else {}),
|
||||
},
|
||||
)
|
||||
response = ModelResponse.model_validate(raw_response.json()["final_result"])
|
||||
final_result = normalize_reasoning_content(raw_response.json()["final_result"])
|
||||
response = ModelResponse.model_validate(final_result)
|
||||
|
||||
# Forward provider_specific_fields from each raw choice onto the
|
||||
# corresponding ModelResponse choice so callers and middleware can
|
||||
# access provider metadata that doesn't fit the OpenAI schema.
|
||||
raw_choices: list[dict] = final_result.get("choices") or []
|
||||
for raw_choice, model_choice in zip(raw_choices, response.choices or []):
|
||||
psf = raw_choice.get("provider_specific_fields")
|
||||
if psf is not None:
|
||||
model_choice.provider_specific_fields = psf # type: ignore[attr-defined]
|
||||
|
||||
# Strip markdown code blocks if JSON response_format was used with Anthropic models
|
||||
# SAP GenAI Hub with Anthropic models sometimes wraps JSON in ```json ... ```
|
||||
|
|
@ -405,6 +514,20 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
|
|||
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def _normalize_reasoning_content(raw: dict[str, object]) -> dict[str, object]:
|
||||
# SAP AI Core returns reasoning tokens as a list: message.reasoning_content = [{"content": ..., "signature": ...}]
|
||||
# ModelResponse.reasoning_content is typed Optional[str], so map the list to
|
||||
# thinking_blocks and collapse the text into a single joined string.
|
||||
return normalize_reasoning_content(raw)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_choice(choice: dict[str, object]) -> dict[str, object]:
|
||||
# Normalize a single choice dict (message or delta shape) in isolation.
|
||||
# Exposed as a static method so tests and callers can exercise one choice
|
||||
# without constructing a full response envelope.
|
||||
return _normalize_choice_fn(choice)
|
||||
|
||||
def _strip_markdown_json(self, response: ModelResponse) -> ModelResponse:
|
||||
"""Strip markdown code block wrapper from JSON content if present.
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class EmbeddingModel(BaseModel):
|
|||
name: str
|
||||
version: str = "latest"
|
||||
params: dict = Field(default_factory=dict)
|
||||
timeout: int | None = Field(default=None, ge=1, le=600)
|
||||
timeout: int | None = Field(default=None, ge=1, le=1200)
|
||||
max_retries: int | None = Field(default=None, ge=0, le=5)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import warnings
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
|
@ -639,3 +640,609 @@ class TestSAPTransformationIntegration:
|
|||
config["config"]["modules"][1]["translation"]["input"]["type"]
|
||||
== "sap_document_translation"
|
||||
)
|
||||
|
||||
class TestGetSupportedOpenaiParams:
|
||||
"""Unit tests for GenAIHubOrchestrationConfig.get_supported_openai_params."""
|
||||
|
||||
@pytest.fixture
|
||||
def config(self):
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
return GenAIHubOrchestrationConfig.__new__(GenAIHubOrchestrationConfig)
|
||||
|
||||
def test_new_params_present_for_standard_models(self, config):
|
||||
for model in ("gpt-4o", "anthropic--claude-4-sonnet", "gemini-2.5-flash"):
|
||||
params = config.get_supported_openai_params(model)
|
||||
assert "reasoning_effort" in params, f"reasoning_effort missing for {model}"
|
||||
assert "thinking" in params, f"thinking missing for {model}"
|
||||
|
||||
def test_response_format_excluded_for_unsupported_models(self, config):
|
||||
for model in ("amazon--titan", "cohere--command", "alephalpha--luminous", "gpt-4"):
|
||||
assert "response_format" not in config.get_supported_openai_params(model)
|
||||
|
||||
def test_tool_choice_excluded_for_gemini_and_amazon(self, config):
|
||||
for model in ("gemini-2.5-flash", "amazon--titan"):
|
||||
assert "tool_choice" not in config.get_supported_openai_params(model)
|
||||
|
||||
def test_tool_choice_present_for_gpt_and_anthropic(self, config):
|
||||
for model in ("gpt-4o", "anthropic--claude-4-sonnet"):
|
||||
assert "tool_choice" in config.get_supported_openai_params(model)
|
||||
|
||||
|
||||
class TestNormalizeReasoningContent:
|
||||
"""Unit tests for GenAIHubOrchestrationConfig._normalize_reasoning_content."""
|
||||
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
_normalize = staticmethod(GenAIHubOrchestrationConfig._normalize_reasoning_content)
|
||||
|
||||
def test_list_reasoning_content_mapped_to_thinking_blocks(self):
|
||||
"""List-shaped reasoning_content is converted to thinking_blocks."""
|
||||
raw = {
|
||||
"choices": [{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Latin.",
|
||||
"reasoning_content": [
|
||||
{"content": "Romans spoke Latin.", "signature": "sig1"},
|
||||
{"content": "That is well known.", "signature": "sig2"},
|
||||
],
|
||||
}
|
||||
}]
|
||||
}
|
||||
out = self._normalize(raw)
|
||||
msg = out["choices"][0]["message"]
|
||||
assert msg["thinking_blocks"] == [
|
||||
{"type": "thinking", "thinking": "Romans spoke Latin.", "signature": "sig1"},
|
||||
{"type": "thinking", "thinking": "That is well known.", "signature": "sig2"},
|
||||
]
|
||||
assert msg["reasoning_content"] == "Romans spoke Latin.\nThat is well known."
|
||||
|
||||
def test_string_reasoning_content_unchanged(self):
|
||||
"""String reasoning_content is left as-is (already the right type)."""
|
||||
raw = {
|
||||
"choices": [{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "42",
|
||||
"reasoning_content": "I thought about it.",
|
||||
}
|
||||
}]
|
||||
}
|
||||
out = self._normalize(raw)
|
||||
msg = out["choices"][0]["message"]
|
||||
assert msg["reasoning_content"] == "I thought about it."
|
||||
assert "thinking_blocks" not in msg
|
||||
|
||||
def test_no_reasoning_content_unchanged(self):
|
||||
"""A message without reasoning_content is not modified."""
|
||||
raw = {"choices": [{"message": {"role": "assistant", "content": "Hi."}}]}
|
||||
out = self._normalize(raw)
|
||||
assert out == raw
|
||||
|
||||
def test_empty_list_reasoning_content_sets_none(self):
|
||||
"""An empty list produces None for reasoning_content and empty thinking_blocks."""
|
||||
raw = {"choices": [{"message": {"reasoning_content": []}}]}
|
||||
out = self._normalize(raw)
|
||||
msg = out["choices"][0]["message"]
|
||||
assert msg["thinking_blocks"] == []
|
||||
assert msg["reasoning_content"] is None
|
||||
|
||||
def test_multiple_choices_all_normalized(self):
|
||||
"""All choices in the response are normalized."""
|
||||
raw = {
|
||||
"choices": [
|
||||
{"message": {"reasoning_content": [{"content": "thought A", "signature": None}]}},
|
||||
{"message": {"reasoning_content": [{"content": "thought B", "signature": "s"}]}},
|
||||
]
|
||||
}
|
||||
out = self._normalize(raw)
|
||||
assert out["choices"][0]["message"]["reasoning_content"] == "thought A"
|
||||
assert out["choices"][1]["message"]["reasoning_content"] == "thought B"
|
||||
|
||||
|
||||
def test_null_content_in_block_uses_empty_string(self):
|
||||
"""Explicit null content value must not leak None into thinking field."""
|
||||
raw = {
|
||||
"choices": [{
|
||||
"message": {
|
||||
"reasoning_content": [{"content": None, "signature": "s"}],
|
||||
}
|
||||
}]
|
||||
}
|
||||
out = self._normalize(raw)
|
||||
block = out["choices"][0]["message"]["thinking_blocks"][0]
|
||||
assert block["thinking"] == ""
|
||||
assert out["choices"][0]["message"]["reasoning_content"] is None
|
||||
|
||||
def test_transform_response_normalizes_list_reasoning_content(self):
|
||||
"""Production path: transform_response must produce a ModelResponse
|
||||
with thinking_blocks populated when the raw payload carries a
|
||||
list-shaped reasoning_content.
|
||||
"""
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
config = GenAIHubOrchestrationConfig()
|
||||
|
||||
final_result = {
|
||||
"id": "chatcmpl-test",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000,
|
||||
"model": "gemini-test",
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "The answer is 42.",
|
||||
"reasoning_content": [
|
||||
{"content": "Let me think.", "signature": "sig1"},
|
||||
{"content": "Yes, 42.", "signature": "sig2"},
|
||||
],
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
|
||||
raw_response = MagicMock()
|
||||
raw_response.text = json.dumps({"final_result": final_result})
|
||||
raw_response.json.return_value = {"final_result": final_result}
|
||||
|
||||
response = config.transform_response(
|
||||
model="gemini-test",
|
||||
raw_response=raw_response,
|
||||
model_response=MagicMock(),
|
||||
logging_obj=MagicMock(),
|
||||
api_key="test",
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
|
||||
choice = response.choices[0]
|
||||
assert hasattr(choice.message, "thinking_blocks"), "thinking_blocks missing from message"
|
||||
assert choice.message.thinking_blocks == [
|
||||
{"type": "thinking", "thinking": "Let me think.", "signature": "sig1"},
|
||||
{"type": "thinking", "thinking": "Yes, 42.", "signature": "sig2"},
|
||||
]
|
||||
assert choice.message.reasoning_content == "Let me think.\nYes, 42."
|
||||
|
||||
|
||||
class TestNormalizeChoice:
|
||||
"""Unit tests for GenAIHubOrchestrationConfig._normalize_choice (message and delta shapes)."""
|
||||
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
_normalize_choice = staticmethod(GenAIHubOrchestrationConfig._normalize_choice)
|
||||
|
||||
def test_message_list_reasoning_content_normalized(self):
|
||||
choice = {
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "hi", "reasoning_content": [{"content": "thought", "signature": "s1"}]},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
result = self._normalize_choice(choice)
|
||||
msg = result["message"]
|
||||
assert msg["reasoning_content"] == "thought"
|
||||
assert msg["thinking_blocks"] == [{"type": "thinking", "thinking": "thought", "signature": "s1"}]
|
||||
|
||||
def test_delta_list_reasoning_content_normalized(self):
|
||||
choice = {
|
||||
"index": 0,
|
||||
"delta": {"role": "assistant", "reasoning_content": [{"content": "delta thought", "signature": None}]},
|
||||
"finish_reason": None,
|
||||
}
|
||||
result = self._normalize_choice(choice)
|
||||
delta = result["delta"]
|
||||
assert delta["reasoning_content"] == "delta thought"
|
||||
assert delta["thinking_blocks"][0]["thinking"] == "delta thought"
|
||||
|
||||
def test_string_reasoning_content_unchanged(self):
|
||||
choice = {"index": 0, "message": {"reasoning_content": "already a string"}}
|
||||
assert self._normalize_choice(choice) == choice
|
||||
|
||||
def test_no_reasoning_content_unchanged(self):
|
||||
choice = {"index": 0, "delta": {"content": "hello"}}
|
||||
assert self._normalize_choice(choice) == choice
|
||||
|
||||
def test_normalize_reasoning_content_covers_delta_path(self):
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
raw = {
|
||||
"id": "c1",
|
||||
"object": "chat.completion.chunk",
|
||||
"choices": [
|
||||
{"index": 0, "delta": {"reasoning_content": [{"content": "stream thought", "signature": "sig"}]}, "finish_reason": None}
|
||||
],
|
||||
}
|
||||
result = GenAIHubOrchestrationConfig._normalize_reasoning_content(raw)
|
||||
delta = result["choices"][0]["delta"]
|
||||
assert delta["reasoning_content"] == "stream thought"
|
||||
assert delta["thinking_blocks"][0]["type"] == "thinking"
|
||||
|
||||
|
||||
class TestMessagesToSapTemplateReasoningContent:
|
||||
"""_messages_to_sap_template must forward reasoning_content on assistant turns."""
|
||||
|
||||
def test_assistant_message_with_reasoning_content_is_preserved(self):
|
||||
from litellm.llms.sap.chat.transformation import _messages_to_sap_template
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "4",
|
||||
"reasoning_content": [
|
||||
{"content": "Simple arithmetic.", "signature": "sig1"}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "Are you sure?"},
|
||||
]
|
||||
result = _messages_to_sap_template(messages)
|
||||
assistant_msg = result[1]
|
||||
assert assistant_msg["reasoning_content"] == [
|
||||
{"content": "Simple arithmetic.", "signature": "sig1"}
|
||||
]
|
||||
|
||||
|
||||
class TestFileContentModel:
|
||||
def test_file_content_serializes_type_alias(self):
|
||||
from litellm.llms.sap.chat.models import FileContent
|
||||
|
||||
fc = FileContent(**{"type": "file", "file_data": "base64=="})
|
||||
dumped = fc.model_dump(by_alias=True, exclude_unset=True)
|
||||
assert dumped["type"] == "file"
|
||||
assert dumped["file_data"] == "base64=="
|
||||
|
||||
def test_file_content_filename_omitted_when_not_set(self):
|
||||
from litellm.llms.sap.chat.models import FileContent
|
||||
|
||||
fc = FileContent(**{"type": "file", "file_data": "abc"})
|
||||
dumped = fc.model_dump(by_alias=True, exclude_unset=True)
|
||||
assert "filename" not in dumped
|
||||
|
||||
def test_file_content_filename_included_when_set(self):
|
||||
from litellm.llms.sap.chat.models import FileContent
|
||||
|
||||
fc = FileContent(**{"type": "file", "file_data": "abc", "filename": "report.pdf"})
|
||||
dumped = fc.model_dump(by_alias=True, exclude_unset=True)
|
||||
assert dumped["filename"] == "report.pdf"
|
||||
|
||||
def test_file_content_requires_file_data(self):
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.llms.sap.chat.models import FileContent
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
FileContent(**{"type": "file"})
|
||||
|
||||
|
||||
class TestSAPUserMessageWithFileContent:
|
||||
def test_user_message_accepts_file_content(self):
|
||||
from litellm.llms.sap.chat.models import FileContent, SAPUserMessage
|
||||
|
||||
fc = FileContent(**{"type": "file", "file_data": "base64=="})
|
||||
msg = SAPUserMessage(role="user", content=fc)
|
||||
dumped = msg.model_dump(by_alias=True, exclude_unset=True)
|
||||
assert dumped["content"]["type"] == "file"
|
||||
assert dumped["content"]["file_data"] == "base64=="
|
||||
|
||||
def test_user_message_accepts_list_with_text_and_file(self):
|
||||
from litellm.llms.sap.chat.models import FileContent, SAPUserMessage, TextContent
|
||||
|
||||
parts = [
|
||||
TextContent(**{"type": "text", "text": "Analyze this:"}),
|
||||
FileContent(**{"type": "file", "file_data": "base64==", "filename": "data.csv"}),
|
||||
]
|
||||
msg = SAPUserMessage(role="user", content=parts)
|
||||
dumped = msg.model_dump(by_alias=True, exclude_unset=True)
|
||||
assert dumped["content"][0]["type"] == "text"
|
||||
assert dumped["content"][1]["type"] == "file"
|
||||
assert dumped["content"][1]["filename"] == "data.csv"
|
||||
|
||||
|
||||
class TestMessagesToSapTemplateWithFileContent:
|
||||
def test_file_content_message_round_trips_through_template(self):
|
||||
from litellm.llms.sap.chat.transformation import _messages_to_sap_template
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": {"type": "file", "file_data": "base64==", "filename": "doc.pdf"},
|
||||
}
|
||||
]
|
||||
result = _messages_to_sap_template(messages)
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[0]["content"]["type"] == "file"
|
||||
assert result[0]["content"]["file_data"] == "base64=="
|
||||
assert result[0]["content"]["filename"] == "doc.pdf"
|
||||
|
||||
def test_file_content_without_filename_omitted_in_template(self):
|
||||
from litellm.llms.sap.chat.transformation import _messages_to_sap_template
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": {"type": "file", "file_data": "xyz"},
|
||||
}
|
||||
]
|
||||
result = _messages_to_sap_template(messages)
|
||||
assert "filename" not in result[0]["content"]
|
||||
|
||||
def test_mixed_list_with_file_and_text_in_template(self):
|
||||
from litellm.llms.sap.chat.transformation import _messages_to_sap_template
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Summarize:"},
|
||||
{"type": "file", "file_data": "base64=="},
|
||||
],
|
||||
}
|
||||
]
|
||||
result = _messages_to_sap_template(messages)
|
||||
content = result[0]["content"]
|
||||
assert content[0]["type"] == "text"
|
||||
assert content[1]["type"] == "file"
|
||||
|
||||
|
||||
class TestProviderSpecificFields:
|
||||
"""Tests for provider_specific_fields propagation through normalize_choice,
|
||||
_StreamParser, and transform_response (spec lines 1161 / 1181)."""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# normalize_choice
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_normalize_choice_preserves_provider_specific_fields(self):
|
||||
"""provider_specific_fields present in a choice must survive normalize_choice."""
|
||||
from litellm.llms.sap.chat.handler import normalize_choice
|
||||
|
||||
choice = {
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "hi"},
|
||||
"finish_reason": "stop",
|
||||
"provider_specific_fields": {"raw_finish_reason": "end_turn", "extra": 42},
|
||||
}
|
||||
result = normalize_choice(choice)
|
||||
assert result["provider_specific_fields"] == {"raw_finish_reason": "end_turn", "extra": 42}
|
||||
|
||||
def test_normalize_choice_no_provider_specific_fields_unchanged(self):
|
||||
"""When provider_specific_fields is absent the key must not be injected."""
|
||||
from litellm.llms.sap.chat.handler import normalize_choice
|
||||
|
||||
choice = {"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}
|
||||
result = normalize_choice(choice)
|
||||
assert "provider_specific_fields" not in result
|
||||
|
||||
def test_normalize_choice_preserves_psf_alongside_reasoning_content(self):
|
||||
"""provider_specific_fields must survive even when reasoning_content is also normalized."""
|
||||
from litellm.llms.sap.chat.handler import normalize_choice
|
||||
|
||||
choice = {
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "answer",
|
||||
"reasoning_content": [{"content": "thought", "signature": "s1"}],
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"provider_specific_fields": {"raw_finish_reason": "end_turn"},
|
||||
}
|
||||
result = normalize_choice(choice)
|
||||
assert result["provider_specific_fields"] == {"raw_finish_reason": "end_turn"}
|
||||
# reasoning normalization must still have happened
|
||||
assert result["message"]["reasoning_content"] == "thought"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# _StreamParser._from_orchestration_result
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_stream_parser_passes_provider_specific_fields(self):
|
||||
"""_from_orchestration_result must forward provider_specific_fields per choice."""
|
||||
from litellm.llms.sap.chat.handler import _StreamParser
|
||||
|
||||
evt = {
|
||||
"orchestration_result": {
|
||||
"id": "chunk-1",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1700000000,
|
||||
"model": "gpt-4o",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": "hello"},
|
||||
"finish_reason": None,
|
||||
"provider_specific_fields": {"raw_finish_reason": None, "extra_flag": True},
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
chunk = _StreamParser.to_openai_chunk(evt)
|
||||
assert chunk is not None
|
||||
choice = chunk.choices[0]
|
||||
assert hasattr(choice, "provider_specific_fields")
|
||||
assert choice.provider_specific_fields == {"raw_finish_reason": None, "extra_flag": True}
|
||||
|
||||
def test_stream_parser_omits_key_when_absent(self):
|
||||
"""When provider_specific_fields is absent the key is not injected into the chunk."""
|
||||
from litellm.llms.sap.chat.handler import _StreamParser
|
||||
|
||||
evt = {
|
||||
"orchestration_result": {
|
||||
"id": "chunk-2",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1700000000,
|
||||
"model": "gpt-4o",
|
||||
"choices": [{"index": 0, "delta": {"content": "world"}, "finish_reason": None}],
|
||||
}
|
||||
}
|
||||
chunk = _StreamParser.to_openai_chunk(evt)
|
||||
assert chunk is not None
|
||||
choice = chunk.choices[0]
|
||||
# either absent or None is acceptable — must not be a non-None populated dict
|
||||
psf = getattr(choice, "provider_specific_fields", None)
|
||||
assert not psf
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# transform_response (non-streaming)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_transform_response_copies_provider_specific_fields(self):
|
||||
"""transform_response must attach provider_specific_fields from each raw choice
|
||||
onto the corresponding ModelResponse choice."""
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
config = GenAIHubOrchestrationConfig()
|
||||
|
||||
final_result = {
|
||||
"id": "chatcmpl-psf-test",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000,
|
||||
"model": "gpt-4o",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello!"},
|
||||
"finish_reason": "stop",
|
||||
"provider_specific_fields": {
|
||||
"finish_reason": "end_turn",
|
||||
"sap_extra": "metadata",
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
||||
}
|
||||
|
||||
raw_response = MagicMock()
|
||||
raw_response.text = json.dumps({"final_result": final_result})
|
||||
raw_response.json.return_value = {"final_result": final_result}
|
||||
raw_response.headers = {}
|
||||
|
||||
response = config.transform_response(
|
||||
model="gpt-4o",
|
||||
raw_response=raw_response,
|
||||
model_response=MagicMock(),
|
||||
logging_obj=MagicMock(),
|
||||
api_key="test",
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
|
||||
choice = response.choices[0]
|
||||
assert hasattr(choice, "provider_specific_fields"), "provider_specific_fields missing from choice"
|
||||
assert choice.provider_specific_fields == {
|
||||
"finish_reason": "end_turn",
|
||||
"sap_extra": "metadata",
|
||||
}
|
||||
|
||||
def test_transform_response_no_provider_specific_fields_skipped(self):
|
||||
"""When provider_specific_fields is absent on raw choices, nothing is attached."""
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
config = GenAIHubOrchestrationConfig()
|
||||
|
||||
final_result = {
|
||||
"id": "chatcmpl-nopsf",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000,
|
||||
"model": "gpt-4o",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hi."},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
|
||||
}
|
||||
|
||||
raw_response = MagicMock()
|
||||
raw_response.text = json.dumps({"final_result": final_result})
|
||||
raw_response.json.return_value = {"final_result": final_result}
|
||||
raw_response.headers = {}
|
||||
|
||||
response = config.transform_response(
|
||||
model="gpt-4o",
|
||||
raw_response=raw_response,
|
||||
model_response=MagicMock(),
|
||||
logging_obj=MagicMock(),
|
||||
api_key="test",
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
|
||||
choice = response.choices[0]
|
||||
psf = getattr(choice, "provider_specific_fields", None)
|
||||
assert not psf
|
||||
|
||||
def test_transform_response_multiple_choices_each_gets_own_psf(self):
|
||||
"""Each choice gets its own provider_specific_fields (different values per choice)."""
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
|
||||
|
||||
config = GenAIHubOrchestrationConfig()
|
||||
|
||||
final_result = {
|
||||
"id": "chatcmpl-multi",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000,
|
||||
"model": "gpt-4o",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "A"},
|
||||
"finish_reason": "stop",
|
||||
"provider_specific_fields": {"raw_finish_reason": "end_turn"},
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"message": {"role": "assistant", "content": "B"},
|
||||
"finish_reason": "length",
|
||||
"provider_specific_fields": {"raw_finish_reason": "max_tokens"},
|
||||
},
|
||||
],
|
||||
"usage": {"prompt_tokens": 4, "completion_tokens": 2, "total_tokens": 6},
|
||||
}
|
||||
|
||||
raw_response = MagicMock()
|
||||
raw_response.text = json.dumps({"final_result": final_result})
|
||||
raw_response.json.return_value = {"final_result": final_result}
|
||||
raw_response.headers = {}
|
||||
|
||||
response = config.transform_response(
|
||||
model="gpt-4o",
|
||||
raw_response=raw_response,
|
||||
model_response=MagicMock(),
|
||||
logging_obj=MagicMock(),
|
||||
api_key="test",
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
|
||||
assert response.choices[0].provider_specific_fields == {"raw_finish_reason": "end_turn"}
|
||||
assert response.choices[1].provider_specific_fields == {"raw_finish_reason": "max_tokens"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue