scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-12 23:01:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

fix(oci): dynamic max_tokens default and filter empty Cohere chat history messages

Browse source 
Two fixes for the OCI Cohere chat integration:

1. Replace hardcoded maxTokens default of 600 with a dynamic lookup
   via get_max_tokens(), following the same pattern used by the Anthropic
   provider. Falls back to DEFAULT_MAX_TOKENS (env-configurable, default
   4096) when the model is not in the cost map. The previous value of 600
   caused responses to be silently truncated well before the model's
   actual output limit (e.g. 4000 for cohere.command-a-03-2025).

2. Skip chat history entries with empty content in
   adapt_messages_to_cohere_standard(). The Cohere API rejects requests
   where any history element has an empty message with a 400 error:
   "all elements in history must have a message". Upstream callers such
   as Open WebUI may include empty assistant placeholders or system
   messages during multi-turn conversations.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rami Salman 2026-03-30 18:38:27 +03:00
parent 58120537af
commit bff16253eb
1 changed files with 34 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
litellm/llms/oci/chat/transformation.py
View file

@ -60,10 +60,12 @@ from litellm.types.utils import (
ModelResponseStream,
StreamingChoices,
)
from litellm.constants import DEFAULT_MAX_TOKENS
from litellm.utils import (
ChatCompletionMessageToolCall,
CustomStreamWrapper,
Usage,
get_max_tokens,
)
if TYPE_CHECKING:
@ -661,7 +663,28 @@ class OCIChatConfig(BaseConfig):
oci_region = optional_params.get("oci_region", "us-ashburn-1")
return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/chat"
def _get_optional_params(self, vendor: OCIVendors, optional_params: dict) -> Dict:
@staticmethod
def _get_max_tokens_for_model(model: Optional[str] = None) -> int:
"""
Get the max output tokens for a given OCI model.
Uses litellm's model cost map for a dynamic lookup, falling back to
DEFAULT_MAX_TOKENS (configurable via the DEFAULT_MAX_TOKENS env var)
when the model is not found.
"""
if model is None:
return DEFAULT_MAX_TOKENS
try:
max_tokens = get_max_tokens(model)
if max_tokens is not None:
return max_tokens
except Exception:
pass
return DEFAULT_MAX_TOKENS
def _get_optional_params(
self, vendor: OCIVendors, optional_params: dict, model: Optional[str] = None
) -> Dict:
selected_params = {}
if vendor == OCIVendors.COHERE:
open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map
@ -669,7 +692,7 @@ class OCIChatConfig(BaseConfig):
open_ai_to_oci_param_map.pop("tool_choice")
# Add default values for Cohere API
selected_params = {
"maxTokens": 600,
"maxTokens": self._get_max_tokens_for_model(model),
"temperature": 1,
"topK": 0,
"topP": 0.75,
@ -781,6 +804,13 @@ class OCIChatConfig(BaseConfig):
)
)
# Cohere requires all chat history elements to have a non-empty
# message. Upstream callers (e.g. Open WebUI) may include
# assistant placeholders or empty system messages. Skip them
# to avoid a 400 "all elements in history must have a message".
if not content and not tool_calls:
continue
if role == "user":
chat_history.append(CohereMessage(role="USER", message=content))
elif role == "assistant":
@ -895,7 +925,7 @@ class OCIChatConfig(BaseConfig):
# Create Cohere-specific chat request
optional_cohere_params = self._get_optional_params(
OCIVendors.COHERE, optional_params
OCIVendors.COHERE, optional_params, model=model
)
chat_request = CohereChatRequest(
apiFormat="COHERE",
@ -918,7 +948,7 @@ class OCIChatConfig(BaseConfig):
chatRequest=OCIChatRequestPayload(
apiFormat=vendor.value,
messages=adapt_messages_to_generic_oci_standard(messages),
**self._get_optional_params(vendor, optional_params),
**self._get_optional_params(vendor, optional_params, model=model),
),
)