From 0c6378a320828696b6df6d0f59f7eef6117e1c0a Mon Sep 17 00:00:00 2001 From: Federico Kamelhar Date: Tue, 9 Jun 2026 05:42:54 -0400 Subject: [PATCH] fix(oci): inject default maxTokens so omitted max_tokens doesn't truncate OCI GenAI applies a tiny server-side maxTokens default (~20 tokens) when the request omits it, so any call that doesn't send max_tokens comes back cut off mid-string with finishReason "length". MLflow judges never send max_tokens, so their JSON responses arrived as unterminated strings and json.loads failed in MLflow's gateway adapter. When no maxTokens/maxCompletionTokens target is set, inject DEFAULT_OCI_CHAT_MAX_TOKENS (env-overridable, defaults 4096), mirroring the Anthropic config's default-max-tokens behaviour. An explicit max_tokens still wins, and reasoning models still route to maxCompletionTokens. Used a fixed default rather than the catalog max_output_tokens because the catalog value is unreliable for some models (grok-4 reports max_output_tokens equal to its context window, not a real output cap, which would risk 400s). Adds TestOCIDefaultMaxTokens covering Cohere and generic injection, the explicit-override case, and the reasoning maxCompletionTokens branch. --- litellm/constants.py | 1 + litellm/llms/oci/chat/transformation.py | 8 ++++ .../oci/chat/test_oci_chat_transformation.py | 40 +++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/litellm/constants.py b/litellm/constants.py index f10cec034f0..1d85183f26c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -418,6 +418,7 @@ REPLICATE_POLLING_DELAY_SECONDS = float( DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS = int( os.getenv("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096) ) +DEFAULT_OCI_CHAT_MAX_TOKENS = int(os.getenv("DEFAULT_OCI_CHAT_MAX_TOKENS", 4096)) TOGETHER_AI_4_B = int(os.getenv("TOGETHER_AI_4_B", 4)) TOGETHER_AI_8_B = int(os.getenv("TOGETHER_AI_8_B", 8)) TOGETHER_AI_21_B = int(os.getenv("TOGETHER_AI_21_B", 21)) diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index f050f9eea36..72d01c34ded 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -25,6 +25,7 @@ from typing import ( import httpx import litellm +from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS from litellm.litellm_core_utils.logging_utils import track_llm_api_timing from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.custom_httpx.http_handler import ( @@ -451,6 +452,13 @@ class OCIChatConfig(BaseConfig): elif oci_alias in optional_params: selected_params[target] = optional_params[oci_alias] # type: ignore[index] + # OCI's server-side default token cap is tiny (~20 tokens), so an + # omitted max_tokens silently truncates the response mid-string. Most + # callers never send a limit (MLflow judges among them), so inject a + # sane default when one is absent, mirroring litellm's Anthropic config. + if max_tokens_key not in selected_params: + selected_params[max_tokens_key] = DEFAULT_OCI_CHAT_MAX_TOKENS + # OCI expects uppercase reasoning levels (LOW/MEDIUM/HIGH/NONE); OpenAI # clients send lowercase. OpenAI's "disable" maps to OCI's "NONE". if "reasoningEffort" in selected_params: diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index e0911e1ef31..94c8b23def3 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -11,6 +11,7 @@ import litellm sys.path.insert(0, os.path.abspath("../../../../..")) from litellm import ModelResponse +from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS from litellm.llms.oci.chat.transformation import ( OCIChatConfig, OCIRequestWrapper, @@ -104,6 +105,7 @@ class TestOCIChatConfig: "chatRequest": { "apiFormat": "GENERIC", "isStream": False, + "maxTokens": DEFAULT_OCI_CHAT_MAX_TOKENS, "messages": [ { "role": "USER", @@ -956,6 +958,44 @@ class TestOCICohereParamMapping: assert result.get("temperature") == 0.5 +class TestOCIDefaultMaxTokens: + """Regression for OCI's tiny server-side token cap (~20 tokens), which + silently truncated responses mid-string whenever the caller omitted + max_tokens (MLflow judges never send it, so their JSON came back cut off). + transform_request injects DEFAULT_OCI_CHAT_MAX_TOKENS when no limit is + supplied, and leaves an explicit limit untouched.""" + + def _chat_request(self, model: str, optional_params: dict) -> dict: + config = OCIChatConfig() + body = config.transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={**BASE_OCI_PARAMS, **optional_params}, + litellm_params={}, + headers={}, + ) + return body["chatRequest"] + + @pytest.mark.parametrize( + "model", ["cohere.command-latest", "meta.llama-3.3-70b-instruct"] + ) + def test_default_injected_when_max_tokens_omitted(self, model): + chat_request = self._chat_request(model, {}) + assert chat_request["maxTokens"] == DEFAULT_OCI_CHAT_MAX_TOKENS + + @pytest.mark.parametrize( + "model", ["cohere.command-latest", "meta.llama-3.3-70b-instruct"] + ) + def test_explicit_max_tokens_not_overridden(self, model): + chat_request = self._chat_request(model, {"max_tokens": 256}) + assert chat_request["maxTokens"] == 256 + + def test_reasoning_model_defaults_max_completion_tokens(self): + chat_request = self._chat_request("openai.gpt-5", {}) + assert chat_request["maxCompletionTokens"] == DEFAULT_OCI_CHAT_MAX_TOKENS + assert "maxTokens" not in chat_request + + class TestOCIReasoningEffort: """ Reasoning-effort handling for GENERIC reasoning models: