From 4ce4af400f0ed56205bbccd88958355de2caf844 Mon Sep 17 00:00:00 2001 From: Federico Kamelhar Date: Thu, 14 May 2026 10:50:17 -0400 Subject: [PATCH] feat(oci): reasoning_effort + reasoning_tokens for OCI GenAI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three small additions for OCI reasoning models, requested by users testing the PR in production fork builds: 1. **reasoning_effort param mapping (GENERIC vendors).** OCI expects uppercase levels ("LOW"/"MEDIUM"/"HIGH"/"NONE") on `reasoningEffort`, but OpenAI-compatible clients send lowercase. Mapped + uppercased in `_get_optional_params`. Marked unsupported on Cohere V1/V2 since OCI Cohere has no reasoning models (avoids Pydantic validation failure on CohereChatRequest). 2. **"disable" → "NONE" mapping.** OpenAI uses "disable" to turn off reasoning; OCI uses "NONE". Without this, callers get a 400. 3. **reasoning_tokens propagated to Usage.** OCI returns `completionTokensDetails.reasoningTokens` but it wasn't being passed to LiteLLM's Usage object. Now flows through to `Usage.completion_tokens_details.reasoning_tokens` so callers can track reasoning token consumption for cost/observability. Tests: 7 new unit tests in TestOCIReasoningEffort covering upper/lower case, "disable"→"NONE", Cohere drop/raise paths, and reasoning_tokens extraction (with and without completionTokensDetails). 5 new live integration tests against xai.grok-3-mini in us-chicago-1 verifying the full request/response loop end-to-end. Existing test_transform_response_simple_text assertion that completion_tokens_details was None has been updated to assert reasoning_tokens flows through. Verified live on xai.grok-3-mini: reasoning_effort=low → OCI accepts "LOW", returns reasoningTokens=316 in usage. reasoning_effort=disable → OCI accepts "NONE". Full suite: 370/370 unit + 51/51 integration. --- litellm/llms/oci/chat/generic.py | 7 + litellm/llms/oci/chat/transformation.py | 22 ++- tests/llm_translation/test_oci_integration.py | 95 +++++++++- .../oci/chat/test_oci_chat_transformation.py | 173 +++++++++++++++++- 4 files changed, 281 insertions(+), 16 deletions(-) diff --git a/litellm/llms/oci/chat/generic.py b/litellm/llms/oci/chat/generic.py index 42fcbcf7eb9..7938e8f3254 100644 --- a/litellm/llms/oci/chat/generic.py +++ b/litellm/llms/oci/chat/generic.py @@ -324,10 +324,17 @@ def handle_generic_response( ) oci_usage = completion_response.chatResponse.usage + reasoning_tokens: Optional[int] = None + if ( + oci_usage.completionTokensDetails + and oci_usage.completionTokensDetails.reasoningTokens + ): + reasoning_tokens = oci_usage.completionTokensDetails.reasoningTokens model_response.usage = Usage( # type: ignore[attr-defined] prompt_tokens=oci_usage.promptTokens, completion_tokens=oci_usage.completionTokens or 0, total_tokens=oci_usage.totalTokens, + reasoning_tokens=reasoning_tokens, ) return model_response diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 10baa1019bf..75a0ba0d0c2 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -151,18 +151,20 @@ class OCIChatConfig(BaseConfig): "response_format": "responseFormat", } - # Cohere param map differs from GENERIC in four ways: + # Cohere param map differs from GENERIC in three ways: # - tool_choice is unsupported # - stop sequences key is "stopSequences" not "stop" # - n (numGenerations) is GENERIC-only - # - reasoning_effort is GENERIC-only (CohereChatRequest v1 has no - # reasoningEffort field; Cohere reasoning models like - # command-a-reasoning use COHEREV2 which is a separate request type) self.openai_to_oci_cohere_param_map = { k: ("stopSequences" if k == "stop" else v) for k, v in self.openai_to_oci_generic_param_map.items() - if k not in ("tool_choice", "max_retries", "n", "reasoning_effort") + if k not in ("tool_choice", "max_retries", "n") } + # OCI Cohere models are not reasoning models; mark reasoning_effort + # explicitly unsupported so callers either get a clear error or have + # the param dropped under drop_params, rather than silently passing + # through and tripping Pydantic validation on CohereChatRequest. + self.openai_to_oci_cohere_param_map["reasoning_effort"] = False def get_supported_openai_params(self, model: str) -> List[str]: param_map = ( @@ -321,6 +323,16 @@ class OCIChatConfig(BaseConfig): ): selected_params[oci_value] = optional_params[oci_value] # type: ignore[index] + # OCI expects uppercase reasoning levels (LOW/MEDIUM/HIGH/NONE); OpenAI + # clients send lowercase. OpenAI's "disable" maps to OCI's "NONE". + if "reasoningEffort" in selected_params: + effort = selected_params["reasoningEffort"] + if isinstance(effort, str): + normalized = effort.upper() + if normalized == "DISABLE": + normalized = "NONE" + selected_params["reasoningEffort"] = normalized + if "tools" in selected_params: if vendor == OCIVendors.COHERE: selected_params["tools"] = adapt_tool_definitions_to_cohere_standard( # type: ignore[assignment] diff --git a/tests/llm_translation/test_oci_integration.py b/tests/llm_translation/test_oci_integration.py index 36510f53747..47d82ff9abe 100644 --- a/tests/llm_translation/test_oci_integration.py +++ b/tests/llm_translation/test_oci_integration.py @@ -40,11 +40,32 @@ pytestmark = pytest.mark.skipif( ) +def _load_oci_config(): + """Load OCI config from the profile named by ``OCI_CONFIG_PROFILE`` env var, + falling back to ``[DEFAULT]``. Lets CI/local runs target a specific profile + without needing a ``[DEFAULT]`` section in ``~/.oci/config``.""" + oci = pytest.importorskip("oci") + profile = os.environ.get("OCI_CONFIG_PROFILE", "DEFAULT") + return oci.config.from_file(profile_name=profile) + + @pytest.fixture(scope="module") def oci_signer(): - """Return an oci.Signer built from ~/.oci/config [DEFAULT].""" + """Return an oci.Signer (or SecurityTokenSigner for session-token profiles) + built from ~/.oci/config — profile chosen via OCI_CONFIG_PROFILE.""" oci = pytest.importorskip("oci") - config = oci.config.from_file() + config = _load_oci_config() + + # Session-token profiles carry a `security_token_file` instead of a user + # OCID; build the corresponding signer in that case. + if "security_token_file" in config: + with open(os.path.expanduser(config["security_token_file"])) as f: + token = f.read().strip() + private_key = oci.signer.load_private_key_from_file( + config["key_file"], config.get("pass_phrase") + ) + return oci.auth.signers.SecurityTokenSigner(token, private_key) + return oci.Signer( tenancy=config["tenancy"], user=config["user"], @@ -56,8 +77,7 @@ def oci_signer(): @pytest.fixture(scope="module") def oci_params(oci_signer) -> dict: """Common OCI call-time parameters shared by all tests.""" - oci = pytest.importorskip("oci") - config = oci.config.from_file() + config = _load_oci_config() compartment_id = os.environ.get("OCI_TEST_COMPARTMENT_ID", config["tenancy"]) region = os.environ.get("OCI_TEST_REGION", "us-chicago-1") return { @@ -307,6 +327,65 @@ async def test_async_tool_use(m: _M, oci_params): _assert_tool_call(resp) +# --------------------------------------------------------------------------- +# Reasoning-effort tests (reasoning models only) +# --------------------------------------------------------------------------- + +# Reasoning model that accepts the `reasoningEffort` parameter on OCI. +# Not every reasoning model does — xai.grok-4-fast-reasoning, for example, +# rejects it with a 400. +_REASONING_MODEL = "xai.grok-3-mini" + + +@pytest.mark.parametrize("effort", ["low", "medium", "high"]) +def test_reasoning_effort_lowercase_accepted(effort, oci_params): + """OpenAI clients send lowercase reasoning_effort; OCI requires uppercase. + The transform layer should uppercase it transparently.""" + import litellm + + resp = litellm.completion( + model=f"oci/{_REASONING_MODEL}", + messages=[{"role": "user", "content": "What is 2+2? One word."}], + max_tokens=200, + reasoning_effort=effort, + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + assert resp.usage.prompt_tokens > 0 + + +def test_reasoning_effort_disable_mapped_to_none(oci_params): + """OpenAI's 'disable' maps to OCI's 'NONE'. Without this mapping the + request 400s.""" + import litellm + + resp = litellm.completion( + model=f"oci/{_REASONING_MODEL}", + messages=[{"role": "user", "content": "What is 2+2? One word."}], + max_tokens=200, + reasoning_effort="disable", + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + + +def test_reasoning_tokens_in_usage(oci_params): + """OCI returns completionTokensDetails.reasoningTokens on reasoning models; + LiteLLM should surface it on Usage.completion_tokens_details.""" + import litellm + + resp = litellm.completion( + model=f"oci/{_REASONING_MODEL}", + messages=[{"role": "user", "content": "What is 2+2? One word."}], + max_tokens=200, + reasoning_effort="low", + **oci_params, + ) + assert resp.usage.completion_tokens_details is not None + assert resp.usage.completion_tokens_details.reasoning_tokens is not None + assert resp.usage.completion_tokens_details.reasoning_tokens > 0 + + # --------------------------------------------------------------------------- # Embedding tests # --------------------------------------------------------------------------- @@ -482,8 +561,8 @@ class TestOCIEnvVarCredentials: def test_completion_via_env_vars(self, monkeypatch): """Completion works when credentials are set through environment variables.""" - oci = pytest.importorskip("oci") - config = oci.config.from_file() + pytest.importorskip("oci") + config = _load_oci_config() key_path = os.path.expanduser(config["key_file"]) with open(key_path) as f: @@ -506,8 +585,8 @@ class TestOCIEnvVarCredentials: assert resp.choices[0].message.content is not None def test_embedding_via_env_vars(self, monkeypatch): - oci = pytest.importorskip("oci") - config = oci.config.from_file() + pytest.importorskip("oci") + config = _load_oci_config() key_path = os.path.expanduser(config["key_file"]) with open(key_path) as f: diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 8b866a8da44..21c06bd23db 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -490,9 +490,10 @@ class TestOCIChatConfig: assert result.usage.prompt_tokens == 10 # type: ignore assert result.usage.completion_tokens == 20 # type: ignore assert result.usage.total_tokens == 30 # type: ignore - # These are not handled in the transformer, TBH no idea why they are here - # but, for now, they seem to be always None - assert result.usage.completion_tokens_details is None + # reasoningTokens from OCI's completionTokensDetails is surfaced on + # Usage.completion_tokens_details.reasoning_tokens. + assert result.usage.completion_tokens_details is not None + assert result.usage.completion_tokens_details.reasoning_tokens == 20 assert result.usage.prompt_tokens_details is None def test_transform_response_with_tool_calls(self): @@ -955,6 +956,172 @@ class TestOCICohereParamMapping: assert result.get("temperature") == 0.5 +class TestOCIReasoningEffort: + """ + Reasoning-effort handling for GENERIC reasoning models: + - OpenAI clients send lowercase ("low"/"medium"/"high"); OCI requires uppercase. + - OpenAI's "disable" maps to OCI's "NONE". + - Cohere on OCI has no reasoning models — the param is unsupported there. + """ + + def _build_chat_request(self, model: str, optional_params: dict) -> dict: + """Drive optional params through map → _get_optional_params and read + the resulting chatRequest body via transform_request.""" + from litellm.llms.oci.chat.transformation import OCIChatConfig + + config = OCIChatConfig() + mapped = config.map_openai_params( + non_default_params=optional_params, + optional_params={}, + model=model, + drop_params=False, + ) + body = config.transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={**BASE_OCI_PARAMS, **mapped}, + litellm_params={}, + headers={}, + ) + return body["chatRequest"] + + def test_reasoning_effort_lowercase_uppercased(self): + chat_request = self._build_chat_request( + "xai.grok-4-fast-reasoning", + {"reasoning_effort": "low"}, + ) + assert chat_request.get("reasoningEffort") == "LOW" + + def test_reasoning_effort_disable_mapped_to_none(self): + chat_request = self._build_chat_request( + "xai.grok-4-fast-reasoning", + {"reasoning_effort": "disable"}, + ) + assert chat_request.get("reasoningEffort") == "NONE" + + def test_reasoning_effort_already_uppercase_preserved(self): + chat_request = self._build_chat_request( + "openai.gpt-5", + {"reasoning_effort": "HIGH"}, + ) + assert chat_request.get("reasoningEffort") == "HIGH" + + def test_reasoning_effort_unsupported_on_cohere_dropped(self): + """drop_params=True → silently drop reasoning_effort for Cohere.""" + from litellm.llms.oci.chat.transformation import OCIChatConfig + + config = OCIChatConfig() + result = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="cohere.command-latest", + drop_params=True, + ) + assert "reasoning_effort" not in result + assert "reasoningEffort" not in result + + def test_reasoning_effort_unsupported_on_cohere_raises(self): + """drop_params=False → raise rather than ship a payload Cohere will reject.""" + from litellm.llms.oci.chat.transformation import OCIChatConfig + from litellm.llms.oci.common_utils import OCIError + + config = OCIChatConfig() + with pytest.raises(OCIError): + config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="cohere.command-latest", + drop_params=False, + ) + + def test_reasoning_tokens_extracted_from_usage(self): + """OCI's completionTokensDetails.reasoningTokens flows into + Usage.completion_tokens_details.reasoning_tokens.""" + from litellm.llms.oci.chat.generic import handle_generic_response + + created_time = ( + datetime.datetime.now(datetime.timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + oci_response = { + "modelId": "xai.grok-4-fast-reasoning", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "choices": [ + { + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [{"type": "TEXT", "text": "ok"}], + }, + "finishReason": "STOP", + } + ], + "timeCreated": created_time, + "usage": { + "promptTokens": 5, + "completionTokens": 12, + "totalTokens": 17, + "completionTokensDetails": {"reasoningTokens": 7}, + }, + }, + } + raw = httpx.Response(status_code=200, json=oci_response) + result = handle_generic_response( + json_data=oci_response, + model="xai.grok-4-fast-reasoning", + model_response=ModelResponse(), + raw_response=raw, + ) + usage = result.usage # type: ignore[attr-defined] + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 7 + + def test_reasoning_tokens_absent_when_no_details(self): + """When OCI omits completionTokensDetails, Usage has no reasoning_tokens.""" + from litellm.llms.oci.chat.generic import handle_generic_response + + created_time = ( + datetime.datetime.now(datetime.timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + oci_response = { + "modelId": "xai.grok-4", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "choices": [ + { + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [{"type": "TEXT", "text": "ok"}], + }, + "finishReason": "STOP", + } + ], + "timeCreated": created_time, + "usage": { + "promptTokens": 5, + "completionTokens": 12, + "totalTokens": 17, + }, + }, + } + raw = httpx.Response(status_code=200, json=oci_response) + result = handle_generic_response( + json_data=oci_response, + model="xai.grok-4", + model_response=ModelResponse(), + raw_response=raw, + ) + usage = result.usage # type: ignore[attr-defined] + assert usage.completion_tokens_details is None + + class TestOCIStreamingSignedBody: """ Unit test for Bug 1: sync and async streaming paths must use signed_json_body