mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-28 01:32:17 +00:00
feat(oci): reasoning_effort + reasoning_tokens for OCI GenAI
Three small additions for OCI reasoning models, requested by users
testing the PR in production fork builds:
1. **reasoning_effort param mapping (GENERIC vendors).** OCI expects
uppercase levels ("LOW"/"MEDIUM"/"HIGH"/"NONE") on `reasoningEffort`,
but OpenAI-compatible clients send lowercase. Mapped + uppercased in
`_get_optional_params`. Marked unsupported on Cohere V1/V2 since OCI
Cohere has no reasoning models (avoids Pydantic validation failure
on CohereChatRequest).
2. **"disable" → "NONE" mapping.** OpenAI uses "disable" to turn off
reasoning; OCI uses "NONE". Without this, callers get a 400.
3. **reasoning_tokens propagated to Usage.** OCI returns
`completionTokensDetails.reasoningTokens` but it wasn't being passed
to LiteLLM's Usage object. Now flows through to
`Usage.completion_tokens_details.reasoning_tokens` so callers can
track reasoning token consumption for cost/observability.
Tests: 7 new unit tests in TestOCIReasoningEffort covering upper/lower
case, "disable"→"NONE", Cohere drop/raise paths, and reasoning_tokens
extraction (with and without completionTokensDetails). 5 new live
integration tests against xai.grok-3-mini in us-chicago-1 verifying the
full request/response loop end-to-end. Existing
test_transform_response_simple_text assertion that
completion_tokens_details was None has been updated to assert
reasoning_tokens flows through.
Verified live on xai.grok-3-mini: reasoning_effort=low → OCI accepts
"LOW", returns reasoningTokens=316 in usage. reasoning_effort=disable
→ OCI accepts "NONE". Full suite: 370/370 unit + 51/51 integration.
This commit is contained in:
Federico Kamelhar
parent
05cb73c16a
commit
4ce4af400f
4 changed files with 281 additions and 16 deletions
|
|
@ -324,10 +324,17 @@ def handle_generic_response(
|
|||
)
|
||||
|
||||
oci_usage = completion_response.chatResponse.usage
|
||||
reasoning_tokens: Optional[int] = None
|
||||
if (
|
||||
oci_usage.completionTokensDetails
|
||||
and oci_usage.completionTokensDetails.reasoningTokens
|
||||
):
|
||||
reasoning_tokens = oci_usage.completionTokensDetails.reasoningTokens
|
||||
model_response.usage = Usage( # type: ignore[attr-defined]
|
||||
prompt_tokens=oci_usage.promptTokens,
|
||||
completion_tokens=oci_usage.completionTokens or 0,
|
||||
total_tokens=oci_usage.totalTokens,
|
||||
reasoning_tokens=reasoning_tokens,
|
||||
)
|
||||
|
||||
return model_response
|
||||
|
|
|
|||
|
|
@ -151,18 +151,20 @@ class OCIChatConfig(BaseConfig):
|
|||
"response_format": "responseFormat",
|
||||
}
|
||||
|
||||
# Cohere param map differs from GENERIC in four ways:
|
||||
# Cohere param map differs from GENERIC in three ways:
|
||||
# - tool_choice is unsupported
|
||||
# - stop sequences key is "stopSequences" not "stop"
|
||||
# - n (numGenerations) is GENERIC-only
|
||||
# - reasoning_effort is GENERIC-only (CohereChatRequest v1 has no
|
||||
# reasoningEffort field; Cohere reasoning models like
|
||||
# command-a-reasoning use COHEREV2 which is a separate request type)
|
||||
self.openai_to_oci_cohere_param_map = {
|
||||
k: ("stopSequences" if k == "stop" else v)
|
||||
for k, v in self.openai_to_oci_generic_param_map.items()
|
||||
if k not in ("tool_choice", "max_retries", "n", "reasoning_effort")
|
||||
if k not in ("tool_choice", "max_retries", "n")
|
||||
}
|
||||
# OCI Cohere models are not reasoning models; mark reasoning_effort
|
||||
# explicitly unsupported so callers either get a clear error or have
|
||||
# the param dropped under drop_params, rather than silently passing
|
||||
# through and tripping Pydantic validation on CohereChatRequest.
|
||||
self.openai_to_oci_cohere_param_map["reasoning_effort"] = False
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> List[str]:
|
||||
param_map = (
|
||||
|
|
@ -321,6 +323,16 @@ class OCIChatConfig(BaseConfig):
|
|||
):
|
||||
selected_params[oci_value] = optional_params[oci_value] # type: ignore[index]
|
||||
|
||||
# OCI expects uppercase reasoning levels (LOW/MEDIUM/HIGH/NONE); OpenAI
|
||||
# clients send lowercase. OpenAI's "disable" maps to OCI's "NONE".
|
||||
if "reasoningEffort" in selected_params:
|
||||
effort = selected_params["reasoningEffort"]
|
||||
if isinstance(effort, str):
|
||||
normalized = effort.upper()
|
||||
if normalized == "DISABLE":
|
||||
normalized = "NONE"
|
||||
selected_params["reasoningEffort"] = normalized
|
||||
|
||||
if "tools" in selected_params:
|
||||
if vendor == OCIVendors.COHERE:
|
||||
selected_params["tools"] = adapt_tool_definitions_to_cohere_standard( # type: ignore[assignment]
|
||||
|
|
|
|||
|
|
@ -40,11 +40,32 @@ pytestmark = pytest.mark.skipif(
|
|||
)
|
||||
|
||||
|
||||
def _load_oci_config():
|
||||
"""Load OCI config from the profile named by ``OCI_CONFIG_PROFILE`` env var,
|
||||
falling back to ``[DEFAULT]``. Lets CI/local runs target a specific profile
|
||||
without needing a ``[DEFAULT]`` section in ``~/.oci/config``."""
|
||||
oci = pytest.importorskip("oci")
|
||||
profile = os.environ.get("OCI_CONFIG_PROFILE", "DEFAULT")
|
||||
return oci.config.from_file(profile_name=profile)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def oci_signer():
|
||||
"""Return an oci.Signer built from ~/.oci/config [DEFAULT]."""
|
||||
"""Return an oci.Signer (or SecurityTokenSigner for session-token profiles)
|
||||
built from ~/.oci/config — profile chosen via OCI_CONFIG_PROFILE."""
|
||||
oci = pytest.importorskip("oci")
|
||||
config = oci.config.from_file()
|
||||
config = _load_oci_config()
|
||||
|
||||
# Session-token profiles carry a `security_token_file` instead of a user
|
||||
# OCID; build the corresponding signer in that case.
|
||||
if "security_token_file" in config:
|
||||
with open(os.path.expanduser(config["security_token_file"])) as f:
|
||||
token = f.read().strip()
|
||||
private_key = oci.signer.load_private_key_from_file(
|
||||
config["key_file"], config.get("pass_phrase")
|
||||
)
|
||||
return oci.auth.signers.SecurityTokenSigner(token, private_key)
|
||||
|
||||
return oci.Signer(
|
||||
tenancy=config["tenancy"],
|
||||
user=config["user"],
|
||||
|
|
@ -56,8 +77,7 @@ def oci_signer():
|
|||
@pytest.fixture(scope="module")
|
||||
def oci_params(oci_signer) -> dict:
|
||||
"""Common OCI call-time parameters shared by all tests."""
|
||||
oci = pytest.importorskip("oci")
|
||||
config = oci.config.from_file()
|
||||
config = _load_oci_config()
|
||||
compartment_id = os.environ.get("OCI_TEST_COMPARTMENT_ID", config["tenancy"])
|
||||
region = os.environ.get("OCI_TEST_REGION", "us-chicago-1")
|
||||
return {
|
||||
|
|
@ -307,6 +327,65 @@ async def test_async_tool_use(m: _M, oci_params):
|
|||
_assert_tool_call(resp)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reasoning-effort tests (reasoning models only)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Reasoning model that accepts the `reasoningEffort` parameter on OCI.
|
||||
# Not every reasoning model does — xai.grok-4-fast-reasoning, for example,
|
||||
# rejects it with a 400.
|
||||
_REASONING_MODEL = "xai.grok-3-mini"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
|
||||
def test_reasoning_effort_lowercase_accepted(effort, oci_params):
|
||||
"""OpenAI clients send lowercase reasoning_effort; OCI requires uppercase.
|
||||
The transform layer should uppercase it transparently."""
|
||||
import litellm
|
||||
|
||||
resp = litellm.completion(
|
||||
model=f"oci/{_REASONING_MODEL}",
|
||||
messages=[{"role": "user", "content": "What is 2+2? One word."}],
|
||||
max_tokens=200,
|
||||
reasoning_effort=effort,
|
||||
**oci_params,
|
||||
)
|
||||
assert resp.choices[0].finish_reason is not None
|
||||
assert resp.usage.prompt_tokens > 0
|
||||
|
||||
|
||||
def test_reasoning_effort_disable_mapped_to_none(oci_params):
|
||||
"""OpenAI's 'disable' maps to OCI's 'NONE'. Without this mapping the
|
||||
request 400s."""
|
||||
import litellm
|
||||
|
||||
resp = litellm.completion(
|
||||
model=f"oci/{_REASONING_MODEL}",
|
||||
messages=[{"role": "user", "content": "What is 2+2? One word."}],
|
||||
max_tokens=200,
|
||||
reasoning_effort="disable",
|
||||
**oci_params,
|
||||
)
|
||||
assert resp.choices[0].finish_reason is not None
|
||||
|
||||
|
||||
def test_reasoning_tokens_in_usage(oci_params):
|
||||
"""OCI returns completionTokensDetails.reasoningTokens on reasoning models;
|
||||
LiteLLM should surface it on Usage.completion_tokens_details."""
|
||||
import litellm
|
||||
|
||||
resp = litellm.completion(
|
||||
model=f"oci/{_REASONING_MODEL}",
|
||||
messages=[{"role": "user", "content": "What is 2+2? One word."}],
|
||||
max_tokens=200,
|
||||
reasoning_effort="low",
|
||||
**oci_params,
|
||||
)
|
||||
assert resp.usage.completion_tokens_details is not None
|
||||
assert resp.usage.completion_tokens_details.reasoning_tokens is not None
|
||||
assert resp.usage.completion_tokens_details.reasoning_tokens > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Embedding tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -482,8 +561,8 @@ class TestOCIEnvVarCredentials:
|
|||
|
||||
def test_completion_via_env_vars(self, monkeypatch):
|
||||
"""Completion works when credentials are set through environment variables."""
|
||||
oci = pytest.importorskip("oci")
|
||||
config = oci.config.from_file()
|
||||
pytest.importorskip("oci")
|
||||
config = _load_oci_config()
|
||||
key_path = os.path.expanduser(config["key_file"])
|
||||
|
||||
with open(key_path) as f:
|
||||
|
|
@ -506,8 +585,8 @@ class TestOCIEnvVarCredentials:
|
|||
assert resp.choices[0].message.content is not None
|
||||
|
||||
def test_embedding_via_env_vars(self, monkeypatch):
|
||||
oci = pytest.importorskip("oci")
|
||||
config = oci.config.from_file()
|
||||
pytest.importorskip("oci")
|
||||
config = _load_oci_config()
|
||||
key_path = os.path.expanduser(config["key_file"])
|
||||
|
||||
with open(key_path) as f:
|
||||
|
|
|
|||
|
|
@ -490,9 +490,10 @@ class TestOCIChatConfig:
|
|||
assert result.usage.prompt_tokens == 10 # type: ignore
|
||||
assert result.usage.completion_tokens == 20 # type: ignore
|
||||
assert result.usage.total_tokens == 30 # type: ignore
|
||||
# These are not handled in the transformer, TBH no idea why they are here
|
||||
# but, for now, they seem to be always None
|
||||
assert result.usage.completion_tokens_details is None
|
||||
# reasoningTokens from OCI's completionTokensDetails is surfaced on
|
||||
# Usage.completion_tokens_details.reasoning_tokens.
|
||||
assert result.usage.completion_tokens_details is not None
|
||||
assert result.usage.completion_tokens_details.reasoning_tokens == 20
|
||||
assert result.usage.prompt_tokens_details is None
|
||||
|
||||
def test_transform_response_with_tool_calls(self):
|
||||
|
|
@ -955,6 +956,172 @@ class TestOCICohereParamMapping:
|
|||
assert result.get("temperature") == 0.5
|
||||
|
||||
|
||||
class TestOCIReasoningEffort:
|
||||
"""
|
||||
Reasoning-effort handling for GENERIC reasoning models:
|
||||
- OpenAI clients send lowercase ("low"/"medium"/"high"); OCI requires uppercase.
|
||||
- OpenAI's "disable" maps to OCI's "NONE".
|
||||
- Cohere on OCI has no reasoning models — the param is unsupported there.
|
||||
"""
|
||||
|
||||
def _build_chat_request(self, model: str, optional_params: dict) -> dict:
|
||||
"""Drive optional params through map → _get_optional_params and read
|
||||
the resulting chatRequest body via transform_request."""
|
||||
from litellm.llms.oci.chat.transformation import OCIChatConfig
|
||||
|
||||
config = OCIChatConfig()
|
||||
mapped = config.map_openai_params(
|
||||
non_default_params=optional_params,
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
body = config.transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={**BASE_OCI_PARAMS, **mapped},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
return body["chatRequest"]
|
||||
|
||||
def test_reasoning_effort_lowercase_uppercased(self):
|
||||
chat_request = self._build_chat_request(
|
||||
"xai.grok-4-fast-reasoning",
|
||||
{"reasoning_effort": "low"},
|
||||
)
|
||||
assert chat_request.get("reasoningEffort") == "LOW"
|
||||
|
||||
def test_reasoning_effort_disable_mapped_to_none(self):
|
||||
chat_request = self._build_chat_request(
|
||||
"xai.grok-4-fast-reasoning",
|
||||
{"reasoning_effort": "disable"},
|
||||
)
|
||||
assert chat_request.get("reasoningEffort") == "NONE"
|
||||
|
||||
def test_reasoning_effort_already_uppercase_preserved(self):
|
||||
chat_request = self._build_chat_request(
|
||||
"openai.gpt-5",
|
||||
{"reasoning_effort": "HIGH"},
|
||||
)
|
||||
assert chat_request.get("reasoningEffort") == "HIGH"
|
||||
|
||||
def test_reasoning_effort_unsupported_on_cohere_dropped(self):
|
||||
"""drop_params=True → silently drop reasoning_effort for Cohere."""
|
||||
from litellm.llms.oci.chat.transformation import OCIChatConfig
|
||||
|
||||
config = OCIChatConfig()
|
||||
result = config.map_openai_params(
|
||||
non_default_params={"reasoning_effort": "low"},
|
||||
optional_params={},
|
||||
model="cohere.command-latest",
|
||||
drop_params=True,
|
||||
)
|
||||
assert "reasoning_effort" not in result
|
||||
assert "reasoningEffort" not in result
|
||||
|
||||
def test_reasoning_effort_unsupported_on_cohere_raises(self):
|
||||
"""drop_params=False → raise rather than ship a payload Cohere will reject."""
|
||||
from litellm.llms.oci.chat.transformation import OCIChatConfig
|
||||
from litellm.llms.oci.common_utils import OCIError
|
||||
|
||||
config = OCIChatConfig()
|
||||
with pytest.raises(OCIError):
|
||||
config.map_openai_params(
|
||||
non_default_params={"reasoning_effort": "low"},
|
||||
optional_params={},
|
||||
model="cohere.command-latest",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
def test_reasoning_tokens_extracted_from_usage(self):
|
||||
"""OCI's completionTokensDetails.reasoningTokens flows into
|
||||
Usage.completion_tokens_details.reasoning_tokens."""
|
||||
from litellm.llms.oci.chat.generic import handle_generic_response
|
||||
|
||||
created_time = (
|
||||
datetime.datetime.now(datetime.timezone.utc)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
oci_response = {
|
||||
"modelId": "xai.grok-4-fast-reasoning",
|
||||
"modelVersion": "1.0",
|
||||
"chatResponse": {
|
||||
"apiFormat": "GENERIC",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "ASSISTANT",
|
||||
"content": [{"type": "TEXT", "text": "ok"}],
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
}
|
||||
],
|
||||
"timeCreated": created_time,
|
||||
"usage": {
|
||||
"promptTokens": 5,
|
||||
"completionTokens": 12,
|
||||
"totalTokens": 17,
|
||||
"completionTokensDetails": {"reasoningTokens": 7},
|
||||
},
|
||||
},
|
||||
}
|
||||
raw = httpx.Response(status_code=200, json=oci_response)
|
||||
result = handle_generic_response(
|
||||
json_data=oci_response,
|
||||
model="xai.grok-4-fast-reasoning",
|
||||
model_response=ModelResponse(),
|
||||
raw_response=raw,
|
||||
)
|
||||
usage = result.usage # type: ignore[attr-defined]
|
||||
assert usage.completion_tokens_details is not None
|
||||
assert usage.completion_tokens_details.reasoning_tokens == 7
|
||||
|
||||
def test_reasoning_tokens_absent_when_no_details(self):
|
||||
"""When OCI omits completionTokensDetails, Usage has no reasoning_tokens."""
|
||||
from litellm.llms.oci.chat.generic import handle_generic_response
|
||||
|
||||
created_time = (
|
||||
datetime.datetime.now(datetime.timezone.utc)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
oci_response = {
|
||||
"modelId": "xai.grok-4",
|
||||
"modelVersion": "1.0",
|
||||
"chatResponse": {
|
||||
"apiFormat": "GENERIC",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "ASSISTANT",
|
||||
"content": [{"type": "TEXT", "text": "ok"}],
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
}
|
||||
],
|
||||
"timeCreated": created_time,
|
||||
"usage": {
|
||||
"promptTokens": 5,
|
||||
"completionTokens": 12,
|
||||
"totalTokens": 17,
|
||||
},
|
||||
},
|
||||
}
|
||||
raw = httpx.Response(status_code=200, json=oci_response)
|
||||
result = handle_generic_response(
|
||||
json_data=oci_response,
|
||||
model="xai.grok-4",
|
||||
model_response=ModelResponse(),
|
||||
raw_response=raw,
|
||||
)
|
||||
usage = result.usage # type: ignore[attr-defined]
|
||||
assert usage.completion_tokens_details is None
|
||||
|
||||
|
||||
class TestOCIStreamingSignedBody:
|
||||
"""
|
||||
Unit test for Bug 1: sync and async streaming paths must use signed_json_body
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue