mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
fix(oci/router): catalog-driven maxCompletionTokens; generic blocked-deployment message
- Drive OCI maxCompletionTokens via supports_reasoning from the model catalog instead of a hardcoded openai.gpt-5 prefix. Add OCI GPT-5 family entries (gpt-5, gpt-5-mini, gpt-5-nano) with supports_reasoning: true. Gate the override to non-Cohere vendor so Cohere reasoning models keep maxTokens (Cohere endpoint does not accept maxCompletionTokens). - Replace proxy-specific 'Contact your proxy admin' phrasing in the four Router blocked-deployment ServiceUnavailableError messages with neutral SDK-appropriate text.
This commit is contained in:
parent
be7ca9a171
commit
1a9e03662e
5 changed files with 149 additions and 23 deletions
|
|
@ -69,6 +69,7 @@ from litellm.types.utils import (
|
|||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
)
|
||||
from litellm.utils import supports_reasoning
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -86,15 +87,16 @@ STREAMING_TIMEOUT = 60 * 5
|
|||
def _model_uses_max_completion_tokens(model: str) -> bool:
|
||||
"""Return True for OCI-hosted models that require ``maxCompletionTokens``.
|
||||
|
||||
GPT-5 family (and related reasoning-mode models) on OCI reject ``maxTokens``
|
||||
Reasoning models on OCI (e.g. the OpenAI GPT-5 family) reject ``maxTokens``
|
||||
with HTTP 400 and require ``maxCompletionTokens`` per OpenAI's reasoning-API
|
||||
convention. The model id we receive is the OCI model id, e.g.
|
||||
``openai.gpt-5``, ``openai.gpt-5-mini``, ``openai.gpt-5.5``.
|
||||
convention. Driven by ``supports_reasoning`` in
|
||||
``model_prices_and_context_window.json`` so new model families are picked
|
||||
up via a catalog update rather than a code change.
|
||||
"""
|
||||
name = (model or "").lower()
|
||||
if name.startswith("oci/"):
|
||||
name = name[4:]
|
||||
return name.startswith("openai.gpt-5") or name == "openai.gpt-5"
|
||||
if not model:
|
||||
return False
|
||||
name = model[4:] if model.lower().startswith("oci/") else model
|
||||
return supports_reasoning(model=name, custom_llm_provider="oci")
|
||||
|
||||
|
||||
def _iter_sse_events(stream: Iterator[str]) -> Iterator[str]:
|
||||
|
|
@ -327,14 +329,15 @@ class OCIChatConfig(BaseConfig):
|
|||
)
|
||||
selected_params: Dict = {}
|
||||
|
||||
# OpenAI GPT-5+ family on OCI rejects "maxTokens" and requires
|
||||
# "maxCompletionTokens" instead. Route to the correct field per OCI's
|
||||
# /20231130/Chat schema for these models. Verified against live OCI:
|
||||
# error reads `Invalid 'maxTokens': Unsupported parameter ... Use
|
||||
# 'maxCompletionTokens' instead.` on openai.gpt-5*, gpt-5.5, etc.
|
||||
# OpenAI reasoning models on OCI (e.g. GPT-5 family) reject "maxTokens"
|
||||
# and require "maxCompletionTokens" per OCI's /20231130/Chat schema.
|
||||
# Driven by the supports_reasoning flag in the model catalog. Cohere's
|
||||
# endpoint uses "maxTokens" regardless, so the override is GENERIC-only.
|
||||
max_tokens_key = (
|
||||
"maxCompletionTokens"
|
||||
if model and _model_uses_max_completion_tokens(model)
|
||||
if vendor != OCIVendors.COHERE
|
||||
and model
|
||||
and _model_uses_max_completion_tokens(model)
|
||||
else "maxTokens"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -25961,6 +25961,48 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"oci/openai.gpt-5": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": false,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"oci/openai.gpt-5-mini": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": false,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"oci/openai.gpt-5-nano": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": false,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"oci/cohere.embed-english-v3.0": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "oci",
|
||||
|
|
|
|||
|
|
@ -10164,7 +10164,7 @@ class Router:
|
|||
if isinstance(healthy_deployments, dict):
|
||||
if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
|
||||
raise litellm.ServiceUnavailableError(
|
||||
message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.",
|
||||
message=f"Model '{model}' is currently paused and cannot accept requests.",
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
|
@ -10428,7 +10428,7 @@ class Router:
|
|||
if isinstance(healthy_deployments, dict):
|
||||
if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
|
||||
raise litellm.ServiceUnavailableError(
|
||||
message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.",
|
||||
message=f"Model '{model}' is currently paused and cannot accept requests.",
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
|
@ -10602,7 +10602,7 @@ class Router:
|
|||
if isinstance(healthy_deployments, dict):
|
||||
if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
|
||||
raise litellm.ServiceUnavailableError(
|
||||
message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.",
|
||||
message=f"Model '{model}' is currently paused and cannot accept requests.",
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
|
@ -10759,7 +10759,7 @@ class Router:
|
|||
if isinstance(healthy_deployments, dict):
|
||||
if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
|
||||
raise litellm.ServiceUnavailableError(
|
||||
message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.",
|
||||
message=f"Model '{model}' is currently paused and cannot accept requests.",
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -25880,6 +25880,48 @@
|
|||
"supports_vision": true,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"oci/openai.gpt-5": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": false,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"oci/openai.gpt-5-mini": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": false,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"oci/openai.gpt-5-nano": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 400000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": false,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"oci/openai.gpt-oss-120b": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "oci",
|
||||
|
|
|
|||
|
|
@ -337,8 +337,35 @@ class TestOCIStreamWrapperChunkCreator:
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _register_oci_gpt5_in_catalog():
|
||||
"""Ensure OCI GPT-5 catalog entries exist for tests run against the live
|
||||
remote map. The bundled backup already contains them — this is a no-op for
|
||||
most environments and only patches in entries when the loaded map omits them.
|
||||
"""
|
||||
import litellm
|
||||
|
||||
needed = {
|
||||
"oci/openai.gpt-5",
|
||||
"oci/openai.gpt-5-mini",
|
||||
"oci/openai.gpt-5-nano",
|
||||
}
|
||||
added = []
|
||||
for key in needed:
|
||||
if key not in litellm.model_cost:
|
||||
litellm.model_cost[key] = {
|
||||
"litellm_provider": "oci",
|
||||
"mode": "chat",
|
||||
"supports_reasoning": True,
|
||||
}
|
||||
added.append(key)
|
||||
yield
|
||||
for key in added:
|
||||
litellm.model_cost.pop(key, None)
|
||||
|
||||
|
||||
class TestGpt5MaxCompletionTokens:
|
||||
def test_helper_detects_gpt5_family(self):
|
||||
def test_helper_detects_gpt5_family(self, _register_oci_gpt5_in_catalog):
|
||||
from litellm.llms.oci.chat.transformation import (
|
||||
_model_uses_max_completion_tokens,
|
||||
)
|
||||
|
|
@ -346,16 +373,16 @@ class TestGpt5MaxCompletionTokens:
|
|||
assert _model_uses_max_completion_tokens("openai.gpt-5") is True
|
||||
assert _model_uses_max_completion_tokens("openai.gpt-5-mini") is True
|
||||
assert _model_uses_max_completion_tokens("openai.gpt-5-nano") is True
|
||||
assert _model_uses_max_completion_tokens("openai.gpt-5.5") is True
|
||||
assert _model_uses_max_completion_tokens("oci/openai.gpt-5") is True
|
||||
|
||||
assert _model_uses_max_completion_tokens("openai.gpt-4o") is False
|
||||
assert _model_uses_max_completion_tokens("openai.gpt-4.1") is False
|
||||
assert _model_uses_max_completion_tokens("openai.gpt-oss-120b") is False
|
||||
assert _model_uses_max_completion_tokens("meta.llama-3.3-70b-instruct") is False
|
||||
assert _model_uses_max_completion_tokens("cohere.command-latest") is False
|
||||
assert _model_uses_max_completion_tokens("") is False
|
||||
|
||||
def test_gpt5_routes_max_tokens_to_max_completion_tokens(self):
|
||||
def test_gpt5_routes_max_tokens_to_max_completion_tokens(
|
||||
self, _register_oci_gpt5_in_catalog
|
||||
):
|
||||
from litellm.llms.oci.chat.transformation import OCIChatConfig, OCIVendors
|
||||
|
||||
cfg = OCIChatConfig()
|
||||
|
|
@ -369,7 +396,7 @@ class TestGpt5MaxCompletionTokens:
|
|||
|
||||
# 2. already pre-translated to OCI alias
|
||||
out_b = cfg._get_optional_params(
|
||||
OCIVendors.GENERIC, {"maxTokens": 64}, model="openai.gpt-5.5"
|
||||
OCIVendors.GENERIC, {"maxTokens": 64}, model="openai.gpt-5-mini"
|
||||
)
|
||||
assert out_b.get("maxCompletionTokens") == 64
|
||||
assert "maxTokens" not in out_b
|
||||
|
|
@ -386,6 +413,18 @@ class TestGpt5MaxCompletionTokens:
|
|||
assert out.get("maxTokens") == 64
|
||||
assert "maxCompletionTokens" not in out
|
||||
|
||||
def test_cohere_reasoning_model_keeps_max_tokens(self):
|
||||
from litellm.llms.oci.chat.transformation import OCIChatConfig, OCIVendors
|
||||
|
||||
cfg = OCIChatConfig()
|
||||
out = cfg._get_optional_params(
|
||||
OCIVendors.COHERE,
|
||||
{"max_tokens": 64},
|
||||
model="cohere.command-a-reasoning",
|
||||
)
|
||||
assert out.get("maxTokens") == 64
|
||||
assert "maxCompletionTokens" not in out
|
||||
|
||||
def test_payload_serializes_max_completion_tokens(self):
|
||||
from litellm.types.llms.oci import OCIChatRequestPayload
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue