From 41924f62e0fa6c0e88cd746c0f010cb94bac6c0e Mon Sep 17 00:00:00 2001 From: Fede Kamelhar Date: Thu, 11 Jun 2026 06:31:15 -0400 Subject: [PATCH] fix(oci): accept default n=1 on Cohere instead of hard-failing (#29705) * fix(oci): accept default n=1 on Cohere instead of hard-failing Cohere on OCI has no numGenerations field, so n was mapped to False and map_openai_params raised "param `n` is not supported on OCI" whenever a client sent n. But n=1 (and None) is the OpenAI default single-generation request, which every OCI model produces anyway, so standard clients that always send n=1 (such as the MLflow gateway) were rejected with a 500. Drop n=1/None silently for Cohere; only n>1 is genuinely unsupported and still raises (or drops under drop_params). Generic models are unaffected and keep numGenerations, including n>1. * docs(oci): explain why n is not advertised for Cohere despite tolerating n=1 * test(oci): gateway integration test for Cohere default n=1 Added to tests/integration/ (the real-network integration suite) reusing the existing OCI proxy harness, not tests/llm_translation/ which is mock-only. --------- Co-authored-by: Sameer Kankute --- litellm/llms/oci/chat/transformation.py | 12 +++++ .../integration/test_oci_proxy_integration.py | 23 ++++++++- .../oci/chat/test_oci_chat_transformation.py | 47 +++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 8812efc3bf6..d487d223bf4 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -333,6 +333,11 @@ class OCIChatConfig(BaseConfig): if get_vendor_from_model(model) == OCIVendors.COHERE else self.openai_to_oci_generic_param_map ) + # `n` is intentionally not advertised for Cohere even though n=1 is + # tolerated: Cohere has no numGenerations field, so n>1 cannot be + # honoured and advertising it would be misleading. Callers that gate on + # this list strip n=1 (a no-op, matching what map_openai_params does); + # callers that bypass it have n=1 dropped there. Both paths converge. return [key for key, value in param_map.items() if value] def map_openai_params( @@ -353,6 +358,13 @@ class OCIChatConfig(BaseConfig): for key, value in {**non_default_params, **optional_params}.items(): alias = param_map.get(key) if alias is False: + # n=1 (or None) is the OpenAI default: a single generation, which + # every OCI model produces anyway. Drop it silently so standard + # clients that always send n=1 (e.g. the MLflow gateway) are not + # rejected; only n>1 is genuinely unsupported on Cohere, which + # has no numGenerations field. + if key == "n" and (value is None or value == 1): + continue if drop_params or litellm.drop_params: continue raise OCIError( diff --git a/tests/integration/test_oci_proxy_integration.py b/tests/integration/test_oci_proxy_integration.py index a3ba844cb8e..b2ad4eb844c 100644 --- a/tests/integration/test_oci_proxy_integration.py +++ b/tests/integration/test_oci_proxy_integration.py @@ -260,7 +260,6 @@ def test_embedding_via_proxy(proxy_url: str) -> None: assert len(embedding) >= 64 assert all(isinstance(x, (int, float)) for x in embedding) - def test_model_list_advertises_oci_models(proxy_url: str) -> None: """The /v1/models registry advertises every OCI alias from the config.""" r = httpx.get( @@ -274,6 +273,26 @@ def test_model_list_advertises_oci_models(proxy_url: str) -> None: assert expected in advertised, f"{expected} missing from /v1/models: {advertised}" +def test_cohere_default_n_via_proxy(proxy_url: str) -> None: + """A Cohere request carrying the default n=1 succeeds through the gateway. + + Regression for the HTTP 500 ``param `n` is not supported on OCI`` that + rejected every client which always sends n=1 (e.g. the MLflow gateway), + since OCI Cohere has no numGenerations field. + """ + payload = {**_chat_payload("oci-cohere-command"), "n": 1} + r = httpx.post( + f"{proxy_url}/v1/chat/completions", + headers=_auth_headers(), + json=payload, + timeout=REQUEST_TIMEOUT_S, + ) + assert r.status_code == 200, f"n=1 -> {r.status_code}: {r.text}" + body = r.json() + assert body["object"] == "chat.completion" + assert body["choices"][0]["message"].get("content") is not None + + @pytest.mark.parametrize("model", ["oci-cohere-command", "oci-llama"]) def test_response_format_json_schema_via_proxy(proxy_url: str, model: str) -> None: """A response_format json_schema succeeds through the gateway for both a @@ -353,4 +372,4 @@ def test_omitted_max_tokens_not_truncated(proxy_url: str) -> None: assert content.strip(), f"empty content: {choice}" # The ~20-token server default truncated well before this; a complete # four-to-five sentence answer comfortably exceeds it. - assert body["usage"]["completion_tokens"] > 50, body["usage"] \ No newline at end of file + assert body["usage"]["completion_tokens"] > 50, body["usage"] diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index aec1d9ed1a9..49f9dfa416e 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -1464,6 +1464,53 @@ class TestOCIChatConfigErrorPaths: ) assert "audio" not in result + def test_map_openai_params_cohere_n_default_dropped(self): + """Cohere has no numGenerations field, but n=1 (and None) is the OpenAI + default single-generation request. It must be dropped silently rather + than raising, so standard clients that always send n=1 (e.g. the MLflow + gateway) are not rejected.""" + config = OCIChatConfig() + for n in (1, None): + result = config.map_openai_params( + non_default_params={"n": n}, + optional_params={}, + model="cohere.command-latest", + drop_params=False, + ) + assert "n" not in result and "numGenerations" not in result + + def test_map_openai_params_cohere_n_gt_1_raises_without_drop(self): + """n>1 is genuinely unsupported on Cohere and must raise without drop.""" + config = OCIChatConfig() + with pytest.raises(Exception, match="not supported on OCI"): + config.map_openai_params( + non_default_params={"n": 3}, + optional_params={}, + model="cohere.command-latest", + drop_params=False, + ) + + def test_map_openai_params_cohere_n_gt_1_dropped_with_drop(self): + config = OCIChatConfig() + result = config.map_openai_params( + non_default_params={"n": 3}, + optional_params={}, + model="cohere.command-latest", + drop_params=True, + ) + assert "n" not in result and "numGenerations" not in result + + def test_map_openai_params_generic_n_maps_to_num_generations(self): + """Generic models keep numGenerations, including n>1.""" + config = OCIChatConfig() + result = config.map_openai_params( + non_default_params={"n": 2}, + optional_params={}, + model=TEST_MODEL_NAME, + drop_params=False, + ) + assert result["numGenerations"] == 2 + def test_transform_request_tool_choice_string_mapped(self): config = OCIChatConfig() result = config.transform_request(