diff --git a/docs/my-website/docs/providers/oci.md b/docs/my-website/docs/providers/oci.md index cea5d6824a0..ce6fe18dd6f 100644 --- a/docs/my-website/docs/providers/oci.md +++ b/docs/my-website/docs/providers/oci.md @@ -58,12 +58,11 @@ This method is an alternative when using the LiteLLM SDK on Oracle Cloud Infrast ## Usage - + Input the parameters obtained from the OCI signing key creation process into the `completion` function: ```python -import os from litellm import completion messages = [{"role": "user", "content": "Hey! how's it going?"}] @@ -86,7 +85,7 @@ print(response) ``` - + Use the OCI SDK `Signer` for authentication: @@ -153,7 +152,6 @@ For applications running on OCI compute instances: from litellm import completion from oci.auth.signers import InstancePrincipalsSecurityTokenSigner -oci.auth.signers.get_oke_workload_identity_resource_principal_signer() # Use instance principal authentication signer = InstancePrincipalsSecurityTokenSigner() @@ -168,7 +166,7 @@ response = completion( print(response) ``` -**Use workload identity authentication** +**Workload Identity Authentication** For applications running in Oracle Kubernetes Engine (OKE): @@ -176,7 +174,7 @@ For applications running in Oracle Kubernetes Engine (OKE): from litellm import completion from oci.auth.signers import get_oke_workload_identity_resource_principal_signer -# Use instance principal authentication +# Use workload identity authentication signer = get_oke_workload_identity_resource_principal_signer() messages = [{"role": "user", "content": "Hey! how's it going?"}] @@ -196,10 +194,9 @@ print(response) Just set `stream=True` when calling completion. - + ```python -import os from litellm import completion messages = [{"role": "user", "content": "Hey! how's it going?"}] @@ -224,7 +221,7 @@ for chunk in response: ``` - + ```python from litellm import completion @@ -258,7 +255,27 @@ for chunk in response: ### Using Cohere Models - + + +```python +from litellm import completion + +messages = [{"role": "user", "content": "Explain quantum computing"}] +response = completion( + model="oci/cohere.command-latest", + messages=messages, + oci_region="us-chicago-1", + oci_user=, + oci_fingerprint=, + oci_tenancy=, + oci_key=, + oci_compartment_id=, +) +print(response) +``` + + + ```python from litellm import completion @@ -283,19 +300,28 @@ print(response) ``` - + + +## Using Dedicated Endpoints + +OCI supports dedicated endpoints for hosting models. Use the `oci_serving_mode="DEDICATED"` parameter along with `oci_endpoint_id` to specify the endpoint ID. + + + ```python from litellm import completion -messages = [{"role": "user", "content": "Explain quantum computing"}] +messages = [{"role": "user", "content": "Hey! how's it going?"}] response = completion( - model="oci/cohere.command-latest", + model="oci/xai.grok-4", # Must match the model type hosted on the endpoint messages=messages, - oci_region="us-chicago-1", + oci_region=, oci_user=, oci_fingerprint=, oci_tenancy=, + oci_serving_mode="DEDICATED", + oci_endpoint_id="ocid1.generativeaiendpoint.oc1...", # Your dedicated endpoint OCID oci_key=, oci_compartment_id=, ) @@ -303,4 +329,69 @@ print(response) ``` - \ No newline at end of file + + +```python +from litellm import completion +from oci.signer import Signer + +signer = Signer( + tenancy="ocid1.tenancy.oc1..", + user="ocid1.user.oc1..", + fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx", + private_key_file_location="~/.oci/key.pem", +) + +messages = [{"role": "user", "content": "Hey! how's it going?"}] +response = completion( + model="oci/xai.grok-4", # Must match the model type hosted on the endpoint + messages=messages, + oci_signer=signer, + oci_region="us-chicago-1", + oci_serving_mode="DEDICATED", + oci_endpoint_id="ocid1.generativeaiendpoint.oc1...", # Your dedicated endpoint OCID + oci_compartment_id="", +) +print(response) +``` + + + + +**Important:** When using `oci_serving_mode="DEDICATED"`: +- The `model` parameter **must match the type of model hosted on your dedicated endpoint** (e.g., use `"oci/cohere.command-latest"` for Cohere models, `"oci/xai.grok-4"` for Grok models) +- The model name determines the API format and vendor-specific handling (Cohere vs Generic) +- The `oci_endpoint_id` parameter specifies your dedicated endpoint's OCID +- If `oci_endpoint_id` is not provided, the `model` parameter will be used as the endpoint ID (for backward compatibility) + +**Example with Cohere Dedicated Endpoint:** +```python +# For a dedicated endpoint hosting a Cohere model +response = completion( + model="oci/cohere.command-latest", # Use Cohere model name to get Cohere API format + messages=messages, + oci_region="us-chicago-1", + oci_user=, + oci_fingerprint=, + oci_tenancy=, + oci_serving_mode="DEDICATED", + oci_endpoint_id="ocid1.generativeaiendpoint.oc1...", # Your Cohere endpoint OCID + oci_key=, + oci_compartment_id=, +) +``` + +## Optional Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `oci_region` | string | `us-ashburn-1` | OCI region where the GenAI service is deployed | +| `oci_serving_mode` | string | `ON_DEMAND` | Service mode: `ON_DEMAND` for managed models or `DEDICATED` for dedicated endpoints | +| `oci_endpoint_id` | string | Same as `model` | (For DEDICATED mode) The OCID of your dedicated endpoint | +| `oci_compartment_id` | string | **Required** | The OCID of the OCI compartment containing your resources | +| `oci_user` | string | - | (Manual auth) The OCID of the OCI user | +| `oci_fingerprint` | string | - | (Manual auth) The fingerprint of the API signing key | +| `oci_tenancy` | string | - | (Manual auth) The OCID of your OCI tenancy | +| `oci_key` | string | - | (Manual auth) The private key content as a string | +| `oci_key_file` | string | - | (Manual auth) Path to the private key file | +| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication | \ No newline at end of file diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 167ba26bacb..f0e2db9a08b 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -765,9 +765,10 @@ class OCIChatConfig(BaseConfig): ) if oci_serving_mode == "DEDICATED": + oci_endpoint_id = optional_params.get("oci_endpoint_id", model) servingMode = OCIServingMode( servingType="DEDICATED", - endpointId=model, + endpointId=oci_endpoint_id, ) else: servingMode = OCIServingMode( diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 9a43a4220ca..3bd46b84e6c 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -150,6 +150,143 @@ class TestOCIChatConfig: assert transformed_request["chatRequest"]["tools"][0]["description"] == "Get the current weather in a given location" assert transformed_request["chatRequest"]["tools"][0]["parameters"] is not None + def test_transform_request_dedicated_mode_with_endpoint_id(self): + """ + Tests if a request with DEDICATED serving mode and explicit oci_endpoint_id is transformed correctly. + """ + config = OCIChatConfig() + test_endpoint_id = "ocid1.generativeaiendpoint.oc1.us-chicago-1.xxxxxx" + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "oci_serving_mode": "DEDICATED", + "oci_endpoint_id": test_endpoint_id, + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + expected_serving_mode = { + "servingType": "DEDICATED", + "endpointId": test_endpoint_id, + } + assert transformed_request["servingMode"] == expected_serving_mode + assert transformed_request["compartmentId"] == TEST_COMPARTMENT_ID + + def test_transform_request_dedicated_mode_without_endpoint_id(self): + """ + Tests if a request with DEDICATED serving mode falls back to model name when oci_endpoint_id is not provided. + """ + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "oci_serving_mode": "DEDICATED", + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + # Should use model name as endpoint ID when oci_endpoint_id is not provided + expected_serving_mode = { + "servingType": "DEDICATED", + "endpointId": TEST_MODEL_NAME, + } + assert transformed_request["servingMode"] == expected_serving_mode + assert transformed_request["compartmentId"] == TEST_COMPARTMENT_ID + + def test_transform_request_on_demand_mode(self): + """ + Tests if a request with ON_DEMAND serving mode uses modelId correctly. + """ + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "oci_serving_mode": "ON_DEMAND", + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + expected_serving_mode = { + "servingType": "ON_DEMAND", + "modelId": TEST_MODEL_NAME, + } + assert transformed_request["servingMode"] == expected_serving_mode + assert transformed_request["compartmentId"] == TEST_COMPARTMENT_ID + + def test_transform_request_invalid_serving_mode(self): + """ + Tests if an invalid serving mode raises an exception. + """ + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "oci_serving_mode": "INVALID_MODE", + } + + with pytest.raises(Exception) as excinfo: + config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "must be either 'ON_DEMAND' or 'DEDICATED'" in str(excinfo.value) + + def test_transform_request_dedicated_cohere_with_endpoint_id(self): + """ + Tests that Cohere vendor detection works correctly with DEDICATED mode and oci_endpoint_id. + This is critical because the model parameter determines the API format even when using oci_endpoint_id. + """ + config = OCIChatConfig() + cohere_model = "cohere.command-latest" + test_endpoint_id = "ocid1.generativeaiendpoint.oc1.us-chicago-1.xxxxxx" + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "oci_serving_mode": "DEDICATED", + "oci_endpoint_id": test_endpoint_id, + } + + messages = [ + {"role": "user", "content": "What is quantum computing?"}, + ] + + transformed_request = config.transform_request( + model=cohere_model, + messages=messages, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + # Verify DEDICATED mode with correct endpoint ID + assert transformed_request["servingMode"]["servingType"] == "DEDICATED" + assert transformed_request["servingMode"]["endpointId"] == test_endpoint_id + + # Verify Cohere API format is used (not GENERIC) + assert transformed_request["chatRequest"]["apiFormat"] == "COHERE" + + # Verify Cohere-specific request structure + assert "message" in transformed_request["chatRequest"] # Cohere uses "message" + assert "chatHistory" in transformed_request["chatRequest"] # Cohere uses "chatHistory" + assert "messages" not in transformed_request["chatRequest"] # Generic uses "messages" + + # Verify the message content + assert transformed_request["chatRequest"]["message"] == "What is quantum computing?" + def test_transform_response_simple_text(self): """ Tests if a simple text response is transformed correctly.