diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index b805cce4d7a..f6fe01ac28f 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -278,7 +278,8 @@ mcp_servers: url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes//invocations" transport: "http" auth_type: "aws_sigv4" - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_role_name: os.environ/AWS_ROLE_ARN # optional — IAM role to assume + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # optional — falls back to IAM role aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 aws_service_name: bedrock-agentcore diff --git a/docs/my-website/docs/mcp_aws_sigv4.md b/docs/my-website/docs/mcp_aws_sigv4.md index 9dc60bce06e..e556ad244f8 100644 --- a/docs/my-website/docs/mcp_aws_sigv4.md +++ b/docs/my-website/docs/mcp_aws_sigv4.md @@ -36,6 +36,8 @@ LiteLLM's `aws_sigv4` auth type handles this automatically: every outgoing MCP r | **AWS Access Key ID** | No | Falls back to boto3 credential chain if blank | | **AWS Secret Access Key** | No | Required if Access Key ID is provided | | **AWS Session Token** | No | Only needed for temporary STS credentials | +| **AWS Role ARN** | No | IAM role ARN for STS AssumeRole (e.g., `arn:aws:iam::123456789012:role/MyRole`). If set, LiteLLM assumes this role before signing | +| **AWS Session Name** | No | Session name for the AssumeRole call — appears in CloudTrail. Auto-generated if omitted | Once created, LiteLLM will sign every outgoing MCP request with SigV4. The server's tools appear automatically in the MCP Tools list. @@ -66,8 +68,8 @@ mcp_servers: url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes//invocations" transport: "http" auth_type: "aws_sigv4" - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID - aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_role_name: os.environ/AWS_ROLE_ARN # IAM role to assume (recommended) + aws_session_name: "litellm-prod" # optional — for CloudTrail auditing aws_region_name: "us-east-1" aws_service_name: "bedrock-agentcore" ``` @@ -128,6 +130,8 @@ curl http://localhost:4000/mcp-rest/tools/call \ | `aws_region_name` | Yes | AWS region (e.g., `us-east-1`) | | `aws_service_name` | No | AWS service name for signing. Defaults to `bedrock-agentcore` | | `aws_session_token` | No | AWS session token for temporary credentials. Supports `os.environ/VAR_NAME` | +| `aws_role_name` | No | IAM role ARN for STS AssumeRole. Supports `os.environ/VAR_NAME`. When set, LiteLLM calls `sts:AssumeRole` to get temporary credentials before signing | +| `aws_session_name` | No | Session name for the AssumeRole call (appears in CloudTrail). Auto-generated if omitted. Supports `os.environ/VAR_NAME` | ## How It Works @@ -157,6 +161,42 @@ mcp_servers: aws_service_name: "bedrock-agentcore" ``` +## Using IAM Role Assumption (AssumeRole) + +For production environments where your LiteLLM instance authenticates via an IAM role (e.g., EKS pod role, EC2 instance profile), you can configure `aws_role_name` to have LiteLLM call `sts:AssumeRole` before signing MCP requests: + +```yaml title="config.yaml with AssumeRole" showLineNumbers +mcp_servers: + my_agentcore_mcp: + url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes//invocations" + transport: "http" + auth_type: "aws_sigv4" + aws_role_name: "arn:aws:iam::123456789012:role/BedrockAgentCoreRole" + aws_session_name: "litellm-prod" # optional + aws_region_name: "us-east-1" + aws_service_name: "bedrock-agentcore" +``` + +LiteLLM uses the ambient credentials (pod role, instance profile, or env vars) to call `sts:AssumeRole`, then signs MCP requests with the assumed role's temporary credentials. + +You can also combine `aws_role_name` with explicit access keys — the keys are then used as the source identity for the AssumeRole call: + +```yaml title="config.yaml with AssumeRole + explicit source keys" showLineNumbers +mcp_servers: + my_agentcore_mcp: + url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes//invocations" + transport: "http" + auth_type: "aws_sigv4" + aws_role_name: os.environ/AWS_ROLE_ARN + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: "us-east-1" +``` + +:::tip +For most Kubernetes deployments, you only need `aws_role_name` and `aws_region_name` — the pod's IAM role provides the source credentials automatically. +::: + ## Troubleshooting ### 403 Forbidden from AWS @@ -166,6 +206,15 @@ mcp_servers: - Ensure `aws_service_name` is set to `bedrock-agentcore` - If using STS credentials, confirm `aws_session_token` is set and not expired +### AssumeRole AccessDenied + +If you get `AccessDenied` when using `aws_role_name`: + +- Verify the role ARN is correct +- Check that the trust policy on the target role allows your source identity to assume it +- If running on EKS, ensure the pod's service account is annotated with the correct IAM role +- Check CloudTrail for the failed `sts:AssumeRole` call to see the exact error + ### Health check errors on startup SigV4-authenticated MCP servers skip the standard health check on proxy startup. This is expected — the proxy will still sign requests correctly when tools are invoked. diff --git a/docs/my-website/docs/providers/oci.md b/docs/my-website/docs/providers/oci.md index ce6fe18dd6f..1d7a0a3d502 100644 --- a/docs/my-website/docs/providers/oci.md +++ b/docs/my-website/docs/providers/oci.md @@ -8,24 +8,54 @@ Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generativ ## Supported Models -### Meta Llama Models +### Chat / Text Generation + +#### Meta Llama Models - `meta.llama-4-maverick-17b-128e-instruct-fp8` - `meta.llama-4-scout-17b-16e-instruct` - `meta.llama-3.3-70b-instruct` +- `meta.llama-3.3-70b-instruct-fp8-dynamic` - `meta.llama-3.2-90b-vision-instruct` +- `meta.llama-3.2-11b-vision-instruct` - `meta.llama-3.1-405b-instruct` +- `meta.llama-3.1-70b-instruct` -### xAI Grok Models +#### xAI Grok Models +- `xai.grok-4.20` +- `xai.grok-4.20-multi-agent` - `xai.grok-4` +- `xai.grok-4-fast` +- `xai.grok-4.1-fast` - `xai.grok-3` - `xai.grok-3-fast` - `xai.grok-3-mini` - `xai.grok-3-mini-fast` +- `xai.grok-code-fast-1` -### Cohere Models +#### Cohere Models - `cohere.command-latest` - `cohere.command-a-03-2025` +- `cohere.command-a-reasoning-08-2025` +- `cohere.command-a-vision-07-2025` +- `cohere.command-a-translate-08-2025` - `cohere.command-plus-latest` +- `cohere.command-r-08-2024` +- `cohere.command-r-plus-08-2024` + +#### Google Gemini Models (via OCI) +- `google.gemini-2.5-pro` +- `google.gemini-2.5-flash` +- `google.gemini-2.5-flash-lite` + +### Embedding Models +- `cohere.embed-english-v3.0` (1024 dimensions) +- `cohere.embed-english-light-v3.0` (384 dimensions) +- `cohere.embed-multilingual-v3.0` (1024 dimensions) +- `cohere.embed-multilingual-light-v3.0` (384 dimensions) +- `cohere.embed-english-image-v3.0` (1024 dimensions, multimodal) +- `cohere.embed-english-light-image-v3.0` (384 dimensions, multimodal) +- `cohere.embed-multilingual-light-image-v3.0` (384 dimensions, multimodal) +- `cohere.embed-v4.0` (1536 dimensions, multimodal) ## Authentication @@ -394,4 +424,75 @@ response = completion( | `oci_tenancy` | string | - | (Manual auth) The OCID of your OCI tenancy | | `oci_key` | string | - | (Manual auth) The private key content as a string | | `oci_key_file` | string | - | (Manual auth) Path to the private key file | -| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication | \ No newline at end of file +| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication | + +## Embeddings + +LiteLLM supports OCI Generative AI embedding models. These models use the same authentication methods described above. + + + + +```python +from litellm import embedding + +response = embedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world", "Goodbye world"], + oci_region="us-ashburn-1", + oci_user=, + oci_fingerprint=, + oci_tenancy=, + oci_key=, + oci_compartment_id=, +) +print(response) +``` + + + + +```python +from litellm import embedding +from oci.signer import Signer + +signer = Signer( + tenancy="ocid1.tenancy.oc1..", + user="ocid1.user.oc1..", + fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx", + private_key_file_location="~/.oci/key.pem", +) + +response = embedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world", "Goodbye world"], + oci_signer=signer, + oci_region="us-ashburn-1", + oci_compartment_id="", +) +print(response) +``` + + + + +### Embedding Optional Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `input_type` | string | - | The type of input: `search_document`, `search_query`, `classification`, `clustering` | +| `truncate` | string | `END` | Truncation strategy when input exceeds max tokens: `END` or `START` | + +### Using Dedicated Embedding Endpoints + +```python +response = embedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world"], + oci_serving_mode="DEDICATED", + oci_endpoint_id="ocid1.generativeaiendpoint.oc1...", + oci_region="us-ashburn-1", + oci_compartment_id="", + # ... auth params +) +``` \ No newline at end of file diff --git a/litellm/__init__.py b/litellm/__init__.py index e45d926e8db..d4418c661a3 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1838,6 +1838,7 @@ if TYPE_CHECKING: ) from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig + from .llms.oci.embed.transformation import OCIEmbeddingConfig as OCIEmbeddingConfig from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig from .llms.lambda_ai.chat.transformation import ( diff --git a/litellm/_logging.py b/litellm/_logging.py index 65e6045b0b1..62283f6f65a 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -26,6 +26,12 @@ _REDACTED = "REDACTED" def _build_secret_patterns() -> re.Pattern: patterns: List[str] = [ + # ── PEM private key / certificate blocks ── + r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----", + # ── GCP OAuth2 access tokens (ya29.*) ── + r"\bya29\.[A-Za-z0-9_.~+/-]+", + # ── Credential %s formatting (space separator, no key= prefix) ── + r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+", # AWS access key IDs r"(?:AKIA|ASIA)[0-9A-Z]{16}", # AWS secrets / session tokens / access key IDs (key=value) @@ -46,7 +52,8 @@ def _build_secret_patterns() -> re.Pattern: # Google API keys r"AIza[0-9A-Za-z\-_]{35}", # Password / secret params (handles key=value and 'key': 'value') - r"\w*(?:password|passwd|client_secret|secret_key|_secret)" + # Word boundary prevents O(n^2) backtracking on long word-char runs. + r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)" r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", # Database connection string credentials (scheme://user:pass@host) r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)", @@ -56,13 +63,21 @@ def _build_secret_patterns() -> re.Pattern: # Catches secrets inside dicts/config dumps by matching on the KEY name # regardless of what the value looks like. # e.g. 'master_key': 'any-value-here', "database_url": "postgres://..." + # private_key with PEM-aware value capture + r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""", r"(?:master_key|database_url|db_url|connection_string|" - r"private_key|signing_key|encryption_key|" + r"signing_key|encryption_key|" r"auth_token|access_token|refresh_token|" r"slack_webhook_url|webhook_url|" r"database_connection_string|" r"huggingface_token|jwt_secret)" r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""", + # ── Raw JWTs (without Bearer prefix) ── + r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*", + # ── Azure SAS tokens in URLs ── + r"[?&]sig=[A-Za-z0-9%+/=]+", + # ── Full JSON service-account blobs (single-line and multi-line) ── + r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', ] return re.compile("|".join(patterns), re.IGNORECASE) @@ -74,6 +89,23 @@ def _redact_string(value: str) -> str: return _SECRET_RE.sub(_REDACTED, value) +def redact_secrets(value: str) -> str: + """Public API: redact known secret/credential patterns from an arbitrary string. + + Use this for code paths that bypass the logging system — e.g. Slack/Teams + alerting, HTTP error response bodies, or any other string that may contain + secrets and will be sent to an external sink. + + Not to be confused with redact_message_input_output_from_logging() in + litellm_core_utils/redact_messages.py, which redacts LLM prompt/response + content for privacy — this function redacts credential patterns (API keys, + PEM blocks, tokens, etc.) by shape. + """ + if not _ENABLE_SECRET_REDACTION: + return value + return _redact_string(value) + + class SecretRedactionFilter(logging.Filter): """Scrubs known secret/credential patterns from log records.""" @@ -441,7 +473,7 @@ def _enable_debugging(): def print_verbose(print_statement): try: if set_verbose: - print(print_statement) # noqa + print(redact_secrets(str(print_statement))) # noqa except Exception: pass diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py index d4c5f6a2985..46253bbcf78 100644 --- a/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py @@ -2,7 +2,7 @@ Pydantic AI provider configuration. """ -from typing import Any, AsyncIterator, Dict +from typing import Any, AsyncIterator, Dict, Optional from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig from litellm.a2a_protocol.providers.pydantic_ai_agents.handler import PydanticAIHandler @@ -20,10 +20,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig): self, request_id: str, params: Dict[str, Any], - api_base: str, + api_base: Optional[str] = None, **kwargs, ) -> Dict[str, Any]: """Handle non-streaming request to Pydantic AI agent.""" + if not api_base: + raise ValueError("api_base is required for Pydantic AI agents") return await PydanticAIHandler.handle_non_streaming( request_id=request_id, params=params, @@ -35,10 +37,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig): self, request_id: str, params: Dict[str, Any], - api_base: str, + api_base: Optional[str] = None, **kwargs, ) -> AsyncIterator[Dict[str, Any]]: """Handle streaming request with fake streaming.""" + if not api_base: + raise ValueError("api_base is required for Pydantic AI agents") async for chunk in PydanticAIHandler.handle_streaming( request_id=request_id, params=params, diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index a638a28aba3..1423617cac0 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -82,6 +82,8 @@ class MCPSigV4Auth(httpx.Auth): aws_session_token: Optional[str] = None, aws_region_name: Optional[str] = None, aws_service_name: Optional[str] = None, + aws_role_name: Optional[str] = None, + aws_session_name: Optional[str] = None, ): try: from botocore.credentials import Credentials @@ -97,7 +99,16 @@ class MCPSigV4Auth(httpx.Auth): # Note: os.environ/ prefixed values are already resolved by # ProxyConfig._check_for_os_environ_vars() at config load time. # Values arrive here as plain strings. - if aws_access_key_id and aws_secret_access_key: + if aws_role_name: + self.credentials = self._assume_role( + aws_role_name=aws_role_name, + aws_session_name=aws_session_name, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + aws_region_name=self.region_name, + ) + elif aws_access_key_id and aws_secret_access_key: self.credentials = Credentials( access_key=aws_access_key_id, secret_key=aws_secret_access_key, @@ -116,6 +127,43 @@ class MCPSigV4Auth(httpx.Auth): "(env vars, ~/.aws/credentials, instance profile)." ) + @staticmethod + def _assume_role( + aws_role_name: str, + aws_session_name: Optional[str], + aws_access_key_id: Optional[str], + aws_secret_access_key: Optional[str], + aws_session_token: Optional[str], + aws_region_name: str, + ): + """Call STS AssumeRole and return temporary credentials.""" + import boto3 + from botocore.credentials import Credentials + + session_name = ( + aws_session_name or f"litellm-mcp-{int(__import__('time').time())}" + ) + + sts_kwargs: dict = {"region_name": aws_region_name} + if aws_access_key_id and aws_secret_access_key: + sts_kwargs["aws_access_key_id"] = aws_access_key_id + sts_kwargs["aws_secret_access_key"] = aws_secret_access_key + if aws_session_token: + sts_kwargs["aws_session_token"] = aws_session_token + + sts_client = boto3.client("sts", **sts_kwargs) + sts_response = sts_client.assume_role( + RoleArn=aws_role_name, + RoleSessionName=session_name, + ) + + sts_creds = sts_response["Credentials"] + return Credentials( + access_key=sts_creds["AccessKeyId"], + secret_key=sts_creds["SecretAccessKey"], + token=sts_creds["SessionToken"], + ) + def auth_flow( self, request: httpx.Request ) -> Generator[httpx.Request, httpx.Response, None]: diff --git a/litellm/integrations/azure_storage/azure_storage.py b/litellm/integrations/azure_storage/azure_storage.py index 6fc7b9c1048..50c1cd9d989 100644 --- a/litellm/integrations/azure_storage/azure_storage.py +++ b/litellm/integrations/azure_storage/azure_storage.py @@ -275,12 +275,11 @@ class AzureBlobStorageLogger(CustomBatchLogger): """ Gets Azure AD token to use for Azure Storage API requests """ - verbose_logger.debug("Getting Azure AD Token from Azure Storage") verbose_logger.debug( - "tenant_id %s, client_id %s, client_secret %s", + "Getting Azure AD Token from Azure Storage, tenant_id=%s, client_id=%s, client_secret=[set=%s]", tenant_id, client_id, - client_secret, + client_secret is not None, ) if tenant_id is None: raise ValueError( diff --git a/litellm/integrations/gcs_bucket/gcs_bucket_base.py b/litellm/integrations/gcs_bucket/gcs_bucket_base.py index 923f613291f..0089e54b1c2 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket_base.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket_base.py @@ -70,7 +70,9 @@ class GCSBucketBase(CustomBatchLogger): custom_llm_provider="vertex_ai", api_base=None, ) - verbose_logger.debug("constructed auth_header %s", auth_header) + verbose_logger.debug( + "constructed auth_header [set=%s]", auth_header is not None + ) headers = { "Authorization": f"Bearer {auth_header}", # auth_header "Content-Type": "application/json", @@ -106,7 +108,9 @@ class GCSBucketBase(CustomBatchLogger): custom_llm_provider="vertex_ai", api_base=None, ) - verbose_logger.debug("constructed auth_header %s", auth_header) + verbose_logger.debug( + "constructed auth_header [set=%s]", auth_header is not None + ) headers = { "Authorization": f"Bearer {auth_header}", # auth_header "Content-Type": "application/json", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 0a2f07bcb25..95bcd4d7186 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -202,8 +202,8 @@ def get_llm_provider( # noqa: PLR0915 ) if dynamic_api_key is not None and not isinstance(dynamic_api_key, str): raise Exception( - "dynamic_api_key needs to be a string. dynamic_api_key={}".format( - dynamic_api_key + "dynamic_api_key needs to be a string. Got type={}".format( + type(dynamic_api_key).__name__ ) ) return model, custom_llm_provider, dynamic_api_key, api_base diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index fcdb3eca23a..4fc1ae960b8 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -101,17 +101,15 @@ def get_azure_ad_token_from_entra_id( _client_secret = client_secret verbose_logger.debug( - "tenant_id %s, client_id %s, client_secret %s", + "tenant_id=%s, client_id=%s, client_secret=[set=%s]", _tenant_id, _client_id, - _client_secret, + _client_secret is not None, ) if _tenant_id is None or _client_id is None or _client_secret is None: raise ValueError("tenant_id, client_id, and client_secret must be provided") credential = ClientSecretCredential(_tenant_id, _client_id, _client_secret) - verbose_logger.debug("credential %s", credential) - token_provider = get_bearer_token_provider(credential, scope) verbose_logger.debug("token_provider %s", token_provider) @@ -140,10 +138,10 @@ def get_azure_ad_token_from_username_password( from azure.identity import UsernamePasswordCredential, get_bearer_token_provider verbose_logger.debug( - "client_id %s, azure_username %s, azure_password %s", + "client_id=%s, azure_username=[set=%s], azure_password=[set=%s]", client_id, - azure_username, - azure_password, + azure_username is not None, + azure_password is not None, ) credential = UsernamePasswordCredential( client_id=client_id, @@ -151,8 +149,6 @@ def get_azure_ad_token_from_username_password( password=azure_password, ) - verbose_logger.debug("credential %s", credential) - token_provider = get_bearer_token_provider(credential, scope) verbose_logger.debug("token_provider %s", token_provider) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index b159d62367d..4157fac53b8 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -156,24 +156,24 @@ class BaseAWSLLM: verbose_logger.debug( "in get credentials\n" - "aws_access_key_id=%s\n" - "aws_secret_access_key=%s\n" - "aws_session_token=%s\n" + "aws_access_key_id=[set=%s]\n" + "aws_secret_access_key=[set=%s]\n" + "aws_session_token=[set=%s]\n" "aws_region_name=%s\n" "aws_session_name=%s\n" "aws_profile_name=%s\n" "aws_role_name=%s\n" - "aws_web_identity_token=%s\n" + "aws_web_identity_token=[set=%s]\n" "aws_sts_endpoint=%s\n" "aws_external_id=%s", - aws_access_key_id, - aws_secret_access_key, - aws_session_token, + aws_access_key_id is not None, + aws_secret_access_key is not None, + aws_session_token is not None, aws_region_name, aws_session_name, aws_profile_name, aws_role_name, - aws_web_identity_token, + aws_web_identity_token is not None, aws_sts_endpoint, aws_external_id, ) diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index b1af7ed2ec3..79cd1c00606 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -174,10 +174,15 @@ def load_private_key_from_file(file_path: str): def get_vendor_from_model(model: str) -> OCIVendors: """ Extracts the vendor from the model name. + + OCI GenAI API uses two apiFormat values: + - "COHERE" for Cohere models (command-r, command-a, etc.) + - "GENERIC" for all other models (Meta Llama, xAI Grok, Google Gemini, etc.) + Args: - model (str): The model name. + model (str): The model name (e.g., "cohere.command-a-03-2025", "meta.llama-3.3-70b-instruct"). Returns: - str: The vendor name. + OCIVendors: The vendor enum value. """ vendor = model.split(".")[0].lower() if vendor == "cohere": diff --git a/litellm/llms/oci/embed/__init__.py b/litellm/llms/oci/embed/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/oci/embed/transformation.py b/litellm/llms/oci/embed/transformation.py new file mode 100644 index 00000000000..1dcd8c5213c --- /dev/null +++ b/litellm/llms/oci/embed/transformation.py @@ -0,0 +1,347 @@ +""" +OCI Generative AI Embedding Configuration + +Supports embedding models available on Oracle Cloud Infrastructure Generative AI service. +Uses the same authentication mechanisms as OCI chat (manual signing or OCI SDK Signer). + +Supported models: +- cohere.embed-english-v3.0 +- cohere.embed-english-light-v3.0 +- cohere.embed-multilingual-v3.0 +- cohere.embed-multilingual-light-v3.0 +- cohere.embed-english-image-v3.0 +- cohere.embed-english-light-image-v3.0 +- cohere.embed-multilingual-light-image-v3.0 +- cohere.embed-v4.0 + +Reference: https://docs.oracle.com/en-us/iaas/api/#/en/generative-ai-inference/latest/EmbedTextResult/EmbedText +""" + +from typing import Any, Dict, List, Optional, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig +from litellm.llms.oci.chat.transformation import OCIChatConfig +from litellm.llms.oci.common_utils import OCIError +from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues +from litellm.types.utils import EmbeddingResponse, Usage + +# Input type mapping from OpenAI conventions to OCI/Cohere conventions +_INPUT_TYPE_MAP = { + "search_document": "SEARCH_DOCUMENT", + "search_query": "SEARCH_QUERY", + "classification": "CLASSIFICATION", + "clustering": "CLUSTERING", +} + + +class OCIEmbeddingConfig(BaseEmbeddingConfig): + """ + Configuration for OCI Generative AI Embedding API. + + The OCI embedding endpoint uses the Cohere embed models hosted on OCI. + Authentication is handled via OCI request signing (manual credentials or OCI SDK Signer). + + Usage: + ```python + import litellm + + response = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world", "Goodbye world"], + oci_compartment_id="ocid1.compartment.oc1..xxx", + oci_region="us-ashburn-1", + oci_user="ocid1.user.oc1..xxx", + oci_fingerprint="xx:xx:xx:xx", + oci_tenancy="ocid1.tenancy.oc1..xxx", + oci_key_file="~/.oci/key.pem", + ) + ``` + """ + + def __init__(self) -> None: + # We reuse OCIChatConfig for signing logic + self._chat_config = OCIChatConfig() + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + if api_base: + return api_base + + oci_region = optional_params.get("oci_region", "us-ashburn-1") + return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/embedText" + + def get_supported_openai_params(self, model: str) -> list: + return [ + "dimensions", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + # Note: OCI Cohere embed does not support custom dimensions natively, + # but we pass it through in case future models support it + if "dimensions" in non_default_params: + optional_params["dimensions"] = non_default_params["dimensions"] + return optional_params + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate OCI credentials for embedding requests. + Supports both OCI SDK Signer and manual credential signing. + """ + oci_signer = optional_params.get("oci_signer") + oci_region = optional_params.get("oci_region", "us-ashburn-1") + + api_base = ( + api_base + or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" + ) + + if oci_signer is None: + oci_user = optional_params.get("oci_user") + oci_fingerprint = optional_params.get("oci_fingerprint") + oci_tenancy = optional_params.get("oci_tenancy") + oci_key = optional_params.get("oci_key") + oci_key_file = optional_params.get("oci_key_file") + oci_compartment_id = optional_params.get("oci_compartment_id") + + if ( + not oci_user + or not oci_fingerprint + or not oci_tenancy + or not (oci_key or oci_key_file) + or not oci_compartment_id + ): + raise Exception( + "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, oci_compartment_id " + "and at least one of oci_key or oci_key_file. " + "Alternatively, provide an oci_signer object from the OCI SDK." + ) + + from litellm.llms.custom_httpx.http_handler import version + + headers.update( + { + "content-type": "application/json", + "user-agent": f"litellm/{version}", + } + ) + + return headers + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ): + """Delegate to OCIChatConfig's signing logic.""" + return self._chat_config.sign_request( + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + api_base: Optional[str] = None, + ) -> dict: + """ + Transform the embedding request to OCI format. + + OCI embedText API expects: + { + "compartmentId": "...", + "servingMode": {"servingType": "ON_DEMAND", "modelId": "..."}, + "inputs": ["text1", "text2"], + "truncate": "END", + "inputType": "SEARCH_DOCUMENT" + } + """ + oci_compartment_id = optional_params.get("oci_compartment_id") + if not oci_compartment_id: + raise Exception( + "kwarg `oci_compartment_id` is required for OCI embedding requests" + ) + + # Build serving mode + oci_serving_mode = optional_params.get("oci_serving_mode", "ON_DEMAND") + if oci_serving_mode == "DEDICATED": + oci_endpoint_id = optional_params.get("oci_endpoint_id", model) + serving_mode = { + "servingType": "DEDICATED", + "endpointId": oci_endpoint_id, + } + else: + serving_mode = { + "servingType": "ON_DEMAND", + "modelId": model, + } + + # Normalize input to list of strings + if isinstance(input, str): + inputs = [input] + elif isinstance(input, list): + inputs = [] + for item in input: + if isinstance(item, str): + inputs.append(item) + elif isinstance(item, list): + raise ValueError( + "OCI embedding does not support token-array inputs. " + "Please convert token lists to strings before calling embedding()." + ) + else: + inputs.append(str(item)) + else: + inputs = [str(input)] + + # Build request data — OCI embedText API expects inputs, truncate, + # and inputType at the top level alongside compartmentId and servingMode + request_data: Dict[str, Any] = { + "compartmentId": oci_compartment_id, + "servingMode": serving_mode, + "inputs": inputs, + "truncate": optional_params.get("truncate", "END"), + } + + # Map input_type if provided + input_type = optional_params.get("input_type") + if input_type: + mapped_type = _INPUT_TYPE_MAP.get(input_type.lower(), input_type.upper()) + request_data["inputType"] = mapped_type + + # Sign the request using the same URL the HTTP handler will POST to + signing_url = self.get_complete_url( + api_base=api_base, + api_key=None, + model=model, + optional_params=optional_params, + litellm_params={}, + ) + + signed_headers, body = self.sign_request( + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=signing_url, + ) + headers.update(signed_headers) + + return request_data + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str] = None, + request_data: dict = {}, + optional_params: dict = {}, + litellm_params: dict = {}, + ) -> EmbeddingResponse: + """ + Transform OCI embedding response to standard EmbeddingResponse format. + + OCI response format: + { + "embeddings": [[0.1, 0.2, ...], [0.3, 0.4, ...]], + "modelId": "cohere.embed-english-v3.0", + "modelVersion": "3.0", + "inputTextTokenCounts": [5, 4] + } + """ + if raw_response.status_code != 200: + raise OCIError( + message=raw_response.text, + status_code=raw_response.status_code, + ) + + try: + raw_response_json = raw_response.json() + except Exception: + raise OCIError( + message=raw_response.text, + status_code=raw_response.status_code, + ) + + embeddings = raw_response_json.get("embeddings", []) + model_id = raw_response_json.get("modelId", model) + + # Build response data in OpenAI format + embedding_data = [] + for idx, embedding in enumerate(embeddings): + embedding_data.append( + { + "object": "embedding", + "index": idx, + "embedding": embedding, + } + ) + + model_response.model = model_id + model_response.data = embedding_data + model_response.object = "list" + + # Calculate token usage + input_token_counts = raw_response_json.get("inputTextTokenCounts", []) + total_tokens = sum(input_token_counts) if input_token_counts else 0 + + usage = Usage( + prompt_tokens=total_tokens, + total_tokens=total_tokens, + ) + model_response.usage = usage + + return model_response + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: Union[dict, httpx.Headers], + ) -> BaseLLMException: + return OCIError( + message=error_message, + status_code=status_code, + headers=headers if isinstance(headers, httpx.Headers) else None, + ) diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 1a29ba82eac..cdabac27af7 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -81,26 +81,26 @@ class VertexBase: ) -> Tuple[Any, str]: if credentials is not None: if isinstance(credentials, str): + _is_path = os.path.exists( + credentials + ) # credentials is from server config (litellm_params), not user input verbose_logger.debug( - "Vertex: Loading vertex credentials from %s", credentials - ) - verbose_logger.debug( - "Vertex: checking if credentials is a valid path, os.path.exists(%s)=%s, current dir %s", - credentials, - os.path.exists(credentials), + "Vertex: Loading vertex credentials, is_file_path=%s, current dir %s", + _is_path, os.getcwd(), ) try: - if os.path.exists(credentials): - json_obj = json.load(open(credentials)) + if _is_path: + with open(credentials) as f: + json_obj = json.load(f) else: json_obj = json.loads(credentials) - except Exception: + except Exception as e: raise Exception( - "Unable to load vertex credentials from environment. Got={}".format( - credentials - ) + "Unable to load vertex credentials from environment. " + "Ensure the JSON is valid (check for unescaped newlines in private_key). " + "Parse error: {}".format(type(e).__name__) ) elif isinstance(credentials, dict): json_obj = credentials @@ -668,8 +668,8 @@ class VertexBase: ## VALIDATION STEP if _credentials.token is None or not isinstance(_credentials.token, str): raise ValueError( - "Could not resolve credentials token. Got None or non-string token - {}".format( - _credentials.token + "Could not resolve credentials token. Got None or non-string token (type={})".format( + type(_credentials.token).__name__ ) ) diff --git a/litellm/main.py b/litellm/main.py index eace9c630ba..cbedd1735c7 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5668,6 +5668,22 @@ def embedding( # noqa: PLR0915 aembedding=aembedding, litellm_params={}, ) + elif custom_llm_provider == "oci": + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + litellm_params=litellm_params_dict, + headers=headers, + ) elif custom_llm_provider in litellm._custom_providers: custom_handler: Optional[CustomLLM] = None for item in litellm.custom_provider_map: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4e1c7f4ac80..8351f084f28 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23803,7 +23803,8 @@ "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_vision": true }, "oci/meta.llama-3.3-70b-instruct": { "input_cost_per_token": 7.2e-07, @@ -23937,6 +23938,287 @@ "supports_function_calling": true, "supports_response_schema": false }, + "oci/cohere.command-a-reasoning-08-2025": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/cohere.command-a-vision-07-2025": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true + }, + "oci/cohere.command-a-translate-08-2025": { + "input_cost_per_token": 9e-08, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 9e-08, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": false, + "supports_response_schema": false + }, + "oci/cohere.command-r-08-2024": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/cohere.command-r-plus-08-2024": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/meta.llama-3.2-11b-vision-instruct": { + "input_cost_per_token": 2e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true + }, + "oci/meta.llama-3.1-70b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/meta.llama-3.3-70b-instruct-fp8-dynamic": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4-fast": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.1-fast": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.20": { + "input_cost_per_token": 3e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.20-multi-agent": { + "input_cost_per_token": 3e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-code-fast-1": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/google.gemini-2.5-pro": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/google.gemini-2.5-flash": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/google.gemini-2.5-flash-lite": { + "input_cost_per_token": 7.5e-08, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/cohere.embed-english-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-english-light-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-multilingual-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-multilingual-light-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-english-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-english-light-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-multilingual-light-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-v4.0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, "ollama/codegeex4": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", @@ -30305,6 +30587,27 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "vertex_ai/claude-haiku-4-5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_native_streaming": true, + "supports_vision": true + }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, "cache_read_input_token_cost": 1e-07, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 1e9d5c5a529..cfbc2c437b0 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -346,6 +346,8 @@ class MCPServerManager: aws_session_token=server_config.get("aws_session_token", None), aws_region_name=server_config.get("aws_region_name", None), aws_service_name=server_config.get("aws_service_name", None), + aws_role_name=server_config.get("aws_role_name", None), + aws_session_name=server_config.get("aws_session_name", None), ) self.config_mcp_servers[server_id] = new_server @@ -686,6 +688,8 @@ class MCPServerManager: aws_session_token=aws_creds.get("aws_session_token"), aws_region_name=aws_creds.get("aws_region_name"), aws_service_name=aws_creds.get("aws_service_name"), + aws_role_name=aws_creds.get("aws_role_name"), + aws_session_name=aws_creds.get("aws_session_name"), ) return new_server @@ -1011,6 +1015,8 @@ class MCPServerManager: aws_session_token=server.aws_session_token, aws_region_name=server.aws_region_name, aws_service_name=server.aws_service_name, + aws_role_name=server.aws_role_name, + aws_session_name=server.aws_session_name, ) return MCPClient( @@ -1571,6 +1577,8 @@ class MCPServerManager: ), "aws_region_name": credentials_dict.get("aws_region_name"), "aws_service_name": credentials_dict.get("aws_service_name"), + "aws_role_name": credentials_dict.get("aws_role_name"), + "aws_session_name": credentials_dict.get("aws_session_name"), } def _extract_scopes(self, scopes_value: Any) -> Optional[List[str]]: diff --git a/litellm/proxy/auth/oauth2_check.py b/litellm/proxy/auth/oauth2_check.py index bb00141ad0a..10b1759b77e 100644 --- a/litellm/proxy/auth/oauth2_check.py +++ b/litellm/proxy/auth/oauth2_check.py @@ -136,7 +136,9 @@ class Oauth2Handler: + CommonProxyErrors.not_premium_user.value ) - verbose_proxy_logger.debug("Oauth2 token validation for token=%s", token) + verbose_proxy_logger.debug( + "Oauth2 token validation for token=[set=%s]", token is not None + ) # Get the token info endpoint from environment variable token_info_endpoint = os.getenv("OAUTH_TOKEN_INFO_ENDPOINT") diff --git a/litellm/proxy/auth/oauth2_proxy_hook.py b/litellm/proxy/auth/oauth2_proxy_hook.py index 7e517092b8a..0dc696bc455 100644 --- a/litellm/proxy/auth/oauth2_proxy_hook.py +++ b/litellm/proxy/auth/oauth2_proxy_hook.py @@ -37,9 +37,13 @@ async def handle_oauth2_proxy_request(request: Request) -> UserAPIKeyAuth: else: auth_data[key] = value verbose_proxy_logger.debug( - f"Auth data before creating UserAPIKeyAuth object: {auth_data}" + "Auth data before creating UserAPIKeyAuth object: keys=%s", + list(auth_data.keys()), ) user_api_key_auth = UserAPIKeyAuth(**auth_data) - verbose_proxy_logger.debug(f"UserAPIKeyAuth object created: {user_api_key_auth}") + verbose_proxy_logger.debug( + "UserAPIKeyAuth object created with keys: %s", + list(user_api_key_auth.__fields_set__), + ) # Create and return UserAPIKeyAuth object return user_api_key_auth diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index fefc6c8af9c..b26d8336191 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -711,7 +711,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): "global_max_parallel_requests", None ) user_api_key = _metadata.get("user_api_key", None) - self.print_verbose(f"user_api_key: {user_api_key}") + self.print_verbose(f"user_api_key: [set={user_api_key is not None}]") if user_api_key is None: return diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 497ddcbbe84..00d8ce182ec 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4228,13 +4228,19 @@ async def list_keys( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), page: int = Query(1, description="Page number", ge=1), size: int = Query(10, description="Page size", ge=1, le=100), - user_id: Optional[str] = Query(None, description="Filter keys by user ID. Supports partial matching (substring, case-insensitive)."), + user_id: Optional[str] = Query( + None, + description="Filter keys by user ID. Supports partial matching (substring, case-insensitive).", + ), team_id: Optional[str] = Query(None, description="Filter keys by team ID"), organization_id: Optional[str] = Query( None, description="Filter keys by organization ID" ), key_hash: Optional[str] = Query(None, description="Filter keys by key hash"), - key_alias: Optional[str] = Query(None, description="Filter keys by key alias. Supports partial matching (substring, case-insensitive)."), + key_alias: Optional[str] = Query( + None, + description="Filter keys by key alias. Supports partial matching (substring, case-insensitive).", + ), return_full_object: bool = Query(False, description="Return full key object"), include_team_keys: bool = Query( False, description="Include all keys for teams that user is an admin of." diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8a122a13506..2a661a8348e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -375,9 +375,7 @@ from litellm.proxy.management_endpoints.fallback_management_endpoints import ( from litellm.proxy.management_endpoints.internal_user_endpoints import ( router as internal_user_router, ) -from litellm.proxy.management_endpoints.internal_user_endpoints import ( - user_update, -) +from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import ( router as jwt_key_mapping_router, ) @@ -446,9 +444,7 @@ from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_route from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, ) -from litellm.proxy.openai_files_endpoints.files_endpoints import ( - set_files_config, -) +from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_config from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( passthrough_endpoint_router, ) @@ -552,9 +548,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import ( LiteLLM_UpperboundKeyGenerateParams, ) from litellm.types.realtime import RealtimeQueryParams -from litellm.types.router import ( - DeploymentTypedDict, -) +from litellm.types.router import DeploymentTypedDict from litellm.types.router import ModelInfo as RouterModelInfo from litellm.types.router import ( RouterGeneralSettings, @@ -9309,20 +9303,17 @@ async def _add_access_group_models_to_team_models( return team_models # Single batch fetch for all access groups - access_group_rows = ( - await prisma_client.db.litellm_accessgrouptable.find_many( - where={"access_group_id": {"in": list(all_access_group_ids)}} - ) + access_group_rows = await prisma_client.db.litellm_accessgrouptable.find_many( + where={"access_group_id": {"in": list(all_access_group_ids)}} ) ag_model_map: Dict[str, List[str]] = { - row.access_group_id: row.access_model_names or [] - for row in access_group_rows + row.access_group_id: row.access_model_names or [] for row in access_group_rows } # Second pass: resolve deployments for each eligible team for team_object in eligible_teams: model_names: Set[str] = set() - for ag_id in team_object.access_group_ids or [] : + for ag_id in team_object.access_group_ids or []: model_names.update(ag_model_map.get(ag_id, [])) for model_name in model_names: @@ -9333,9 +9324,7 @@ async def _add_access_group_models_to_team_models( for deployment in deployments: model_id = deployment.get("model_info", {}).get("id", None) if model_id is not None: - team_models.setdefault(model_id, set()).add( - team_object.team_id - ) + team_models.setdefault(model_id, set()).add(team_object.team_id) return team_models diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 97fa6c39fb5..3c1b7cfd10c 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1013,7 +1013,9 @@ async def get_global_spend_report( "/spend/report endpoint " + CommonProxyErrors.not_premium_user.value ) if api_key is not None: - verbose_proxy_logger.debug("Getting /spend for api_key: %s", api_key) + verbose_proxy_logger.debug( + "Getting /spend for api_key: [set=%s]", api_key is not None + ) if api_key.startswith("sk-"): api_key = hash_token(token=api_key) sql_query = """ diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 845919e9120..34c22d2deeb 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3293,7 +3293,7 @@ class PrismaClient: if update_key_values is not None: update_key_values = self.jsonify_object(data=update_key_values) if token is not None: - print_verbose(f"token: {token}") + print_verbose(f"token: [set={token is not None}]") # check if plain text or hash token = _hash_token_if_needed(token=token) db_data["token"] = token diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 9075373f1cf..2207acbb37a 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2123,9 +2123,9 @@ class LiteLLMCompletionResponsesConfig: hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None ): - output_details_dict["reasoning_tokens"] = ( - completion_details.reasoning_tokens - ) + output_details_dict[ + "reasoning_tokens" + ] = completion_details.reasoning_tokens else: output_details_dict["reasoning_tokens"] = 0 diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 1309846c102..1188ce9d592 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -147,6 +147,13 @@ async def get_deployments_for_tag( ) return healthy_deployments + # Tag filtering applies only when there is at least one deployment to evaluate. + if isinstance(healthy_deployments, list) and len(healthy_deployments) == 0: + verbose_logger.debug( + "get_deployments_for_tag: empty candidate set; skipping tag filter" + ) + return healthy_deployments + verbose_logger.debug( "request metadata: %s", request_kwargs.get(metadata_variable_name) ) diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index 63231923f1a..c23e6ce473a 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, Any, Optional, Union -from litellm._logging import verbose_router_logger +from litellm._logging import redact_secrets, verbose_router_logger from litellm.constants import MAX_EXCEPTION_MESSAGE_LENGTH from litellm.router_utils.cooldown_handlers import ( _async_get_cooldown_deployments_with_debug_info, @@ -57,6 +57,9 @@ async def send_llm_exception_alert( exception_str += litellm_debug_info exception_str += f"\n\n{error_traceback_str[:MAX_EXCEPTION_MESSAGE_LENGTH]}" + # Redact secrets before sending to external service (Slack / MS Teams) + exception_str = redact_secrets(exception_str) + await litellm_router_instance.slack_alerting_logger.send_alert( message=f"LLM API call failed: `{exception_str}`", level="High", diff --git a/litellm/secret_managers/secret_manager_handler.py b/litellm/secret_managers/secret_manager_handler.py index eb90dda0e99..0b16f7e10ad 100644 --- a/litellm/secret_managers/secret_manager_handler.py +++ b/litellm/secret_managers/secret_manager_handler.py @@ -117,12 +117,14 @@ def get_secret_from_manager( # noqa: PLR0915 secret_name=secret_name, primary_secret_name=primary_secret_name, ) - print_verbose(f"get_secret_value_response: {secret}") + print_verbose(f"get_secret_value_response: [set={secret is not None}]") elif key_manager == KeyManagementSystem.GOOGLE_SECRET_MANAGER.value: try: secret = client.get_secret_from_google_secret_manager(secret_name) - print_verbose(f"secret from google secret manager: {secret}") + print_verbose( + f"secret from google secret manager: [set={secret is not None}]" + ) if secret is None: raise ValueError( f"No secret found in Google Secret Manager for {secret_name}" diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index af91926de2f..ebabf3fb6f8 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -111,6 +111,12 @@ class MCPCredentials(TypedDict, total=False): aws_service_name: Optional[str] """AWS service name for SigV4 signing (e.g., 'bedrock-agentcore'). Not a secret — stored unencrypted.""" + aws_role_name: Optional[str] + """IAM role ARN for STS AssumeRole (e.g., 'arn:aws:iam::123456789012:role/MyRole'). Not a secret — stored unencrypted.""" + + aws_session_name: Optional[str] + """Session name for STS AssumeRole (used in CloudTrail). Not a secret — stored unencrypted.""" + class MCPServerCostInfo(TypedDict, total=False): default_cost_per_query: Optional[float] diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index ed391f0af68..db7657a0174 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -54,6 +54,8 @@ class MCPServer(BaseModel): aws_session_token: Optional[str] = None aws_region_name: Optional[str] = None aws_service_name: Optional[str] = None # defaults to "bedrock-agentcore" + aws_role_name: Optional[str] = None # IAM role ARN for STS AssumeRole + aws_session_name: Optional[str] = None # session name for CloudTrail auditing # Stdio-specific fields command: Optional[str] = None args: Optional[List[str]] = None diff --git a/litellm/utils.py b/litellm/utils.py index 37bc35af299..6806961bf51 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8314,6 +8314,10 @@ class ProviderConfigManager: return SagemakerEmbeddingConfig.get_model_config(model) elif litellm.LlmProviders.PERPLEXITY == provider: return litellm.PerplexityEmbeddingConfig() + elif litellm.LlmProviders.OCI == provider: + from litellm.llms.oci.embed.transformation import OCIEmbeddingConfig + + return OCIEmbeddingConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 35fa2206761..6da9a004b8a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23788,7 +23788,8 @@ "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_vision": true }, "oci/meta.llama-3.3-70b-instruct": { "input_cost_per_token": 7.2e-07, @@ -23922,6 +23923,287 @@ "supports_function_calling": true, "supports_response_schema": false }, + "oci/cohere.command-a-reasoning-08-2025": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/cohere.command-a-vision-07-2025": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true + }, + "oci/cohere.command-a-translate-08-2025": { + "input_cost_per_token": 9e-08, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 9e-08, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": false, + "supports_response_schema": false + }, + "oci/cohere.command-r-08-2024": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/cohere.command-r-plus-08-2024": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/meta.llama-3.2-11b-vision-instruct": { + "input_cost_per_token": 2e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true + }, + "oci/meta.llama-3.1-70b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/meta.llama-3.3-70b-instruct-fp8-dynamic": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4-fast": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.1-fast": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.20": { + "input_cost_per_token": 3e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.20-multi-agent": { + "input_cost_per_token": 3e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-code-fast-1": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/google.gemini-2.5-pro": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/google.gemini-2.5-flash": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/google.gemini-2.5-flash-lite": { + "input_cost_per_token": 7.5e-08, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/cohere.embed-english-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-english-light-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-multilingual-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-multilingual-light-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-english-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-english-light-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-multilingual-light-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-v4.0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, "ollama/codegeex4": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", @@ -30290,6 +30572,27 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "vertex_ai/claude-haiku-4-5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_native_streaming": true, + "supports_vision": true + }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, "cache_read_input_token_cost": 1e-07, diff --git a/tests/test_litellm/llms/oci/embed/__init__.py b/tests/test_litellm/llms/oci/embed/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py new file mode 100644 index 00000000000..4ecca377e63 --- /dev/null +++ b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py @@ -0,0 +1,369 @@ +import json +import os +import sys +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.oci.embed.transformation import OCIEmbeddingConfig +from litellm.types.utils import EmbeddingResponse + +# Test constants +TEST_MODEL_NAME = "cohere.embed-english-v3.0" +TEST_MODEL = f"oci/{TEST_MODEL_NAME}" +TEST_COMPARTMENT_ID = "ocid1.compartment.oc1..xxxxxx" +BASE_OCI_PARAMS = { + "oci_region": "us-ashburn-1", + "oci_user": "ocid1.user.oc1..xxxxxxEXAMPLExxxxxx", + "oci_fingerprint": "4f:29:77:cc:b1:3e:55:ab:61:2a:de:47:f1:38:4c:90", + "oci_tenancy": "ocid1.tenancy.oc1..xxxxxxEXAMPLExxxxxx", + "oci_compartment_id": TEST_COMPARTMENT_ID, +} + +TEST_OCI_PARAMS_KEY = { + **BASE_OCI_PARAMS, + "oci_key": "", +} + +TEST_OCI_PARAMS_KEY_FILE = { + **BASE_OCI_PARAMS, + "oci_key_file": "", +} + +# Mock OCI embedding response +MOCK_OCI_EMBEDDING_RESPONSE = { + "embeddings": [[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8]], + "modelId": "cohere.embed-english-v3.0", + "modelVersion": "3.0", + "inputTextTokenCounts": [5, 4], +} + + +@pytest.fixture(params=[TEST_OCI_PARAMS_KEY, TEST_OCI_PARAMS_KEY_FILE]) +def supplied_params(request): + """Fixture for passing in optional_parameters""" + return request.param + + +class TestOCIEmbeddingConfig: + def test_get_complete_url_default_region(self): + """test_get_complete_url returns URL with us-ashburn-1 when no api_base is given.""" + config = OCIEmbeddingConfig() + url = config.get_complete_url( + api_base=None, + api_key=None, + model=TEST_MODEL_NAME, + optional_params={}, + litellm_params={}, + ) + assert "us-ashburn-1" in url + assert "embedText" in url + + def test_get_complete_url_custom_region(self): + """test_get_complete_url uses region from optional_params.""" + config = OCIEmbeddingConfig() + url = config.get_complete_url( + api_base=None, + api_key=None, + model=TEST_MODEL_NAME, + optional_params={"oci_region": "us-chicago-1"}, + litellm_params={}, + ) + assert "us-chicago-1" in url + assert "embedText" in url + + def test_get_complete_url_custom_api_base(self): + """test_get_complete_url returns api_base as-is when provided.""" + config = OCIEmbeddingConfig() + custom_base = "https://custom.oci.example.com/embed" + url = config.get_complete_url( + api_base=custom_base, + api_key=None, + model=TEST_MODEL_NAME, + optional_params={}, + litellm_params={}, + ) + assert url == custom_base + + def test_get_supported_openai_params(self): + """test_get_supported_openai_params returns expected params list.""" + config = OCIEmbeddingConfig() + params = config.get_supported_openai_params(model=TEST_MODEL_NAME) + assert "dimensions" in params + assert "encoding_format" not in params + + def test_map_openai_params_dimensions(self): + """test dimensions is mapped correctly.""" + config = OCIEmbeddingConfig() + optional_params = {} + result = config.map_openai_params( + non_default_params={"dimensions": 512}, + optional_params=optional_params, + model=TEST_MODEL_NAME, + drop_params=False, + ) + assert result["dimensions"] == 512 + + def test_validate_environment_with_credentials(self, supplied_params): + """test validate_environment returns content-type and user-agent headers when credentials are supplied.""" + config = OCIEmbeddingConfig() + headers = {} + result = config.validate_environment( + headers=headers, + model=TEST_MODEL, + messages=[], + optional_params=supplied_params, + litellm_params={}, + ) + assert result["content-type"] == "application/json" + assert "litellm" in result["user-agent"] + + def test_validate_environment_missing_credentials(self): + """test validate_environment raises Exception with 'Missing required parameters' when credentials are incomplete.""" + config = OCIEmbeddingConfig() + incomplete_params = { + "oci_user": "ocid1.user.oc1..xxx", + # Missing oci_fingerprint, oci_tenancy, oci_key/oci_key_file, oci_compartment_id + } + with pytest.raises(Exception) as excinfo: + config.validate_environment( + headers={}, + model=TEST_MODEL, + messages=[], + optional_params=incomplete_params, + litellm_params={}, + ) + assert "Missing required parameters" in str(excinfo.value) + + def test_validate_environment_with_signer(self): + """test validate_environment passes when oci_signer is provided.""" + config = OCIEmbeddingConfig() + + class MockSigner: + def do_request_sign(self, request, enforce_content_headers=True): + request.headers["authorization"] = 'Signature version="1"' + + optional_params = { + "oci_signer": MockSigner(), + "oci_region": "us-ashburn-1", + } + result = config.validate_environment( + headers={}, + model=TEST_MODEL, + messages=[], + optional_params=optional_params, + litellm_params={}, + ) + assert result["content-type"] == "application/json" + + def test_transform_embedding_request_on_demand(self): + """test transform_embedding_request builds correct ON_DEMAND OCI request body.""" + config = OCIEmbeddingConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + } + with patch.object(config, "sign_request", return_value=({}, "{}")): + result = config.transform_embedding_request( + model=TEST_MODEL_NAME, + input=["Hello world", "Goodbye world"], + optional_params=optional_params, + headers={}, + ) + + assert result["compartmentId"] == TEST_COMPARTMENT_ID + assert result["servingMode"]["servingType"] == "ON_DEMAND" + assert result["servingMode"]["modelId"] == TEST_MODEL_NAME + assert result["inputs"] == ["Hello world", "Goodbye world"] + assert result["truncate"] == "END" + + def test_transform_embedding_request_dedicated(self): + """test transform_embedding_request builds DEDICATED servingMode with endpointId.""" + config = OCIEmbeddingConfig() + test_endpoint_id = "ocid1.generativeaiendpoint.oc1.us-chicago-1.xxxxxx" + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "oci_serving_mode": "DEDICATED", + "oci_endpoint_id": test_endpoint_id, + } + with patch.object(config, "sign_request", return_value=({}, "{}")): + result = config.transform_embedding_request( + model=TEST_MODEL_NAME, + input=["Hello world"], + optional_params=optional_params, + headers={}, + ) + + assert result["servingMode"]["servingType"] == "DEDICATED" + assert result["servingMode"]["endpointId"] == test_endpoint_id + + def test_transform_embedding_request_input_type(self): + """test input_type=search_query is mapped to SEARCH_QUERY in request data.""" + config = OCIEmbeddingConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "input_type": "search_query", + } + with patch.object(config, "sign_request", return_value=({}, "{}")): + result = config.transform_embedding_request( + model=TEST_MODEL_NAME, + input=["What is the capital of Brazil?"], + optional_params=optional_params, + headers={}, + ) + + assert result["inputType"] == "SEARCH_QUERY" + + def test_transform_embedding_request_string_input(self): + """test single string input is wrapped in a list.""" + config = OCIEmbeddingConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + } + with patch.object(config, "sign_request", return_value=({}, "{}")): + result = config.transform_embedding_request( + model=TEST_MODEL_NAME, + input="Hello world", + optional_params=optional_params, + headers={}, + ) + + assert isinstance(result["inputs"], list) + assert result["inputs"] == ["Hello world"] + + def test_transform_embedding_request_token_list_raises(self): + """test token-array inputs raise ValueError instead of silent conversion.""" + config = OCIEmbeddingConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + } + with patch.object(config, "sign_request", return_value=({}, "{}")): + with pytest.raises(ValueError, match="does not support token-array"): + config.transform_embedding_request( + model=TEST_MODEL_NAME, + input=[[1234, 5678]], + optional_params=optional_params, + headers={}, + ) + + def test_transform_embedding_response(self): + """test OCI embedding response is correctly transformed into EmbeddingResponse.""" + config = OCIEmbeddingConfig() + mock_response = httpx.Response( + status_code=200, + json=MOCK_OCI_EMBEDDING_RESPONSE, + request=httpx.Request("POST", "https://test.com"), + ) + mock_logging = MagicMock() + model_response = EmbeddingResponse() + + result = config.transform_embedding_response( + model=TEST_MODEL_NAME, + raw_response=mock_response, + model_response=model_response, + logging_obj=mock_logging, + ) + + assert isinstance(result, EmbeddingResponse) + assert result.model == "cohere.embed-english-v3.0" + assert len(result.data) == 2 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3, 0.4] + assert result.data[1]["embedding"] == [0.5, 0.6, 0.7, 0.8] + assert result.data[0]["index"] == 0 + assert result.data[1]["index"] == 1 + # Total tokens: 5 + 4 = 9 + assert result.usage.prompt_tokens == 9 + assert result.usage.total_tokens == 9 + + def test_transform_embedding_response_error(self): + """test non-200 status code raises OCIError.""" + from litellm.llms.oci.common_utils import OCIError + + config = OCIEmbeddingConfig() + mock_response = httpx.Response( + status_code=400, + text="Bad Request", + request=httpx.Request("POST", "https://test.com"), + ) + mock_logging = MagicMock() + model_response = EmbeddingResponse() + + with pytest.raises(OCIError): + config.transform_embedding_response( + model=TEST_MODEL_NAME, + raw_response=mock_response, + model_response=model_response, + logging_obj=mock_logging, + ) + + def test_model_prices_embedding_models(self): + """test all 8 OCI embedding models exist in model_prices_and_context_window.json with mode=embedding.""" + model_prices_path = os.path.join( + os.path.dirname(__file__), + "..", + "..", + "..", + "..", + "..", + "model_prices_and_context_window.json", + ) + with open(model_prices_path) as f: + model_prices = json.load(f) + + expected_embedding_models = [ + "oci/cohere.embed-english-v3.0", + "oci/cohere.embed-english-light-v3.0", + "oci/cohere.embed-multilingual-v3.0", + "oci/cohere.embed-multilingual-light-v3.0", + "oci/cohere.embed-english-image-v3.0", + "oci/cohere.embed-english-light-image-v3.0", + "oci/cohere.embed-multilingual-light-image-v3.0", + "oci/cohere.embed-v4.0", + ] + + for model_key in expected_embedding_models: + assert model_key in model_prices, f"Missing model: {model_key}" + assert ( + model_prices[model_key].get("mode") == "embedding" + ), f"Model {model_key} does not have mode='embedding'" + + def test_model_prices_new_chat_models(self): + """test the 16 new OCI chat models exist in model_prices_and_context_window.json with mode=chat.""" + model_prices_path = os.path.join( + os.path.dirname(__file__), + "..", + "..", + "..", + "..", + "..", + "model_prices_and_context_window.json", + ) + with open(model_prices_path) as f: + model_prices = json.load(f) + + expected_chat_models = [ + "oci/xai.grok-3", + "oci/xai.grok-3-fast", + "oci/xai.grok-3-mini", + "oci/xai.grok-3-mini-fast", + "oci/xai.grok-4", + "oci/xai.grok-4-fast", + "oci/xai.grok-4.1-fast", + "oci/xai.grok-4.20", + "oci/xai.grok-4.20-multi-agent", + "oci/xai.grok-code-fast-1", + "oci/cohere.command-a-03-2025", + "oci/cohere.command-a-reasoning-08-2025", + "oci/cohere.command-a-vision-07-2025", + "oci/cohere.command-a-translate-08-2025", + "oci/google.gemini-2.5-pro", + "oci/google.gemini-2.5-flash", + ] + + for model_key in expected_chat_models: + assert model_key in model_prices, f"Missing model: {model_key}" + assert ( + model_prices[model_key].get("mode") == "chat" + ), f"Model {model_key} does not have mode='chat'" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py index a2295e1271e..7c142e3a771 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py @@ -162,6 +162,140 @@ class TestMCPSigV4Auth: assert "x-amz-security-token" in signed_request.headers +class TestMCPSigV4AssumeRole: + """Tests for STS AssumeRole credential resolution in MCPSigV4Auth.""" + + def test_assume_role_with_ambient_credentials(self): + """MCPSigV4Auth calls STS AssumeRole when aws_role_name is provided (no explicit keys).""" + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + with patch("boto3.client", return_value=mock_sts) as mock_boto3: + auth = MCPSigV4Auth( + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_region_name="us-east-1", + ) + + mock_boto3.assert_called_once_with("sts", region_name="us-east-1") + mock_sts.assume_role.assert_called_once() + call_kwargs = mock_sts.assume_role.call_args[1] + assert call_kwargs["RoleArn"] == "arn:aws:iam::123456789012:role/TestRole" + assert call_kwargs["RoleSessionName"].startswith("litellm-mcp-") + assert auth.credentials.access_key == "ASSUMED_KEY" + assert auth.credentials.secret_key == "ASSUMED_SECRET" + assert auth.credentials.token == "ASSUMED_TOKEN" + + def test_assume_role_with_explicit_source_credentials(self): + """When aws_role_name + explicit keys are provided, keys are used as STS source identity.""" + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + with patch("boto3.client", return_value=mock_sts) as mock_boto3: + auth = MCPSigV4Auth( + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_access_key_id="SOURCE_KEY", + aws_secret_access_key="SOURCE_SECRET", + aws_region_name="us-west-2", + ) + + mock_boto3.assert_called_once_with( + "sts", + region_name="us-west-2", + aws_access_key_id="SOURCE_KEY", + aws_secret_access_key="SOURCE_SECRET", + ) + assert auth.credentials.access_key == "ASSUMED_KEY" + + def test_assume_role_with_custom_session_name(self): + """Custom aws_session_name is used in the AssumeRole call.""" + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + with patch("boto3.client", return_value=mock_sts): + MCPSigV4Auth( + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_session_name="regeneron-litellm-prod", + ) + + call_kwargs = mock_sts.assume_role.call_args[1] + assert call_kwargs["RoleSessionName"] == "regeneron-litellm-prod" + + def test_assume_role_signing_works(self): + """Requests are signed correctly with STS-derived credentials.""" + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "AKIAIOSFODNN7EXAMPLE", + "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "SessionToken": "STS_SESSION_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + with patch("boto3.client", return_value=mock_sts): + auth = MCPSigV4Auth( + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_region_name="us-east-1", + aws_service_name="bedrock-agentcore", + ) + + request = httpx.Request( + method="POST", + url="https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/test/invocations", + headers={"Content-Type": "application/json"}, + content=b'{"jsonrpc":"2.0","method":"tools/list","id":1}', + ) + + signed_request = next(auth.auth_flow(request)) + assert "Authorization" in signed_request.headers + assert "AWS4-HMAC-SHA256" in signed_request.headers["Authorization"] + assert "x-amz-security-token" in signed_request.headers + + def test_assume_role_takes_precedence_over_explicit_keys(self): + """When both aws_role_name and explicit keys are provided, AssumeRole is used (keys become source identity).""" + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + with patch("boto3.client", return_value=mock_sts): + auth = MCPSigV4Auth( + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_access_key_id="EXPLICIT_KEY", + aws_secret_access_key="EXPLICIT_SECRET", + ) + + # Credentials should be from AssumeRole, not the explicit keys + assert auth.credentials.access_key == "ASSUMED_KEY" + assert auth.credentials.secret_key == "ASSUMED_SECRET" + + class TestMCPClientSigV4Integration: """Tests for MCPClient with SigV4 auth wired through.""" @@ -319,6 +453,86 @@ class TestMCPServerManagerSigV4: assert client._aws_auth is None + @pytest.mark.asyncio + async def test_load_config_with_aws_role_name(self): + """Config loading correctly parses aws_role_name and aws_session_name.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + + config = { + "agentcore_tools": { + "url": "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/test/invocations", + "transport": "http", + "auth_type": "aws_sigv4", + "aws_role_name": "arn:aws:iam::123456789012:role/TestRole", + "aws_session_name": "litellm-prod", + "aws_region_name": "us-east-1", + } + } + + manager = MCPServerManager() + await manager.load_servers_from_config(config) + + server = next(iter(manager.config_mcp_servers.values())) + assert server.aws_role_name == "arn:aws:iam::123456789012:role/TestRole" + assert server.aws_session_name == "litellm-prod" + + @pytest.mark.asyncio + async def test_create_mcp_client_with_role_assumption(self): + """_create_mcp_client passes aws_role_name to MCPSigV4Auth.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + server = MCPServer( + server_id="test-sigv4-role", + name="test_sigv4_role", + server_name="test_sigv4_role", + url="https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/test/invocations", + transport=MCPTransport.http, + auth_type=MCPAuth.aws_sigv4, + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_region_name="us-east-1", + ) + + manager = MCPServerManager() + with patch("boto3.client", return_value=mock_sts): + client = await manager._create_mcp_client(server=server) + + assert client._aws_auth is not None + assert isinstance(client._aws_auth, MCPSigV4Auth) + mock_sts.assume_role.assert_called_once() + + def test_extract_aws_credentials_includes_role_fields(self): + """_extract_aws_credentials extracts aws_role_name and aws_session_name.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + + manager = MCPServerManager() + creds = { + "aws_access_key_id": "KEY", + "aws_region_name": "us-east-1", + "aws_role_name": "arn:aws:iam::123456789012:role/TestRole", + "aws_session_name": "my-session", + } + + result = manager._extract_aws_credentials(creds, credentials_are_encrypted=False) + assert result["aws_role_name"] == "arn:aws:iam::123456789012:role/TestRole" + assert result["aws_session_name"] == "my-session" + class TestSigV4CredentialEncryption: """Test encrypt/decrypt round-trip for AWS SigV4 credentials.""" diff --git a/tests/test_litellm/proxy/db/test_check_migration.py b/tests/test_litellm/proxy/db/test_check_migration.py index ad72a0d1195..f2468d95820 100644 --- a/tests/test_litellm/proxy/db/test_check_migration.py +++ b/tests/test_litellm/proxy/db/test_check_migration.py @@ -27,12 +27,14 @@ def test_check_migration_out_of_sync(mocker): - 🚨 [IMPORTANT] Does NOT Raise an Exception when the Prisma schema is out of sync with the database. - logs an error when the Prisma schema is out of sync with the database. """ - # Mock the logger BEFORE importing the function - mock_logger = mocker.patch("litellm._logging.verbose_logger") - - # Import the function after mocking the logger + # Import the function first so check_migration module is in sys.modules, + # then patch the logger reference in that module directly (not the source + # module) so the patch works regardless of import order or xdist worker + # assignment. from litellm.proxy.db.check_migration import check_prisma_schema_diff + mock_logger = mocker.patch("litellm.proxy.db.check_migration.verbose_logger") + # Mock the helper function to simulate out-of-sync state mock_diff_helper = mocker.patch( "litellm.proxy.db.check_migration.check_prisma_schema_diff_helper", diff --git a/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py b/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py index 6c7cfa61b58..dca2bd84f92 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py @@ -188,6 +188,26 @@ async def test_tag_filtering_disabled_returns_all_deployments(): assert result == ALL_DEPLOYMENTS +@pytest.mark.asyncio +async def test_empty_healthy_deployments_with_request_tags_returns_empty_list(): + """ + With an empty candidate list, return [] even when the request includes metadata tags. + + Tag-based filtering runs only against non-empty healthy_deployments; an empty list is + returned unchanged for the router's standard handling. + """ + router = _make_router_mock() + result = await get_deployments_for_tag( + llm_router_instance=router, + model="gpt-5.2", + healthy_deployments=[], + request_kwargs={ + "metadata": {"tags": ["client_id:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]} + }, + ) + assert result == [] + + @pytest.mark.asyncio async def test_explicit_tag_match_takes_precedence_over_regex(): """A deployment with both tags and tag_regex: exact tag match fires first.""" diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index 1d575195194..6cbb2fd7b8f 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -262,3 +262,80 @@ def test_key_name_redaction_in_general_settings_dict(): assert "REDACTED" in output # Non-sensitive values should survive assert "enable_jwt_auth" in output + + +# ── GCP service-account / Vertex credential redaction ── + + +_SAMPLE_SA_JSON = ( + '{"type": "service_account", "project_id": "my-proj-123", ' + '"private_key_id": "abc123def", ' + '"private_key": "-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkq\\n-----END PRIVATE KEY-----\\n", ' + '"client_email": "sa@my-proj.iam.gserviceaccount.com", ' + '"client_id": "123456789"}' +) + + +def test_pem_private_key_redacted_in_json(): + result = _redact_string(_SAMPLE_SA_JSON) + assert "MIIEvQIBADA" not in result + assert "-----BEGIN" not in result + + +def test_pem_private_key_redacted_in_dict_repr(): + import json + + sa = json.loads(_SAMPLE_SA_JSON) + result = _redact_string(str(sa)) + assert "MIIEvQIBADA" not in result + + +def test_service_account_blob_fully_redacted(): + result = _redact_string(f"Got={_SAMPLE_SA_JSON}") + assert "my-proj-123" not in result + assert "sa@my-proj.iam.gserviceaccount.com" not in result + assert "abc123def" not in result + assert "MIIEvQIBADA" not in result + + +def test_vertex_error_message_no_credential_leak(): + """The old Vertex error format leaked the full credential JSON. + The new format must not contain any credential material.""" + new_msg = ( + "Unable to load vertex credentials from environment. " + "Ensure the JSON is valid (check for unescaped newlines in private_key). " + "Parse error: JSONDecodeError" + ) + result = _redact_string(new_msg) + assert result == new_msg # nothing to redact + + +def test_vertex_traceback_redacts_pem(): + traceback_text = ( + "Traceback (most recent call last):\n" + ' File "vertex_llm_base.py", line 95\n' + " json_obj = json.loads(credentials)\n" + "json.decoder.JSONDecodeError: Invalid control character\n" + "Failed to load vertex credentials. Error: " + "Unable to load vertex credentials from environment. " + f"Got={_SAMPLE_SA_JSON}" + ) + result = _redact_string(traceback_text) + assert "MIIEvQIBADA" not in result + assert "-----BEGIN" not in result + + +def test_gcp_oauth_token_redacted(): + result = _redact_string("access token ya29.c.c0ASRK0GZvXlongtokenhere") + assert "ya29." not in result + assert "REDACTED" in result + + +def test_non_pem_private_key_value_redacted(): + result = _redact_string("'private_key': 'some-non-pem-secret-value'") + assert "some-non-pem-secret" not in result + + +def test_normal_vertex_log_not_redacted(): + msg = "Vertex: Loading vertex credentials, is_file_path=True, current dir /app" + assert _redact_string(msg) == msg diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index aaddf53fd7e..4c824fcee0b 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -934,6 +934,38 @@ const CreateMCPServer: React.FC = ({ className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500" /> + + AWS Role ARN + + + + + } + name={["credentials", "aws_role_name"]} + > + + + + AWS Session Name + + + + + } + name={["credentials", "aws_session_name"]} + > + + )} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 118461f0043..e81d2f3960e 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -977,6 +977,38 @@ const MCPServerEdit: React.FC = ({ className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500" /> + + AWS Role ARN + + + + + } + name={["credentials", "aws_role_name"]} + > + + + + AWS Session Name + + + + + } + name={["credentials", "aws_session_name"]} + > + + )}