From b53cfe729abc3210b245a204bf2577713704dd85 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Sat, 4 Apr 2026 14:44:07 -0700 Subject: [PATCH] Litellm ishaan march30 (#24887) (#25151) * fix(pricing): add unversioned vertex_ai/claude-haiku-4-5 entry Missing unversioned entry causes cost tracking to return $0.00 for all requests using vertex_ai/claude-haiku-4-5. All other Vertex AI Claude models have both versioned and unversioned entries. * fix(router): skip misleading tags error when no candidates (e.g. cooldown) Return early from get_deployments_for_tag when healthy_deployments is empty so tag-based routing does not raise no_deployments_with_tag_routing after cooldown filters all deployments. Adds regression test. Made-with: Cursor * feat(oci): add embedding support and update model catalog - Add OCIEmbeddingConfig for OCI GenAI embedding models - Add 16 new chat models (Cohere, Meta Llama, xAI Grok, Google Gemini) - Add 8 embedding models (Cohere embed v3.0, v4.0) - Update documentation with embedding examples - Update pricing for all new models * test(oci): add unit tests for OCI embedding support - 17 unit tests covering OCIEmbeddingConfig - Tests for URL generation, param mapping, request/response transform - Tests for model pricing JSON completeness * style(oci): format with black and ruff * fix(oci): correct embedding request body format OCI embedText API expects inputs, truncate, and inputType at the top level of the request body, not nested under embedTextDetails. Fixed transformation and updated tests accordingly. Verified with real OCI API: 3/3 embedding models working. * docs: clarify tag routing early return and test intent Made-with: Cursor * fix(oci): address code review findings from Greptile - P1: Fix signing URL mismatch with custom api_base by accepting api_base parameter in transform_embedding_request - P2: Remove encoding_format from supported params (OCI does not support it, was silently dropped) - P2: Raise ValueError for token-array inputs instead of silently converting to string representation - Add test for token-list rejection * fix(mcp): add STS AssumeRole support for MCP SigV4 authentication MCPSigV4Auth only supported static AWS credentials or the boto3 default credential chain. Production Kubernetes environments typically authenticate via IAM role assumption (sts:AssumeRole), which was not possible. Add aws_role_name and aws_session_name parameters to the MCP SigV4 auth stack. When aws_role_name is provided, MCPSigV4Auth calls sts:AssumeRole to obtain temporary credentials before signing requests. Explicit keys, if also provided, are used as the source identity for the STS call; otherwise ambient credentials (pod role, instance profile) are used. * fix: stop logging credential values and add missing redaction patterns Replaces raw credential values in debug/error log messages with boolean presence checks or type names. Adds PEM block, GCP token, JWT, SAS token, and service-account blob patterns to the redaction filter. Fixes private_key pattern to capture full PEM blocks instead of stopping at the first whitespace. Addresses: Vertex AI credential JSON (including RSA private key) being logged to stderr on health check failures. * fix: log only field names for UserAPIKeyAuth, not full object * style: apply black formatting to experimental_mcp_client/client.py * style: fix black/isort formatting and mypy error in proxy_server.py - Fix black formatting in experimental_mcp_client/client.py (done in prev commit) - Fix black/isort formatting in key_management_endpoints.py, proxy_server.py, transformation.py - Fix mypy: iterate over optional list safely (access_group_ids or []) in proxy_server.py * fix(test): patch check_migration.verbose_logger directly to fix xdist ordering issue When test_proxy_cli.py tests run before test_check_migration.py in the same xdist worker, litellm.proxy.db.check_migration is already in sys.modules. Patching litellm._logging.verbose_logger has no effect on the already-bound reference. Patch the correct target (check_migration.verbose_logger) and import the module before patching so the order doesn't matter. * fix(mypy): make api_base Optional in PydanticAIProviderConfig to match base class signature --------- Co-authored-by: Ihsan Soydemir Co-authored-by: Milan Co-authored-by: Daniel Gandolfi Co-authored-by: Claude Sonnet 4.6 Co-authored-by: michelligabriele Co-authored-by: user <70670632+stuxf@users.noreply.github.com> Co-authored-by: Ishaan Jaffer --- docs/my-website/docs/mcp.md | 3 +- docs/my-website/docs/mcp_aws_sigv4.md | 53 ++- docs/my-website/docs/providers/oci.md | 109 +++++- litellm/__init__.py | 1 + litellm/_logging.py | 38 +- .../providers/pydantic_ai_agents/config.py | 10 +- litellm/experimental_mcp_client/client.py | 50 ++- .../azure_storage/azure_storage.py | 5 +- .../gcs_bucket/gcs_bucket_base.py | 8 +- .../get_llm_provider_logic.py | 4 +- litellm/llms/azure/common_utils.py | 14 +- litellm/llms/bedrock/base_aws_llm.py | 16 +- litellm/llms/oci/chat/transformation.py | 9 +- litellm/llms/oci/embed/__init__.py | 0 litellm/llms/oci/embed/transformation.py | 347 ++++++++++++++++ litellm/llms/vertex_ai/vertex_llm_base.py | 28 +- litellm/main.py | 16 + ...odel_prices_and_context_window_backup.json | 305 ++++++++++++++- .../mcp_server/mcp_server_manager.py | 8 + litellm/proxy/auth/oauth2_check.py | 4 +- litellm/proxy/auth/oauth2_proxy_hook.py | 8 +- .../proxy/hooks/parallel_request_limiter.py | 2 +- .../key_management_endpoints.py | 10 +- litellm/proxy/proxy_server.py | 27 +- .../spend_management_endpoints.py | 4 +- litellm/proxy/utils.py | 2 +- .../transformation.py | 6 +- litellm/router_strategy/tag_based_routing.py | 7 + litellm/router_utils/handle_error.py | 5 +- .../secret_managers/secret_manager_handler.py | 6 +- litellm/types/mcp.py | 6 + .../types/mcp_server/mcp_server_manager.py | 2 + litellm/utils.py | 4 + model_prices_and_context_window.json | 305 ++++++++++++++- tests/test_litellm/llms/oci/embed/__init__.py | 0 .../llms/oci/embed/test_oci_embedding.py | 369 ++++++++++++++++++ .../mcp_server/test_mcp_sigv4_auth.py | 214 ++++++++++ .../proxy/db/test_check_migration.py | 10 +- .../test_router_tag_regex_routing.py | 20 + tests/test_litellm/test_secret_redaction.py | 77 ++++ .../mcp_tools/create_mcp_server.tsx | 32 ++ .../components/mcp_tools/mcp_server_edit.tsx | 32 ++ 42 files changed, 2083 insertions(+), 93 deletions(-) create mode 100644 litellm/llms/oci/embed/__init__.py create mode 100644 litellm/llms/oci/embed/transformation.py create mode 100644 tests/test_litellm/llms/oci/embed/__init__.py create mode 100644 tests/test_litellm/llms/oci/embed/test_oci_embedding.py diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index b805cce4d7a..f6fe01ac28f 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -278,7 +278,8 @@ mcp_servers: url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes//invocations" transport: "http" auth_type: "aws_sigv4" - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_role_name: os.environ/AWS_ROLE_ARN # optional — IAM role to assume + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # optional — falls back to IAM role aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 aws_service_name: bedrock-agentcore diff --git a/docs/my-website/docs/mcp_aws_sigv4.md b/docs/my-website/docs/mcp_aws_sigv4.md index 9dc60bce06e..e556ad244f8 100644 --- a/docs/my-website/docs/mcp_aws_sigv4.md +++ b/docs/my-website/docs/mcp_aws_sigv4.md @@ -36,6 +36,8 @@ LiteLLM's `aws_sigv4` auth type handles this automatically: every outgoing MCP r | **AWS Access Key ID** | No | Falls back to boto3 credential chain if blank | | **AWS Secret Access Key** | No | Required if Access Key ID is provided | | **AWS Session Token** | No | Only needed for temporary STS credentials | +| **AWS Role ARN** | No | IAM role ARN for STS AssumeRole (e.g., `arn:aws:iam::123456789012:role/MyRole`). If set, LiteLLM assumes this role before signing | +| **AWS Session Name** | No | Session name for the AssumeRole call — appears in CloudTrail. Auto-generated if omitted | Once created, LiteLLM will sign every outgoing MCP request with SigV4. The server's tools appear automatically in the MCP Tools list. @@ -66,8 +68,8 @@ mcp_servers: url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes//invocations" transport: "http" auth_type: "aws_sigv4" - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID - aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_role_name: os.environ/AWS_ROLE_ARN # IAM role to assume (recommended) + aws_session_name: "litellm-prod" # optional — for CloudTrail auditing aws_region_name: "us-east-1" aws_service_name: "bedrock-agentcore" ``` @@ -128,6 +130,8 @@ curl http://localhost:4000/mcp-rest/tools/call \ | `aws_region_name` | Yes | AWS region (e.g., `us-east-1`) | | `aws_service_name` | No | AWS service name for signing. Defaults to `bedrock-agentcore` | | `aws_session_token` | No | AWS session token for temporary credentials. Supports `os.environ/VAR_NAME` | +| `aws_role_name` | No | IAM role ARN for STS AssumeRole. Supports `os.environ/VAR_NAME`. When set, LiteLLM calls `sts:AssumeRole` to get temporary credentials before signing | +| `aws_session_name` | No | Session name for the AssumeRole call (appears in CloudTrail). Auto-generated if omitted. Supports `os.environ/VAR_NAME` | ## How It Works @@ -157,6 +161,42 @@ mcp_servers: aws_service_name: "bedrock-agentcore" ``` +## Using IAM Role Assumption (AssumeRole) + +For production environments where your LiteLLM instance authenticates via an IAM role (e.g., EKS pod role, EC2 instance profile), you can configure `aws_role_name` to have LiteLLM call `sts:AssumeRole` before signing MCP requests: + +```yaml title="config.yaml with AssumeRole" showLineNumbers +mcp_servers: + my_agentcore_mcp: + url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes//invocations" + transport: "http" + auth_type: "aws_sigv4" + aws_role_name: "arn:aws:iam::123456789012:role/BedrockAgentCoreRole" + aws_session_name: "litellm-prod" # optional + aws_region_name: "us-east-1" + aws_service_name: "bedrock-agentcore" +``` + +LiteLLM uses the ambient credentials (pod role, instance profile, or env vars) to call `sts:AssumeRole`, then signs MCP requests with the assumed role's temporary credentials. + +You can also combine `aws_role_name` with explicit access keys — the keys are then used as the source identity for the AssumeRole call: + +```yaml title="config.yaml with AssumeRole + explicit source keys" showLineNumbers +mcp_servers: + my_agentcore_mcp: + url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes//invocations" + transport: "http" + auth_type: "aws_sigv4" + aws_role_name: os.environ/AWS_ROLE_ARN + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: "us-east-1" +``` + +:::tip +For most Kubernetes deployments, you only need `aws_role_name` and `aws_region_name` — the pod's IAM role provides the source credentials automatically. +::: + ## Troubleshooting ### 403 Forbidden from AWS @@ -166,6 +206,15 @@ mcp_servers: - Ensure `aws_service_name` is set to `bedrock-agentcore` - If using STS credentials, confirm `aws_session_token` is set and not expired +### AssumeRole AccessDenied + +If you get `AccessDenied` when using `aws_role_name`: + +- Verify the role ARN is correct +- Check that the trust policy on the target role allows your source identity to assume it +- If running on EKS, ensure the pod's service account is annotated with the correct IAM role +- Check CloudTrail for the failed `sts:AssumeRole` call to see the exact error + ### Health check errors on startup SigV4-authenticated MCP servers skip the standard health check on proxy startup. This is expected — the proxy will still sign requests correctly when tools are invoked. diff --git a/docs/my-website/docs/providers/oci.md b/docs/my-website/docs/providers/oci.md index ce6fe18dd6f..1d7a0a3d502 100644 --- a/docs/my-website/docs/providers/oci.md +++ b/docs/my-website/docs/providers/oci.md @@ -8,24 +8,54 @@ Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generativ ## Supported Models -### Meta Llama Models +### Chat / Text Generation + +#### Meta Llama Models - `meta.llama-4-maverick-17b-128e-instruct-fp8` - `meta.llama-4-scout-17b-16e-instruct` - `meta.llama-3.3-70b-instruct` +- `meta.llama-3.3-70b-instruct-fp8-dynamic` - `meta.llama-3.2-90b-vision-instruct` +- `meta.llama-3.2-11b-vision-instruct` - `meta.llama-3.1-405b-instruct` +- `meta.llama-3.1-70b-instruct` -### xAI Grok Models +#### xAI Grok Models +- `xai.grok-4.20` +- `xai.grok-4.20-multi-agent` - `xai.grok-4` +- `xai.grok-4-fast` +- `xai.grok-4.1-fast` - `xai.grok-3` - `xai.grok-3-fast` - `xai.grok-3-mini` - `xai.grok-3-mini-fast` +- `xai.grok-code-fast-1` -### Cohere Models +#### Cohere Models - `cohere.command-latest` - `cohere.command-a-03-2025` +- `cohere.command-a-reasoning-08-2025` +- `cohere.command-a-vision-07-2025` +- `cohere.command-a-translate-08-2025` - `cohere.command-plus-latest` +- `cohere.command-r-08-2024` +- `cohere.command-r-plus-08-2024` + +#### Google Gemini Models (via OCI) +- `google.gemini-2.5-pro` +- `google.gemini-2.5-flash` +- `google.gemini-2.5-flash-lite` + +### Embedding Models +- `cohere.embed-english-v3.0` (1024 dimensions) +- `cohere.embed-english-light-v3.0` (384 dimensions) +- `cohere.embed-multilingual-v3.0` (1024 dimensions) +- `cohere.embed-multilingual-light-v3.0` (384 dimensions) +- `cohere.embed-english-image-v3.0` (1024 dimensions, multimodal) +- `cohere.embed-english-light-image-v3.0` (384 dimensions, multimodal) +- `cohere.embed-multilingual-light-image-v3.0` (384 dimensions, multimodal) +- `cohere.embed-v4.0` (1536 dimensions, multimodal) ## Authentication @@ -394,4 +424,75 @@ response = completion( | `oci_tenancy` | string | - | (Manual auth) The OCID of your OCI tenancy | | `oci_key` | string | - | (Manual auth) The private key content as a string | | `oci_key_file` | string | - | (Manual auth) Path to the private key file | -| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication | \ No newline at end of file +| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication | + +## Embeddings + +LiteLLM supports OCI Generative AI embedding models. These models use the same authentication methods described above. + + + + +```python +from litellm import embedding + +response = embedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world", "Goodbye world"], + oci_region="us-ashburn-1", + oci_user=, + oci_fingerprint=, + oci_tenancy=, + oci_key=, + oci_compartment_id=, +) +print(response) +``` + + + + +```python +from litellm import embedding +from oci.signer import Signer + +signer = Signer( + tenancy="ocid1.tenancy.oc1..", + user="ocid1.user.oc1..", + fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx", + private_key_file_location="~/.oci/key.pem", +) + +response = embedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world", "Goodbye world"], + oci_signer=signer, + oci_region="us-ashburn-1", + oci_compartment_id="", +) +print(response) +``` + + + + +### Embedding Optional Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `input_type` | string | - | The type of input: `search_document`, `search_query`, `classification`, `clustering` | +| `truncate` | string | `END` | Truncation strategy when input exceeds max tokens: `END` or `START` | + +### Using Dedicated Embedding Endpoints + +```python +response = embedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world"], + oci_serving_mode="DEDICATED", + oci_endpoint_id="ocid1.generativeaiendpoint.oc1...", + oci_region="us-ashburn-1", + oci_compartment_id="", + # ... auth params +) +``` \ No newline at end of file diff --git a/litellm/__init__.py b/litellm/__init__.py index e45d926e8db..d4418c661a3 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1838,6 +1838,7 @@ if TYPE_CHECKING: ) from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig + from .llms.oci.embed.transformation import OCIEmbeddingConfig as OCIEmbeddingConfig from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig from .llms.lambda_ai.chat.transformation import ( diff --git a/litellm/_logging.py b/litellm/_logging.py index 65e6045b0b1..62283f6f65a 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -26,6 +26,12 @@ _REDACTED = "REDACTED" def _build_secret_patterns() -> re.Pattern: patterns: List[str] = [ + # ── PEM private key / certificate blocks ── + r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----", + # ── GCP OAuth2 access tokens (ya29.*) ── + r"\bya29\.[A-Za-z0-9_.~+/-]+", + # ── Credential %s formatting (space separator, no key= prefix) ── + r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+", # AWS access key IDs r"(?:AKIA|ASIA)[0-9A-Z]{16}", # AWS secrets / session tokens / access key IDs (key=value) @@ -46,7 +52,8 @@ def _build_secret_patterns() -> re.Pattern: # Google API keys r"AIza[0-9A-Za-z\-_]{35}", # Password / secret params (handles key=value and 'key': 'value') - r"\w*(?:password|passwd|client_secret|secret_key|_secret)" + # Word boundary prevents O(n^2) backtracking on long word-char runs. + r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)" r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", # Database connection string credentials (scheme://user:pass@host) r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)", @@ -56,13 +63,21 @@ def _build_secret_patterns() -> re.Pattern: # Catches secrets inside dicts/config dumps by matching on the KEY name # regardless of what the value looks like. # e.g. 'master_key': 'any-value-here', "database_url": "postgres://..." + # private_key with PEM-aware value capture + r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""", r"(?:master_key|database_url|db_url|connection_string|" - r"private_key|signing_key|encryption_key|" + r"signing_key|encryption_key|" r"auth_token|access_token|refresh_token|" r"slack_webhook_url|webhook_url|" r"database_connection_string|" r"huggingface_token|jwt_secret)" r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""", + # ── Raw JWTs (without Bearer prefix) ── + r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*", + # ── Azure SAS tokens in URLs ── + r"[?&]sig=[A-Za-z0-9%+/=]+", + # ── Full JSON service-account blobs (single-line and multi-line) ── + r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', ] return re.compile("|".join(patterns), re.IGNORECASE) @@ -74,6 +89,23 @@ def _redact_string(value: str) -> str: return _SECRET_RE.sub(_REDACTED, value) +def redact_secrets(value: str) -> str: + """Public API: redact known secret/credential patterns from an arbitrary string. + + Use this for code paths that bypass the logging system — e.g. Slack/Teams + alerting, HTTP error response bodies, or any other string that may contain + secrets and will be sent to an external sink. + + Not to be confused with redact_message_input_output_from_logging() in + litellm_core_utils/redact_messages.py, which redacts LLM prompt/response + content for privacy — this function redacts credential patterns (API keys, + PEM blocks, tokens, etc.) by shape. + """ + if not _ENABLE_SECRET_REDACTION: + return value + return _redact_string(value) + + class SecretRedactionFilter(logging.Filter): """Scrubs known secret/credential patterns from log records.""" @@ -441,7 +473,7 @@ def _enable_debugging(): def print_verbose(print_statement): try: if set_verbose: - print(print_statement) # noqa + print(redact_secrets(str(print_statement))) # noqa except Exception: pass diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py index d4c5f6a2985..46253bbcf78 100644 --- a/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py @@ -2,7 +2,7 @@ Pydantic AI provider configuration. """ -from typing import Any, AsyncIterator, Dict +from typing import Any, AsyncIterator, Dict, Optional from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig from litellm.a2a_protocol.providers.pydantic_ai_agents.handler import PydanticAIHandler @@ -20,10 +20,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig): self, request_id: str, params: Dict[str, Any], - api_base: str, + api_base: Optional[str] = None, **kwargs, ) -> Dict[str, Any]: """Handle non-streaming request to Pydantic AI agent.""" + if not api_base: + raise ValueError("api_base is required for Pydantic AI agents") return await PydanticAIHandler.handle_non_streaming( request_id=request_id, params=params, @@ -35,10 +37,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig): self, request_id: str, params: Dict[str, Any], - api_base: str, + api_base: Optional[str] = None, **kwargs, ) -> AsyncIterator[Dict[str, Any]]: """Handle streaming request with fake streaming.""" + if not api_base: + raise ValueError("api_base is required for Pydantic AI agents") async for chunk in PydanticAIHandler.handle_streaming( request_id=request_id, params=params, diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index a638a28aba3..1423617cac0 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -82,6 +82,8 @@ class MCPSigV4Auth(httpx.Auth): aws_session_token: Optional[str] = None, aws_region_name: Optional[str] = None, aws_service_name: Optional[str] = None, + aws_role_name: Optional[str] = None, + aws_session_name: Optional[str] = None, ): try: from botocore.credentials import Credentials @@ -97,7 +99,16 @@ class MCPSigV4Auth(httpx.Auth): # Note: os.environ/ prefixed values are already resolved by # ProxyConfig._check_for_os_environ_vars() at config load time. # Values arrive here as plain strings. - if aws_access_key_id and aws_secret_access_key: + if aws_role_name: + self.credentials = self._assume_role( + aws_role_name=aws_role_name, + aws_session_name=aws_session_name, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + aws_region_name=self.region_name, + ) + elif aws_access_key_id and aws_secret_access_key: self.credentials = Credentials( access_key=aws_access_key_id, secret_key=aws_secret_access_key, @@ -116,6 +127,43 @@ class MCPSigV4Auth(httpx.Auth): "(env vars, ~/.aws/credentials, instance profile)." ) + @staticmethod + def _assume_role( + aws_role_name: str, + aws_session_name: Optional[str], + aws_access_key_id: Optional[str], + aws_secret_access_key: Optional[str], + aws_session_token: Optional[str], + aws_region_name: str, + ): + """Call STS AssumeRole and return temporary credentials.""" + import boto3 + from botocore.credentials import Credentials + + session_name = ( + aws_session_name or f"litellm-mcp-{int(__import__('time').time())}" + ) + + sts_kwargs: dict = {"region_name": aws_region_name} + if aws_access_key_id and aws_secret_access_key: + sts_kwargs["aws_access_key_id"] = aws_access_key_id + sts_kwargs["aws_secret_access_key"] = aws_secret_access_key + if aws_session_token: + sts_kwargs["aws_session_token"] = aws_session_token + + sts_client = boto3.client("sts", **sts_kwargs) + sts_response = sts_client.assume_role( + RoleArn=aws_role_name, + RoleSessionName=session_name, + ) + + sts_creds = sts_response["Credentials"] + return Credentials( + access_key=sts_creds["AccessKeyId"], + secret_key=sts_creds["SecretAccessKey"], + token=sts_creds["SessionToken"], + ) + def auth_flow( self, request: httpx.Request ) -> Generator[httpx.Request, httpx.Response, None]: diff --git a/litellm/integrations/azure_storage/azure_storage.py b/litellm/integrations/azure_storage/azure_storage.py index 6fc7b9c1048..50c1cd9d989 100644 --- a/litellm/integrations/azure_storage/azure_storage.py +++ b/litellm/integrations/azure_storage/azure_storage.py @@ -275,12 +275,11 @@ class AzureBlobStorageLogger(CustomBatchLogger): """ Gets Azure AD token to use for Azure Storage API requests """ - verbose_logger.debug("Getting Azure AD Token from Azure Storage") verbose_logger.debug( - "tenant_id %s, client_id %s, client_secret %s", + "Getting Azure AD Token from Azure Storage, tenant_id=%s, client_id=%s, client_secret=[set=%s]", tenant_id, client_id, - client_secret, + client_secret is not None, ) if tenant_id is None: raise ValueError( diff --git a/litellm/integrations/gcs_bucket/gcs_bucket_base.py b/litellm/integrations/gcs_bucket/gcs_bucket_base.py index 923f613291f..0089e54b1c2 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket_base.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket_base.py @@ -70,7 +70,9 @@ class GCSBucketBase(CustomBatchLogger): custom_llm_provider="vertex_ai", api_base=None, ) - verbose_logger.debug("constructed auth_header %s", auth_header) + verbose_logger.debug( + "constructed auth_header [set=%s]", auth_header is not None + ) headers = { "Authorization": f"Bearer {auth_header}", # auth_header "Content-Type": "application/json", @@ -106,7 +108,9 @@ class GCSBucketBase(CustomBatchLogger): custom_llm_provider="vertex_ai", api_base=None, ) - verbose_logger.debug("constructed auth_header %s", auth_header) + verbose_logger.debug( + "constructed auth_header [set=%s]", auth_header is not None + ) headers = { "Authorization": f"Bearer {auth_header}", # auth_header "Content-Type": "application/json", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 0a2f07bcb25..95bcd4d7186 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -202,8 +202,8 @@ def get_llm_provider( # noqa: PLR0915 ) if dynamic_api_key is not None and not isinstance(dynamic_api_key, str): raise Exception( - "dynamic_api_key needs to be a string. dynamic_api_key={}".format( - dynamic_api_key + "dynamic_api_key needs to be a string. Got type={}".format( + type(dynamic_api_key).__name__ ) ) return model, custom_llm_provider, dynamic_api_key, api_base diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index fcdb3eca23a..4fc1ae960b8 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -101,17 +101,15 @@ def get_azure_ad_token_from_entra_id( _client_secret = client_secret verbose_logger.debug( - "tenant_id %s, client_id %s, client_secret %s", + "tenant_id=%s, client_id=%s, client_secret=[set=%s]", _tenant_id, _client_id, - _client_secret, + _client_secret is not None, ) if _tenant_id is None or _client_id is None or _client_secret is None: raise ValueError("tenant_id, client_id, and client_secret must be provided") credential = ClientSecretCredential(_tenant_id, _client_id, _client_secret) - verbose_logger.debug("credential %s", credential) - token_provider = get_bearer_token_provider(credential, scope) verbose_logger.debug("token_provider %s", token_provider) @@ -140,10 +138,10 @@ def get_azure_ad_token_from_username_password( from azure.identity import UsernamePasswordCredential, get_bearer_token_provider verbose_logger.debug( - "client_id %s, azure_username %s, azure_password %s", + "client_id=%s, azure_username=[set=%s], azure_password=[set=%s]", client_id, - azure_username, - azure_password, + azure_username is not None, + azure_password is not None, ) credential = UsernamePasswordCredential( client_id=client_id, @@ -151,8 +149,6 @@ def get_azure_ad_token_from_username_password( password=azure_password, ) - verbose_logger.debug("credential %s", credential) - token_provider = get_bearer_token_provider(credential, scope) verbose_logger.debug("token_provider %s", token_provider) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index b159d62367d..4157fac53b8 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -156,24 +156,24 @@ class BaseAWSLLM: verbose_logger.debug( "in get credentials\n" - "aws_access_key_id=%s\n" - "aws_secret_access_key=%s\n" - "aws_session_token=%s\n" + "aws_access_key_id=[set=%s]\n" + "aws_secret_access_key=[set=%s]\n" + "aws_session_token=[set=%s]\n" "aws_region_name=%s\n" "aws_session_name=%s\n" "aws_profile_name=%s\n" "aws_role_name=%s\n" - "aws_web_identity_token=%s\n" + "aws_web_identity_token=[set=%s]\n" "aws_sts_endpoint=%s\n" "aws_external_id=%s", - aws_access_key_id, - aws_secret_access_key, - aws_session_token, + aws_access_key_id is not None, + aws_secret_access_key is not None, + aws_session_token is not None, aws_region_name, aws_session_name, aws_profile_name, aws_role_name, - aws_web_identity_token, + aws_web_identity_token is not None, aws_sts_endpoint, aws_external_id, ) diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index b1af7ed2ec3..79cd1c00606 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -174,10 +174,15 @@ def load_private_key_from_file(file_path: str): def get_vendor_from_model(model: str) -> OCIVendors: """ Extracts the vendor from the model name. + + OCI GenAI API uses two apiFormat values: + - "COHERE" for Cohere models (command-r, command-a, etc.) + - "GENERIC" for all other models (Meta Llama, xAI Grok, Google Gemini, etc.) + Args: - model (str): The model name. + model (str): The model name (e.g., "cohere.command-a-03-2025", "meta.llama-3.3-70b-instruct"). Returns: - str: The vendor name. + OCIVendors: The vendor enum value. """ vendor = model.split(".")[0].lower() if vendor == "cohere": diff --git a/litellm/llms/oci/embed/__init__.py b/litellm/llms/oci/embed/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/oci/embed/transformation.py b/litellm/llms/oci/embed/transformation.py new file mode 100644 index 00000000000..1dcd8c5213c --- /dev/null +++ b/litellm/llms/oci/embed/transformation.py @@ -0,0 +1,347 @@ +""" +OCI Generative AI Embedding Configuration + +Supports embedding models available on Oracle Cloud Infrastructure Generative AI service. +Uses the same authentication mechanisms as OCI chat (manual signing or OCI SDK Signer). + +Supported models: +- cohere.embed-english-v3.0 +- cohere.embed-english-light-v3.0 +- cohere.embed-multilingual-v3.0 +- cohere.embed-multilingual-light-v3.0 +- cohere.embed-english-image-v3.0 +- cohere.embed-english-light-image-v3.0 +- cohere.embed-multilingual-light-image-v3.0 +- cohere.embed-v4.0 + +Reference: https://docs.oracle.com/en-us/iaas/api/#/en/generative-ai-inference/latest/EmbedTextResult/EmbedText +""" + +from typing import Any, Dict, List, Optional, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig +from litellm.llms.oci.chat.transformation import OCIChatConfig +from litellm.llms.oci.common_utils import OCIError +from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues +from litellm.types.utils import EmbeddingResponse, Usage + +# Input type mapping from OpenAI conventions to OCI/Cohere conventions +_INPUT_TYPE_MAP = { + "search_document": "SEARCH_DOCUMENT", + "search_query": "SEARCH_QUERY", + "classification": "CLASSIFICATION", + "clustering": "CLUSTERING", +} + + +class OCIEmbeddingConfig(BaseEmbeddingConfig): + """ + Configuration for OCI Generative AI Embedding API. + + The OCI embedding endpoint uses the Cohere embed models hosted on OCI. + Authentication is handled via OCI request signing (manual credentials or OCI SDK Signer). + + Usage: + ```python + import litellm + + response = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world", "Goodbye world"], + oci_compartment_id="ocid1.compartment.oc1..xxx", + oci_region="us-ashburn-1", + oci_user="ocid1.user.oc1..xxx", + oci_fingerprint="xx:xx:xx:xx", + oci_tenancy="ocid1.tenancy.oc1..xxx", + oci_key_file="~/.oci/key.pem", + ) + ``` + """ + + def __init__(self) -> None: + # We reuse OCIChatConfig for signing logic + self._chat_config = OCIChatConfig() + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + if api_base: + return api_base + + oci_region = optional_params.get("oci_region", "us-ashburn-1") + return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/embedText" + + def get_supported_openai_params(self, model: str) -> list: + return [ + "dimensions", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + # Note: OCI Cohere embed does not support custom dimensions natively, + # but we pass it through in case future models support it + if "dimensions" in non_default_params: + optional_params["dimensions"] = non_default_params["dimensions"] + return optional_params + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate OCI credentials for embedding requests. + Supports both OCI SDK Signer and manual credential signing. + """ + oci_signer = optional_params.get("oci_signer") + oci_region = optional_params.get("oci_region", "us-ashburn-1") + + api_base = ( + api_base + or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" + ) + + if oci_signer is None: + oci_user = optional_params.get("oci_user") + oci_fingerprint = optional_params.get("oci_fingerprint") + oci_tenancy = optional_params.get("oci_tenancy") + oci_key = optional_params.get("oci_key") + oci_key_file = optional_params.get("oci_key_file") + oci_compartment_id = optional_params.get("oci_compartment_id") + + if ( + not oci_user + or not oci_fingerprint + or not oci_tenancy + or not (oci_key or oci_key_file) + or not oci_compartment_id + ): + raise Exception( + "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, oci_compartment_id " + "and at least one of oci_key or oci_key_file. " + "Alternatively, provide an oci_signer object from the OCI SDK." + ) + + from litellm.llms.custom_httpx.http_handler import version + + headers.update( + { + "content-type": "application/json", + "user-agent": f"litellm/{version}", + } + ) + + return headers + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ): + """Delegate to OCIChatConfig's signing logic.""" + return self._chat_config.sign_request( + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + api_base: Optional[str] = None, + ) -> dict: + """ + Transform the embedding request to OCI format. + + OCI embedText API expects: + { + "compartmentId": "...", + "servingMode": {"servingType": "ON_DEMAND", "modelId": "..."}, + "inputs": ["text1", "text2"], + "truncate": "END", + "inputType": "SEARCH_DOCUMENT" + } + """ + oci_compartment_id = optional_params.get("oci_compartment_id") + if not oci_compartment_id: + raise Exception( + "kwarg `oci_compartment_id` is required for OCI embedding requests" + ) + + # Build serving mode + oci_serving_mode = optional_params.get("oci_serving_mode", "ON_DEMAND") + if oci_serving_mode == "DEDICATED": + oci_endpoint_id = optional_params.get("oci_endpoint_id", model) + serving_mode = { + "servingType": "DEDICATED", + "endpointId": oci_endpoint_id, + } + else: + serving_mode = { + "servingType": "ON_DEMAND", + "modelId": model, + } + + # Normalize input to list of strings + if isinstance(input, str): + inputs = [input] + elif isinstance(input, list): + inputs = [] + for item in input: + if isinstance(item, str): + inputs.append(item) + elif isinstance(item, list): + raise ValueError( + "OCI embedding does not support token-array inputs. " + "Please convert token lists to strings before calling embedding()." + ) + else: + inputs.append(str(item)) + else: + inputs = [str(input)] + + # Build request data — OCI embedText API expects inputs, truncate, + # and inputType at the top level alongside compartmentId and servingMode + request_data: Dict[str, Any] = { + "compartmentId": oci_compartment_id, + "servingMode": serving_mode, + "inputs": inputs, + "truncate": optional_params.get("truncate", "END"), + } + + # Map input_type if provided + input_type = optional_params.get("input_type") + if input_type: + mapped_type = _INPUT_TYPE_MAP.get(input_type.lower(), input_type.upper()) + request_data["inputType"] = mapped_type + + # Sign the request using the same URL the HTTP handler will POST to + signing_url = self.get_complete_url( + api_base=api_base, + api_key=None, + model=model, + optional_params=optional_params, + litellm_params={}, + ) + + signed_headers, body = self.sign_request( + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=signing_url, + ) + headers.update(signed_headers) + + return request_data + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str] = None, + request_data: dict = {}, + optional_params: dict = {}, + litellm_params: dict = {}, + ) -> EmbeddingResponse: + """ + Transform OCI embedding response to standard EmbeddingResponse format. + + OCI response format: + { + "embeddings": [[0.1, 0.2, ...], [0.3, 0.4, ...]], + "modelId": "cohere.embed-english-v3.0", + "modelVersion": "3.0", + "inputTextTokenCounts": [5, 4] + } + """ + if raw_response.status_code != 200: + raise OCIError( + message=raw_response.text, + status_code=raw_response.status_code, + ) + + try: + raw_response_json = raw_response.json() + except Exception: + raise OCIError( + message=raw_response.text, + status_code=raw_response.status_code, + ) + + embeddings = raw_response_json.get("embeddings", []) + model_id = raw_response_json.get("modelId", model) + + # Build response data in OpenAI format + embedding_data = [] + for idx, embedding in enumerate(embeddings): + embedding_data.append( + { + "object": "embedding", + "index": idx, + "embedding": embedding, + } + ) + + model_response.model = model_id + model_response.data = embedding_data + model_response.object = "list" + + # Calculate token usage + input_token_counts = raw_response_json.get("inputTextTokenCounts", []) + total_tokens = sum(input_token_counts) if input_token_counts else 0 + + usage = Usage( + prompt_tokens=total_tokens, + total_tokens=total_tokens, + ) + model_response.usage = usage + + return model_response + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: Union[dict, httpx.Headers], + ) -> BaseLLMException: + return OCIError( + message=error_message, + status_code=status_code, + headers=headers if isinstance(headers, httpx.Headers) else None, + ) diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 1a29ba82eac..cdabac27af7 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -81,26 +81,26 @@ class VertexBase: ) -> Tuple[Any, str]: if credentials is not None: if isinstance(credentials, str): + _is_path = os.path.exists( + credentials + ) # credentials is from server config (litellm_params), not user input verbose_logger.debug( - "Vertex: Loading vertex credentials from %s", credentials - ) - verbose_logger.debug( - "Vertex: checking if credentials is a valid path, os.path.exists(%s)=%s, current dir %s", - credentials, - os.path.exists(credentials), + "Vertex: Loading vertex credentials, is_file_path=%s, current dir %s", + _is_path, os.getcwd(), ) try: - if os.path.exists(credentials): - json_obj = json.load(open(credentials)) + if _is_path: + with open(credentials) as f: + json_obj = json.load(f) else: json_obj = json.loads(credentials) - except Exception: + except Exception as e: raise Exception( - "Unable to load vertex credentials from environment. Got={}".format( - credentials - ) + "Unable to load vertex credentials from environment. " + "Ensure the JSON is valid (check for unescaped newlines in private_key). " + "Parse error: {}".format(type(e).__name__) ) elif isinstance(credentials, dict): json_obj = credentials @@ -668,8 +668,8 @@ class VertexBase: ## VALIDATION STEP if _credentials.token is None or not isinstance(_credentials.token, str): raise ValueError( - "Could not resolve credentials token. Got None or non-string token - {}".format( - _credentials.token + "Could not resolve credentials token. Got None or non-string token (type={})".format( + type(_credentials.token).__name__ ) ) diff --git a/litellm/main.py b/litellm/main.py index eace9c630ba..cbedd1735c7 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5668,6 +5668,22 @@ def embedding( # noqa: PLR0915 aembedding=aembedding, litellm_params={}, ) + elif custom_llm_provider == "oci": + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + litellm_params=litellm_params_dict, + headers=headers, + ) elif custom_llm_provider in litellm._custom_providers: custom_handler: Optional[CustomLLM] = None for item in litellm.custom_provider_map: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4e1c7f4ac80..8351f084f28 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23803,7 +23803,8 @@ "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_vision": true }, "oci/meta.llama-3.3-70b-instruct": { "input_cost_per_token": 7.2e-07, @@ -23937,6 +23938,287 @@ "supports_function_calling": true, "supports_response_schema": false }, + "oci/cohere.command-a-reasoning-08-2025": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/cohere.command-a-vision-07-2025": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true + }, + "oci/cohere.command-a-translate-08-2025": { + "input_cost_per_token": 9e-08, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 9e-08, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": false, + "supports_response_schema": false + }, + "oci/cohere.command-r-08-2024": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/cohere.command-r-plus-08-2024": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/meta.llama-3.2-11b-vision-instruct": { + "input_cost_per_token": 2e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true + }, + "oci/meta.llama-3.1-70b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/meta.llama-3.3-70b-instruct-fp8-dynamic": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4-fast": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.1-fast": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.20": { + "input_cost_per_token": 3e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.20-multi-agent": { + "input_cost_per_token": 3e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-code-fast-1": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/google.gemini-2.5-pro": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/google.gemini-2.5-flash": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/google.gemini-2.5-flash-lite": { + "input_cost_per_token": 7.5e-08, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/cohere.embed-english-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-english-light-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-multilingual-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-multilingual-light-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-english-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-english-light-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-multilingual-light-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-v4.0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, "ollama/codegeex4": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", @@ -30305,6 +30587,27 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "vertex_ai/claude-haiku-4-5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_native_streaming": true, + "supports_vision": true + }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, "cache_read_input_token_cost": 1e-07, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 1e9d5c5a529..cfbc2c437b0 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -346,6 +346,8 @@ class MCPServerManager: aws_session_token=server_config.get("aws_session_token", None), aws_region_name=server_config.get("aws_region_name", None), aws_service_name=server_config.get("aws_service_name", None), + aws_role_name=server_config.get("aws_role_name", None), + aws_session_name=server_config.get("aws_session_name", None), ) self.config_mcp_servers[server_id] = new_server @@ -686,6 +688,8 @@ class MCPServerManager: aws_session_token=aws_creds.get("aws_session_token"), aws_region_name=aws_creds.get("aws_region_name"), aws_service_name=aws_creds.get("aws_service_name"), + aws_role_name=aws_creds.get("aws_role_name"), + aws_session_name=aws_creds.get("aws_session_name"), ) return new_server @@ -1011,6 +1015,8 @@ class MCPServerManager: aws_session_token=server.aws_session_token, aws_region_name=server.aws_region_name, aws_service_name=server.aws_service_name, + aws_role_name=server.aws_role_name, + aws_session_name=server.aws_session_name, ) return MCPClient( @@ -1571,6 +1577,8 @@ class MCPServerManager: ), "aws_region_name": credentials_dict.get("aws_region_name"), "aws_service_name": credentials_dict.get("aws_service_name"), + "aws_role_name": credentials_dict.get("aws_role_name"), + "aws_session_name": credentials_dict.get("aws_session_name"), } def _extract_scopes(self, scopes_value: Any) -> Optional[List[str]]: diff --git a/litellm/proxy/auth/oauth2_check.py b/litellm/proxy/auth/oauth2_check.py index bb00141ad0a..10b1759b77e 100644 --- a/litellm/proxy/auth/oauth2_check.py +++ b/litellm/proxy/auth/oauth2_check.py @@ -136,7 +136,9 @@ class Oauth2Handler: + CommonProxyErrors.not_premium_user.value ) - verbose_proxy_logger.debug("Oauth2 token validation for token=%s", token) + verbose_proxy_logger.debug( + "Oauth2 token validation for token=[set=%s]", token is not None + ) # Get the token info endpoint from environment variable token_info_endpoint = os.getenv("OAUTH_TOKEN_INFO_ENDPOINT") diff --git a/litellm/proxy/auth/oauth2_proxy_hook.py b/litellm/proxy/auth/oauth2_proxy_hook.py index 7e517092b8a..0dc696bc455 100644 --- a/litellm/proxy/auth/oauth2_proxy_hook.py +++ b/litellm/proxy/auth/oauth2_proxy_hook.py @@ -37,9 +37,13 @@ async def handle_oauth2_proxy_request(request: Request) -> UserAPIKeyAuth: else: auth_data[key] = value verbose_proxy_logger.debug( - f"Auth data before creating UserAPIKeyAuth object: {auth_data}" + "Auth data before creating UserAPIKeyAuth object: keys=%s", + list(auth_data.keys()), ) user_api_key_auth = UserAPIKeyAuth(**auth_data) - verbose_proxy_logger.debug(f"UserAPIKeyAuth object created: {user_api_key_auth}") + verbose_proxy_logger.debug( + "UserAPIKeyAuth object created with keys: %s", + list(user_api_key_auth.__fields_set__), + ) # Create and return UserAPIKeyAuth object return user_api_key_auth diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index fefc6c8af9c..b26d8336191 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -711,7 +711,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): "global_max_parallel_requests", None ) user_api_key = _metadata.get("user_api_key", None) - self.print_verbose(f"user_api_key: {user_api_key}") + self.print_verbose(f"user_api_key: [set={user_api_key is not None}]") if user_api_key is None: return diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 497ddcbbe84..00d8ce182ec 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4228,13 +4228,19 @@ async def list_keys( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), page: int = Query(1, description="Page number", ge=1), size: int = Query(10, description="Page size", ge=1, le=100), - user_id: Optional[str] = Query(None, description="Filter keys by user ID. Supports partial matching (substring, case-insensitive)."), + user_id: Optional[str] = Query( + None, + description="Filter keys by user ID. Supports partial matching (substring, case-insensitive).", + ), team_id: Optional[str] = Query(None, description="Filter keys by team ID"), organization_id: Optional[str] = Query( None, description="Filter keys by organization ID" ), key_hash: Optional[str] = Query(None, description="Filter keys by key hash"), - key_alias: Optional[str] = Query(None, description="Filter keys by key alias. Supports partial matching (substring, case-insensitive)."), + key_alias: Optional[str] = Query( + None, + description="Filter keys by key alias. Supports partial matching (substring, case-insensitive).", + ), return_full_object: bool = Query(False, description="Return full key object"), include_team_keys: bool = Query( False, description="Include all keys for teams that user is an admin of." diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8a122a13506..2a661a8348e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -375,9 +375,7 @@ from litellm.proxy.management_endpoints.fallback_management_endpoints import ( from litellm.proxy.management_endpoints.internal_user_endpoints import ( router as internal_user_router, ) -from litellm.proxy.management_endpoints.internal_user_endpoints import ( - user_update, -) +from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import ( router as jwt_key_mapping_router, ) @@ -446,9 +444,7 @@ from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_route from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, ) -from litellm.proxy.openai_files_endpoints.files_endpoints import ( - set_files_config, -) +from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_config from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( passthrough_endpoint_router, ) @@ -552,9 +548,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import ( LiteLLM_UpperboundKeyGenerateParams, ) from litellm.types.realtime import RealtimeQueryParams -from litellm.types.router import ( - DeploymentTypedDict, -) +from litellm.types.router import DeploymentTypedDict from litellm.types.router import ModelInfo as RouterModelInfo from litellm.types.router import ( RouterGeneralSettings, @@ -9309,20 +9303,17 @@ async def _add_access_group_models_to_team_models( return team_models # Single batch fetch for all access groups - access_group_rows = ( - await prisma_client.db.litellm_accessgrouptable.find_many( - where={"access_group_id": {"in": list(all_access_group_ids)}} - ) + access_group_rows = await prisma_client.db.litellm_accessgrouptable.find_many( + where={"access_group_id": {"in": list(all_access_group_ids)}} ) ag_model_map: Dict[str, List[str]] = { - row.access_group_id: row.access_model_names or [] - for row in access_group_rows + row.access_group_id: row.access_model_names or [] for row in access_group_rows } # Second pass: resolve deployments for each eligible team for team_object in eligible_teams: model_names: Set[str] = set() - for ag_id in team_object.access_group_ids or [] : + for ag_id in team_object.access_group_ids or []: model_names.update(ag_model_map.get(ag_id, [])) for model_name in model_names: @@ -9333,9 +9324,7 @@ async def _add_access_group_models_to_team_models( for deployment in deployments: model_id = deployment.get("model_info", {}).get("id", None) if model_id is not None: - team_models.setdefault(model_id, set()).add( - team_object.team_id - ) + team_models.setdefault(model_id, set()).add(team_object.team_id) return team_models diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 97fa6c39fb5..3c1b7cfd10c 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1013,7 +1013,9 @@ async def get_global_spend_report( "/spend/report endpoint " + CommonProxyErrors.not_premium_user.value ) if api_key is not None: - verbose_proxy_logger.debug("Getting /spend for api_key: %s", api_key) + verbose_proxy_logger.debug( + "Getting /spend for api_key: [set=%s]", api_key is not None + ) if api_key.startswith("sk-"): api_key = hash_token(token=api_key) sql_query = """ diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 845919e9120..34c22d2deeb 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3293,7 +3293,7 @@ class PrismaClient: if update_key_values is not None: update_key_values = self.jsonify_object(data=update_key_values) if token is not None: - print_verbose(f"token: {token}") + print_verbose(f"token: [set={token is not None}]") # check if plain text or hash token = _hash_token_if_needed(token=token) db_data["token"] = token diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 9075373f1cf..2207acbb37a 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2123,9 +2123,9 @@ class LiteLLMCompletionResponsesConfig: hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None ): - output_details_dict["reasoning_tokens"] = ( - completion_details.reasoning_tokens - ) + output_details_dict[ + "reasoning_tokens" + ] = completion_details.reasoning_tokens else: output_details_dict["reasoning_tokens"] = 0 diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 1309846c102..1188ce9d592 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -147,6 +147,13 @@ async def get_deployments_for_tag( ) return healthy_deployments + # Tag filtering applies only when there is at least one deployment to evaluate. + if isinstance(healthy_deployments, list) and len(healthy_deployments) == 0: + verbose_logger.debug( + "get_deployments_for_tag: empty candidate set; skipping tag filter" + ) + return healthy_deployments + verbose_logger.debug( "request metadata: %s", request_kwargs.get(metadata_variable_name) ) diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index 63231923f1a..c23e6ce473a 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, Any, Optional, Union -from litellm._logging import verbose_router_logger +from litellm._logging import redact_secrets, verbose_router_logger from litellm.constants import MAX_EXCEPTION_MESSAGE_LENGTH from litellm.router_utils.cooldown_handlers import ( _async_get_cooldown_deployments_with_debug_info, @@ -57,6 +57,9 @@ async def send_llm_exception_alert( exception_str += litellm_debug_info exception_str += f"\n\n{error_traceback_str[:MAX_EXCEPTION_MESSAGE_LENGTH]}" + # Redact secrets before sending to external service (Slack / MS Teams) + exception_str = redact_secrets(exception_str) + await litellm_router_instance.slack_alerting_logger.send_alert( message=f"LLM API call failed: `{exception_str}`", level="High", diff --git a/litellm/secret_managers/secret_manager_handler.py b/litellm/secret_managers/secret_manager_handler.py index eb90dda0e99..0b16f7e10ad 100644 --- a/litellm/secret_managers/secret_manager_handler.py +++ b/litellm/secret_managers/secret_manager_handler.py @@ -117,12 +117,14 @@ def get_secret_from_manager( # noqa: PLR0915 secret_name=secret_name, primary_secret_name=primary_secret_name, ) - print_verbose(f"get_secret_value_response: {secret}") + print_verbose(f"get_secret_value_response: [set={secret is not None}]") elif key_manager == KeyManagementSystem.GOOGLE_SECRET_MANAGER.value: try: secret = client.get_secret_from_google_secret_manager(secret_name) - print_verbose(f"secret from google secret manager: {secret}") + print_verbose( + f"secret from google secret manager: [set={secret is not None}]" + ) if secret is None: raise ValueError( f"No secret found in Google Secret Manager for {secret_name}" diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index af91926de2f..ebabf3fb6f8 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -111,6 +111,12 @@ class MCPCredentials(TypedDict, total=False): aws_service_name: Optional[str] """AWS service name for SigV4 signing (e.g., 'bedrock-agentcore'). Not a secret — stored unencrypted.""" + aws_role_name: Optional[str] + """IAM role ARN for STS AssumeRole (e.g., 'arn:aws:iam::123456789012:role/MyRole'). Not a secret — stored unencrypted.""" + + aws_session_name: Optional[str] + """Session name for STS AssumeRole (used in CloudTrail). Not a secret — stored unencrypted.""" + class MCPServerCostInfo(TypedDict, total=False): default_cost_per_query: Optional[float] diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index ed391f0af68..db7657a0174 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -54,6 +54,8 @@ class MCPServer(BaseModel): aws_session_token: Optional[str] = None aws_region_name: Optional[str] = None aws_service_name: Optional[str] = None # defaults to "bedrock-agentcore" + aws_role_name: Optional[str] = None # IAM role ARN for STS AssumeRole + aws_session_name: Optional[str] = None # session name for CloudTrail auditing # Stdio-specific fields command: Optional[str] = None args: Optional[List[str]] = None diff --git a/litellm/utils.py b/litellm/utils.py index 37bc35af299..6806961bf51 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8314,6 +8314,10 @@ class ProviderConfigManager: return SagemakerEmbeddingConfig.get_model_config(model) elif litellm.LlmProviders.PERPLEXITY == provider: return litellm.PerplexityEmbeddingConfig() + elif litellm.LlmProviders.OCI == provider: + from litellm.llms.oci.embed.transformation import OCIEmbeddingConfig + + return OCIEmbeddingConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 35fa2206761..6da9a004b8a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23788,7 +23788,8 @@ "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_vision": true }, "oci/meta.llama-3.3-70b-instruct": { "input_cost_per_token": 7.2e-07, @@ -23922,6 +23923,287 @@ "supports_function_calling": true, "supports_response_schema": false }, + "oci/cohere.command-a-reasoning-08-2025": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/cohere.command-a-vision-07-2025": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true + }, + "oci/cohere.command-a-translate-08-2025": { + "input_cost_per_token": 9e-08, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 9e-08, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": false, + "supports_response_schema": false + }, + "oci/cohere.command-r-08-2024": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/cohere.command-r-plus-08-2024": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/meta.llama-3.2-11b-vision-instruct": { + "input_cost_per_token": 2e-06, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true + }, + "oci/meta.llama-3.1-70b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/meta.llama-3.3-70b-instruct-fp8-dynamic": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4-fast": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.1-fast": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.20": { + "input_cost_per_token": 3e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-4.20-multi-agent": { + "input_cost_per_token": 3e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/xai.grok-code-fast-1": { + "input_cost_per_token": 5e-06, + "litellm_provider": "oci", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false + }, + "oci/google.gemini-2.5-pro": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/google.gemini-2.5-flash": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/google.gemini-2.5-flash-lite": { + "input_cost_per_token": 7.5e-08, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/cohere.embed-english-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-english-light-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-multilingual-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-multilingual-light-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/cohere.embed-english-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-english-light-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-multilingual-light-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "max_tokens": 512, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 384, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, + "oci/cohere.embed-v4.0": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_tokens": 128000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1536, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_embedding_image_input": true + }, "ollama/codegeex4": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", @@ -30290,6 +30572,27 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "vertex_ai/claude-haiku-4-5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_native_streaming": true, + "supports_vision": true + }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, "cache_read_input_token_cost": 1e-07, diff --git a/tests/test_litellm/llms/oci/embed/__init__.py b/tests/test_litellm/llms/oci/embed/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py new file mode 100644 index 00000000000..4ecca377e63 --- /dev/null +++ b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py @@ -0,0 +1,369 @@ +import json +import os +import sys +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.oci.embed.transformation import OCIEmbeddingConfig +from litellm.types.utils import EmbeddingResponse + +# Test constants +TEST_MODEL_NAME = "cohere.embed-english-v3.0" +TEST_MODEL = f"oci/{TEST_MODEL_NAME}" +TEST_COMPARTMENT_ID = "ocid1.compartment.oc1..xxxxxx" +BASE_OCI_PARAMS = { + "oci_region": "us-ashburn-1", + "oci_user": "ocid1.user.oc1..xxxxxxEXAMPLExxxxxx", + "oci_fingerprint": "4f:29:77:cc:b1:3e:55:ab:61:2a:de:47:f1:38:4c:90", + "oci_tenancy": "ocid1.tenancy.oc1..xxxxxxEXAMPLExxxxxx", + "oci_compartment_id": TEST_COMPARTMENT_ID, +} + +TEST_OCI_PARAMS_KEY = { + **BASE_OCI_PARAMS, + "oci_key": "", +} + +TEST_OCI_PARAMS_KEY_FILE = { + **BASE_OCI_PARAMS, + "oci_key_file": "", +} + +# Mock OCI embedding response +MOCK_OCI_EMBEDDING_RESPONSE = { + "embeddings": [[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8]], + "modelId": "cohere.embed-english-v3.0", + "modelVersion": "3.0", + "inputTextTokenCounts": [5, 4], +} + + +@pytest.fixture(params=[TEST_OCI_PARAMS_KEY, TEST_OCI_PARAMS_KEY_FILE]) +def supplied_params(request): + """Fixture for passing in optional_parameters""" + return request.param + + +class TestOCIEmbeddingConfig: + def test_get_complete_url_default_region(self): + """test_get_complete_url returns URL with us-ashburn-1 when no api_base is given.""" + config = OCIEmbeddingConfig() + url = config.get_complete_url( + api_base=None, + api_key=None, + model=TEST_MODEL_NAME, + optional_params={}, + litellm_params={}, + ) + assert "us-ashburn-1" in url + assert "embedText" in url + + def test_get_complete_url_custom_region(self): + """test_get_complete_url uses region from optional_params.""" + config = OCIEmbeddingConfig() + url = config.get_complete_url( + api_base=None, + api_key=None, + model=TEST_MODEL_NAME, + optional_params={"oci_region": "us-chicago-1"}, + litellm_params={}, + ) + assert "us-chicago-1" in url + assert "embedText" in url + + def test_get_complete_url_custom_api_base(self): + """test_get_complete_url returns api_base as-is when provided.""" + config = OCIEmbeddingConfig() + custom_base = "https://custom.oci.example.com/embed" + url = config.get_complete_url( + api_base=custom_base, + api_key=None, + model=TEST_MODEL_NAME, + optional_params={}, + litellm_params={}, + ) + assert url == custom_base + + def test_get_supported_openai_params(self): + """test_get_supported_openai_params returns expected params list.""" + config = OCIEmbeddingConfig() + params = config.get_supported_openai_params(model=TEST_MODEL_NAME) + assert "dimensions" in params + assert "encoding_format" not in params + + def test_map_openai_params_dimensions(self): + """test dimensions is mapped correctly.""" + config = OCIEmbeddingConfig() + optional_params = {} + result = config.map_openai_params( + non_default_params={"dimensions": 512}, + optional_params=optional_params, + model=TEST_MODEL_NAME, + drop_params=False, + ) + assert result["dimensions"] == 512 + + def test_validate_environment_with_credentials(self, supplied_params): + """test validate_environment returns content-type and user-agent headers when credentials are supplied.""" + config = OCIEmbeddingConfig() + headers = {} + result = config.validate_environment( + headers=headers, + model=TEST_MODEL, + messages=[], + optional_params=supplied_params, + litellm_params={}, + ) + assert result["content-type"] == "application/json" + assert "litellm" in result["user-agent"] + + def test_validate_environment_missing_credentials(self): + """test validate_environment raises Exception with 'Missing required parameters' when credentials are incomplete.""" + config = OCIEmbeddingConfig() + incomplete_params = { + "oci_user": "ocid1.user.oc1..xxx", + # Missing oci_fingerprint, oci_tenancy, oci_key/oci_key_file, oci_compartment_id + } + with pytest.raises(Exception) as excinfo: + config.validate_environment( + headers={}, + model=TEST_MODEL, + messages=[], + optional_params=incomplete_params, + litellm_params={}, + ) + assert "Missing required parameters" in str(excinfo.value) + + def test_validate_environment_with_signer(self): + """test validate_environment passes when oci_signer is provided.""" + config = OCIEmbeddingConfig() + + class MockSigner: + def do_request_sign(self, request, enforce_content_headers=True): + request.headers["authorization"] = 'Signature version="1"' + + optional_params = { + "oci_signer": MockSigner(), + "oci_region": "us-ashburn-1", + } + result = config.validate_environment( + headers={}, + model=TEST_MODEL, + messages=[], + optional_params=optional_params, + litellm_params={}, + ) + assert result["content-type"] == "application/json" + + def test_transform_embedding_request_on_demand(self): + """test transform_embedding_request builds correct ON_DEMAND OCI request body.""" + config = OCIEmbeddingConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + } + with patch.object(config, "sign_request", return_value=({}, "{}")): + result = config.transform_embedding_request( + model=TEST_MODEL_NAME, + input=["Hello world", "Goodbye world"], + optional_params=optional_params, + headers={}, + ) + + assert result["compartmentId"] == TEST_COMPARTMENT_ID + assert result["servingMode"]["servingType"] == "ON_DEMAND" + assert result["servingMode"]["modelId"] == TEST_MODEL_NAME + assert result["inputs"] == ["Hello world", "Goodbye world"] + assert result["truncate"] == "END" + + def test_transform_embedding_request_dedicated(self): + """test transform_embedding_request builds DEDICATED servingMode with endpointId.""" + config = OCIEmbeddingConfig() + test_endpoint_id = "ocid1.generativeaiendpoint.oc1.us-chicago-1.xxxxxx" + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "oci_serving_mode": "DEDICATED", + "oci_endpoint_id": test_endpoint_id, + } + with patch.object(config, "sign_request", return_value=({}, "{}")): + result = config.transform_embedding_request( + model=TEST_MODEL_NAME, + input=["Hello world"], + optional_params=optional_params, + headers={}, + ) + + assert result["servingMode"]["servingType"] == "DEDICATED" + assert result["servingMode"]["endpointId"] == test_endpoint_id + + def test_transform_embedding_request_input_type(self): + """test input_type=search_query is mapped to SEARCH_QUERY in request data.""" + config = OCIEmbeddingConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "input_type": "search_query", + } + with patch.object(config, "sign_request", return_value=({}, "{}")): + result = config.transform_embedding_request( + model=TEST_MODEL_NAME, + input=["What is the capital of Brazil?"], + optional_params=optional_params, + headers={}, + ) + + assert result["inputType"] == "SEARCH_QUERY" + + def test_transform_embedding_request_string_input(self): + """test single string input is wrapped in a list.""" + config = OCIEmbeddingConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + } + with patch.object(config, "sign_request", return_value=({}, "{}")): + result = config.transform_embedding_request( + model=TEST_MODEL_NAME, + input="Hello world", + optional_params=optional_params, + headers={}, + ) + + assert isinstance(result["inputs"], list) + assert result["inputs"] == ["Hello world"] + + def test_transform_embedding_request_token_list_raises(self): + """test token-array inputs raise ValueError instead of silent conversion.""" + config = OCIEmbeddingConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + } + with patch.object(config, "sign_request", return_value=({}, "{}")): + with pytest.raises(ValueError, match="does not support token-array"): + config.transform_embedding_request( + model=TEST_MODEL_NAME, + input=[[1234, 5678]], + optional_params=optional_params, + headers={}, + ) + + def test_transform_embedding_response(self): + """test OCI embedding response is correctly transformed into EmbeddingResponse.""" + config = OCIEmbeddingConfig() + mock_response = httpx.Response( + status_code=200, + json=MOCK_OCI_EMBEDDING_RESPONSE, + request=httpx.Request("POST", "https://test.com"), + ) + mock_logging = MagicMock() + model_response = EmbeddingResponse() + + result = config.transform_embedding_response( + model=TEST_MODEL_NAME, + raw_response=mock_response, + model_response=model_response, + logging_obj=mock_logging, + ) + + assert isinstance(result, EmbeddingResponse) + assert result.model == "cohere.embed-english-v3.0" + assert len(result.data) == 2 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3, 0.4] + assert result.data[1]["embedding"] == [0.5, 0.6, 0.7, 0.8] + assert result.data[0]["index"] == 0 + assert result.data[1]["index"] == 1 + # Total tokens: 5 + 4 = 9 + assert result.usage.prompt_tokens == 9 + assert result.usage.total_tokens == 9 + + def test_transform_embedding_response_error(self): + """test non-200 status code raises OCIError.""" + from litellm.llms.oci.common_utils import OCIError + + config = OCIEmbeddingConfig() + mock_response = httpx.Response( + status_code=400, + text="Bad Request", + request=httpx.Request("POST", "https://test.com"), + ) + mock_logging = MagicMock() + model_response = EmbeddingResponse() + + with pytest.raises(OCIError): + config.transform_embedding_response( + model=TEST_MODEL_NAME, + raw_response=mock_response, + model_response=model_response, + logging_obj=mock_logging, + ) + + def test_model_prices_embedding_models(self): + """test all 8 OCI embedding models exist in model_prices_and_context_window.json with mode=embedding.""" + model_prices_path = os.path.join( + os.path.dirname(__file__), + "..", + "..", + "..", + "..", + "..", + "model_prices_and_context_window.json", + ) + with open(model_prices_path) as f: + model_prices = json.load(f) + + expected_embedding_models = [ + "oci/cohere.embed-english-v3.0", + "oci/cohere.embed-english-light-v3.0", + "oci/cohere.embed-multilingual-v3.0", + "oci/cohere.embed-multilingual-light-v3.0", + "oci/cohere.embed-english-image-v3.0", + "oci/cohere.embed-english-light-image-v3.0", + "oci/cohere.embed-multilingual-light-image-v3.0", + "oci/cohere.embed-v4.0", + ] + + for model_key in expected_embedding_models: + assert model_key in model_prices, f"Missing model: {model_key}" + assert ( + model_prices[model_key].get("mode") == "embedding" + ), f"Model {model_key} does not have mode='embedding'" + + def test_model_prices_new_chat_models(self): + """test the 16 new OCI chat models exist in model_prices_and_context_window.json with mode=chat.""" + model_prices_path = os.path.join( + os.path.dirname(__file__), + "..", + "..", + "..", + "..", + "..", + "model_prices_and_context_window.json", + ) + with open(model_prices_path) as f: + model_prices = json.load(f) + + expected_chat_models = [ + "oci/xai.grok-3", + "oci/xai.grok-3-fast", + "oci/xai.grok-3-mini", + "oci/xai.grok-3-mini-fast", + "oci/xai.grok-4", + "oci/xai.grok-4-fast", + "oci/xai.grok-4.1-fast", + "oci/xai.grok-4.20", + "oci/xai.grok-4.20-multi-agent", + "oci/xai.grok-code-fast-1", + "oci/cohere.command-a-03-2025", + "oci/cohere.command-a-reasoning-08-2025", + "oci/cohere.command-a-vision-07-2025", + "oci/cohere.command-a-translate-08-2025", + "oci/google.gemini-2.5-pro", + "oci/google.gemini-2.5-flash", + ] + + for model_key in expected_chat_models: + assert model_key in model_prices, f"Missing model: {model_key}" + assert ( + model_prices[model_key].get("mode") == "chat" + ), f"Model {model_key} does not have mode='chat'" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py index a2295e1271e..7c142e3a771 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py @@ -162,6 +162,140 @@ class TestMCPSigV4Auth: assert "x-amz-security-token" in signed_request.headers +class TestMCPSigV4AssumeRole: + """Tests for STS AssumeRole credential resolution in MCPSigV4Auth.""" + + def test_assume_role_with_ambient_credentials(self): + """MCPSigV4Auth calls STS AssumeRole when aws_role_name is provided (no explicit keys).""" + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + with patch("boto3.client", return_value=mock_sts) as mock_boto3: + auth = MCPSigV4Auth( + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_region_name="us-east-1", + ) + + mock_boto3.assert_called_once_with("sts", region_name="us-east-1") + mock_sts.assume_role.assert_called_once() + call_kwargs = mock_sts.assume_role.call_args[1] + assert call_kwargs["RoleArn"] == "arn:aws:iam::123456789012:role/TestRole" + assert call_kwargs["RoleSessionName"].startswith("litellm-mcp-") + assert auth.credentials.access_key == "ASSUMED_KEY" + assert auth.credentials.secret_key == "ASSUMED_SECRET" + assert auth.credentials.token == "ASSUMED_TOKEN" + + def test_assume_role_with_explicit_source_credentials(self): + """When aws_role_name + explicit keys are provided, keys are used as STS source identity.""" + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + with patch("boto3.client", return_value=mock_sts) as mock_boto3: + auth = MCPSigV4Auth( + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_access_key_id="SOURCE_KEY", + aws_secret_access_key="SOURCE_SECRET", + aws_region_name="us-west-2", + ) + + mock_boto3.assert_called_once_with( + "sts", + region_name="us-west-2", + aws_access_key_id="SOURCE_KEY", + aws_secret_access_key="SOURCE_SECRET", + ) + assert auth.credentials.access_key == "ASSUMED_KEY" + + def test_assume_role_with_custom_session_name(self): + """Custom aws_session_name is used in the AssumeRole call.""" + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + with patch("boto3.client", return_value=mock_sts): + MCPSigV4Auth( + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_session_name="regeneron-litellm-prod", + ) + + call_kwargs = mock_sts.assume_role.call_args[1] + assert call_kwargs["RoleSessionName"] == "regeneron-litellm-prod" + + def test_assume_role_signing_works(self): + """Requests are signed correctly with STS-derived credentials.""" + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "AKIAIOSFODNN7EXAMPLE", + "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "SessionToken": "STS_SESSION_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + with patch("boto3.client", return_value=mock_sts): + auth = MCPSigV4Auth( + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_region_name="us-east-1", + aws_service_name="bedrock-agentcore", + ) + + request = httpx.Request( + method="POST", + url="https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/test/invocations", + headers={"Content-Type": "application/json"}, + content=b'{"jsonrpc":"2.0","method":"tools/list","id":1}', + ) + + signed_request = next(auth.auth_flow(request)) + assert "Authorization" in signed_request.headers + assert "AWS4-HMAC-SHA256" in signed_request.headers["Authorization"] + assert "x-amz-security-token" in signed_request.headers + + def test_assume_role_takes_precedence_over_explicit_keys(self): + """When both aws_role_name and explicit keys are provided, AssumeRole is used (keys become source identity).""" + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + with patch("boto3.client", return_value=mock_sts): + auth = MCPSigV4Auth( + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_access_key_id="EXPLICIT_KEY", + aws_secret_access_key="EXPLICIT_SECRET", + ) + + # Credentials should be from AssumeRole, not the explicit keys + assert auth.credentials.access_key == "ASSUMED_KEY" + assert auth.credentials.secret_key == "ASSUMED_SECRET" + + class TestMCPClientSigV4Integration: """Tests for MCPClient with SigV4 auth wired through.""" @@ -319,6 +453,86 @@ class TestMCPServerManagerSigV4: assert client._aws_auth is None + @pytest.mark.asyncio + async def test_load_config_with_aws_role_name(self): + """Config loading correctly parses aws_role_name and aws_session_name.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + + config = { + "agentcore_tools": { + "url": "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/test/invocations", + "transport": "http", + "auth_type": "aws_sigv4", + "aws_role_name": "arn:aws:iam::123456789012:role/TestRole", + "aws_session_name": "litellm-prod", + "aws_region_name": "us-east-1", + } + } + + manager = MCPServerManager() + await manager.load_servers_from_config(config) + + server = next(iter(manager.config_mcp_servers.values())) + assert server.aws_role_name == "arn:aws:iam::123456789012:role/TestRole" + assert server.aws_session_name == "litellm-prod" + + @pytest.mark.asyncio + async def test_create_mcp_client_with_role_assumption(self): + """_create_mcp_client passes aws_role_name to MCPSigV4Auth.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + mock_sts = MagicMock() + mock_sts.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": "2026-03-30T12:00:00Z", + } + } + + server = MCPServer( + server_id="test-sigv4-role", + name="test_sigv4_role", + server_name="test_sigv4_role", + url="https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/test/invocations", + transport=MCPTransport.http, + auth_type=MCPAuth.aws_sigv4, + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_region_name="us-east-1", + ) + + manager = MCPServerManager() + with patch("boto3.client", return_value=mock_sts): + client = await manager._create_mcp_client(server=server) + + assert client._aws_auth is not None + assert isinstance(client._aws_auth, MCPSigV4Auth) + mock_sts.assume_role.assert_called_once() + + def test_extract_aws_credentials_includes_role_fields(self): + """_extract_aws_credentials extracts aws_role_name and aws_session_name.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + + manager = MCPServerManager() + creds = { + "aws_access_key_id": "KEY", + "aws_region_name": "us-east-1", + "aws_role_name": "arn:aws:iam::123456789012:role/TestRole", + "aws_session_name": "my-session", + } + + result = manager._extract_aws_credentials(creds, credentials_are_encrypted=False) + assert result["aws_role_name"] == "arn:aws:iam::123456789012:role/TestRole" + assert result["aws_session_name"] == "my-session" + class TestSigV4CredentialEncryption: """Test encrypt/decrypt round-trip for AWS SigV4 credentials.""" diff --git a/tests/test_litellm/proxy/db/test_check_migration.py b/tests/test_litellm/proxy/db/test_check_migration.py index ad72a0d1195..f2468d95820 100644 --- a/tests/test_litellm/proxy/db/test_check_migration.py +++ b/tests/test_litellm/proxy/db/test_check_migration.py @@ -27,12 +27,14 @@ def test_check_migration_out_of_sync(mocker): - 🚨 [IMPORTANT] Does NOT Raise an Exception when the Prisma schema is out of sync with the database. - logs an error when the Prisma schema is out of sync with the database. """ - # Mock the logger BEFORE importing the function - mock_logger = mocker.patch("litellm._logging.verbose_logger") - - # Import the function after mocking the logger + # Import the function first so check_migration module is in sys.modules, + # then patch the logger reference in that module directly (not the source + # module) so the patch works regardless of import order or xdist worker + # assignment. from litellm.proxy.db.check_migration import check_prisma_schema_diff + mock_logger = mocker.patch("litellm.proxy.db.check_migration.verbose_logger") + # Mock the helper function to simulate out-of-sync state mock_diff_helper = mocker.patch( "litellm.proxy.db.check_migration.check_prisma_schema_diff_helper", diff --git a/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py b/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py index 6c7cfa61b58..dca2bd84f92 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py @@ -188,6 +188,26 @@ async def test_tag_filtering_disabled_returns_all_deployments(): assert result == ALL_DEPLOYMENTS +@pytest.mark.asyncio +async def test_empty_healthy_deployments_with_request_tags_returns_empty_list(): + """ + With an empty candidate list, return [] even when the request includes metadata tags. + + Tag-based filtering runs only against non-empty healthy_deployments; an empty list is + returned unchanged for the router's standard handling. + """ + router = _make_router_mock() + result = await get_deployments_for_tag( + llm_router_instance=router, + model="gpt-5.2", + healthy_deployments=[], + request_kwargs={ + "metadata": {"tags": ["client_id:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]} + }, + ) + assert result == [] + + @pytest.mark.asyncio async def test_explicit_tag_match_takes_precedence_over_regex(): """A deployment with both tags and tag_regex: exact tag match fires first.""" diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index 1d575195194..6cbb2fd7b8f 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -262,3 +262,80 @@ def test_key_name_redaction_in_general_settings_dict(): assert "REDACTED" in output # Non-sensitive values should survive assert "enable_jwt_auth" in output + + +# ── GCP service-account / Vertex credential redaction ── + + +_SAMPLE_SA_JSON = ( + '{"type": "service_account", "project_id": "my-proj-123", ' + '"private_key_id": "abc123def", ' + '"private_key": "-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkq\\n-----END PRIVATE KEY-----\\n", ' + '"client_email": "sa@my-proj.iam.gserviceaccount.com", ' + '"client_id": "123456789"}' +) + + +def test_pem_private_key_redacted_in_json(): + result = _redact_string(_SAMPLE_SA_JSON) + assert "MIIEvQIBADA" not in result + assert "-----BEGIN" not in result + + +def test_pem_private_key_redacted_in_dict_repr(): + import json + + sa = json.loads(_SAMPLE_SA_JSON) + result = _redact_string(str(sa)) + assert "MIIEvQIBADA" not in result + + +def test_service_account_blob_fully_redacted(): + result = _redact_string(f"Got={_SAMPLE_SA_JSON}") + assert "my-proj-123" not in result + assert "sa@my-proj.iam.gserviceaccount.com" not in result + assert "abc123def" not in result + assert "MIIEvQIBADA" not in result + + +def test_vertex_error_message_no_credential_leak(): + """The old Vertex error format leaked the full credential JSON. + The new format must not contain any credential material.""" + new_msg = ( + "Unable to load vertex credentials from environment. " + "Ensure the JSON is valid (check for unescaped newlines in private_key). " + "Parse error: JSONDecodeError" + ) + result = _redact_string(new_msg) + assert result == new_msg # nothing to redact + + +def test_vertex_traceback_redacts_pem(): + traceback_text = ( + "Traceback (most recent call last):\n" + ' File "vertex_llm_base.py", line 95\n' + " json_obj = json.loads(credentials)\n" + "json.decoder.JSONDecodeError: Invalid control character\n" + "Failed to load vertex credentials. Error: " + "Unable to load vertex credentials from environment. " + f"Got={_SAMPLE_SA_JSON}" + ) + result = _redact_string(traceback_text) + assert "MIIEvQIBADA" not in result + assert "-----BEGIN" not in result + + +def test_gcp_oauth_token_redacted(): + result = _redact_string("access token ya29.c.c0ASRK0GZvXlongtokenhere") + assert "ya29." not in result + assert "REDACTED" in result + + +def test_non_pem_private_key_value_redacted(): + result = _redact_string("'private_key': 'some-non-pem-secret-value'") + assert "some-non-pem-secret" not in result + + +def test_normal_vertex_log_not_redacted(): + msg = "Vertex: Loading vertex credentials, is_file_path=True, current dir /app" + assert _redact_string(msg) == msg diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index aaddf53fd7e..4c824fcee0b 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -934,6 +934,38 @@ const CreateMCPServer: React.FC = ({ className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500" /> + + AWS Role ARN + + + + + } + name={["credentials", "aws_role_name"]} + > + + + + AWS Session Name + + + + + } + name={["credentials", "aws_session_name"]} + > + + )} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 118461f0043..e81d2f3960e 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -977,6 +977,38 @@ const MCPServerEdit: React.FC = ({ className="rounded-lg border-gray-300 focus:border-blue-500 focus:ring-blue-500" /> + + AWS Role ARN + + + + + } + name={["credentials", "aws_role_name"]} + > + + + + AWS Session Name + + + + + } + name={["credentials", "aws_session_name"]} + > + + )}