diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index dab838133e9..a1e5cb99416 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -10,6 +10,7 @@ anthropic orjson==3.10.12 # fast /embedding responses pydantic==2.10.2 google-cloud-aiplatform==1.43.0 +google-cloud-iam==2.19.1 fastapi-sso==0.16.0 uvloop==0.21.0 mcp==1.10.1 # for MCP server diff --git a/cookbook/misc/test_responses_api.py b/cookbook/misc/test_responses_api.py new file mode 100644 index 00000000000..5fd19c6f66f --- /dev/null +++ b/cookbook/misc/test_responses_api.py @@ -0,0 +1,53 @@ +import base64 +from openai import OpenAI +import time +client = OpenAI( + base_url="http://0.0.0.0:4001", + api_key="sk-1234" +) + +# Function to encode the image +def encode_image(image_path): + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode("utf-8") + + +# Path to your image +image_path = "litellm/proxy/logo.jpg" + +# Getting the Base64 string +base64_image = encode_image(image_path) + + +response = client.responses.create( + model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + input=[ + { + "role": "user", + "content": [ + { "type": "input_text", "text": "what color is the image"}, + { + "type": "input_image", + "image_url": f"data:image/jpeg;base64,{base64_image}", + }, + ], + } + ], +) + + + +print(response.output_text) +print("response1 id===", response.id) +print("sleeping for 20 seconds...") +time.sleep(20) +print("making follow up request for existing id") +response2 = client.responses.create( + model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + previous_response_id=response.id, + input="ok, and what objects are in the image?" +) + +print(response2.output_text) + + diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 143e62fceb3..cf10be0a76b 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -1,9 +1,11 @@ {{- if .Values.migrationJob.enabled }} -# This job runs the prisma migrations for the LiteLLM DB. +# This job runs the Prisma migrations for the LiteLLM DB. apiVersion: batch/v1 kind: Job metadata: name: {{ include "litellm.fullname" . }}-migrations + labels: + {{- include "litellm.labels" . | nindent 4 }} annotations: {{- if .Values.migrationJob.hooks.argocd.enabled }} argocd.argoproj.io/hook: PreSync @@ -18,6 +20,8 @@ metadata: spec: template: metadata: + labels: + {{- include "litellm.labels" . | nindent 8 }} annotations: {{- with .Values.migrationJob.annotations }} {{- toYaml . | nindent 8 }} diff --git a/docs/my-website/docs/caching/all_caches.md b/docs/my-website/docs/caching/all_caches.md index a6be3396291..0548c331f80 100644 --- a/docs/my-website/docs/caching/all_caches.md +++ b/docs/my-website/docs/caching/all_caches.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Caching - In-Memory, Redis, s3, Redis Semantic Cache, Disk +# Caching - In-Memory, Redis, s3, gcs, Redis Semantic Cache, Disk [**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm/caching/caching.py) @@ -14,7 +14,7 @@ import TabItem from '@theme/TabItem'; ::: -## Initialize Cache - In Memory, Redis, s3 Bucket, Redis Semantic, Disk Cache, Qdrant Semantic +## Initialize Cache - In Memory, Redis, s3 Bucket, gcs Bucket, Redis Semantic, Disk Cache, Qdrant Semantic @@ -28,6 +28,8 @@ pip install redis For the hosted version you can setup your own Redis DB here: https://redis.io/try-free/ +**Basic Redis Cache** + ```python import litellm from litellm import completion @@ -48,6 +50,91 @@ response2 = completion( # response1 == response2, response 1 is cached ``` +**GCP IAM Redis Authentication** + +For GCP Memorystore Redis with IAM authentication: + +```shell +pip install google-cloud-iam +``` + +```python +import litellm +from litellm import completion +# For Redis Cluster with GCP IAM +from litellm.caching.redis_cluster_cache import RedisClusterCache + +litellm.cache = RedisClusterCache( + startup_nodes=[ + {"host": "10.128.0.2", "port": 6379}, + {"host": "10.128.0.2", "port": 11008}, + ], + gcp_service_account="projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com", + ssl=True, + ssl_cert_reqs=None, + ssl_check_hostname=False, +) + +# Make completion calls +response1 = completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Tell me a joke."}] +) +response2 = completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Tell me a joke."}] +) + +# response1 == response2, response 1 is cached +``` + +**Environment Variables for GCP IAM Redis** + +You can also set these as environment variables: + +```shell +export REDIS_HOST="10.128.0.2" +export REDIS_PORT="6379" +export REDIS_GCP_SERVICE_ACCOUNT="projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" +export REDIS_SSL="False" +``` + +Then simply initialize: + +```python +litellm.cache = Cache(type="redis") +``` + + + + + +Set environment variables + +```shell +GCS_BUCKET_NAME="my-cache-bucket" +GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json" +``` + +```python +import litellm +from litellm import completion +from litellm.caching.caching import Cache + +litellm.cache = Cache(type="gcs", gcs_bucket_name="my-cache-bucket", gcs_path_service_account="/path/to/service_account.json") + +response1 = completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Tell me a joke."}] +) +response2 = completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Tell me a joke."}] +) + +# response1 == response2, response 1 is cached +``` + @@ -405,7 +492,7 @@ Advanced Params ```python litellm.enable_cache( - type: Optional[Literal["local", "redis", "s3", "disk"]] = "local", + type: Optional[Literal["local", "redis", "s3", "gcs", "disk"]] = "local", host: Optional[str] = None, port: Optional[str] = None, password: Optional[str] = None, @@ -429,7 +516,7 @@ Update the Cache params ```python litellm.update_cache( - type: Optional[Literal["local", "redis", "s3", "disk"]] = "local", + type: Optional[Literal["local", "redis", "s3", "gcs", "disk"]] = "local", host: Optional[str] = None, port: Optional[str] = None, password: Optional[str] = None, @@ -490,7 +577,7 @@ cache.get_cache = get_cache ```python def __init__( self, - type: Optional[Literal["local", "redis", "redis-semantic", "s3", "disk"]] = "local", + type: Optional[Literal["local", "redis", "redis-semantic", "s3", "gcs", "disk"]] = "local", supported_call_types: Optional[ List[Literal["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"]] ] = ["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"], @@ -504,6 +591,13 @@ def __init__( namespace: Optional[str] = None, default_in_redis_ttl: Optional[float] = None, redis_flush_size=None, + + # GCP IAM Redis authentication params + gcp_service_account: Optional[str] = None, + gcp_ssl_ca_certs: Optional[str] = None, + ssl: Optional[bool] = None, + ssl_cert_reqs: Optional[Union[str, None]] = None, + ssl_check_hostname: Optional[bool] = None, # redis semantic cache params similarity_threshold: Optional[float] = None, diff --git a/docs/my-website/docs/observability/braintrust.md b/docs/my-website/docs/observability/braintrust.md index 79f3cf13be2..eb26680b18a 100644 --- a/docs/my-website/docs/observability/braintrust.md +++ b/docs/my-website/docs/observability/braintrust.md @@ -15,6 +15,7 @@ import os # set env os.environ["BRAINTRUST_API_KEY"] = "" +os.environ["BRAINTRUST_API_BASE"] = "https://api.braintrustdata.com/v1" os.environ['OPENAI_API_KEY']="" # set braintrust as a callback, litellm will send the data to braintrust @@ -35,6 +36,7 @@ response = litellm.completion( ```env BRAINTRUST_API_KEY="" +BRAINTRUST_API_BASE="https://api.braintrustdata.com/v1" ``` 2. Add braintrust to callbacks @@ -157,6 +159,8 @@ For more examples, [**Click Here**](../proxy/user_keys.md#chatcompletions) +You can use `BRAINTRUST_API_BASE` to point to your self-hosted Braintrust data plane. Read more about this [here](https://www.braintrust.dev/docs/guides/self-hosting). + ## Full API Spec Here's everything you can pass in metadata for a braintrust request diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 4b4f53a8fcf..a7a9dc30013 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -4,6 +4,7 @@ import TabItem from '@theme/TabItem'; # Anthropic LiteLLM supports all anthropic models. +- `claude-opus-4-1-20250805` - `claude-4` (`claude-opus-4-20250514`, `claude-sonnet-4-20250514`) - `claude-3.7` (`claude-3-7-sonnet-20250219`) - `claude-3.5` (`claude-3-5-sonnet-20240620`) diff --git a/docs/my-website/docs/providers/oci.md b/docs/my-website/docs/providers/oci.md new file mode 100644 index 00000000000..28beb71094a --- /dev/null +++ b/docs/my-website/docs/providers/oci.md @@ -0,0 +1,75 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Oracle Cloud Infrastructure (OCI) +LiteLLM supports the following models for OCI on-demand GenAI API. + +Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm) to see if the model is available for your region. + +- `meta.llama-4-maverick-17b-128e-instruct-fp8` +- `meta.llama-4-scout-17b-16e-instruct` +- `meta.llama-3.3-70b-instruct` +- `meta.llama-3.2-90b-vision-instruct` +- `meta.llama-3.1-405b-instruct` + +- `xai.grok-4` +- `xai.grok-3` +- `xai.grok-3-fast` +- `xai.grok-3-mini` +- `xai.grok-3-mini-fast` + +## Authentication + +LiteLLM uses OCI signing key authentication. Follow the [official Oracle tutorial](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm) to create a signing key and obtain the following parameters: + +- `user` +- `fingerprint` +- `tenancy` +- `region` +- `key_file` + +## Usage + +Input the parameters obtained from the OCI signing key creation process into the `completion` function. + +```python +import os +from litellm import completion + +messages = [{"role": "user", "content": "Hey! how's it going?"}] +response = completion( + model="oci/xai.grok-4", + messages=messages, + oci_region=, + oci_user=, + oci_fingerprint=, + oci_tenancy=, + oci_key=, + oci_compartment_id=, +) +print(response) +``` + + +## Usage - Streaming +Just set `stream=True` when calling completion. + +```python +import os +from litellm import completion + +messages = [{"role": "user", "content": "Hey! how's it going?"}] +response = completion( + model="oci/xai.grok-4", + messages=messages, + stream=True, + oci_region=, + oci_user=, + oci_fingerprint=, + oci_tenancy=, + oci_key=, + oci_compartment_id=, +) +for chunk in response: + print(chunk["choices"][0]["delta"]["content"]) # same as openai format +``` diff --git a/docs/my-website/docs/proxy/caching.md b/docs/my-website/docs/proxy/caching.md index aec734e9142..1fb7385f689 100644 --- a/docs/my-website/docs/proxy/caching.md +++ b/docs/my-website/docs/proxy/caching.md @@ -204,7 +204,71 @@ For quick testing, you can also use REDIS_URL, eg.: REDIS_URL="rediss://.." ``` -but we **don't** recommend using REDIS_URL in prod. We've noticed a performance difference between using it vs. redis_host, port, etc. +but we **don't** recommend using REDIS_URL in prod. We've noticed a performance difference between using it vs. redis_host, port, etc. + +#### GCP IAM Authentication + +For GCP Memorystore Redis with IAM authentication, install the required dependency: + +:::info +IAM authentication for redis is only supported via GCP and only on Redis Clusters for now. +::: + +```shell +pip install google-cloud-iam +``` + + + + + +For Redis Cluster with GCP IAM: + +```yaml +litellm_settings: + cache: True + cache_params: + type: redis + redis_startup_nodes: [{"host": "10.128.0.2", "port": 6379}, {"host": "10.128.0.2", "port": 11008}] + gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" + ssl: true + ssl_cert_reqs: null + ssl_check_hostname: false +``` + + + + + +You can configure GCP IAM Redis authentication in your .env: + + +For Redis Cluster: + +```env +REDIS_CLUSTER_NODES='[{"host": "10.128.0.2", "port": 6379}, {"host": "10.128.0.2", "port": 11008}]' +REDIS_GCP_SERVICE_ACCOUNT="projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" +REDIS_GCP_SSL_CA_CERTS="./server-ca.pem" +REDIS_SSL="True" +REDIS_SSL_CERT_REQS="None" +REDIS_SSL_CHECK_HOSTNAME="False" +``` + +**GCP Authentication Setup** + +Make sure your GCP credentials are configured: + +```shell +# Option 1: Service account key file +export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json" + +# Option 2: If running on GCP compute instance with service account attached +# No additional setup needed +``` + + + + #### Step 2: Add Redis Credentials to .env Set either `REDIS_URL` or the `REDIS_HOST` in your os environment, to enable caching. @@ -917,6 +981,13 @@ cache_params: password: secret_password # Redis server password namespace: Optional[str] = None, + # GCP IAM Authentication for Redis + gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" # GCP service account for IAM authentication + gcp_ssl_ca_certs: "./server-ca.pem" # Path to SSL CA certificate file for GCP Memorystore Redis + ssl: true # Enable SSL for secure connections + ssl_cert_reqs: null # Set to null for self-signed certificates + ssl_check_hostname: false # Set to false for self-signed certificates + # S3 cache parameters s3_bucket_name: your_s3_bucket_name # Name of the S3 bucket diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 853322bd19b..3b903935a04 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -58,6 +58,13 @@ litellm_settings: service_name: "mymaster" sentinel_nodes: [["localhost", 26379]] + # Optional - GCP IAM Authentication for Redis + gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" # GCP service account for IAM authentication + gcp_ssl_ca_certs: "./server-ca.pem" # Path to SSL CA certificate file for GCP Memorystore Redis + ssl: true # Enable SSL for secure connections + ssl_cert_reqs: null # Set to null for self-signed certificates + ssl_check_hostname: false # Set to false for self-signed certificates + # Optional - Qdrant Semantic Cache Settings qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list qdrant_collection_name: test_collection @@ -362,6 +369,7 @@ router_settings: | BEDROCK_MAX_POLICY_SIZE | Maximum size for Bedrock policy. Default is 75 | BERRISPEND_ACCOUNT_ID | Account ID for BerriSpend service | BRAINTRUST_API_KEY | API key for Braintrust integration +| BRAINTRUST_API_BASE | Base URL for Braintrust API. Default is https://api.braintrustdata.com/v1 | CACHED_STREAMING_CHUNK_DELAY | Delay in seconds for cached streaming chunks. Default is 0.02 | CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI | CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI @@ -649,6 +657,8 @@ router_settings: | REDIS_PASSWORD | Password for Redis service | REDIS_PORT | Port number for Redis server | REDIS_SOCKET_TIMEOUT | Timeout in seconds for Redis socket operations. Default is 0.1 +| REDIS_GCP_SERVICE_ACCOUNT | GCP service account for IAM authentication with Redis. Format: "projects/-/serviceAccounts/name@project.iam.gserviceaccount.com" +| REDIS_GCP_SSL_CA_CERTS | Path to SSL CA certificate file for secure GCP Memorystore Redis connections | REDOC_URL | The path to the Redoc Fast API documentation. **By default this is "/redoc"** | REPEATED_STREAMING_CHUNK_LIMIT | Limit for repeated streaming chunks to detect looping. Default is 100 | REPLICATE_MODEL_NAME_WITH_ID_LENGTH | Length of Replicate model names with ID. Default is 64 diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index b1aae1da7c6..dc7030949bd 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -261,7 +261,12 @@ model_list: litellm_settings: callbacks: ["prometheus"] custom_prometheus_metadata_labels: ["metadata.foo", "metadata.bar"] - custom_prometheus_tags: ["prod", "staging", "batch-job"] + custom_prometheus_tags: + - "prod" + - "staging" + - "batch-job" + - "User-Agent: RooCode/*" + - "User-Agent: claude-cli/*" ``` 2. Make a request with tags @@ -297,16 +302,26 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ ``` **How Custom Tags Work:** -- Each configured tag becomes a boolean label in prometheus metrics -- If a tag is present in the request, the label value is `"true"` -- If a tag is not present in the request, the label value is `"false"` +- Each configured tag becomes a boolean label in prometheus metrics +- If a tag matches (exact or wildcard), the label value is `"true"`, otherwise `"false"` - Tag names are sanitized for prometheus compatibility (e.g., `"batch-job"` becomes `"tag_batch_job"`) +- **Wildcard patterns** supported using `*` (e.g., `"User-Agent: RooCode/*"` matches `"User-Agent: RooCode/1.0.0"`) + +**Example with wildcards:** +```yaml +litellm_settings: + callbacks: ["prometheus"] + custom_prometheus_tags: + - "User-Agent: RooCode/*" + - "User-Agent: claude-cli/*" +``` **Use Cases:** - Environment tracking (`prod`, `staging`, `dev`) - Request type classification (`batch-job`, `user-facing`, `background`) - Feature flags (`new-feature`, `beta-users`) - Team or service identification (`team-a`, `service-xyz`) +- User-Agent Tracking - use this to track how much Roo Code, Claude Code, Gemini CLI are used (`User-Agent: RooCode/*`, `User-Agent: claude-cli/*`, `User-Agent: gemini-cli/*`) ## Configuring Metrics and Labels diff --git a/docs/my-website/docs/proxy/user_keys.md b/docs/my-website/docs/proxy/user_keys.md index e56cc6867df..ecf6f2d0532 100644 --- a/docs/my-website/docs/proxy/user_keys.md +++ b/docs/my-website/docs/proxy/user_keys.md @@ -86,6 +86,11 @@ response = client.chat.completions.create( print(response) ``` + + + +[**šŸ‘‰ Go Here**](../providers/litellm_proxy#send-all-sdk-requests-to-litellm-proxy) + diff --git a/docs/my-website/release_notes/v1.74.15-stable/index.md b/docs/my-website/release_notes/v1.74.15-stable/index.md index a94359e3403..4fbb76bdbc4 100644 --- a/docs/my-website/release_notes/v1.74.15-stable/index.md +++ b/docs/my-website/release_notes/v1.74.15-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.15.rc.1 +ghcr.io/berriai/litellm:1.74.15.rc.1 ``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 83ebfe29dc4..bffa8a91b6d 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -469,7 +469,8 @@ const sidebars = { "providers/featherless_ai", "providers/nebius", "providers/dashscope", - "providers/bytez" + "providers/bytez", + "providers/oci", ], }, { diff --git a/enterprise/dist/litellm_enterprise-0.1.17-py3-none-any.whl b/enterprise/dist/litellm_enterprise-0.1.17-py3-none-any.whl new file mode 100644 index 00000000000..9c2856b4652 Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.17-py3-none-any.whl differ diff --git a/enterprise/dist/litellm_enterprise-0.1.17.tar.gz b/enterprise/dist/litellm_enterprise-0.1.17.tar.gz new file mode 100644 index 00000000000..92d4a6ee92f Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.17.tar.gz differ diff --git a/enterprise/dist/litellm_enterprise-0.1.19-py3-none-any.whl b/enterprise/dist/litellm_enterprise-0.1.19-py3-none-any.whl new file mode 100644 index 00000000000..5b48b65e4d2 Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.19-py3-none-any.whl differ diff --git a/enterprise/dist/litellm_enterprise-0.1.19.tar.gz b/enterprise/dist/litellm_enterprise-0.1.19.tar.gz new file mode 100644 index 00000000000..2f99960bdeb Binary files /dev/null and b/enterprise/dist/litellm_enterprise-0.1.19.tar.gz differ diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/session_handler.py b/enterprise/litellm_enterprise/enterprise_callbacks/session_handler.py deleted file mode 100644 index 1a08a8f9101..00000000000 --- a/enterprise/litellm_enterprise/enterprise_callbacks/session_handler.py +++ /dev/null @@ -1,160 +0,0 @@ -import json -from typing import TYPE_CHECKING, Any, List, Optional, Union, cast - -from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import SpendLogsPayload -from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.llms.openai import ( - AllMessageValues, - ChatCompletionResponseMessage, - GenericChatCompletionMessage, - ResponseInputParam, -) -from litellm.types.utils import ChatCompletionMessageToolCall, Message, ModelResponse - -if TYPE_CHECKING: - from litellm.responses.litellm_completion_transformation.transformation import ( - ChatCompletionSession, - ) -else: - ChatCompletionSession = Any - - -class _ENTERPRISE_ResponsesSessionHandler: - @staticmethod - async def get_chat_completion_message_history_for_previous_response_id( - previous_response_id: str, - ) -> ChatCompletionSession: - """ - Return the chat completion message history for a previous response id - """ - from litellm.responses.litellm_completion_transformation.transformation import ( - ChatCompletionSession, - LiteLLMCompletionResponsesConfig, - ) - - verbose_proxy_logger.debug( - "inside get_chat_completion_message_history_for_previous_response_id" - ) - all_spend_logs: List[ - SpendLogsPayload - ] = await _ENTERPRISE_ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( - previous_response_id - ) - verbose_proxy_logger.debug( - "found %s spend logs for this response id", len(all_spend_logs) - ) - - litellm_session_id: Optional[str] = None - if len(all_spend_logs) > 0: - litellm_session_id = all_spend_logs[0].get("session_id") - - chat_completion_message_history: List[ - Union[ - AllMessageValues, - GenericChatCompletionMessage, - ChatCompletionMessageToolCall, - ChatCompletionResponseMessage, - Message, - ] - ] = [] - for spend_log in all_spend_logs: - proxy_server_request: Union[str, dict] = ( - spend_log.get("proxy_server_request") or "{}" - ) - proxy_server_request_dict: Optional[dict] = None - response_input_param: Optional[Union[str, ResponseInputParam]] = None - if isinstance(proxy_server_request, dict): - proxy_server_request_dict = proxy_server_request - else: - proxy_server_request_dict = json.loads(proxy_server_request) - - ############################################################ - # Add Input messages for this Spend Log - ############################################################ - if proxy_server_request_dict: - _response_input_param = proxy_server_request_dict.get("input", None) - if isinstance(_response_input_param, str): - response_input_param = _response_input_param - elif isinstance(_response_input_param, dict): - response_input_param = cast( - ResponseInputParam, _response_input_param - ) - - if response_input_param: - chat_completion_messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( - input=response_input_param, - responses_api_request=proxy_server_request_dict or {}, - ) - chat_completion_message_history.extend(chat_completion_messages) - - ############################################################ - # Add Output messages for this Spend Log - ############################################################ - _response_output = spend_log.get("response", "{}") - if isinstance(_response_output, dict): - # transform `ChatCompletion Response` to `ResponsesAPIResponse` - model_response = ModelResponse(**_response_output) - for choice in model_response.choices: - if hasattr(choice, "message"): - chat_completion_message_history.append( - getattr(choice, "message") - ) - - verbose_proxy_logger.debug( - "chat_completion_message_history %s", - json.dumps(chat_completion_message_history, indent=4, default=str), - ) - return ChatCompletionSession( - messages=chat_completion_message_history, - litellm_session_id=litellm_session_id, - ) - - @staticmethod - async def get_all_spend_logs_for_previous_response_id( - previous_response_id: str, - ) -> List[SpendLogsPayload]: - """ - Get all spend logs for a previous response id - - - SQL query - - SELECT session_id FROM spend_logs WHERE response_id = previous_response_id, SELECT * FROM spend_logs WHERE session_id = session_id - """ - from litellm.proxy.proxy_server import prisma_client - - verbose_proxy_logger.debug("decoding response id=%s", previous_response_id) - - decoded_response_id = ( - ResponsesAPIRequestUtils._decode_responses_api_response_id( - previous_response_id - ) - ) - previous_response_id = decoded_response_id.get( - "response_id", previous_response_id - ) - if prisma_client is None: - return [] - - query = """ - WITH matching_session AS ( - SELECT session_id - FROM "LiteLLM_SpendLogs" - WHERE request_id = $1 - ) - SELECT * - FROM "LiteLLM_SpendLogs" - WHERE session_id IN (SELECT session_id FROM matching_session) - ORDER BY "endTime" ASC; - """ - - spend_logs = await prisma_client.db.query_raw(query, previous_response_id) - - verbose_proxy_logger.debug( - "Found the following spend logs for previous response id %s: %s", - previous_response_id, - json.dumps(spend_logs, indent=4, default=str), - ) - - return spend_logs diff --git a/enterprise/litellm_enterprise/integrations/prometheus.py b/enterprise/litellm_enterprise/integrations/prometheus.py index ddbcf948c85..a2d781fa1c4 100644 --- a/enterprise/litellm_enterprise/integrations/prometheus.py +++ b/enterprise/litellm_enterprise/integrations/prometheus.py @@ -122,19 +122,19 @@ class PrometheusLogger(CustomLogger): # Counter for total_output_tokens self.litellm_tokens_metric = self._counter_factory( - "litellm_total_tokens", + "litellm_total_tokens_metric", "Total number of input + output tokens from LLM requests", labelnames=self.get_labels_for_metric("litellm_total_tokens_metric"), ) self.litellm_input_tokens_metric = self._counter_factory( - "litellm_input_tokens", + "litellm_input_tokens_metric", "Total number of input tokens from LLM requests", labelnames=self.get_labels_for_metric("litellm_input_tokens_metric"), ) self.litellm_output_tokens_metric = self._counter_factory( - "litellm_output_tokens", + "litellm_output_tokens_metric", "Total number of output tokens from LLM requests", labelnames=self.get_labels_for_metric("litellm_output_tokens_metric"), ) @@ -2293,10 +2293,60 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]: return result +def _tag_matches_wildcard_configured_pattern(tags: List[str], configured_tag: str) -> bool: + """ + Check if any of the request tags matches a wildcard configured pattern + + Args: + tags: List[str] - The request tags + configured_tag: str - The configured tag + + Returns: + bool - True if any of the request tags matches the configured tag, False otherwise + + e.g. + tags = ["User-Agent: curl/7.68.0", "User-Agent: python-requests/2.28.1", "prod"] + configured_tag = "User-Agent: curl/*" + _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True + + configured_tag = "User-Agent: python-requests/*" + _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True + + configured_tag = "gm" + _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # False + """ + import re + + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter + pattern_router = PatternMatchRouter() + regex_pattern = pattern_router._pattern_to_regex(configured_tag) + return any(re.match(pattern=regex_pattern, string=tag) for tag in tags) + + def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]: """ - Get custom labels from tags based on admin configuration + Get custom labels from tags based on admin configuration. + + Supports both exact matches and wildcard patterns: + - Exact match: "prod" matches "prod" exactly + - Wildcard pattern: "User-Agent: curl/*" matches "User-Agent: curl/7.68.0" + + Reuses PatternMatchRouter for wildcard pattern matching. + + Returns dict of label_name: "true" if the tag matches the configured tag, "false" otherwise + + { + "tag_User-Agent_curl": "true", + "tag_User-Agent_python_requests": "false", + "tag_Environment_prod": "true", + "tag_Environment_dev": "false", + "tag_Service_api_gateway_v2": "true", + "tag_Service_web_app_v1": "false", + } """ + import re + + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.integrations.prometheus import _sanitize_prometheus_label_name configured_tags = litellm.custom_prometheus_tags @@ -2304,16 +2354,22 @@ def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]: return {} result: Dict[str, str] = {} + pattern_router = PatternMatchRouter() - # Map each configured tag to its presence in the request tags for configured_tag in configured_tags: - # Create a safe prometheus label name label_name = _sanitize_prometheus_label_name(f"tag_{configured_tag}") - - # Check if this tag is present in the request tags + + # Check for exact match first (backwards compatibility) if configured_tag in tags: result[label_name] = "true" - else: - result[label_name] = "false" + continue + + # Use PatternMatchRouter for wildcard pattern matching + if "*" in configured_tag and _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag): + result[label_name] = "true" + continue + + # No match found + result[label_name] = "false" return result diff --git a/enterprise/litellm_enterprise/proxy/auth/route_checks.py b/enterprise/litellm_enterprise/proxy/auth/route_checks.py index 1d4bfc664d5..6cce781faf3 100644 --- a/enterprise/litellm_enterprise/proxy/auth/route_checks.py +++ b/enterprise/litellm_enterprise/proxy/auth/route_checks.py @@ -20,7 +20,6 @@ class EnterpriseRouteChecks: status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"šŸšØšŸšØšŸšØ DISABLING LLM API ENDPOINTS is an Enterprise feature\nšŸšØ {CommonProxyErrors.not_premium_user.value}", ) - return False return get_secret_bool("DISABLE_LLM_API_ENDPOINTS") is True diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 182001e3d50..217bb753f42 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-enterprise" -version = "0.1.16" +version = "0.1.19" description = "Package for LiteLLM Enterprise features" authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.1.16" +version = "0.1.19" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-enterprise==", diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507161526_add_mcp_table_to_db/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507161526_add_mcp_table_to_db/migration.sql index fb0cb661a75..6b8adc6e7e8 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507161526_add_mcp_table_to_db/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507161526_add_mcp_table_to_db/migration.sql @@ -15,13 +15,3 @@ CREATE TABLE "LiteLLM_MCPServerTable" ( CONSTRAINT "LiteLLM_MCPServerTable_pkey" PRIMARY KEY ("server_id") ); --- Migration for existing tables: rename alias to server_name if upgrading -DO $$ -BEGIN - IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'LiteLLM_MCPServerTable' AND column_name = 'alias') THEN - ALTER TABLE "LiteLLM_MCPServerTable" RENAME COLUMN "alias" TO "server_name"; - END IF; -END $$; --- Migration for existing tables: add alias column if upgrading -ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "alias" TEXT; - diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250806095134_rename_alias_to_server_name_mcp_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250806095134_rename_alias_to_server_name_mcp_table/migration.sql new file mode 100644 index 00000000000..11463d44b0e --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250806095134_rename_alias_to_server_name_mcp_table/migration.sql @@ -0,0 +1,10 @@ +-- Migration for existing tables: rename alias to server_name if upgrading +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'LiteLLM_MCPServerTable' AND column_name = 'alias') THEN + ALTER TABLE "LiteLLM_MCPServerTable" RENAME COLUMN "alias" TO "server_name"; + END IF; +END $$; + +-- Migration for existing tables: add alias column if upgrading +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "alias" TEXT; \ No newline at end of file diff --git a/litellm/__init__.py b/litellm/__init__.py index 8cf7d3922a9..17fc6c00e12 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -269,6 +269,7 @@ blocked_user_list: Optional[Union[str, List]] = None banned_keywords_list: Optional[Union[str, List]] = None llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all" guardrail_name_config_map: Dict[str, GuardrailItem] = {} +include_cost_in_streaming_usage: bool = False ### PROMPTS ### from litellm.types.prompts.init_prompts import PromptSpec @@ -429,6 +430,9 @@ project = None config_path = None vertex_ai_safety_settings: Optional[dict] = None BEDROCK_CONVERSE_MODELS = [ + "openai.gpt-oss-20b-1:0", + "openai.gpt-oss-120b-1:0", + "anthropic.claude-opus-4-1-20250805-v1:0", "anthropic.claude-opus-4-20250514-v1:0", "anthropic.claude-sonnet-4-20250514-v1:0", "anthropic.claude-3-7-sonnet-20250219-v1:0", @@ -1199,6 +1203,7 @@ from .llms.nebius.chat.transformation import NebiusConfig from .llms.dashscope.chat.transformation import DashScopeChatConfig from .llms.moonshot.chat.transformation import MoonshotChatConfig from .llms.v0.chat.transformation import V0ChatConfig +from .llms.oci.chat.transformation import OCIChatConfig from .llms.morph.chat.transformation import MorphChatConfig from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig diff --git a/litellm/_logging.py b/litellm/_logging.py index d1084900dee..8c23994f92a 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -108,9 +108,20 @@ verbose_router_logger.addHandler(handler) verbose_proxy_logger.addHandler(handler) verbose_logger.addHandler(handler) -# Suppress httpx request logging at INFO level -httpx_logger = logging.getLogger("httpx") -httpx_logger.setLevel(logging.WARNING) +def _suppress_loggers(): + """Suppress noisy loggers at INFO level""" + # Suppress httpx request logging at INFO level + httpx_logger = logging.getLogger("httpx") + httpx_logger.setLevel(logging.WARNING) + + # Suppress APScheduler logging at INFO level + apscheduler_executors_logger = logging.getLogger("apscheduler.executors.default") + apscheduler_executors_logger.setLevel(logging.WARNING) + apscheduler_scheduler_logger = logging.getLogger("apscheduler.scheduler") + apscheduler_scheduler_logger.setLevel(logging.WARNING) + +# Call the suppression function +_suppress_loggers() ALL_LOGGERS = [ logging.getLogger(), diff --git a/litellm/_redis.py b/litellm/_redis.py index cb01064f413..8371ef5bbc7 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -12,7 +12,7 @@ import json # s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation import os -from typing import List, Optional, Union +from typing import Callable, List, Optional, Union import redis # type: ignore import redis.asyncio as async_redis # type: ignore @@ -34,7 +34,7 @@ def _get_redis_kwargs(): "retry", } - include_args = ["url"] + include_args = ["url", "redis_connect_func", "gcp_service_account", "gcp_ssl_ca_certs"] available_args = [x for x in arg_spec.args if x not in exclude_args] + include_args @@ -72,6 +72,12 @@ def _get_redis_cluster_kwargs(client=None): available_args.append("password") available_args.append("username") available_args.append("ssl") + available_args.append("ssl_cert_reqs") + available_args.append("ssl_check_hostname") + available_args.append("ssl_ca_certs") + available_args.append("redis_connect_func") # Needed for sync clusters and IAM detection + available_args.append("gcp_service_account") + available_args.append("gcp_ssl_ca_certs") return available_args @@ -93,6 +99,73 @@ def _redis_kwargs_from_environment(): return return_dict +def _generate_gcp_iam_access_token(service_account: str) -> str: + """ + Generate GCP IAM access token for Redis authentication. + + Args: + service_account: GCP service account in format 'projects/-/serviceAccounts/name@project.iam.gserviceaccount.com' + + Returns: + Access token string for GCP IAM authentication + """ + try: + from google.cloud import iam_credentials_v1 + except ImportError: + raise ImportError( + "google-cloud-iam is required for GCP IAM Redis authentication. " + "Install it with: pip install google-cloud-iam" + ) + + client = iam_credentials_v1.IAMCredentialsClient() + request = iam_credentials_v1.GenerateAccessTokenRequest( + name=service_account, + scope=['https://www.googleapis.com/auth/cloud-platform'], + ) + response = client.generate_access_token(request=request) + return str(response.access_token) + + +def create_gcp_iam_redis_connect_func( + service_account: str, + ssl_ca_certs: Optional[str] = None, +) -> Callable: + """ + Creates a custom Redis connection function for GCP IAM authentication. + + Args: + service_account: GCP service account in format 'projects/-/serviceAccounts/name@project.iam.gserviceaccount.com' + ssl_ca_certs: Path to SSL CA certificate file for secure connections + + Returns: + A connection function that can be used with Redis clients + """ + def iam_connect(self): + """Initialize the connection and authenticate using GCP IAM""" + from redis.exceptions import AuthenticationError, AuthenticationWrongNumberOfArgsError + from redis.utils import str_if_bytes + + self._parser.on_connect(self) + + auth_args = (_generate_gcp_iam_access_token(service_account),) + self.send_command("AUTH", *auth_args, check_health=False) + + try: + auth_response = self.read_response() + except AuthenticationWrongNumberOfArgsError: + # Fallback to password auth if IAM fails + if hasattr(self, 'password') and self.password: + self.send_command("AUTH", self.password, check_health=False) + auth_response = self.read_response() + else: + raise + + if str_if_bytes(auth_response) != "OK": + raise AuthenticationError("GCP IAM authentication failed") + + return iam_connect + + def get_redis_url_from_environment(): if "REDIS_URL" in os.environ: return os.environ["REDIS_URL"] @@ -156,6 +229,27 @@ def _get_redis_client_logic(**env_overrides): if _service_name is not None: redis_kwargs["service_name"] = _service_name + # Handle GCP IAM authentication + _gcp_service_account = redis_kwargs.get("gcp_service_account") or get_secret_str("REDIS_GCP_SERVICE_ACCOUNT") + _gcp_ssl_ca_certs = redis_kwargs.get("gcp_ssl_ca_certs") or get_secret_str("REDIS_GCP_SSL_CA_CERTS") + + if _gcp_service_account is not None: + verbose_logger.debug("Setting up GCP IAM authentication for Redis with service account.") + redis_kwargs["redis_connect_func"] = create_gcp_iam_redis_connect_func( + service_account=_gcp_service_account, + ssl_ca_certs=_gcp_ssl_ca_certs + ) + # Store GCP service account in redis_connect_func for async cluster access + redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account + + # Remove GCP-specific kwargs that shouldn't be passed to Redis client + redis_kwargs.pop("gcp_service_account", None) + redis_kwargs.pop("gcp_ssl_ca_certs", None) + + # Only enable SSL if explicitly requested AND SSL CA certs are provided + if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False): + redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs + if "url" in redis_kwargs and redis_kwargs["url"] is not None: redis_kwargs.pop("host", None) redis_kwargs.pop("port", None) @@ -198,7 +292,7 @@ def init_redis_cluster(redis_kwargs) -> redis.RedisCluster: for item in redis_kwargs["startup_nodes"]: new_startup_nodes.append(ClusterNode(**item)) - redis_kwargs.pop("startup_nodes") + cluster_kwargs.pop("startup_nodes", None) return redis.RedisCluster(startup_nodes=new_startup_nodes, **cluster_kwargs) # type: ignore @@ -273,7 +367,7 @@ def get_redis_client(**env_overrides): def get_redis_async_client( **env_overrides, -) -> async_redis.Redis: +) -> Union[async_redis.Redis, async_redis.RedisCluster]: redis_kwargs = _get_redis_client_logic(**env_overrides) if "url" in redis_kwargs and redis_kwargs["url"] is not None: args = _get_redis_url_kwargs(client=async_redis.Redis.from_url) @@ -298,14 +392,46 @@ def get_redis_async_client( if arg in args: cluster_kwargs[arg] = redis_kwargs[arg] + # Handle GCP IAM authentication for async clusters + redis_connect_func = cluster_kwargs.pop("redis_connect_func", None) + from litellm import get_secret_str + + # Get GCP service account - first try from redis_connect_func, then from environment + gcp_service_account = None + if redis_connect_func and hasattr(redis_connect_func, '_gcp_service_account'): + gcp_service_account = redis_connect_func._gcp_service_account + else: + gcp_service_account = redis_kwargs.get("gcp_service_account") or get_secret_str("REDIS_GCP_SERVICE_ACCOUNT") + + verbose_logger.info(f"DEBUG: Redis cluster kwargs: redis_connect_func={redis_connect_func is not None}, gcp_service_account_provided={gcp_service_account is not None}") + + # If GCP IAM is configured (indicated by redis_connect_func), generate access token and use as password + if redis_connect_func and gcp_service_account: + verbose_logger.info("DEBUG: Generating IAM token for service account (value not logged for security reasons)") + try: + # Generate IAM access token using the helper function + access_token = _generate_gcp_iam_access_token(gcp_service_account) + cluster_kwargs["password"] = access_token + verbose_logger.info("DEBUG: Successfully generated GCP IAM access token for async Redis cluster") + except Exception as e: + verbose_logger.error(f"Failed to generate GCP IAM access token: {e}") + from redis.exceptions import AuthenticationError + raise AuthenticationError("Failed to generate GCP IAM access token") + else: + verbose_logger.info(f"DEBUG: Not using GCP IAM auth - redis_connect_func={redis_connect_func is not None}, gcp_service_account={gcp_service_account}") + new_startup_nodes: List[ClusterNode] = [] for item in redis_kwargs["startup_nodes"]: new_startup_nodes.append(ClusterNode(**item)) - redis_kwargs.pop("startup_nodes") - return async_redis.RedisCluster( + cluster_kwargs.pop("startup_nodes", None) + + # Create async RedisCluster with IAM token as password if available + cluster_client = async_redis.RedisCluster( startup_nodes=new_startup_nodes, **cluster_kwargs # type: ignore ) + + return cluster_client # Check for Redis Sentinel if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs: diff --git a/litellm/caching/__init__.py b/litellm/caching/__init__.py index badc462e09b..bbe90b04121 100644 --- a/litellm/caching/__init__.py +++ b/litellm/caching/__init__.py @@ -7,4 +7,5 @@ from .qdrant_semantic_cache import QdrantSemanticCache from .redis_cache import RedisCache from .redis_cluster_cache import RedisClusterCache from .redis_semantic_cache import RedisSemanticCache -from .s3_cache import S3Cache \ No newline at end of file +from .s3_cache import S3Cache +from .gcs_cache import GCSCache diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 6959467cddd..1455e011bc5 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -34,6 +34,7 @@ from .redis_cache import RedisCache from .redis_cluster_cache import RedisClusterCache from .redis_semantic_cache import RedisSemanticCache from .s3_cache import S3Cache +from .gcs_cache import GCSCache def print_verbose(print_statement): @@ -92,6 +93,9 @@ class Cache: s3_aws_session_token: Optional[str] = None, s3_config: Optional[Any] = None, s3_path: Optional[str] = None, + gcs_bucket_name: Optional[str] = None, + gcs_path_service_account: Optional[str] = None, + gcs_path: Optional[str] = None, redis_semantic_cache_embedding_model: str = "text-embedding-ada-002", redis_semantic_cache_index_name: Optional[str] = None, redis_flush_size: Optional[int] = None, @@ -102,6 +106,9 @@ class Cache: qdrant_collection_name: Optional[str] = None, qdrant_quantization_config: Optional[str] = None, qdrant_semantic_cache_embedding_model: str = "text-embedding-ada-002", + # GCP IAM authentication parameters + gcp_service_account: Optional[str] = None, + gcp_ssl_ca_certs: Optional[str] = None, **kwargs, ): """ @@ -140,6 +147,11 @@ class Cache: s3_aws_session_token (str, optional): The aws session token for the s3 cache. Defaults to None. s3_config (dict, optional): The config for the s3 cache. Defaults to None. + # GCS Cache Args + gcs_bucket_name (str, optional): The bucket name for the gcs cache. Defaults to None. + gcs_path_service_account (str, optional): Path to the service account json. + gcs_path (str, optional): Folder path inside the bucket to store cache files. + # Common Cache Args supported_call_types (list, optional): List of call types to cache for. Defaults to cache == on for all call types. **kwargs: Additional keyword arguments for redis.Redis() cache @@ -152,14 +164,21 @@ class Cache: """ if type == LiteLLMCacheType.REDIS: if redis_startup_nodes: - self.cache: BaseCache = RedisClusterCache( - host=host, - port=port, - password=password, - redis_flush_size=redis_flush_size, - startup_nodes=redis_startup_nodes, + # Only pass GCP parameters if they are provided + cluster_kwargs = { + "host": host, + "port": port, + "password": password, + "redis_flush_size": redis_flush_size, + "startup_nodes": redis_startup_nodes, **kwargs, - ) + } + if gcp_service_account is not None: + cluster_kwargs["gcp_service_account"] = gcp_service_account + if gcp_ssl_ca_certs is not None: + cluster_kwargs["gcp_ssl_ca_certs"] = gcp_ssl_ca_certs + + self.cache: BaseCache = RedisClusterCache(**cluster_kwargs) else: self.cache = RedisCache( host=host, @@ -204,6 +223,12 @@ class Cache: s3_path=s3_path, **kwargs, ) + elif type == LiteLLMCacheType.GCS: + self.cache = GCSCache( + bucket_name=gcs_bucket_name, + path_service_account=gcs_path_service_account, + gcs_path=gcs_path, + ) elif type == LiteLLMCacheType.AZURE_BLOB: self.cache = AzureBlobCache( account_url=azure_account_url, diff --git a/litellm/caching/gcs_cache.py b/litellm/caching/gcs_cache.py new file mode 100644 index 00000000000..88857ba0e70 --- /dev/null +++ b/litellm/caching/gcs_cache.py @@ -0,0 +1,97 @@ +"""GCS Cache implementation +Supports syncing responses to Google Cloud Storage Buckets using HTTP requests. +""" +import json +import asyncio +from typing import Optional + +from litellm._logging import print_verbose, verbose_logger +from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + _get_httpx_client, + httpxSpecialProvider, +) +from .base_cache import BaseCache + + +class GCSCache(BaseCache): + def __init__(self, bucket_name: Optional[str] = None, path_service_account: Optional[str] = None, gcs_path: Optional[str] = None) -> None: + super().__init__() + self.bucket_name = bucket_name or GCSBucketBase(bucket_name=None).BUCKET_NAME + self.path_service_account = path_service_account or GCSBucketBase(bucket_name=None).path_service_account_json + self.key_prefix = gcs_path.rstrip("/") + "/" if gcs_path else "" + # create httpx clients + self.async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) + self.sync_client = _get_httpx_client() + + def _construct_headers(self) -> dict: + base = GCSBucketBase(bucket_name=self.bucket_name) + base.path_service_account_json = self.path_service_account + base.BUCKET_NAME = self.bucket_name + return base.sync_construct_request_headers() + + def set_cache(self, key, value, **kwargs): + try: + print_verbose(f"LiteLLM SET Cache - GCS. Key={key}. Value={value}") + headers = self._construct_headers() + object_name = self.key_prefix + key + bucket_name = self.bucket_name + url = f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={object_name}" + data = json.dumps(value) + self.sync_client.post(url=url, data=data, headers=headers) + except Exception as e: + print_verbose(f"GCS Caching: set_cache() - Got exception from GCS: {e}") + + async def async_set_cache(self, key, value, **kwargs): + try: + headers = self._construct_headers() + object_name = self.key_prefix + key + bucket_name = self.bucket_name + url = f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={object_name}" + data = json.dumps(value) + await self.async_client.post(url=url, data=data, headers=headers) + except Exception as e: + print_verbose(f"GCS Caching: async_set_cache() - Got exception from GCS: {e}") + + def get_cache(self, key, **kwargs): + try: + headers = self._construct_headers() + object_name = self.key_prefix + key + bucket_name = self.bucket_name + url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{object_name}?alt=media" + response = self.sync_client.get(url=url, headers=headers) + if response.status_code == 200: + cached_response = json.loads(response.text) + verbose_logger.debug( + f"Got GCS Cache: key: {key}, cached_response {cached_response}. Type Response {type(cached_response)}" + ) + return cached_response + return None + except Exception as e: + verbose_logger.error(f"GCS Caching: get_cache() - Got exception from GCS: {e}") + + async def async_get_cache(self, key, **kwargs): + try: + headers = self._construct_headers() + object_name = self.key_prefix + key + bucket_name = self.bucket_name + url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{object_name}?alt=media" + response = await self.async_client.get(url=url, headers=headers) + if response.status_code == 200: + return json.loads(response.text) + return None + except Exception as e: + verbose_logger.error(f"GCS Caching: async_get_cache() - Got exception from GCS: {e}") + + def flush_cache(self): + pass + + async def disconnect(self): + pass + + async def async_set_cache_pipeline(self, cache_list, **kwargs): + tasks = [] + for val in cache_list: + tasks.append(self.async_set_cache(val[0], val[1], **kwargs)) + await asyncio.gather(*tasks) diff --git a/litellm/constants.py b/litellm/constants.py index b25e2ff0fb9..c7404f10a78 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -279,6 +279,7 @@ LITELLM_CHAT_PROVIDERS = [ "dashscope", "moonshot", "v0", + "oci", "morph", "lambda_ai", ] @@ -765,6 +766,7 @@ MAXIMUM_TRACEBACK_LINES_TO_LOG = int(os.getenv("MAXIMUM_TRACEBACK_LINES_TO_LOG", X_LITELLM_DISABLE_CALLBACKS = "x-litellm-disable-callbacks" LITELLM_METADATA_FIELD = "litellm_metadata" OLD_LITELLM_METADATA_FIELD = "metadata" +LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated" ########################### LiteLLM Proxy Specific Constants ########################### ######################################################################################## diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6fecc7fa976..9956a9d314a 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1265,7 +1265,7 @@ class BaseTokenUsageProcessor: Combine multiple Usage objects into a single Usage object, checking model keys for nested values. """ from litellm.types.utils import ( - CompletionTokensDetails, + CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, Usage, ) @@ -1320,7 +1320,7 @@ class BaseTokenUsageProcessor: not hasattr(combined, "completion_tokens_details") or not combined.completion_tokens_details ): - combined.completion_tokens_details = CompletionTokensDetails() + combined.completion_tokens_details = CompletionTokensDetailsWrapper() # Check what keys exist in the model's completion_tokens_details for attr in usage.completion_tokens_details.model_fields: diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index c68674f77ba..8149a6131e8 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -42,7 +42,7 @@ class BraintrustLogger(CustomLogger): ) -> None: super().__init__() self.validate_environment(api_key=api_key) - self.api_base = api_base or API_BASE + self.api_base = api_base or os.getenv("BRAINTRUST_API_BASE") or API_BASE self.default_project_id = None self.api_key: str = api_key or os.getenv("BRAINTRUST_API_KEY") # type: ignore self.headers = { diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index ded5ccca766..ee7e771faa6 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -34,8 +34,6 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import UserAPIKeyAuth from litellm.types.mcp import ( - MCPDuringCallRequestObject, - MCPDuringCallResponseObject, MCPPostCallResponseObject, MCPPreCallRequestObject, MCPPreCallResponseObject, @@ -412,59 +410,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac ######################################################### # MCP TOOL CALL HOOKS ######################################################### - async def async_pre_mcp_tool_call_hook( - self, - kwargs, - request_obj: MCPPreCallRequestObject, - start_time, - end_time - ) -> Optional[MCPPreCallResponseObject]: - """ - This hook gets called before the MCP tool call is made. - Useful for: - - Validating tool calls before execution - - Modifying arguments before they are sent to the MCP server - - Implementing access control and rate limiting - - Adding custom metadata or tracking information - - Args: - kwargs: The logging kwargs containing model call details - request_obj: MCPPreCallRequestObject containing tool name, arguments, and metadata - start_time: Start time of the request - end_time: End time of the request - - Returns: - MCPPreCallResponseObject with validation results and any modifications - """ - return None - - async def async_during_mcp_tool_call_hook( - self, - kwargs, - request_obj: MCPDuringCallRequestObject, - start_time, - end_time - ) -> Optional[MCPDuringCallResponseObject]: - """ - This hook gets called during the MCP tool call execution. - - Useful for: - - Concurrent monitoring and validation during tool execution - - Implementing timeouts and cancellation logic - - Real-time cost tracking and billing - - Performance monitoring and metrics collection - - Args: - kwargs: The logging kwargs containing model call details - request_obj: MCPDuringCallRequestObject containing tool execution context - start_time: Start time of the request - end_time: End time of the request - - Returns: - MCPDuringCallResponseObject with execution control decisions - """ - return None async def async_post_mcp_tool_call_hook( self, kwargs, response_obj: MCPPostCallResponseObject, start_time, end_time @@ -595,3 +541,14 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac model_call_details_copy["standard_logging_object"] = standard_logging_object_copy return model_call_details_copy + + + + async def get_proxy_server_request_from_cold_storage_with_object_key( + self, + object_key: str, + ) -> Optional[dict]: + """ + Get the proxy server request from cold storage using the object key directly. + """ + pass diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 7df3e58b2da..efe18cb68ad 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -304,7 +304,10 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): data=prepped.body, headers=prepped.headers, ) - SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request) + aws_region_name = self.get_aws_region_name_for_non_llm_api_calls( + aws_region_name=self.s3_region_name + ) + SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request) # Prepare the signed headers signed_headers = dict(aws_request.headers.items()) @@ -444,7 +447,10 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): data=prepped.body, headers=prepped.headers, ) - SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request) + aws_region_name = self.get_aws_region_name_for_non_llm_api_calls( + aws_region_name=self.s3_region_name + ) + SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request) # Prepare the signed headers signed_headers = dict(aws_request.headers.items()) @@ -455,3 +461,108 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): response.raise_for_status() except Exception as e: verbose_logger.exception(f"Error uploading to s3: {str(e)}") + + + async def _download_object_from_s3(self, s3_object_key: str) -> Optional[dict]: + """ + Download and parse JSON object from S3. + + Args: + s3_object_key: The S3 object key to download + + Returns: + Optional[dict]: The parsed JSON object or None if not found/error + """ + try: + import hashlib + + import requests + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + except ImportError: + raise ImportError("Missing boto3 to call S3. Run 'pip install boto3'.") + + try: + from litellm.litellm_core_utils.asyncify import asyncify + + # Get AWS credentials + asyncified_get_credentials = asyncify(self.get_credentials) + credentials = await asyncified_get_credentials( + aws_access_key_id=self.s3_aws_access_key_id, + aws_secret_access_key=self.s3_aws_secret_access_key, + aws_session_token=self.s3_aws_session_token, + aws_region_name=self.s3_region_name, + aws_session_name=self.s3_aws_session_name, + aws_profile_name=self.s3_aws_profile_name, + aws_role_name=self.s3_aws_role_name, + aws_web_identity_token=self.s3_aws_web_identity_token, + aws_sts_endpoint=self.s3_aws_sts_endpoint, + ) + + verbose_logger.debug( + f"s3_v2 logger - downloading data from s3 - {s3_object_key}" + ) + + # Prepare the URL + url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}" + + if self.s3_endpoint_url: + url = self.s3_endpoint_url + "/" + s3_object_key + + # Prepare the request for GET operation + # For GET requests, we need x-amz-content-sha256 with hash of empty string + empty_string_hash = hashlib.sha256(b"").hexdigest() + headers = { + "x-amz-content-sha256": empty_string_hash, + } + req = requests.Request("GET", url, headers=headers) + prepped = req.prepare() + + # Sign the request + aws_request = AWSRequest( + method=prepped.method, + url=prepped.url, + headers=prepped.headers, + ) + SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request) + + # Prepare the signed headers + signed_headers = dict(aws_request.headers.items()) + + # Make the request + response = await self.async_httpx_client.get(url, headers=signed_headers) + + if response.status_code != 200: + verbose_logger.exception("S3 object not found, saw response=", response.text) + return None + + # Parse JSON response + return response.json() + + except Exception as e: + verbose_logger.exception(f"Error downloading from S3: {str(e)}") + return None + + async def get_proxy_server_request_from_cold_storage_with_object_key( + self, + object_key: str, + ) -> Optional[dict]: + """ + Get the proxy server request from cold storage + + Allows fetching a dict of the proxy server request from s3 or GCS bucket. + + Args: + request_id: The unique request ID to search for + start_time: The start time of the request (datetime or ISO string) + + Returns: + Optional[dict]: The request data dictionary or None if not found + """ + try: + # Download and return the object from S3 + downloaded_object = await self._download_object_from_s3(object_key) + return downloaded_object + except Exception as e: + verbose_logger.exception(f"Error retrieving object {object_key} from cold storage: {str(e)}") + return None \ No newline at end of file diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index 9606b47b9b8..fd82ecdf2b2 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -10,6 +10,7 @@ Example: from typing import Union +from litellm import _custom_logger_compatible_callbacks_literal from litellm.integrations.agentops import AgentOps from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.integrations.argilla import ArgillaLogger @@ -150,3 +151,14 @@ class CustomLoggerRegistry: if callback_class == class_type: callback_strs.append(callback_str) return callback_strs + + + @classmethod + def get_class_type_for_custom_logger_name( + cls, + custom_logger_name: _custom_logger_compatible_callbacks_literal, + ) -> type: + """ + Get the class type for a given custom logger name + """ + return cls.CALLBACK_CLASS_STR_TO_CLASS_TYPE[custom_logger_name] diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 4e0a2efb0c6..702196a7f05 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -356,6 +356,8 @@ def get_llm_provider( # noqa: PLR0915 # bytez models elif model.startswith("bytez/"): custom_llm_provider = "bytez" + elif model.startswith("oci/"): + custom_llm_provider = "oci" if not custom_llm_provider: if litellm.suppress_debug_info is False: print() # noqa diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index f1901fa2ce9..b2a2c364240 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -146,7 +146,9 @@ def get_supported_openai_params( # noqa: PLR0915 return litellm.HuggingFaceChatConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "jina_ai": if request_type == "embeddings": - return litellm.JinaAIEmbeddingConfig().get_supported_openai_params() + return litellm.JinaAIEmbeddingConfig().get_supported_openai_params( + model=model + ) elif custom_llm_provider == "together_ai": return litellm.TogetherAIConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "databricks": diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index dfa941d8301..5462257c9b7 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3830,6 +3830,8 @@ class StandardLoggingPayloadSetup: ] = None, usage_object: Optional[dict] = None, proxy_server_request: Optional[dict] = None, + start_time: Optional[dt_object] = None, + response_id: Optional[str] = None, ) -> StandardLoggingMetadata: """ Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata. @@ -3881,6 +3883,7 @@ class StandardLoggingPayloadSetup: usage_object=usage_object, requester_custom_headers=None, user_api_key_request_route=None, + cold_storage_object_key=None, ) if isinstance(metadata, dict): # Filter the metadata dictionary to include only the specified keys @@ -3913,6 +3916,16 @@ class StandardLoggingPayloadSetup: proxy_server_request=proxy_server_request, ) + # Generate cold storage object key if cold storage is configured + if start_time is not None and response_id is not None: + cold_storage_object_key = StandardLoggingPayloadSetup._generate_cold_storage_object_key( + start_time=start_time, + response_id=response_id, + team_alias=clean_metadata.get("user_api_key_team_alias"), + ) + if cold_storage_object_key: + clean_metadata["cold_storage_object_key"] = cold_storage_object_key + return clean_metadata @staticmethod @@ -4071,6 +4084,49 @@ class StandardLoggingPayloadSetup: return api_base.rstrip("/") return api_base + @staticmethod + def _generate_cold_storage_object_key( + start_time: dt_object, + response_id: str, + team_alias: Optional[str] = None, + ) -> Optional[str]: + """ + Generate cold storage object key in the same format as S3Logger. + + Args: + start_time: The start time of the request + response_id: The response ID + team_alias: Optional team alias for team-based prefixing + + Returns: + Optional[str]: The generated object key or None if cold storage not configured + """ + # Generate object key in same format as S3Logger + from litellm.integrations.s3 import get_s3_object_key + from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler + + # Only generate object key if cold storage is configured + configured_cold_storage_logger = ColdStorageHandler._get_configured_cold_storage_custom_logger() + if configured_cold_storage_logger is None: + return None + + try: + # Generate file name in same format as litellm.utils.get_logging_id + s3_file_name = f"time-{start_time.strftime('%H-%M-%S-%f')}_{response_id}" + + + s3_object_key = get_s3_object_key( + s3_path="", # Use empty path as default + team_alias_prefix="", # Don't split by team alias for cold storage + start_time=start_time, + s3_file_name=s3_file_name, + ) + + return s3_object_key + except Exception: + # If any error occurs in generating the key, return None + return None + @staticmethod def get_error_information( original_exception: Optional[Exception], @@ -4322,6 +4378,8 @@ def get_standard_logging_object_payload( ), usage_object=usage.model_dump(), proxy_server_request=proxy_server_request, + start_time=start_time, + response_id=id, ) _request_body = proxy_server_request.get("body", {}) @@ -4469,6 +4527,7 @@ def get_standard_logging_metadata( usage_object=None, requester_custom_headers=None, user_api_key_request_route=None, + cold_storage_object_key=None, ) if isinstance(metadata, dict): # Filter the metadata dictionary to include only the specified keys diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 44cb146f91a..9ec346c20a1 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -1,4 +1,4 @@ -from typing import Callable, List, Set, Type, Union +from typing import TYPE_CHECKING, Callable, List, Optional, Set, Type, Union import litellm from litellm._logging import verbose_logger @@ -6,6 +6,11 @@ from litellm.integrations.additional_logging_utils import AdditionalLoggingUtils from litellm.integrations.custom_logger import CustomLogger from litellm.types.utils import CallbacksByType +if TYPE_CHECKING: + from litellm import _custom_logger_compatible_callbacks_literal +else: + _custom_logger_compatible_callbacks_literal = str + class LoggingCallbackManager: """ @@ -343,3 +348,26 @@ class LoggingCallbackManager: elif callable(callback): return getattr(callback, "__name__", str(callback)) return str(callback) + + + def get_active_custom_logger_for_callback_name( + self, + callback_name: _custom_logger_compatible_callbacks_literal, + ) -> Optional[CustomLogger]: + """ + Get the active custom logger for a given callback name + """ + from litellm.litellm_core_utils.custom_logger_registry import ( + CustomLoggerRegistry, + ) + + # get the custom logger class type + custom_logger_class_type = CustomLoggerRegistry.get_class_type_for_custom_logger_name(callback_name) + + # get the active custom logger + custom_logger = self.get_custom_loggers_for_type(custom_logger_class_type) + + if len(custom_logger) == 0: + raise ValueError(f"No active custom logger found for callback name: {callback_name}") + + return custom_logger[0] diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 827d28598ec..9ba547b3600 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -519,25 +519,25 @@ def unpack_defs(schema: dict, defs: dict) -> None: } # Use iterative approach with queue to avoid recursion - # Each item in queue is (node, parent_container, key/index, active_defs, seen_ids) + # Each item in queue is (node, parent_container, key/index, active_defs, ref_chain) queue: deque[ tuple[Any, Union[dict, list, None], Union[str, int, None], dict, set] ] = deque([(schema, None, None, root_defs, set())]) while queue: - node, parent, key, active_defs, seen = queue.popleft() - - # Avoid infinite loops on self-referential schemas - if id(node) in seen: - continue - seen = seen.copy() # Create new set for this branch - seen.add(id(node)) + node, parent, key, active_defs, ref_chain = queue.popleft() # ----------------------------- dict ----------------------------- if isinstance(node, dict): # --- Case 1: this node *is* a reference --- if "$ref" in node: ref_name = node["$ref"].split("/")[-1] + + # Check for circular reference in the resolution chain + if ref_name in ref_chain: + # Circular reference detected - leave as-is to prevent infinite recursion + continue + target_schema = active_defs.get(ref_name) # Unknown reference ā€“ leave untouched if target_schema is None: @@ -563,8 +563,12 @@ def unpack_defs(schema: dict, defs: dict) -> None: schema.update(resolved) resolved = schema + # Add to ref chain to track circular references + new_ref_chain = ref_chain.copy() + new_ref_chain.add(ref_name) + # Add resolved node to queue for further processing - queue.append((resolved, parent, key, child_defs, seen)) + queue.append((resolved, parent, key, child_defs, new_ref_chain)) continue # --- Case 2: regular dict ā€“ process its values --- @@ -577,13 +581,13 @@ def unpack_defs(schema: dict, defs: dict) -> None: # Add all dict values to queue for k, v in node.items(): - queue.append((v, node, k, current_defs, seen)) + queue.append((v, node, k, current_defs, ref_chain)) # ---------------------------- list ------------------------------ elif isinstance(node, list): # Add all list items to queue for idx, item in enumerate(node): - queue.append((item, node, idx, active_defs, seen)) + queue.append((item, node, idx, active_defs, ref_chain)) def _get_image_mime_type_from_url(url: str) -> Optional[str]: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index b4ace1545d2..26388dc2362 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1,5 +1,6 @@ import copy import json +import mimetypes import re import uuid import xml.etree.ElementTree as ET @@ -13,6 +14,7 @@ import litellm.types import litellm.types.llms from litellm import verbose_logger from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client +from litellm.types.files import get_file_extension_from_mime_type from litellm.types.llms.anthropic import * from litellm.types.llms.bedrock import MessageBlock as BedrockMessageBlock from litellm.types.llms.custom_http import httpxSpecialProvider @@ -2351,7 +2353,6 @@ def stringify_json_tool_call_content(messages: List) -> List: ###### AMAZON BEDROCK ####### import base64 -import mimetypes from email.message import Message import httpx @@ -2479,20 +2480,11 @@ class BedrockImageProcessor: ) if is_document: - potential_extensions = mimetypes.guess_all_extensions(mime_type) - valid_extensions = [ - ext[1:] - for ext in potential_extensions - if ext[1:] in supported_doc_formats - ] + return BedrockImageProcessor._get_document_format( + mime_type=mime_type, + supported_doc_formats=supported_doc_formats + ) - if not valid_extensions: - raise ValueError( - f"No supported extensions for MIME type: {mime_type}. Supported formats: {supported_doc_formats}" - ) - - # Use first valid extension instead of provided image_format - return valid_extensions[0] else: ######################################################### # Check if image_format is an image or video @@ -2502,6 +2494,60 @@ class BedrockImageProcessor: f"Unsupported image format: {image_format}. Supported formats: {supported_image_and_video_formats}" ) return image_format + + @staticmethod + def _get_document_format( + mime_type: str, + supported_doc_formats: List[str] + ) -> str: + """ + Get the document format from the mime type + + - Primary method - uses `mimetypes.guess_all_extensions` + - Fallback method - uses `get_file_extension_from_mime_type` + + Relevant Issue: https://github.com/BerriAI/litellm/issues/12260 + + `mimetypes` is not available in docker containers, so we fallback to `get_file_extension_from_mime_type` + + Args: + mime_type: The mime type of the document + supported_doc_formats: The supported document formats for the current model + + Returns: + The document format + """ + valid_extensions: Optional[List[str]] = None + potential_extensions = mimetypes.guess_all_extensions( + mime_type, strict=False + ) + valid_extensions = [ + ext[1:] + for ext in potential_extensions + if ext[1:] in supported_doc_formats + ] + + # Fallback to types/files.py if mimetypes doesn't return valid extensions + ################# + # litellm runs on docker containers and `mimetypes` depends on the installed mimetypes of the OS + # we fallback to well known mime types in types/files.py if mimetypes doesn't return valid extensions + if not valid_extensions: + try: + fallback_extension = get_file_extension_from_mime_type(mime_type) + if fallback_extension in supported_doc_formats: + valid_extensions = [fallback_extension] + except ValueError: + # Neither mimetypes nor files.py could handle this MIME type + # get_file_extension_from_mime_type raises ValueError if the mime type is not supported + pass + + if not valid_extensions: + raise ValueError( + f"No supported extensions for MIME type: {mime_type}. Supported formats: {supported_doc_formats}" + ) + + # Use first valid extension instead of provided image_format + return valid_extensions[0] @staticmethod def _create_bedrock_block( @@ -2950,7 +2996,10 @@ def process_empty_text_blocks( ] modified_message = message.copy() - modified_message["content"] = modified_content_block + modified_message["content"] = cast( + Union[List[ChatCompletionTextObject], List[ChatCompletionThinkingBlock]], + modified_content_block, + ) return modified_message diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index fb919afd49d..2f85c7aef60 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -527,7 +527,12 @@ class ChunkProcessor: returned_usage, "cache_read_input_tokens", cache_read_input_tokens ) # for anthropic if completion_tokens_details is not None: - returned_usage.completion_tokens_details = completion_tokens_details + if isinstance(completion_tokens_details, CompletionTokensDetails): + returned_usage.completion_tokens_details = CompletionTokensDetailsWrapper( + **completion_tokens_details.model_dump() + ) + else: + returned_usage.completion_tokens_details = completion_tokens_details if reasoning_tokens is not None: if returned_usage.completion_tokens_details is None: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 2e9e6770a1d..3721851a38f 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1584,7 +1584,9 @@ class CustomStreamWrapper: except StopIteration: if self.sent_last_chunk is True: complete_streaming_response = litellm.stream_chunk_builder( - chunks=self.chunks, messages=self.messages + chunks=self.chunks, + messages=self.messages, + logging_obj=self.logging_obj, ) response = self.model_response_creator() @@ -1768,7 +1770,9 @@ class CustomStreamWrapper: if self.sent_last_chunk is True: # log the final chunk with accurate streaming values complete_streaming_response = litellm.stream_chunk_builder( - chunks=self.chunks, messages=self.messages + chunks=self.chunks, + messages=self.messages, + logging_obj=self.logging_obj, ) response = self.model_response_creator() if complete_streaming_response is not None: diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index c263d903188..3845dc5a46e 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -2,7 +2,7 @@ This file contains common utils for anthropic calls. """ -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union import httpx @@ -229,6 +229,60 @@ class AnthropicModelInfo(BaseLLMModelInfo): litellm_model_names.append(litellm_model_name) return litellm_model_names + def get_token_counter(self) -> Optional["AnthropicTokenCounter"]: + """ + Factory method to create an Anthropic token counter. + + Returns: + AnthropicTokenCounter instance for this provider. + """ + return AnthropicTokenCounter() + + +class AnthropicTokenCounter: + """Token counter implementation for Anthropic provider.""" + + def supports_provider( + self, + deployment: Optional[Dict[str, Any]] = None, + from_endpoint: bool = False + ) -> bool: + if not from_endpoint: + return False + + if deployment is None: + return False + + full_model = deployment.get("litellm_params", {}).get("model", "") + is_anthropic_provider = full_model.startswith("anthropic/") or "anthropic" in full_model.lower() + + return is_anthropic_provider + + async def count_tokens( + self, + model_to_use: str, + messages: Optional[List[Dict[str, Any]]], + deployment: Optional[Dict[str, Any]] = None, + request_model: str = "", + ) -> Optional[Dict[str, Any]]: + from litellm.proxy.utils import count_tokens_with_anthropic_api + + result = await count_tokens_with_anthropic_api( + model_to_use=model_to_use, + messages=messages, + deployment=deployment, + ) + + if result is not None: + return { + "total_tokens": result["total_tokens"], + "request_model": request_model, + "model_used": model_to_use, + "tokenizer_type": result["tokenizer_used"], + } + + return None + def process_anthropic_headers(headers: Union[httpx.Headers, dict]) -> dict: openai_headers = {} diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index 35959f0d083..3961ee2b9e9 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -70,6 +70,16 @@ class BaseLLMModelInfo(ABC): """ pass + def get_token_counter(self): + """ + Factory method to create a token counter for this provider. + + Returns: + Optional TokenCounterInterface implementation for this provider, + or None if token counting is not supported. + """ + return None + def _convert_tool_response_to_message( tool_calls: List[ChatCompletionToolCallChunk], diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index e7d7920769f..908419f7193 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -371,8 +371,8 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): reasoning_content += sum["text"] thinking_block = ChatCompletionThinkingBlock( type="thinking", - thinking=sum["text"], - signature=sum["signature"], + thinking=sum.get("text", ""), + signature=sum.get("signature", ""), ) if thinking_blocks is None: thinking_blocks = [] diff --git a/litellm/llms/jina_ai/common_utils.py b/litellm/llms/jina_ai/common_utils.py new file mode 100644 index 00000000000..cd9fd402afb --- /dev/null +++ b/litellm/llms/jina_ai/common_utils.py @@ -0,0 +1,6 @@ +from litellm.llms.base_llm.chat.transformation import BaseLLMException + + +class JinaAIError(BaseLLMException): + def __init__(self, status_code, message): + super().__init__(status_code=status_code, message=message) diff --git a/litellm/llms/jina_ai/embedding/transformation.py b/litellm/llms/jina_ai/embedding/transformation.py index 5263be900fa..7a634903005 100644 --- a/litellm/llms/jina_ai/embedding/transformation.py +++ b/litellm/llms/jina_ai/embedding/transformation.py @@ -1,5 +1,5 @@ """ -Transformation logic from OpenAI /v1/embeddings format to Jina AI's `/v1/embeddings` format. +Transformation logic from OpenAI /v1/embeddings format to Jina AI's `/v1/embeddings` format. Why separate file? Make it easy to see how transformation works @@ -7,13 +7,23 @@ Docs - https://jina.ai/embeddings/ """ import types -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Union, cast + +import httpx from litellm import LlmProviders from litellm.secret_managers.main import get_secret_str +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm import BaseEmbeddingConfig +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues +from litellm.types.utils import EmbeddingResponse +from litellm.utils import is_base64_encoded + +from ..common_utils import JinaAIError -class JinaAIEmbeddingConfig: +class JinaAIEmbeddingConfig(BaseEmbeddingConfig): """ Reference: https://jina.ai/embeddings/ """ @@ -44,11 +54,15 @@ class JinaAIEmbeddingConfig: and v is not None } - def get_supported_openai_params(self) -> List[str]: + def get_supported_openai_params(self, model: str) -> List[str]: return ["dimensions"] def map_openai_params( - self, non_default_params: dict, optional_params: dict + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, ) -> dict: if "dimensions" in non_default_params: optional_params["dimensions"] = non_default_params["dimensions"] @@ -76,3 +90,88 @@ class JinaAIEmbeddingConfig: or get_secret_str("JINA_AI_TOKEN") ) return LlmProviders.JINA_AI.value, api_base, dynamic_api_key + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + return ( + f"{api_base}/embeddings" + if api_base + else "https://api.jina.ai/v1/embeddings" + ) + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + ) -> dict: + data = {"model": model, **optional_params} + input = cast(List[str], input) if isinstance(input, List) else [input] + if any((is_base64_encoded(x) for x in input)): + transformed_input = [] + for value in input: + if isinstance(value, str): + if is_base64_encoded(value): + img_data = value.split(",")[1] + transformed_input.append({"image": img_data}) + else: + transformed_input.append({"text": value}) + data["input"] = transformed_input + else: + data["input"] = input + return data + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + request_data: dict, + optional_params: dict, + litellm_params: dict, + ) -> EmbeddingResponse: + response_json = raw_response.json() + ## LOGGING + logging_obj.post_call( + input=input, + api_key=api_key, + additional_args={"complete_input_dict": request_data}, + original_response=response_json, + ) + return EmbeddingResponse(**response_json) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + default_headers = { + "Content-Type": "application/json", + } + if api_key: + default_headers["Authorization"] = f"Bearer {api_key}" + headers = {**default_headers, **headers} + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return JinaAIError( + status_code=status_code, + message=error_message, + ) diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py new file mode 100644 index 00000000000..3524817da63 --- /dev/null +++ b/litellm/llms/oci/chat/transformation.py @@ -0,0 +1,868 @@ +import base64 +import datetime +import hashlib +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from urllib.parse import urlparse + +import httpx + +import litellm +from litellm.litellm_core_utils.logging_utils import track_llm_api_timing +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, + get_async_httpx_client, + version, +) +from litellm.llms.oci.common_utils import OCIError +from litellm.types.llms.oci import ( + OCIChatRequestPayload, + OCICompletionPayload, + OCICompletionResponse, + OCIContentPartUnion, + OCIImageContentPart, + OCIMessage, + OCIRoles, + OCIServingMode, + OCIStreamChunk, + OCITextContentPart, + OCIToolCall, + OCIToolDefinition, + OCIVendors, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ( + Delta, + LlmProviders, + ModelResponseStream, + StreamingChoices, +) +from litellm.utils import ( + ChatCompletionMessageToolCall, + CustomStreamWrapper, + ModelResponse, + Usage, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +def sha256_base64(data: bytes) -> str: + digest = hashlib.sha256(data).digest() + return base64.b64encode(digest).decode() + + +def build_signature_string(method, path, headers, signed_headers): + lines = [] + for header in signed_headers: + if header == "(request-target)": + value = f"{method.lower()} {path}" + else: + value = headers[header] + lines.append(f"{header}: {value}") + return "\n".join(lines) + + +def load_private_key_from_str(key_str: str): + try: + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import rsa + except ImportError as e: + raise ImportError( + "cryptography package is required for OCI authentication. " + "Please install it with: pip install cryptography" + ) from e + + key = serialization.load_pem_private_key( + key_str.encode("utf-8"), + password=None, + ) + if not isinstance(key, rsa.RSAPrivateKey): + raise TypeError( + "The provided private key is not an RSA key, which is required for OCI signing." + ) + return key + + +def get_vendor_from_model(model: str) -> OCIVendors: + """ + Extracts the vendor from the model name. + Args: + model (str): The model name. + Returns: + str: The vendor name. + """ + vendor = model.split(".")[0].lower() + if vendor == "cohere": + return OCIVendors.COHERE + else: + return OCIVendors.GENERIC + + +# 5 minute timeout (models may need to load) +STREAMING_TIMEOUT = 60 * 5 + + +class OCIChatConfig(BaseConfig): + """ + Configuration class for OCI's API interface. + """ + + def __init__( + self, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + # mark the class as using a custom stream wrapper because the default only iterates on lines + setattr(self.__class__, "has_custom_stream_wrapper", True) + + self.openai_to_oci_generic_param_map = { + "stream": "isStream", + "max_tokens": "maxTokens", + "max_completion_tokens": "maxTokens", + "temperature": "temperature", + "tools": "tools", + "frequency_penalty": "frequencyPenalty", + "logprobs": "logProbs", + "logit_bias": "logitBias", + "n": "numGenerations", + "presence_penalty": "presencePenalty", + "seed": "seed", + "stop": "stop", + "tool_choice": "toolChoice", + "top_p": "topP", + "max_retries": False, + "top_logprobs": False, + "modalities": False, + "prediction": False, + "stream_options": False, + "function_call": False, + "functions": False, + "extra_headers": False, + "parallel_tool_calls": False, + "audio": False, + "web_search_options": False, + } + + def get_supported_openai_params(self, model: str) -> List[str]: + supported_params = [] + vendor = get_vendor_from_model(model) + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map + for key, value in open_ai_to_oci_param_map.items(): + if value: + supported_params.append(key) + + return supported_params + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + adapted_params = {} + vendor = get_vendor_from_model(model) + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map + + all_params = {**non_default_params, **optional_params} + + for key, value in all_params.items(): + alias = open_ai_to_oci_param_map.get(key) + + if alias is False: + if drop_params: + continue + + raise Exception(f"param `{key}` is not supported on OCI") + + if alias is None: + adapted_params[key] = value + continue + + adapted_params[alias] = value + + return adapted_params + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + """ + Some providers like Bedrock require signing the request. The sign request funtion needs access to `request_data` and `complete_url` + Args: + headers: dict + optional_params: dict + request_data: dict - the request body being sent in http request + api_base: str - the complete url being sent in http request + Returns: + dict - the signed headers + """ + import json + + oci_region = optional_params.get("oci_region", "us-ashburn-1") + api_base = ( + api_base + or litellm.api_base + or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" + ) + oci_user = optional_params.get("oci_user") + oci_fingerprint = optional_params.get("oci_fingerprint") + oci_tenancy = optional_params.get("oci_tenancy") + oci_key = optional_params.get("oci_key") + + if not oci_user or not oci_fingerprint or not oci_tenancy or not oci_key: + raise Exception( + "Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key" + ) + + method = str(optional_params.get("method", "POST")).upper() + body = json.dumps(request_data).encode("utf-8") + parsed = urlparse(api_base) + path = parsed.path or "/" + host = parsed.netloc + + date = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") + content_type = headers.get("content-type", "application/json") + content_length = str(len(body)) + x_content_sha256 = sha256_base64(body) + + headers_to_sign = { + "date": date, + "host": host, + "content-type": content_type, + "content-length": content_length, + "x-content-sha256": x_content_sha256, + } + + signed_headers = [ + "date", + "(request-target)", + "host", + "content-length", + "content-type", + "x-content-sha256", + ] + signing_string = build_signature_string( + method, path, headers_to_sign, signed_headers + ) + + try: + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.asymmetric import padding + except ImportError as e: + raise ImportError( + "cryptography package is required for OCI authentication. " + "Please install it with: pip install cryptography" + ) from e + + private_key = load_private_key_from_str(oci_key) + signature = private_key.sign( + signing_string.encode("utf-8"), + padding.PKCS1v15(), + hashes.SHA256(), + ) + signature_b64 = base64.b64encode(signature).decode() + + key_id = f"{oci_tenancy}/{oci_user}/{oci_fingerprint}" + + authorization = ( + 'Signature version="1",' + f'keyId="{key_id}",' + 'algorithm="rsa-sha256",' + f'headers="{" ".join(signed_headers)}",' + f'signature="{signature_b64}"' + ) + + headers.update( + { + "authorization": authorization, + "date": date, + "host": host, + "content-type": content_type, + "content-length": content_length, + "x-content-sha256": x_content_sha256, + } + ) + + return headers, None + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + oci_region = optional_params.get("oci_region", "us-ashburn-1") + api_base = ( + api_base + or litellm.api_base + or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" + ) + oci_user = optional_params.get("oci_user") + oci_fingerprint = optional_params.get("oci_fingerprint") + oci_tenancy = optional_params.get("oci_tenancy") + oci_key = optional_params.get("oci_key") + oci_compartment_id = optional_params.get("oci_compartment_id") + + if ( + not oci_user + or not oci_fingerprint + or not oci_tenancy + or not oci_key + or not oci_compartment_id + ): + raise Exception( + "Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key, oci_compartment_id" + ) + + if not api_base: + raise Exception( + "Either `api_base` must be provided or `litellm.api_base` must be set. Alternatively, you can set the `oci_region` optional parameter to use the default OCI region." + ) + + headers.update( + { + "content-type": "application/json", + "user-agent": f"litellm/{version}", + } + ) + + if not messages: + raise Exception( + "kwarg `messages` must be an array of messages that follow the openai chat standard" + ) + + return headers + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + oci_region = optional_params.get("oci_region", "us-ashburn-1") + return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/chat" + + def _get_optional_params(self, vendor: OCIVendors, optional_params: dict) -> Dict: + selected_params = {} + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map + + for value in open_ai_to_oci_param_map.values(): + if value in optional_params: + selected_params[value] = optional_params[value] + if "tools" in selected_params: + selected_params["tools"] = adapt_tool_definition_to_oci_standard( + selected_params["tools"], vendor + ) + return selected_params + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + oci_compartment_id = optional_params.get("oci_compartment_id", None) + if not oci_compartment_id: + raise Exception("kwarg `oci_compartment_id` is required for OCI requests") + + vendor = get_vendor_from_model(model) + + if vendor == OCIVendors.COHERE: + raise Exception( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + data = OCICompletionPayload( + compartmentId=oci_compartment_id, + servingMode=OCIServingMode( + servingType="ON_DEMAND", + modelId=model, + ), + chatRequest=OCIChatRequestPayload( + apiFormat=vendor.value, + messages=adapt_messages_to_generic_oci_standard(messages), + **self._get_optional_params(vendor, optional_params), + ), + ) + + return data.model_dump(exclude_none=True) + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + json = raw_response.json() # noqa: F811 + + error = json.get("error") + + if error is not None: + raise OCIError( + message=str(json["error"]), + status_code=raw_response.status_code, + ) + + if not isinstance(json, dict): + raise OCIError( + message="Invalid response format from OCI", + status_code=raw_response.status_code, + ) + + try: + completion_response = OCICompletionResponse(**json) + except TypeError as e: + raise OCIError( + message=f"Response cannot be casted to OCICompletionResponse: {str(e)}", + status_code=raw_response.status_code, + ) + + vendor = get_vendor_from_model(model) + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + iso_str = completion_response.chatResponse.timeCreated + dt = datetime.datetime.fromisoformat(iso_str.replace("Z", "+00:00")) + model_response.created = int(dt.timestamp()) + + model_response.model = completion_response.modelId + + message = model_response.choices[0].message # type: ignore + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + response_message = completion_response.chatResponse.choices[0].message + if response_message.content and response_message.content[0].type == "TEXT": + message.content = response_message.content[0].text + if response_message.toolCalls: + message.tool_calls = adapt_tools_to_openai_standard( + response_message.toolCalls + ) + + usage = Usage( + prompt_tokens=completion_response.chatResponse.usage.promptTokens, + completion_tokens=completion_response.chatResponse.usage.completionTokens, + total_tokens=completion_response.chatResponse.usage.totalTokens, + ) + model_response.usage = usage # type: ignore + + model_response._hidden_params["additional_headers"] = raw_response.headers + + return model_response + + @track_llm_api_timing() + def get_sync_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> "OCIStreamWrapper": + if "stream" in data: + del data["stream"] + if client is None or isinstance(client, AsyncHTTPHandler): + client = _get_httpx_client(params={}) + + try: + response = client.post( + api_base, + headers=headers, + data=json.dumps(data), + stream=True, + logging_obj=logging_obj, + timeout=STREAMING_TIMEOUT, + ) + except httpx.HTTPStatusError as e: + raise OCIError(status_code=e.response.status_code, message=e.response.text) + + if response.status_code != 200: + raise OCIError(status_code=response.status_code, message=response.text) + + completion_stream = response.iter_text() + + streaming_response = OCIStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + return streaming_response + + @track_llm_api_timing() + async def get_async_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> "OCIStreamWrapper": + if "stream" in data: + del data["stream"] + + if client is None or isinstance(client, HTTPHandler): + client = get_async_httpx_client(llm_provider=LlmProviders.BYTEZ, params={}) + + try: + response = await client.post( + api_base, + headers=headers, + data=json.dumps(data), + stream=True, + logging_obj=logging_obj, + timeout=STREAMING_TIMEOUT, + ) + except httpx.HTTPStatusError as e: + raise OCIError(status_code=e.response.status_code, message=e.response.text) + + if response.status_code != 200: + raise OCIError(status_code=response.status_code, message=response.text) + + completion_stream = response.aiter_text() + + streaming_response = OCIStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + return streaming_response + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return OCIError(status_code=status_code, message=error_message) + + +open_ai_to_generic_oci_role_map: Dict[str, OCIRoles] = { + "system": "SYSTEM", + "user": "USER", + "assistant": "ASSISTANT", + "tool": "TOOL", +} + + +def adapt_messages_to_generic_oci_standard_content_message( + role: str, content: Union[str, list] +) -> OCIMessage: + new_content: List[OCIContentPartUnion] = [] + if isinstance(content, str): + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=[OCITextContentPart(text=content)], + toolCalls=None, + toolCallId=None, + ) + + # content is a list of content items: + # [ + # {"type": "text", "text": "Hello"}, + # {"type": "image_url", "image_url": "https://example.com/image.png"} + # ] + for content_item in content: + if not isinstance(content_item, dict): + raise Exception("Each content item must be a dictionary") + + type = content_item.get("type") + if not isinstance(type, str): + raise Exception("Prop `type` is not a string") + + if type not in ["text", "image_url"]: + raise Exception(f"Prop `{type}` is not supported") + + if type == "text": + text = content_item.get("text") + if not isinstance(text, str): + raise Exception("Prop `text` is not a string") + new_content.append(OCITextContentPart(text=text)) + + elif type == "image_url": + image_url = content_item.get("image_url") + if not isinstance(image_url, str): + raise Exception("Prop `image_url` is not a string") + new_content.append(OCIImageContentPart(imageUrl=image_url)) + + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=new_content, + toolCalls=None, + toolCallId=None, + ) + + +def adapt_messages_to_generic_oci_standard_tool_call( + role: str, tool_calls: list +) -> OCIMessage: + tool_calls_formated = [] + for tool_call in tool_calls: + if not isinstance(tool_call, dict): + raise Exception("Each tool call must be a dictionary") + + if tool_call.get("type") != "function": + raise Exception("OCI only supports function tools") + + tool_call_id = tool_call.get("id") + if not isinstance(tool_call_id, str): + raise Exception("Prop `id` is not a string") + + tool_function = tool_call.get("function") + if not isinstance(tool_function, dict): + raise Exception("Prop `function` is not a dictionary") + + function_name = tool_function.get("name") + if not isinstance(function_name, str): + raise Exception("Prop `name` is not a string") + + arguments = tool_call["function"].get("arguments", "{}") + if not isinstance(arguments, str): + raise Exception("Prop `arguments` is not a string") + + # tool_calls_formated.append(OCIToolCall( + # id=tool_call_id, + # type="FUNCTION", + # function=OCIFunction( + # name=function_name, + # arguments=arguments + # ) + # )) + + tool_calls_formated.append( + OCIToolCall( + id=tool_call_id, + type="FUNCTION", + name=function_name, + arguments=arguments, + ) + ) + + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=None, + toolCalls=tool_calls_formated, + toolCallId=None, + ) + + +def adapt_messages_to_generic_oci_standard_tool_response( + role: str, tool_call_id: str, content: str +) -> OCIMessage: + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=[OCITextContentPart(text=content)], + toolCalls=None, + toolCallId=tool_call_id, + ) + + +def adapt_messages_to_generic_oci_standard( + messages: List[AllMessageValues], +) -> List[OCIMessage]: + new_messages = [] + for message in messages: + role = message["role"] + content = message.get("content") + tool_calls = message.get("tool_calls") + tool_call_id = message.get("tool_call_id") + + if role in ["system", "user", "assistant"] and content is not None: + if not isinstance(content, (str, list)): + raise Exception( + "Prop `content` must be a string or a list of content items" + ) + new_messages.append( + adapt_messages_to_generic_oci_standard_content_message(role, content) + ) + + elif role == "assistant" and tool_calls is not None: + if not isinstance(tool_calls, list): + raise Exception("Prop `tool_calls` must be a list of tool calls") + new_messages.append( + adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls) + ) + + elif role == "tool": + if not isinstance(tool_call_id, str): + raise Exception("Prop `tool_call_id` is required and must be a string") + if not isinstance(content, str): + raise Exception("Prop `content` is not a string") + new_messages.append( + adapt_messages_to_generic_oci_standard_tool_response( + role, tool_call_id, content + ) + ) + + return new_messages + + +def adapt_tool_definition_to_oci_standard(tools: List[Dict], vendor: OCIVendors): + new_tools = [] + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + for tool in tools: + if tool["type"] != "function": + raise Exception("OCI only supports function tools") + + tool_function = tool.get("function") + if not isinstance(tool_function, dict): + raise Exception("Prop `function` is not a dictionary") + + new_tool = OCIToolDefinition( + type="FUNCTION", + name=tool_function.get("name"), + description=tool_function.get("description", ""), + parameters=tool_function.get("parameters", {}), + ) + new_tools.append(new_tool) + + return new_tools + + +def adapt_tools_to_openai_standard( + tools: List[OCIToolCall], +) -> List[ChatCompletionMessageToolCall]: + new_tools = [] + for tool in tools: + new_tool = ChatCompletionMessageToolCall( + id=tool.id, + type="function", + function={ + "name": tool.name, + "arguments": tool.arguments, + }, + ) + new_tools.append(new_tool) + return new_tools + + +class OCIStreamWrapper(CustomStreamWrapper): + """ + Custom stream wrapper for OCI responses. + This class is used to handle streaming responses from OCI's API. + """ + + def __init__( + self, + **kwargs: Any, + ): + super().__init__(**kwargs) + + def chunk_creator(self, chunk: Any): + if not isinstance(chunk, str): + raise ValueError(f"Chunk is not a string: {chunk}") + if not chunk.startswith("data:"): + raise ValueError(f"Chunk does not start with 'data:': {chunk}") + dict_chunk = json.loads(chunk[5:]) # Remove 'data: ' prefix and parse JSON + try: + typed_chunk = OCIStreamChunk(**dict_chunk) + except TypeError as e: + raise ValueError(f"Chunk cannot be casted to OCIStreamChunk: {str(e)}") + + if typed_chunk.index is None: + typed_chunk.index = 0 + + text = "" + if typed_chunk.message and typed_chunk.message.content: + for item in typed_chunk.message.content: + if isinstance(item, OCITextContentPart): + text += item.text + elif isinstance(item, OCIImageContentPart): + raise ValueError( + "OCI does not support image content in streaming responses" + ) + else: + raise ValueError( + f"Unsupported content type in OCI response: {item.type}" + ) + + tool_calls = None + if typed_chunk.message and typed_chunk.message.toolCalls: + tool_calls = adapt_tools_to_openai_standard(typed_chunk.message.toolCalls) + + return ModelResponseStream( + choices=[ + StreamingChoices( + index=typed_chunk.index if typed_chunk.index else 0, + delta=Delta( + content=text, + tool_calls=( + [tool.model_dump() for tool in tool_calls] + if tool_calls + else None + ), + provider_specific_fields=None, # OCI does not have provider specific fields in the response + thinking_blocks=None, # OCI does not have thinking blocks in the response + reasoning_content=None, # OCI does not have reasoning content in the response + ), + finish_reason=typed_chunk.finishReason, + ) + ] + ) diff --git a/litellm/llms/oci/common_utils.py b/litellm/llms/oci/common_utils.py new file mode 100644 index 00000000000..661a6c89e4b --- /dev/null +++ b/litellm/llms/oci/common_utils.py @@ -0,0 +1,19 @@ +from typing import Optional + +import httpx + +from litellm.llms.base_llm.chat.transformation import BaseLLMException + + +class OCIError(BaseLLMException): + def __init__( + self, + status_code: int, + message: str, + headers: Optional[httpx.Headers] = None, + ): + super().__init__( + status_code=status_code, + message=message, + headers=headers, + ) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 527ae4a9d49..12814286f6c 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -4,6 +4,9 @@ import httpx import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + _safe_convert_created_field, +) from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import * @@ -11,7 +14,6 @@ from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams from ..common_utils import OpenAIError -from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import _safe_convert_created_field if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -47,6 +49,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): "top_p", "truncation", "user", + "service_tier", + "safety_identifier", "extra_headers", "extra_query", "extra_body", diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py index 0e890f0fd51..76cd12be8ee 100644 --- a/litellm/llms/openai/vector_stores/transformation.py +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -14,6 +14,7 @@ from litellm.types.vector_stores import ( VectorStoreSearchRequest, VectorStoreSearchResponse, ) +from litellm.utils import add_openai_metadata if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -119,12 +120,13 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): api_base: str, ) -> Tuple[str, Dict]: url = api_base # Base URL for creating vector stores + metadata = vector_store_create_optional_params.get("metadata", None) typed_request_body = VectorStoreCreateRequest( name=vector_store_create_optional_params.get("name", None), file_ids=vector_store_create_optional_params.get("file_ids", None), expires_after=vector_store_create_optional_params.get("expires_after", None), chunking_strategy=vector_store_create_optional_params.get("chunking_strategy", None), - metadata=vector_store_create_optional_params.get("metadata", None), + metadata=add_openai_metadata(metadata) if metadata is not None else None, ) dict_request_body = cast(dict, typed_request_body) diff --git a/litellm/main.py b/litellm/main.py index 4fc81cce6db..6bedf8f7ea5 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -107,6 +107,7 @@ from litellm.utils import ( supports_httpx_timeout, token_counter, validate_and_fix_openai_messages, + validate_and_fix_openai_tools, validate_chat_completion_tool_choice, ) @@ -151,6 +152,7 @@ from .llms.gemini.common_utils import get_api_key_from_env from .llms.groq.chat.handler import GroqChatCompletion from .llms.huggingface.embedding.handler import HuggingFaceEmbedding from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion +from .llms.oci.chat.transformation import OCIChatConfig from .llms.ollama.completion import handler as ollama from .llms.oobabooga.chat import oobabooga from .llms.openai.completion.handler import OpenAITextCompletion @@ -252,6 +254,7 @@ base_llm_http_handler = BaseLLMHTTPHandler() base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler() sagemaker_chat_completion = SagemakerChatHandler() bytez_transformation = BytezChatConfig() +oci_transformation = OCIChatConfig() ####### COMPLETION ENDPOINTS ################ @@ -963,6 +966,7 @@ def completion( # type: ignore # noqa: PLR0915 raise ValueError("model param not passed in.") # validate messages messages = validate_and_fix_openai_messages(messages=messages) + tools = validate_and_fix_openai_tools(tools=tools) # validate tool_choice tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice) ######### unpacking kwargs ##################### @@ -2399,6 +2403,24 @@ def completion( # type: ignore # noqa: PLR0915 encoding=encoding, stream=stream, ) + elif custom_llm_provider == "oci": + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=encoding, + stream=stream, + ) elif custom_llm_provider == "oobabooga": custom_llm_provider = "oobabooga" model_response = oobabooga.completion( @@ -3880,7 +3902,6 @@ def embedding( # noqa: PLR0915 ) elif ( custom_llm_provider == "openai_like" - or custom_llm_provider == "jina_ai" or custom_llm_provider == "hosted_vllm" or custom_llm_provider == "llamafile" or custom_llm_provider == "lm_studio" @@ -4307,6 +4328,25 @@ def embedding( # noqa: PLR0915 client=client, aembedding=aembedding, ) + elif custom_llm_provider == "jina_ai": + if isinstance(input, str): + transformed_input = [input] + else: + transformed_input = input + response = base_llm_http_handler.embedding( + model=model, + input=transformed_input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + litellm_params={}, + client=client, + aembedding=aembedding, + ) elif custom_llm_provider in litellm._custom_providers: custom_handler: Optional[CustomLLM] = None for item in litellm.custom_provider_map: @@ -5682,7 +5722,11 @@ def stream_chunk_builder_text_completion( def stream_chunk_builder( # noqa: PLR0915 - chunks: list, messages: Optional[list] = None, start_time=None, end_time=None + chunks: list, + messages: Optional[list] = None, + start_time=None, + end_time=None, + logging_obj: Optional[Logging] = None, ) -> Optional[Union[ModelResponse, TextCompletionResponse]]: try: if chunks is None: @@ -5807,6 +5851,12 @@ def stream_chunk_builder( # noqa: PLR0915 setattr(response, "usage", usage) + # Add cost to usage object if include_cost_in_streaming_usage is True + if litellm.include_cost_in_streaming_usage and logging_obj is not None: + setattr( + usage, "cost", logging_obj._response_cost_calculator(result=response) + ) + return response except Exception as e: verbose_logger.exception( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ef4891c2686..8281cb424fd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -607,9 +607,9 @@ "supports_system_messages": true, "supports_tool_choice": true, "search_context_cost_per_query": { - "search_context_size_low": 30.0, - "search_context_size_medium": 35.0, - "search_context_size_high": 50.0 + "search_context_size_low": 0.025, + "search_context_size_medium": 0.0275, + "search_context_size_high": 0.03 } }, "codex-mini-latest": { @@ -3750,7 +3750,7 @@ "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3.3e-06, - "output_cost_per_token": 16.5e-06, + "output_cost_per_token": 1.65e-05, "litellm_provider": "azure_ai", "mode": "chat", "supports_function_calling": true, @@ -3764,7 +3764,7 @@ "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-06, - "output_cost_per_token": 15e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "azure_ai", "mode": "chat", "supports_function_calling": true, @@ -3777,7 +3777,7 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.25e-06, + "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.27e-06, "litellm_provider": "azure_ai", "mode": "chat", @@ -3792,7 +3792,7 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.275e-06, + "input_cost_per_token": 2.75e-07, "output_cost_per_token": 1.38e-06, "litellm_provider": "azure_ai", "mode": "chat", @@ -5486,6 +5486,36 @@ "litellm_provider": "groq", "mode": "audio_transcription" }, + "groq/openai/gpt-oss-20b": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_web_search": true + }, + "groq/openai/gpt-oss-120b": { + "max_tokens": 32766, + "max_input_tokens": 131072, + "max_output_tokens": 32766, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 7.5e-07, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_web_search": true + }, "cerebras/llama3.1-8b": { "max_tokens": 128000, "max_input_tokens": 128000, @@ -5741,6 +5771,32 @@ "supports_reasoning": true, "supports_computer_use": true }, + "claude-opus-4-1-20250805": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "anthropic", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "claude-sonnet-4-20250514": { "max_tokens": 64000, "max_input_tokens": 200000, @@ -7337,12 +7393,12 @@ "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, "max_pdf_size_mb": 30, - "input_cost_per_token": 3.5e-07, + "input_cost_per_token": 3.5e-07, "input_cost_per_audio_token": 2.1e-06, "input_cost_per_image": 2.1e-06, "input_cost_per_video_per_second": 2.1e-06, "output_cost_per_token": 1.5e-06, - "output_cost_per_audio_token": 8.5e-06, + "output_cost_per_audio_token": 8.5e-06, "litellm_provider": "gemini", "mode": "chat", "rpm": 10, @@ -8690,6 +8746,40 @@ "source": "https://aistudio.google.com", "supports_tool_choice": true }, + "vertex_ai/claude-opus-4-1": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 15e-06, + "output_cost_per_token": 75e-06, + "input_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_batches": 37.5e-06, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_assistant_prefill": true, + "supports_tool_choice": true + }, + "vertex_ai/claude-opus-4-1@20250805": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 15e-06, + "output_cost_per_token": 75e-06, + "input_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_batches": 37.5e-06, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_assistant_prefill": true, + "supports_tool_choice": true + }, "vertex_ai/claude-3-sonnet": { "max_tokens": 4096, "max_input_tokens": 200000, @@ -9039,9 +9129,9 @@ "supports_tool_choice": true }, "vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas": { - "max_tokens": 10000000.0, - "max_input_tokens": 10000000.0, - "max_output_tokens": 10000000.0, + "max_tokens": 10000000, + "max_input_tokens": 10000000, + "max_output_tokens": 10000000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 7e-07, "litellm_provider": "vertex_ai-llama_models", @@ -9059,9 +9149,9 @@ ] }, "vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": { - "max_tokens": 10000000.0, - "max_input_tokens": 10000000.0, - "max_output_tokens": 10000000.0, + "max_tokens": 10000000, + "max_input_tokens": 10000000, + "max_output_tokens": 10000000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 7e-07, "litellm_provider": "vertex_ai-llama_models", @@ -9079,9 +9169,9 @@ ] }, "vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas": { - "max_tokens": 1000000.0, - "max_input_tokens": 1000000.0, - "max_output_tokens": 1000000.0, + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, "input_cost_per_token": 3.5e-07, "output_cost_per_token": 1.15e-06, "litellm_provider": "vertex_ai-llama_models", @@ -9099,9 +9189,9 @@ ] }, "vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": { - "max_tokens": 1000000.0, - "max_input_tokens": 1000000.0, - "max_output_tokens": 1000000.0, + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, "input_cost_per_token": 3.5e-07, "output_cost_per_token": 1.15e-06, "litellm_provider": "vertex_ai-llama_models", @@ -9174,7 +9264,7 @@ "max_input_tokens": 128000, "max_output_tokens": 2048, "input_cost_per_token": 5e-06, - "output_cost_per_token": 16e-06, + "output_cost_per_token": 1.6e-05, "litellm_provider": "vertex_ai-llama_models", "mode": "chat", "supports_system_messages": true, @@ -10480,7 +10570,7 @@ "supports_tool_choice": true, "supports_prompt_caching": true }, - "openrouter/x-ai/grok-4":{ + "openrouter/x-ai/grok-4": { "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, @@ -10494,12 +10584,12 @@ "source": "https://openrouter.ai/x-ai/grok-4", "supports_web_search": true }, - "openrouter/bytedance/ui-tars-1.5-7b":{ + "openrouter/bytedance/ui-tars-1.5-7b": { "max_tokens": 2048, "max_input_tokens": 131072, "max_output_tokens": 2048, - "input_cost_per_token": 0.1e-06, - "output_cost_per_token": 0.2e-06, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "openrouter", "mode": "chat", "source": "https://openrouter.ai/api/v1/models/bytedance/ui-tars-1.5-7b", @@ -11178,8 +11268,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 2048, - "input_cost_per_token": 0.21e-06, - "output_cost_per_token": 0.63e-06, + "input_cost_per_token": 2.1e-07, + "output_cost_per_token": 6.3e-07, "litellm_provider": "openrouter", "mode": "chat", "supports_tool_choice": true @@ -11891,6 +11981,60 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "openai.gpt-oss-20b-1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "openai.gpt-oss-120b-1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "anthropic.claude-opus-4-1-20250805-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "anthropic.claude-opus-4-20250514-v1:0": { "max_tokens": 32000, "max_input_tokens": 200000, @@ -12093,6 +12237,32 @@ "supports_tool_choice": true, "supports_reasoning": true }, + "us.anthropic.claude-opus-4-1-20250805-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "us.anthropic.claude-opus-4-20250514-v1:0": { "max_tokens": 32000, "max_input_tokens": 200000, @@ -12266,6 +12436,32 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "eu.anthropic.claude-opus-4-1-20250805-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "max_tokens": 32000, "max_input_tokens": 200000, @@ -14763,7 +14959,7 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 16384, - "input_cost_per_token": 0.6e-06, + "input_cost_per_token": 6e-07, "output_cost_per_token": 2.5e-06, "litellm_provider": "fireworks_ai", "mode": "chat", @@ -14809,6 +15005,58 @@ "source": "https://fireworks.ai/pricing", "supports_tool_choice": false }, + "fireworks_ai/accounts/fireworks/models/glm-4p5": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 96000, + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2.19e-06, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://fireworks.ai/models/fireworks/glm-4p5" + }, + "fireworks_ai/accounts/fireworks/models/glm-4p5-air": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 96000, + "input_cost_per_token": 2.2e-07, + "output_cost_per_token": 8.8e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://artificialanalysis.ai/models/glm-4-5-air" + }, + "fireworks_ai/accounts/fireworks/models/gpt-oss-120b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://fireworks.ai/pricing" + }, + "fireworks_ai/accounts/fireworks/models/gpt-oss-20b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://fireworks.ai/pricing" + }, "fireworks_ai/nomic-ai/nomic-embed-text-v1.5": { "max_tokens": 8192, "max_input_tokens": 8192, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index e1ea9918e68..15891b53f4e 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -492,7 +492,10 @@ class MCPServerManager: ) tools = await self._fetch_tools_with_timeout(client, server.name) - return self._create_prefixed_tools(tools, server) + + prefixed_tools = self._create_prefixed_tools(tools, server) + + return prefixed_tools except Exception as e: verbose_logger.warning( @@ -523,6 +526,7 @@ class MCPServerManager: async def _list_tools_task(): try: await client.connect() + tools = await client.list_tools() verbose_logger.debug(f"Tools from {server_name}: {tools}") return tools @@ -644,6 +648,7 @@ class MCPServerManager: ######################################################### # Pre MCP Tool Call Hook # Allow validation and modification of tool calls before execution + # Using standard pre_call_hook with call_type="mcp_call" ######################################################### if proxy_logging_obj: pre_hook_kwargs = { @@ -651,24 +656,32 @@ class MCPServerManager: "arguments": arguments, "server_name": server_name_from_prefix, "user_api_key_auth": user_api_key_auth, + "user_api_key_user_id": getattr(user_api_key_auth, 'user_id', None) if user_api_key_auth else None, + "user_api_key_team_id": getattr(user_api_key_auth, 'team_id', None) if user_api_key_auth else None, + "user_api_key_end_user_id": getattr(user_api_key_auth, 'end_user_id', None) if user_api_key_auth else None, + "user_api_key_hash": getattr(user_api_key_auth, 'api_key_hash', None) if user_api_key_auth else None, } + + # Create MCP request object for processing + mcp_request_obj = proxy_logging_obj._create_mcp_request_object_from_kwargs(pre_hook_kwargs) + + # Convert to LLM format for existing guardrail compatibility + synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(mcp_request_obj, pre_hook_kwargs) + try: - pre_hook_result = await proxy_logging_obj.async_pre_mcp_tool_call_hook( - kwargs=pre_hook_kwargs, - request_obj=None, # Will be created in the hook - start_time=start_time, - end_time=start_time, + # Use standard pre_call_hook with call_type="mcp_call" + modified_data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_auth, #type: ignore + data=synthetic_llm_data, + call_type="mcp_call" #type: ignore ) - - if pre_hook_result: - # Apply any argument modifications - if pre_hook_result.get("modified_arguments"): - arguments = pre_hook_result["modified_arguments"] - except ( - BlockedPiiEntityError, - GuardrailRaisedException, - HTTPException, - ) as e: + if modified_data: + # Convert response back to MCP format and apply modifications + modified_kwargs = proxy_logging_obj._convert_mcp_hook_response_to_kwargs(modified_data, pre_hook_kwargs) + if modified_kwargs.get("arguments") != arguments: + arguments = modified_kwargs["arguments"] + + except (BlockedPiiEntityError, GuardrailRaisedException, HTTPException) as e: # Re-raise guardrail exceptions to properly fail the MCP call verbose_logger.error( f"Guardrail blocked MCP tool call pre call: {str(e)}" @@ -699,22 +712,34 @@ class MCPServerManager: name=original_tool_name, arguments=arguments, ) - - # Initialize during_hook_task as None - during_hook_task = None tasks = [] - # Start during hook if proxy_logging_obj is available if proxy_logging_obj: + # Create synthetic LLM data for during hook processing + from litellm.types.mcp import MCPDuringCallRequestObject + from litellm.types.llms.base import HiddenParams + + request_obj = MCPDuringCallRequestObject( + tool_name=name, + arguments=arguments, + server_name=server_name_from_prefix, + start_time=start_time.timestamp() if start_time else None, + hidden_params=HiddenParams(), + ) + + during_hook_kwargs = { + "name": name, + "arguments": arguments, + "server_name": server_name_from_prefix, + "user_api_key_auth": user_api_key_auth, + } + + synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(request_obj, during_hook_kwargs) + during_hook_task = asyncio.create_task( - proxy_logging_obj.async_during_mcp_tool_call_hook( - kwargs={ - "name": name, - "arguments": arguments, - "server_name": server_name_from_prefix, - }, - request_obj=None, # Will be created in the hook - start_time=start_time, - end_time=start_time, + proxy_logging_obj.during_call_hook( + user_api_key_dict=user_api_key_auth, + data=synthetic_llm_data, + call_type="mcp_call" #type: ignore ) ) tasks.append(during_hook_task) @@ -809,20 +834,28 @@ class MCPServerManager: get_prisma_client_or_throw, ) + verbose_logger.info("Loading MCP servers from database into registry...") + # perform authz check to filter the mcp servers user has access to prisma_client = get_prisma_client_or_throw( "Database not connected. Connect a database to your proxy" ) db_mcp_servers = await get_all_mcp_servers(prisma_client) + verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database") + # ensure the global_mcp_server_manager is up to date with the db for server in db_mcp_servers: + verbose_logger.debug(f"Adding server to registry: {server.server_id} ({server.server_name})") self.add_update_server(server) + + verbose_logger.info(f"Registry now contains {len(self.get_registry())} servers") def get_mcp_server_by_id(self, server_id: str) -> Optional[MCPServer]: """ Get the MCP Server from the server id """ - for server in self.get_registry().values(): + registry = self.get_registry() + for server in registry.values(): if server.server_id == server_id: return server return None @@ -1096,5 +1129,12 @@ class MCPServerManager: for server in list_mcp_servers ] + async def reload_servers_from_database(self): + """ + Public method to reload all MCP servers from database into registry. + This can be called from management endpoints to ensure registry is up to date. + """ + await self._add_mcp_servers_from_db_to_in_memory_registry() + global_mcp_server_manager: MCPServerManager = MCPServerManager() diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index a31fabf57cf..6624eb7e64e 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1,5 +1,5 @@ import importlib -from typing import Optional +from typing import Optional, Dict from fastapi import APIRouter, Depends, Query, Request @@ -32,9 +32,49 @@ if MCP_AVAILABLE: ######################################################## ############ MCP Server REST API Routes ################# + def _get_server_auth_header( + server, mcp_server_auth_headers: Optional[Dict[str, str]], mcp_auth_header: Optional[str] + ) -> Optional[str]: + """Helper function to get server-specific auth header with case-insensitive matching.""" + if mcp_server_auth_headers and server.alias: + normalized_server_alias = server.alias.lower() + normalized_headers = {k.lower(): v for k, v in mcp_server_auth_headers.items()} + server_auth = normalized_headers.get(normalized_server_alias) + if server_auth is not None: + return server_auth + elif mcp_server_auth_headers and server.server_name: + normalized_server_name = server.server_name.lower() + normalized_headers = {k.lower(): v for k, v in mcp_server_auth_headers.items()} + server_auth = normalized_headers.get(normalized_server_name) + if server_auth is not None: + return server_auth + return mcp_auth_header + + def _create_tool_response_objects(tools, server_mcp_info): + """Helper function to create tool response objects.""" + return [ + ListMCPToolsRestAPIResponseObject( + name=tool.name, + description=tool.description, + inputSchema=tool.inputSchema, + mcp_info=server_mcp_info, + ) + for tool in tools + ] + + async def _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version): + """Helper function to get tools for a single server.""" + tools = await global_mcp_server_manager._get_tools_from_server( + server=server, + mcp_auth_header=server_auth_header, + mcp_protocol_version=mcp_protocol_version, + ) + return _create_tool_response_objects(tools, server.mcp_info) + ######################################################## @router.get("/tools/list", dependencies=[Depends(user_api_key_auth)]) async def list_tool_rest_api( + request: Request, server_id: Optional[str] = Query( None, description="The server id to list tools for" ), @@ -60,7 +100,15 @@ if MCP_AVAILABLE: "message": "Successfully retrieved tools" } """ + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + try: + # Extract auth headers from request + headers = request.headers + mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(headers) + mcp_server_auth_headers = MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers) + mcp_protocol_version = headers.get(MCPRequestHandler.MCP_PROTOCOL_VERSION_HEADER_NAME) + list_tools_result = [] error_message = None @@ -73,19 +121,11 @@ if MCP_AVAILABLE: "error": "server_not_found", "message": f"Server with id {server_id} not found" } + + server_auth_header = _get_server_auth_header(server, mcp_server_auth_headers, mcp_auth_header) + try: - tools = await global_mcp_server_manager._get_tools_from_server( - server=server, - ) - for tool in tools: - list_tools_result.append( - ListMCPToolsRestAPIResponseObject( - name=tool.name, - description=tool.description, - inputSchema=tool.inputSchema, - mcp_info=server.mcp_info, - ) - ) + list_tools_result = await _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version) except Exception as e: verbose_logger.exception(f"Error getting tools from {server.name}: {e}") return { @@ -97,19 +137,11 @@ if MCP_AVAILABLE: # Query all servers errors = [] for server in global_mcp_server_manager.get_registry().values(): + server_auth_header = _get_server_auth_header(server, mcp_server_auth_headers, mcp_auth_header) + try: - tools = await global_mcp_server_manager._get_tools_from_server( - server=server, - ) - for tool in tools: - list_tools_result.append( - ListMCPToolsRestAPIResponseObject( - name=tool.name, - description=tool.description, - inputSchema=tool.inputSchema, - mcp_info=server.mcp_info, - ) - ) + tools_result = await _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version) + list_tools_result.extend(tools_result) except Exception as e: verbose_logger.exception(f"Error getting tools from {server.name}: {e}") errors.append(f"{server.name}: {str(e)}") diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index f30076137a2..00000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index ad4dabf9d04..b587913a08a 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -2,27 +2,4 @@ model_list: - model_name: openai-test litellm_params: model: gpt-3.5-turbo - api_key: os.environ/OPENAI_API_KEY - -guardrails: - - guardrail_name: azure-text-moderation - litellm_params: - guardrail: azure/text_moderations - mode: "post_call" - api_key: os.environ/AZURE_GUARDRAIL_API_KEY - api_base: os.environ/AZURE_GUARDRAIL_API_BASE - -prompts: - - prompt_id: test_my_json_prompt - litellm_params: - prompt_integration: dotprompt - prompt_id: test_hello_world_prompt - prompt_file: /Users/krrishdholakia/Documents/litellm/litellm/proxy/test_prompts/test_hello_world_prompt.prompt - - prompt_id: test_hello_world_prompt_2 - litellm_params: - prompt_integration: dotprompt - prompt_id: test_hello_world_prompt_2 - prompt_file: /Users/krrishdholakia/Documents/litellm/litellm/proxy/test_prompts/test_hello_world_prompt.prompt - -litellm_settings: - callbacks: ["datadog_llm_observability"] \ No newline at end of file + api_key: os.environ/OPENAI_API_KEY \ No newline at end of file diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9012dee8fc0..bb59f2e94b6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -380,6 +380,8 @@ class LiteLLMRoutes(enum.Enum): "/health", "/key/list", "/user/filter/ui", + "/models", + "/v1/models", ] # NOTE: ROUTES ONLY FOR MASTER KEY - only the Master Key should be able to Reset Spend @@ -492,6 +494,8 @@ class LiteLLMRoutes(enum.Enum): "/global/spend/end_users", "/global/activity", "/global/activity/model", + "/v1/models/{model_id}", + "/models/{model_id}", ] + spend_tracking_routes + key_management_routes @@ -2206,7 +2210,7 @@ class AllCallbacks(LiteLLMPydanticObjectBase): braintrust: CallbackOnUI = CallbackOnUI( litellm_callback_name="braintrust", - litellm_callback_params=["BRAINTRUST_API_KEY"], + litellm_callback_params=["BRAINTRUST_API_KEY","BRAINTRUST_API_BASE"], ui_callback_name="Braintrust", ) @@ -2260,6 +2264,7 @@ class SpendLogsMetadata(TypedDict): error_information: Optional[StandardLoggingPayloadErrorInformation] usage_object: Optional[dict] model_map_information: Optional[StandardLoggingModelInformation] + cold_storage_object_key: Optional[str] # S3/GCS object key for cold storage retrieval class SpendLogsPayload(TypedDict): diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 8af9e4d1c18..8965ae7715d 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -200,3 +200,80 @@ async def anthropic_response( # noqa: PLR0915 param=getattr(e, "param", "None"), code=getattr(e, "status_code", 500), ) + + +@router.post( + "/v1/messages/count_tokens", + tags=["[beta] Anthropic Messages Token Counting"], + dependencies=[Depends(user_api_key_auth)], +) +async def count_tokens( + request: Request, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # Used for auth +): + """ + Count tokens for Anthropic Messages API format. + + This endpoint follows the Anthropic Messages API token counting specification. + It accepts the same parameters as the /v1/messages endpoint but returns + token counts instead of generating a response. + + Example usage: + ``` + curl -X POST "http://localhost:4000/v1/messages/count_tokens?beta=true" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-key" \ + -d '{ + "model": "claude-3-sonnet-20240229", + "messages": [{"role": "user", "content": "Hello Claude!"}] + }' + ``` + + Returns: {"input_tokens": } + """ + from litellm.proxy.proxy_server import token_counter as internal_token_counter + + try: + request_data = await _read_request_body(request=request) + data: dict = {**request_data} + + # Extract required fields + model_name = data.get("model") + messages = data.get("messages", []) + + if not model_name: + raise HTTPException( + status_code=400, + detail={"error": "model parameter is required"} + ) + + if not messages: + raise HTTPException( + status_code=400, + detail={"error": "messages parameter is required"} + ) + + # Create TokenCountRequest for the internal endpoint + from litellm.proxy._types import TokenCountRequest + + token_request = TokenCountRequest( + model=model_name, + messages=messages + ) + + # Call the internal token counter function with direct request flag set to False + token_response = await internal_token_counter(token_request, is_direct_request=False) + + # Convert the internal response to Anthropic API format + return {"input_tokens": token_response.total_tokens} + + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception( + "litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - {}".format(str(e)) + ) + raise HTTPException( + status_code=500, + detail={"error": f"Internal server error: {str(e)}"} + ) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 32ba2080a36..b306512847f 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -288,9 +288,6 @@ def _is_api_route_allowed( if valid_token is None: raise Exception("Invalid proxy server token passed. valid_token=None.") - # Check if management routes are disabled and raise exception if they are - RouteChecks.should_call_route(route=route, valid_token=valid_token) - if not _is_user_proxy_admin(user_obj=user_obj): # if non-admin RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, @@ -440,6 +437,7 @@ async def get_end_user_object( end_user_id: Optional[str], prisma_client: Optional[PrismaClient], user_api_key_cache: DualCache, + route: str, parent_otel_span: Optional[Span] = None, proxy_logging_obj: Optional[ProxyLogging] = None, ) -> Optional[LiteLLM_EndUserTable]: @@ -456,6 +454,8 @@ async def get_end_user_object( _key = "end_user_id:{}".format(end_user_id) def check_in_budget(end_user_obj: LiteLLM_EndUserTable): + if route in LiteLLMRoutes.info_routes.value: # allow calling info routes + return if end_user_obj.litellm_budget_table is None: return end_user_budget = end_user_obj.litellm_budget_table.max_budget diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 12146f0e866..60529f1e2f4 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -83,7 +83,7 @@ class JWTHandler: self.user_api_key_cache = user_api_key_cache self.litellm_jwtauth = litellm_jwtauth self.leeway = leeway - + @staticmethod def is_jwt(token: str): parts = token.split(".") @@ -844,6 +844,7 @@ class JWTAuthManager: user_api_key_cache: DualCache, parent_otel_span: Optional[Span], proxy_logging_obj: ProxyLogging, + route: str, ) -> Tuple[ Optional[LiteLLM_UserTable], Optional[LiteLLM_OrganizationTable], @@ -892,6 +893,7 @@ class JWTAuthManager: user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, proxy_logging_obj=proxy_logging_obj, + route=route, ) if end_user_id else None @@ -1133,6 +1135,7 @@ class JWTAuthManager: user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, proxy_logging_obj=proxy_logging_obj, + route=route, ) await JWTAuthManager.sync_user_role_and_teams( diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index cd158fd70a7..f6b088d15a6 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -25,6 +25,8 @@ class RouteChecks: from litellm_enterprise.proxy.auth.route_checks import EnterpriseRouteChecks EnterpriseRouteChecks.should_call_route(route=route) + except HTTPException as e: + raise e except Exception: pass @@ -208,7 +210,7 @@ class RouteChecks: route=route, allowed_routes=LiteLLMRoutes.self_managed_routes.value ): # routes that manage their own allowed/disallowed logic pass - elif route.startswith("/v1/mcp/"): + elif route.startswith("/v1/mcp/") or route.startswith("/mcp-rest/"): pass # authN/authZ handled by api itself else: user_role = "unknown" @@ -386,7 +388,7 @@ class RouteChecks: if "thread" in request.url.path or "assistant" in request.url.path: return True return False - + @staticmethod def is_generate_content_route(route: str) -> bool: """ diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 31395fe7da0..9efa904574a 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -49,6 +49,7 @@ from litellm.proxy.auth.auth_utils import ( from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler from litellm.proxy.auth.oauth2_check import check_oauth2_token from litellm.proxy.auth.oauth2_proxy_hook import handle_oauth2_proxy_request +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, _safe_get_request_headers, @@ -259,6 +260,7 @@ def get_api_key( from litellm.proxy.common_utils.http_parsing_utils import ( _safe_get_request_query_params, ) + api_key = api_key passed_in_key: Optional[str] = None if isinstance(custom_litellm_key_header, str): @@ -279,7 +281,11 @@ def get_api_key( elif isinstance(azure_apim_header, str): passed_in_key = azure_apim_header api_key = azure_apim_header - elif RouteChecks.is_generate_content_route(route=route) and request is not None and _safe_get_request_query_params(request).get("key"): + elif ( + RouteChecks.is_generate_content_route(route=route) + and request is not None + and _safe_get_request_query_params(request).get("key") + ): google_auth_key: str = _safe_get_request_query_params(request).get("key") or "" passed_in_key = google_auth_key api_key = google_auth_key @@ -609,6 +615,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, proxy_logging_obj=proxy_logging_obj, + route=route, ) if _end_user_object is not None: end_user_params["allowed_model_region"] = ( @@ -1141,6 +1148,8 @@ async def user_api_key_auth( request_data = await _read_request_body(request=request) route: str = get_request_route(request=request) + ## CHECK IF ROUTE IS ALLOWED + user_api_key_auth_obj = await _user_api_key_auth_builder( request=request, api_key=api_key, @@ -1152,6 +1161,9 @@ async def user_api_key_auth( custom_litellm_key_header=custom_litellm_key_header, ) + ## ENSURE DISABLE ROUTE WORKS ACROSS ALL USER AUTH FLOWS ## + RouteChecks.should_call_route(route=route, valid_token=user_api_key_auth_obj) + end_user_id = get_end_user_id_from_request_body( request_data, _safe_get_request_headers(request) ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 384d958946b..15953f4229b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -5,6 +5,7 @@ # +-------------------------------------------------------------+ # Thank you users! We ā¤ļø you! - Krrish & Ishaan +import copy import os import sys @@ -50,6 +51,51 @@ from litellm.types.utils import ( GUARDRAIL_NAME = "bedrock" +def _redact_pii_matches(response_json: dict) -> dict: + try: + # Create a deep copy to avoid modifying the original response + redacted_response = copy.deepcopy(response_json) + + # Get assessments from the response + assessments = redacted_response.get("assessments", []) + if not assessments: + return redacted_response + + for assessment in assessments: + # Redact PII entities in sensitive information policy + sensitive_info_policy = assessment.get("sensitiveInformationPolicy") + if sensitive_info_policy: + pii_entities = sensitive_info_policy.get("piiEntities", []) + for pii_entity in pii_entities: + if "match" in pii_entity: + pii_entity["match"] = "[REDACTED]" + + # Redact regex matches + regexes = sensitive_info_policy.get("regexes", []) + for regex_match in regexes: + if "match" in regex_match: + regex_match["match"] = "[REDACTED]" + + # Redact custom word matches in word policy + word_policy = assessment.get("wordPolicy") + if word_policy: + custom_words = word_policy.get("customWords", []) + for custom_word in custom_words: + if "match" in custom_word: + custom_word["match"] = "[REDACTED]" + + managed_words = word_policy.get("managedWordLists", []) + for managed_word in managed_words: + if "match" in managed_word: + managed_word["match"] = "[REDACTED]" + + return redacted_response + except Exception as e: + # We do not want to fail in any case so this is just a warning + verbose_proxy_logger.warning("Guardrail log redaction failed: %s", str(e)) + return response_json + + class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): def __init__( self, @@ -271,10 +317,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): data=prepared_request.body, # type: ignore headers=prepared_request.headers, # type: ignore ) - verbose_proxy_logger.debug("Bedrock AI response: %s", response.text) if response.status_code == 200: # check if the response was flagged _json_response = response.json() + redacted_response = _redact_pii_matches(_json_response) + verbose_proxy_logger.debug("Bedrock AI response : %s", redacted_response) bedrock_guardrail_response = BedrockGuardrailResponse(**_json_response) if self._should_raise_guardrail_blocked_exception( bedrock_guardrail_response diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 90f61c60596..be265c73bfc 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -403,6 +403,9 @@ if MCP_AVAILABLE: touched_by=user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME, ) global_mcp_server_manager.add_update_server(new_mcp_server) + + # Ensure registry is up to date by reloading from database + await global_mcp_server_manager.reload_servers_from_database() except Exception as e: verbose_proxy_logger.exception(f"Error creating mcp server: {str(e)}") raise HTTPException( @@ -461,6 +464,9 @@ if MCP_AVAILABLE: detail={"error": f"MCP Server not found, passed server_id={server_id}"}, ) global_mcp_server_manager.remove_server(mcp_server_record_deleted) + + # Ensure registry is up to date by reloading from database + await global_mcp_server_manager.reload_servers_from_database() # TODO: Enterprise: Finish audit log trail if litellm.store_audit_logs: @@ -533,6 +539,9 @@ if MCP_AVAILABLE: }, ) global_mcp_server_manager.add_update_server(mcp_server_record_updated) + + # Ensure registry is up to date by reloading from database + await global_mcp_server_manager.reload_servers_from_database() # TODO: Enterprise: Finish audit log trail if litellm.store_audit_logs: diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 14f273f3269..ee38eb6515d 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -116,6 +116,14 @@ async def serve_login_page( missing_env_vars = show_missing_vars_in_env() if missing_env_vars is not None: return missing_env_vars + ######################################################### + # Construct Redirect URL + base_url_to_redirect_to: Optional[str] = None + base_url_to_redirect_to = os.getenv("PROXY_BASE_URL", "") + server_root_path = os.getenv("SERVER_ROOT_PATH", "") + if server_root_path != "": + base_url_to_redirect_to += server_root_path + ######################################################### # Build the unified login page HTML error_message = "" @@ -137,7 +145,13 @@ async def serve_login_page( sso_button = "" if sso_available: - sso_button = """ + sso_login_url = base_url_to_redirect_to + if sso_login_url.endswith("/"): + sso_login_url += "sso/login" + else: + sso_login_url += "/sso/login" + + sso_button = f"""
or

- diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 7aababa79d4..aa63369ef2b 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,6 +1,15 @@ model_list: - - model_name: vertex_ai/* + - model_name: bedrock/* litellm_params: - model: vertex_ai/* + model: bedrock/* +litellm_settings: + callbacks: ["s3_v2"] + s3_callback_params: + s3_bucket_name: litellm-logs # AWS Bucket Name for S3 + s3_region_name: us-west-2 + +general_settings: + cold_storage_custom_logger: s3_v2 + store_prompts_in_cold_storage: true \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index af9f5d363da..931459c1386 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -126,6 +126,7 @@ import litellm from litellm import Router from litellm._logging import verbose_proxy_logger, verbose_router_logger from litellm.caching.caching import DualCache, RedisCache +from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.constants import ( DAYS_IN_A_MONTH, DEFAULT_HEALTH_CHECK_INTERVAL, @@ -1591,7 +1592,9 @@ class ProxyConfig: litellm.cache = Cache(**cache_params) - if litellm.cache is not None and isinstance(litellm.cache.cache, RedisCache): + if litellm.cache is not None and isinstance( + litellm.cache.cache, (RedisCache, RedisClusterCache) + ): ## INIT PROXY REDIS USAGE CLIENT ## redis_usage_cache = litellm.cache.cache @@ -1728,7 +1731,7 @@ class ProxyConfig: self._load_environment_variables(config=config) ## Callback settings - callback_settings = config.get("callback_settings", None) + callback_settings = config.get("callback_settings", {}) ## LITELLM MODULE SETTINGS (e.g. litellm.drop_params=True,..) litellm_settings = config.get("litellm_settings", None) @@ -2670,7 +2673,7 @@ class ProxyConfig: proxy_logging_obj: ProxyLogging """ _general_settings = config_data.get("general_settings", {}) - if "alerting" in _general_settings: + if _general_settings is not None and "alerting" in _general_settings: if ( general_settings is not None and general_settings.get("alerting", None) is not None @@ -2702,14 +2705,14 @@ class ProxyConfig: "alerting" ] - if "alert_types" in _general_settings: + if _general_settings is not None and "alert_types" in _general_settings: general_settings["alert_types"] = _general_settings["alert_types"] proxy_logging_obj.alert_types = general_settings["alert_types"] proxy_logging_obj.slack_alerting_instance.update_values( alert_types=general_settings["alert_types"], llm_router=llm_router ) - if "alert_to_webhook_url" in _general_settings: + if _general_settings is not None and "alert_to_webhook_url" in _general_settings: general_settings["alert_to_webhook_url"] = _general_settings[ "alert_to_webhook_url" ] @@ -3766,100 +3769,37 @@ async def model_list( Defaults to "general" when include_metadata=true """ global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj - all_models = [] - model_access_groups: Dict[str, List[str]] = defaultdict(list) - ## CHECK IF MODEL RESTRICTIONS ARE SET AT KEY/TEAM LEVEL ## - if llm_router is None: - proxy_model_list = [] - else: - proxy_model_list = llm_router.get_model_names() - model_access_groups = llm_router.get_model_access_groups() - ## if only_model_access_groups is True, - """ - 1. Get all models key/user/team has access to - 2. Filter out models that are not model access groups - 3. Return the models - """ - if only_model_access_groups is True: - include_model_access_groups = True + from litellm.proxy.utils import ( + create_model_info_response, + get_available_models_for_user, + ) - key_models = get_key_models( + # Get available models for the user + all_models = await get_available_models_for_user( user_api_key_dict=user_api_key_dict, - proxy_model_list=proxy_model_list, - model_access_groups=model_access_groups, - include_model_access_groups=include_model_access_groups, - ) - - team_models: List[str] = user_api_key_dict.team_models - - if team_id: - key_models = [] - team_object = await get_team_object( - team_id=team_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - validate_membership(user_api_key_dict=user_api_key_dict, team_table=team_object) - team_models = team_object.models - - team_models = get_team_models( - team_models=team_models, - proxy_model_list=proxy_model_list, - model_access_groups=model_access_groups, - include_model_access_groups=include_model_access_groups, - ) - - all_models = get_complete_model_list( - key_models=key_models, - team_models=team_models, - proxy_model_list=proxy_model_list, - user_model=user_model, - infer_model_from_keys=general_settings.get("infer_model_from_keys", False), - return_wildcard_routes=return_wildcard_routes, llm_router=llm_router, - model_access_groups=model_access_groups, - include_model_access_groups=include_model_access_groups, - only_model_access_groups=only_model_access_groups, + general_settings=general_settings, + user_model=user_model, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + team_id=team_id, + include_model_access_groups=include_model_access_groups or False, + only_model_access_groups=only_model_access_groups or False, + return_wildcard_routes=return_wildcard_routes or False, + user_api_key_cache=user_api_key_cache, ) # Build response data model_data = [] for model in all_models: - model_info = { - "id": model, - "object": "model", - "created": DEFAULT_MODEL_CREATED_AT_TIME, - "owned_by": "openai", - } - - # Add metadata if requested - if include_metadata: - metadata = {} - - # Default fallback_type to "general" if include_metadata is true - effective_fallback_type = ( - fallback_type if fallback_type is not None else "general" - ) - - # Validate fallback_type - valid_fallback_types = ["general", "context_window", "content_policy"] - if effective_fallback_type not in valid_fallback_types: - raise HTTPException( - status_code=400, - detail=f"Invalid fallback_type. Must be one of: {valid_fallback_types}", - ) - - fallbacks = get_all_fallbacks( - model=model, - llm_router=llm_router, - fallback_type=effective_fallback_type, - ) - metadata["fallbacks"] = fallbacks - - model_info["metadata"] = metadata - + model_info = create_model_info_response( + model_id=model, + provider="openai", + include_metadata=include_metadata or False, + fallback_type=fallback_type, + llm_router=llm_router, + ) model_data.append(model_info) return dict( @@ -3868,6 +3808,68 @@ async def model_list( ) +@router.get( + "/v1/models/{model_id}", + dependencies=[Depends(user_api_key_auth)], + tags=["model management"], +) +@router.get( + "/models/{model_id}", + dependencies=[Depends(user_api_key_auth)], + tags=["model management"], +) +async def model_info( + model_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Retrieve information about a specific model accessible to your API key. + + Returns model details only if the model is available to your API key/team. + Returns 404 if the model doesn't exist or is not accessible. + + Follows OpenAI API specification for individual model retrieval. + https://platform.openai.com/docs/api-reference/models/retrieve + """ + global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj + + from litellm.proxy.utils import ( + create_model_info_response, + get_available_models_for_user, + validate_model_access, + ) + + # Get available models for the user + all_models = await get_available_models_for_user( + user_api_key_dict=user_api_key_dict, + llm_router=llm_router, + general_settings=general_settings, + user_model=user_model, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + team_id=None, + include_model_access_groups=False, + only_model_access_groups=False, + return_wildcard_routes=False, + user_api_key_cache=user_api_key_cache, + ) + + # Validate that the requested model is accessible + validate_model_access(model_id=model_id, available_models=all_models) + + # Get provider information + _, provider, _, _ = litellm.get_llm_provider(model=model_id) + + # Return the model information in the same format as the list endpoint + return create_model_info_response( + model_id=model_id, + provider=provider, + include_metadata=False, + fallback_type=None, + llm_router=llm_router, + ) + + @router.post( "/v1/chat/completions", dependencies=[Depends(user_api_key_auth)], @@ -3923,7 +3925,7 @@ async def chat_completion( # noqa: PLR0915 data = await _read_request_body(request=request) base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) try: - return await base_llm_response_processor.base_process_llm_request( + result = await base_llm_response_processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -3941,6 +3943,10 @@ async def chat_completion( # noqa: PLR0915 user_api_base=user_api_base, version=version, ) + if isinstance(result, BaseModel): + return result.model_dump(exclude_none=True, exclude_unset=True) + else: + return result except RejectedRequestError as e: _data = e.request_data await proxy_logging_obj.post_call_failure_hook( @@ -5614,13 +5620,62 @@ async def run_thread( # async def get_available_routes(user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)): +def _get_provider_token_counter(deployment: dict, model_to_use: str): + """ + Auto-route to the correct provider's token counter based on model/deployment. + Uses the existing get_provider_model_info infrastructure with switch-case pattern. + """ + if deployment is None: + return None + + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + full_model = deployment.get("litellm_params", {}).get("model", "") + + try: + # Use existing LiteLLM logic to determine provider + model, provider, dynamic_api_key, api_base = get_llm_provider( + model=full_model, + custom_llm_provider=deployment.get("litellm_params", {}).get( + "custom_llm_provider" + ), + api_base=deployment.get("litellm_params", {}).get("api_base"), + api_key=deployment.get("litellm_params", {}).get("api_key"), + ) + + # Switch case pattern using existing get_provider_model_info + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + # Convert string provider to LlmProviders enum + llm_provider_enum = LlmProviders(provider) + # Add more provider mappings as needed + + if llm_provider_enum: + provider_model_info = ProviderConfigManager.get_provider_model_info( + model=full_model, provider=llm_provider_enum + ) + if provider_model_info is not None: + return provider_model_info.get_token_counter() + + except Exception: + # If provider detection fails, fall back to manual checks + if full_model.startswith("anthropic/") or "anthropic" in full_model.lower(): + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + anthropic_model_info = AnthropicModelInfo() + return anthropic_model_info.get_token_counter() + + return None + + @router.post( "/utils/token_counter", tags=["llm utils"], dependencies=[Depends(user_api_key_auth)], response_model=TokenCountResponse, ) -async def token_counter(request: TokenCountRequest): +async def token_counter(request: TokenCountRequest, is_direct_request: bool = True): """ """ from litellm import token_counter @@ -5653,6 +5708,31 @@ async def token_counter(request: TokenCountRequest): litellm_model_name or request.model ) # use litellm model name, if it's not avalable then fallback to request.model + # Try provider-specific token counting first - only for non-direct requests (from provider endpoints) + provider_counter = None + if deployment is not None and not is_direct_request: + # Auto-route to the correct provider based on model + provider_counter = _get_provider_token_counter(deployment, model_to_use) + + if provider_counter is not None and provider_counter.supports_provider( + deployment=deployment, from_endpoint=not is_direct_request + ): + result = await provider_counter.count_tokens( + model_to_use=model_to_use, + messages=messages, # type: ignore + deployment=deployment, + request_model=request.model, + ) + + if result is not None: + return TokenCountResponse( + total_tokens=result["total_tokens"], + request_model=result["request_model"], + model_used=result["model_used"], + tokenizer_type=result["tokenizer_type"], + ) + + # Default LiteLLM token counting custom_tokenizer: Optional[CustomHuggingfaceTokenizer] = None if model_info is not None: custom_tokenizer = cast( @@ -6930,56 +7010,21 @@ async def model_group_info( status_code=500, detail={"error": "LLM Router is not loaded in"} ) - ## CHECK IF MODEL RESTRICTIONS ARE SET AT KEY/TEAM LEVEL ## - model_access_groups: Dict[str, List[str]] = defaultdict(list) - if llm_router is None: - proxy_model_list = [] - else: - proxy_model_list = llm_router.get_model_names() - model_access_groups = llm_router.get_model_access_groups() + from litellm.proxy.utils import get_available_models_for_user - key_models = get_key_models( + # Get available models for the user + all_models_str = await get_available_models_for_user( user_api_key_dict=user_api_key_dict, - proxy_model_list=proxy_model_list, - model_access_groups=model_access_groups, - ) - team_models = [] - if ( - not user_api_key_dict.team_id - and user_api_key_dict.user_id is not None - and not _user_has_admin_view(user_api_key_dict) - ): - if prisma_client is None: - raise HTTPException( - status_code=500, - detail={"error": CommonProxyErrors.db_not_connected_error.value}, - ) - user_object = await prisma_client.db.litellm_usertable.find_first( - where={"user_id": user_api_key_dict.user_id} - ) - user_object_typed = LiteLLM_UserTable(**user_object.model_dump()) - user_models = [] - if user_object is not None: - user_models = get_team_models( - team_models=user_object_typed.models, - proxy_model_list=proxy_model_list, - model_access_groups=model_access_groups, - ) - team_models = user_models - else: - team_models = get_team_models( - team_models=user_api_key_dict.team_models, - proxy_model_list=proxy_model_list, - model_access_groups=model_access_groups, - ) - - all_models_str = get_complete_model_list( - key_models=key_models, - team_models=team_models, - proxy_model_list=proxy_model_list, - user_model=user_model, - infer_model_from_keys=general_settings.get("infer_model_from_keys", False), llm_router=llm_router, + general_settings=general_settings, + user_model=user_model, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + team_id=None, + include_model_access_groups=False, + only_model_access_groups=False, + return_wildcard_routes=False, + user_api_key_cache=user_api_key_cache, ) model_groups: List[ModelGroupInfoProxy] = _get_model_group_info( llm_router=llm_router, all_models_str=all_models_str, model_group=model_group @@ -7727,6 +7772,13 @@ async def claim_onboarding_link(data: InvitationClaim): return user_obj +@app.get("/get_logo_url", include_in_schema=False) +def get_logo_url(): + """Get the current logo URL from environment""" + logo_path = os.getenv("UI_LOGO_PATH", "") + return {"logo_url": logo_path} + + @app.get("/get_image", include_in_schema=False) def get_image(): """Get logo to show on admin UI""" diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index e57539a612b..eb48f7de2dc 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -153,6 +153,8 @@ async def route_request( "aget_responses", "adelete_responses", "alist_input_items", + "avector_store_create", + "avector_store_search", ]: # moderation endpoint does not require `model` parameter return getattr(llm_router, f"{route_type}")(**data) diff --git a/litellm/proxy/spend_tracking/cold_storage_handler.py b/litellm/proxy/spend_tracking/cold_storage_handler.py new file mode 100644 index 00000000000..21e785425ad --- /dev/null +++ b/litellm/proxy/spend_tracking/cold_storage_handler.py @@ -0,0 +1,73 @@ +""" +This module is responsible for handling Getting/Setting the proxy server request from cold storage. + +It allows fetching a dict of the proxy server request from s3 or GCS bucket. +""" +from typing import Optional, cast + +import litellm +from litellm import _custom_logger_compatible_callbacks_literal +from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_logger import CustomLogger + + +class ColdStorageHandler: + """ + This class is responsible for handling Getting/Setting the proxy server request from cold storage. + + It allows fetching a dict of the proxy server request from s3 or GCS bucket. + """ + + async def get_proxy_server_request_from_cold_storage_with_object_key( + self, + object_key: str, + ) -> Optional[dict]: + """ + Get the proxy server request from cold storage using the object key directly. + + Args: + object_key: The S3/GCS object key to retrieve + + Returns: + Optional[dict]: The proxy server request dict or None if not found + """ + + # select the custom logger to use for cold storage + custom_logger_name: Optional[_custom_logger_compatible_callbacks_literal] = self._select_custom_logger_for_cold_storage() + + # if no custom logger name is configured, return None + if custom_logger_name is None: + return None + + # get the active/initialized custom logger + custom_logger: Optional[CustomLogger] = litellm.logging_callback_manager.get_active_custom_logger_for_callback_name(custom_logger_name) + + # if no custom logger is found, return None + if custom_logger is None: + return None + + proxy_server_request = await custom_logger.get_proxy_server_request_from_cold_storage_with_object_key( + object_key=object_key, + ) + + return proxy_server_request + + + + def _select_custom_logger_for_cold_storage( + self, + ) -> Optional[_custom_logger_compatible_callbacks_literal]: + cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = ColdStorageHandler._get_configured_cold_storage_custom_logger() + + return cold_storage_custom_logger + + + @staticmethod + def _get_configured_cold_storage_custom_logger() -> Optional[_custom_logger_compatible_callbacks_literal]: + from litellm.proxy.proxy_server import general_settings + cold_storage_custom_logger: Optional[str] = general_settings.get("cold_storage_custom_logger") + if not cold_storage_custom_logger: + verbose_proxy_logger.debug("No cold storage custom logger found in general settings") + return None + + return cast(_custom_logger_compatible_callbacks_literal, cold_storage_custom_logger) \ No newline at end of file diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index ad5cad29e60..653426e8bd2 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -53,6 +53,7 @@ def _get_spend_logs_metadata( guardrail_information: Optional[StandardLoggingGuardrailInformation] = None, usage_object: Optional[dict] = None, model_map_information: Optional[StandardLoggingModelInformation] = None, + cold_storage_object_key: Optional[str] = None ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -75,6 +76,7 @@ def _get_spend_logs_metadata( model_map_information=None, usage_object=None, guardrail_information=None, + cold_storage_object_key=cold_storage_object_key, ) verbose_proxy_logger.debug( "getting payload for SpendLogs, available keys in metadata: " @@ -98,6 +100,8 @@ def _get_spend_logs_metadata( clean_metadata["guardrail_information"] = guardrail_information clean_metadata["usage_object"] = usage_object clean_metadata["model_map_information"] = model_map_information + clean_metadata["cold_storage_object_key"] = cold_storage_object_key + return clean_metadata @@ -267,6 +271,11 @@ def get_logging_payload( # noqa: PLR0915 if standard_logging_payload is not None else None ), + cold_storage_object_key=( + standard_logging_payload["metadata"].get("cold_storage_object_key", None) + if standard_logging_payload is not None + else None + ), ) special_usage_fields = ["completion_tokens", "prompt_tokens", "total_tokens"] @@ -474,6 +483,7 @@ def _sanitize_request_body_for_spend_logs_payload( Recursively sanitize request body to prevent logging large base64 strings or other large values. Truncates strings longer than 1000 characters and handles nested dictionaries. """ + from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD MAX_STRING_LENGTH = 1000 if visited is None: @@ -492,7 +502,7 @@ def _sanitize_request_body_for_spend_logs_payload( return [_sanitize_value(item) for item in value] elif isinstance(value, str): if len(value) > MAX_STRING_LENGTH: - return f"{value[:MAX_STRING_LENGTH]}... (truncated {len(value) - MAX_STRING_LENGTH} chars)" + return f"{value[:MAX_STRING_LENGTH]}... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} {len(value) - MAX_STRING_LENGTH} chars)" return value return value diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 8557bd2bafb..83acf0e8226 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -1,7 +1,7 @@ #### CRUD ENDPOINTS for UI Settings ##### -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Union, Optional -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, UploadFile, File import litellm from litellm._logging import verbose_proxy_logger @@ -19,6 +19,16 @@ class IPAddress(BaseModel): ip: str +class UIThemeConfig(BaseModel): + """Configuration for UI theme customization""" + + # Logo configuration + logo_url: Optional[str] = Field( + default=None, + description="URL or path to custom logo image. Can be a local file path or HTTP/HTTPS URL" + ) + + class SettingsResponse(BaseModel): """Base response model for settings with values and schema information""" @@ -47,6 +57,12 @@ class DefaultTeamSettingsResponse(SettingsResponse): pass +class UIThemeSettingsResponse(SettingsResponse): + """Response model for UI theme settings""" + + pass + + @router.get( "/get/allowed_ips", tags=["Budget & Spend Tracking"], @@ -507,3 +523,155 @@ async def update_sso_settings(sso_config: SSOConfig): "status": "success", "settings": sso_data, } + + +@router.get( + "/get/ui_theme_settings", + tags=["UI Theme Settings"], + dependencies=[Depends(user_api_key_auth)], + response_model=UIThemeSettingsResponse, +) +async def get_ui_theme_settings(): + """ + Get UI theme configuration from the litellm_settings. + Returns current logo settings for UI customization. + """ + from litellm.proxy.proxy_server import proxy_config + + # Load existing config + config = await proxy_config.get_config() + + return await _get_settings_with_schema( + settings_key="ui_theme_config", + settings_class=UIThemeConfig, + config=config, + ) + + +@router.patch( + "/update/ui_theme_settings", + tags=["UI Theme Settings"], + dependencies=[Depends(user_api_key_auth)], +) +async def update_ui_theme_settings(theme_config: UIThemeConfig): + """ + Update UI theme configuration. + Updates logo settings for the admin UI. + """ + from litellm.proxy.proxy_server import proxy_config, store_model_in_db + import os + + if store_model_in_db is not True: + raise HTTPException( + status_code=500, + detail={ + "error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature." + }, + ) + + # Load existing config + config = await proxy_config.get_config() + + # Update config with UI theme settings + if "general_settings" not in config: + config["general_settings"] = {} + + if "environment_variables" not in config: + config["environment_variables"] = {} + + # Convert theme config to dict + theme_data = theme_config.model_dump(exclude_none=True) + + # Store UI theme config in litellm_settings (where it's retrieved from) + if "litellm_settings" not in config: + config["litellm_settings"] = {} + config["litellm_settings"]["ui_theme_config"] = theme_data + + # Update UI_LOGO_PATH environment variable if logo_url is provided + # If logo_url is empty string, None, or null, remove the environment variable to use default + logo_url = theme_data.get("logo_url") + verbose_proxy_logger.debug(f"Updating logo_url: {logo_url}") + + if logo_url and isinstance(logo_url, str) and logo_url.strip(): # Check if logo_url exists and is not empty/whitespace + config["environment_variables"]["UI_LOGO_PATH"] = logo_url + os.environ["UI_LOGO_PATH"] = logo_url + verbose_proxy_logger.debug(f"Set UI_LOGO_PATH to: {logo_url}") + else: + # Remove the environment variable to restore default logo + if "UI_LOGO_PATH" in config.get("environment_variables", {}): + del config["environment_variables"]["UI_LOGO_PATH"] + verbose_proxy_logger.debug("Removed UI_LOGO_PATH from config") + if "UI_LOGO_PATH" in os.environ: + del os.environ["UI_LOGO_PATH"] + verbose_proxy_logger.debug("Removed UI_LOGO_PATH from environment") + + # Handle environment variable encryption if needed + stored_config = config.copy() + if "environment_variables" in stored_config and len(stored_config["environment_variables"]) > 0: + # Only encrypt if there are environment variables to encrypt + stored_config["environment_variables"] = proxy_config._encrypt_env_variables( + environment_variables=stored_config["environment_variables"] + ) + + # Save the updated config + await proxy_config.save_config(new_config=stored_config) + + return { + "message": "Logo settings updated successfully.", + "status": "success", + "theme_config": theme_data, + } + + +@router.post( + "/upload/logo", + tags=["UI Theme Settings"], + dependencies=[Depends(user_api_key_auth)], +) +async def upload_logo(file: UploadFile = File(...)): + """ + Upload a custom logo for the admin UI. + Accepts image files (PNG, JPG, JPEG, SVG) and stores them for use in the UI. + """ + import os + from pathlib import Path + + # Validate file type + allowed_extensions = {".png", ".jpg", ".jpeg", ".svg"} + file_extension = Path(file.filename or "").suffix.lower() + + if file_extension not in allowed_extensions: + raise HTTPException( + status_code=400, + detail=f"Invalid file type. Allowed types: {', '.join(allowed_extensions)}" + ) + + # Validate file size (max 5MB) + file_content = await file.read() + if len(file_content) > 5 * 1024 * 1024: # 5MB + raise HTTPException( + status_code=400, + detail="File size too large. Maximum size is 5MB." + ) + + # Create uploads directory if it doesn't exist + current_dir = os.path.dirname(os.path.abspath(__file__)) + upload_dir = os.path.join(current_dir, "..", "uploads") + os.makedirs(upload_dir, exist_ok=True) + + # Generate unique filename + import uuid + unique_filename = f"logo_{uuid.uuid4().hex}{file_extension}" + file_path = os.path.join(upload_dir, unique_filename) + + # Save the file + with open(file_path, "wb") as buffer: + buffer.write(file_content) + + return { + "message": "Logo uploaded successfully", + "status": "success", + "file_path": file_path, + "filename": unique_filename, + "file_size": len(file_content), + } diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index f1e113d8ded..0e6b242bdb6 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -22,7 +22,7 @@ from typing import ( overload, ) -from litellm.constants import MAX_TEAM_LIST_LIMIT +from litellm.constants import MAX_TEAM_LIST_LIMIT, DEFAULT_MODEL_CREATED_AT_TIME from litellm.proxy._types import ( DB_CONNECTION_ERROR_TYPES, CommonProxyErrors, @@ -55,11 +55,7 @@ from litellm import ( from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceLogging, ServiceTypes from litellm.caching.caching import DualCache, RedisCache -from litellm.exceptions import ( - BlockedPiiEntityError, - GuardrailRaisedException, - RejectedRequestError, -) +from litellm.exceptions import RejectedRequestError from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting @@ -452,108 +448,6 @@ class ProxyLogging: litellm_parent_otel_span=None, ) - async def async_pre_mcp_tool_call_hook( - self, - kwargs: dict, - request_obj: Any, - start_time: datetime, - end_time: datetime, - ) -> Optional[Any]: - """ - Pre MCP Tool Call Hook - - Use this to validate and modify MCP tool calls before execution. - Reuses existing LLM guardrail logic by converting MCP calls to message format. - """ - from litellm.types.llms.base import HiddenParams - from litellm.types.mcp import MCPPreCallRequestObject - - callbacks = self.get_combined_callback_list( - dynamic_success_callbacks=getattr(self, "dynamic_success_callbacks", None), - global_callbacks=litellm.success_callback, - ) - - # Create the request object if it's not already one - if not isinstance(request_obj, MCPPreCallRequestObject): - # Convert UserAPIKeyAuth object to dict if needed - user_api_key_auth_dict = self._convert_user_api_key_auth_to_dict( - kwargs.get("user_api_key_auth") - ) - - request_obj = MCPPreCallRequestObject( - tool_name=kwargs.get("name", ""), - arguments=kwargs.get("arguments", {}), - server_name=kwargs.get("server_name"), - user_api_key_auth=user_api_key_auth_dict, - hidden_params=HiddenParams(), - ) - - for callback in callbacks: - try: - _callback: Optional[CustomLogger] = None - if isinstance(callback, str): - from typing import cast - - from litellm import _custom_logger_compatible_callbacks_literal - - _callback = litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class( - cast(_custom_logger_compatible_callbacks_literal, callback) - ) - else: - _callback = callback # type: ignore - - if _callback is not None and isinstance(_callback, CustomGuardrail): - from litellm.types.guardrails import GuardrailEventHooks - - # Check if guardrail should be run for pre_call hook (reusing existing logic) - if ( - _callback.should_run_guardrail( - data=kwargs, event_type=GuardrailEventHooks.pre_mcp_call - ) - is not True - ): - continue - - # Convert MCP tool call to LLM message format for existing guardrail logic - synthetic_llm_data = self._convert_mcp_to_llm_format( - request_obj, kwargs - ) - # Reuse existing LLM guardrail logic - user_api_key_auth_dict = self._convert_user_api_key_auth_to_dict( - kwargs.get("user_api_key_auth") - ) - - result = await _callback.async_pre_call_hook( - user_api_key_dict=user_api_key_auth_dict, # type: ignore - cache=self.call_details["user_api_key_cache"], - data=synthetic_llm_data, - call_type="mcp_call", - ) - - # Convert result back to MCP response format if blocked/modified - if result is not None: - mcp_response = self._convert_llm_result_to_mcp_response( - result, request_obj - ) - if mcp_response is not None: - return self._parse_pre_mcp_call_hook_response( - response=mcp_response, original_request=request_obj - ) - - except ( - BlockedPiiEntityError, - GuardrailRaisedException, - HTTPException, - ) as e: - # Re-raise guardrail exceptions so they can be properly handled - raise e - except Exception as e: - verbose_proxy_logger.exception( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {}".format( - str(e) - ) - ) - return None def _convert_user_api_key_auth_to_dict(self, user_api_key_auth_obj): """ @@ -567,7 +461,7 @@ class ProxyLogging: elif hasattr(user_api_key_auth_obj, "__dict__"): # If it's a regular object, convert to dict return user_api_key_auth_obj.__dict__ - return user_api_key_auth_obj + return {} def _convert_mcp_to_llm_format(self, request_obj, kwargs: dict) -> dict: """ @@ -765,8 +659,6 @@ class ProxyLogging: """ Convert LLM guardrail result back to MCP during call response format. """ - from litellm.types.mcp import MCPDuringCallResponseObject - # If result is an exception, it means the guardrail wants to stop execution if isinstance(llm_result, Exception): return MCPDuringCallResponseObject( @@ -836,112 +728,39 @@ class ProxyLogging: } return result - async def async_during_mcp_tool_call_hook( - self, - kwargs: dict, - request_obj: Any, - start_time: datetime, - end_time: datetime, - ) -> Optional[Any]: + def _create_mcp_request_object_from_kwargs(self, kwargs: dict) -> "MCPPreCallRequestObject": """ - During MCP Tool Call Hook - - Use this for concurrent monitoring and validation during tool execution. - Reuses existing LLM guardrail logic by converting MCP calls to message format. + Helper function to create MCPPreCallRequestObject from kwargs for standard pre_call_hook. """ from litellm.types.llms.base import HiddenParams - from litellm.types.mcp import MCPDuringCallRequestObject + from litellm.types.mcp import MCPPreCallRequestObject - callbacks = self.get_combined_callback_list( - dynamic_success_callbacks=getattr(self, "dynamic_success_callbacks", None), - global_callbacks=litellm.success_callback, + user_api_key_auth_dict = self._convert_user_api_key_auth_to_dict(kwargs.get("user_api_key_auth")) + + return MCPPreCallRequestObject( + tool_name=kwargs.get("name", ""), + arguments=kwargs.get("arguments", {}), + server_name=kwargs.get("server_name"), + user_api_key_auth=user_api_key_auth_dict, + hidden_params=HiddenParams(), ) - # Create the request object if it's not already one - if not isinstance(request_obj, MCPDuringCallRequestObject): - request_obj = MCPDuringCallRequestObject( - tool_name=kwargs.get("name", ""), - arguments=kwargs.get("arguments", {}), - server_name=kwargs.get("server_name"), - start_time=start_time.timestamp() if start_time else None, - hidden_params=HiddenParams(), - ) - - for callback in callbacks: - try: - _callback: Optional[CustomLogger] = None - if isinstance(callback, str): - from typing import cast - - from litellm import _custom_logger_compatible_callbacks_literal - - _callback = litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class( - cast(_custom_logger_compatible_callbacks_literal, callback) - ) - else: - _callback = callback # type: ignore - - if _callback is not None and isinstance(_callback, CustomGuardrail): - from litellm.types.guardrails import GuardrailEventHooks - - # Check if guardrail should be run for during_call hook (reusing existing logic) - if ( - _callback.should_run_guardrail( - data=kwargs, event_type=GuardrailEventHooks.during_mcp_call - ) - is not True - ): - continue - # Convert MCP tool call to LLM message format for existing guardrail logic - synthetic_llm_data = self._convert_mcp_to_llm_format( - request_obj, kwargs - ) - - # Reuse existing LLM guardrail logic for during call - user_api_key_auth_dict = self._convert_user_api_key_auth_to_dict( - kwargs.get("user_api_key_auth") - ) - - result = await _callback.async_moderation_hook( - data=synthetic_llm_data, - user_api_key_dict=user_api_key_auth_dict, # type: ignore - call_type="mcp_call", - ) - # Convert result back to MCP response format if blocked/modified - if result is not None: - mcp_response = self._convert_llm_result_to_mcp_during_response( - result, request_obj - ) - if mcp_response is not None: - return self._parse_during_mcp_call_hook_response( - response=mcp_response - ) - - except Exception as e: - raise e - verbose_proxy_logger.exception( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {}".format( - str(e) - ) - ) - return None - - def _parse_during_mcp_call_hook_response( - self, response: MCPDuringCallResponseObject - ) -> Dict[str, Any]: + def _convert_mcp_hook_response_to_kwargs(self, response_data: Optional[dict], original_kwargs: dict) -> dict: """ - Parse the response from the during_mcp_tool_call_hook - - 1. Check if execution should continue - 2. Handle any error messages - 3. Apply any hidden parameter updates + Helper function to convert pre_call_hook response back to kwargs for MCP usage. """ - result = { - "should_continue": response.should_continue, - "error_message": response.error_message, - "hidden_params": response.hidden_params, - } - return result + if not response_data: + return original_kwargs + + # Apply any argument modifications from the hook response + modified_kwargs = original_kwargs.copy() + + # If the response contains modified arguments, apply them + if response_data.get("modified_arguments"): + modified_kwargs["arguments"] = response_data["modified_arguments"] + + return modified_kwargs + async def process_pre_call_hook_response(self, response, data, call_type): if isinstance(response, Exception): @@ -975,6 +794,7 @@ class ProxyLogging: "audio_transcription", "pass_through_endpoint", "rerank", + "mcp_call", ], ) -> None: pass @@ -993,6 +813,7 @@ class ProxyLogging: "audio_transcription", "pass_through_endpoint", "rerank", + "mcp_call", ], ) -> dict: pass @@ -1010,6 +831,7 @@ class ProxyLogging: "audio_transcription", "pass_through_endpoint", "rerank", + "mcp_call", ], ) -> Optional[dict]: """ @@ -1081,10 +903,14 @@ class ProxyLogging: _callback = callback # type: ignore if _callback is not None and isinstance(_callback, CustomGuardrail): from litellm.types.guardrails import GuardrailEventHooks - + + event_type = GuardrailEventHooks.pre_call + if call_type == "mcp_call": + event_type = GuardrailEventHooks.pre_mcp_call + if ( _callback.should_run_guardrail( - data=data, event_type=GuardrailEventHooks.pre_call + data=data, event_type=event_type ) is not True ): @@ -1108,6 +934,9 @@ class ProxyLogging: and _callback.__class__.async_pre_call_hook != CustomLogger.async_pre_call_hook ): + if call_type == "mcp_call" and user_api_key_dict is None: + continue + response = await _callback.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=self.call_details["user_api_key_cache"], @@ -1126,7 +955,7 @@ class ProxyLogging: async def during_call_hook( self, data: dict, - user_api_key_dict: UserAPIKeyAuth, + user_api_key_dict: Optional[UserAPIKeyAuth], call_type: Literal[ "completion", "responses", @@ -1134,6 +963,7 @@ class ProxyLogging: "image_generation", "moderation", "audio_transcription", + "mcp_call", ], ): """ @@ -1156,16 +986,26 @@ class ProxyLogging: # Main - V2 Guardrails implementation from litellm.types.guardrails import GuardrailEventHooks + event_type = GuardrailEventHooks.during_call + if call_type == "mcp_call": + event_type = GuardrailEventHooks.during_mcp_call + if ( callback.should_run_guardrail( - data=data, event_type=GuardrailEventHooks.during_call + data=data, event_type=event_type ) is not True ): continue + # Convert user_api_key_dict to proper format for async_moderation_hook + if call_type == "mcp_call": + user_api_key_auth_dict = self._convert_user_api_key_auth_to_dict(user_api_key_dict) + else: + user_api_key_auth_dict = user_api_key_dict + await callback.async_moderation_hook( data=data, - user_api_key_dict=user_api_key_dict, + user_api_key_dict=user_api_key_auth_dict, # type: ignore call_type=call_type, ) except Exception as e: @@ -3802,7 +3642,6 @@ def is_valid_api_key(key: str) -> bool: def construct_database_url_from_env_vars() -> Optional[str]: """ Construct a DATABASE_URL from individual environment variables. - Returns: Optional[str]: The constructed DATABASE_URL or None if required variables are missing """ @@ -3829,5 +3668,234 @@ def construct_database_url_from_env_vars() -> Optional[str]: database_url = f"postgresql://{database_username_enc}@{database_host}/{database_name_enc}" return database_url - + return None + +async def count_tokens_with_anthropic_api( + model_to_use: str, + messages: Optional[List[Dict[str, Any]]], + deployment: Optional[Dict[str, Any]] = None, +) -> Optional[Dict[str, Any]]: + """ + Helper function to count tokens using Anthropic API directly. + + Args: + model_to_use: The model name to use for token counting + messages: The messages to count tokens for + deployment: Optional deployment configuration containing API key + + Returns: + Optional dict with token count and tokenizer info, or None if failed + """ + if not messages: + return None + + try: + import anthropic + import os + + # Get Anthropic API key from deployment config + anthropic_api_key = None + if deployment is not None: + anthropic_api_key = deployment.get("litellm_params", {}).get("api_key") + + # Fallback to environment variable + if not anthropic_api_key: + anthropic_api_key = os.getenv("ANTHROPIC_API_KEY") + + if anthropic_api_key and messages: + # Call Anthropic API directly for more accurate token counting + client = anthropic.Anthropic(api_key=anthropic_api_key) + + # Call with explicit parameters to satisfy type checking + # Type ignore for now since messages come from generic dict input + response = client.beta.messages.count_tokens( + model=model_to_use, + messages=messages, # type: ignore + betas=["token-counting-2024-11-01"] + ) + total_tokens = response.input_tokens + tokenizer_used = "anthropic_api" + + return { + "total_tokens": total_tokens, + "tokenizer_used": tokenizer_used, + } + + except ImportError: + verbose_proxy_logger.warning("Anthropic library not available, falling back to LiteLLM tokenizer") + except Exception as e: + verbose_proxy_logger.warning(f"Error calling Anthropic API: {e}, falling back to LiteLLM tokenizer") + return None + +async def get_available_models_for_user( + user_api_key_dict: "UserAPIKeyAuth", + llm_router: Optional["Router"], + general_settings: dict, + user_model: Optional[str], + prisma_client: Optional["PrismaClient"] = None, + proxy_logging_obj: Optional["ProxyLogging"] = None, + team_id: Optional[str] = None, + include_model_access_groups: bool = False, + only_model_access_groups: bool = False, + return_wildcard_routes: bool = False, + user_api_key_cache: Optional["DualCache"] = None, +) -> List[str]: + """ + Get the list of models available to a user based on their API key and team permissions. + + Args: + user_api_key_dict: User API key authentication object + llm_router: LiteLLM router instance + general_settings: General settings from config + user_model: User-specific model + prisma_client: Prisma client for database operations + proxy_logging_obj: Proxy logging object + team_id: Specific team ID to check (optional) + include_model_access_groups: Whether to include model access groups + only_model_access_groups: Whether to only return model access groups + return_wildcard_routes: Whether to return wildcard routes + + Returns: + List of model names available to the user + """ + from litellm.proxy.auth.model_checks import ( + get_key_models, + get_team_models, + get_complete_model_list, + ) + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.management_endpoints.team_endpoints import validate_membership + + # Get proxy model list and access groups + if llm_router is None: + proxy_model_list = [] + model_access_groups = {} + else: + proxy_model_list = llm_router.get_model_names() + model_access_groups = llm_router.get_model_access_groups() + + # Get key models + key_models = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + ) + + # Get team models + team_models: List[str] = user_api_key_dict.team_models + + # If specific team_id is provided, validate and get team models + if team_id and prisma_client and proxy_logging_obj and user_api_key_cache: + key_models = [] + team_object = await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + validate_membership(user_api_key_dict=user_api_key_dict, team_table=team_object) + team_models = team_object.models + + team_models = get_team_models( + team_models=team_models, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + ) + + # Get complete model list + all_models = get_complete_model_list( + key_models=key_models, + team_models=team_models, + proxy_model_list=proxy_model_list, + user_model=user_model, + infer_model_from_keys=general_settings.get("infer_model_from_keys", False), + return_wildcard_routes=return_wildcard_routes, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + only_model_access_groups=only_model_access_groups, + ) + + return all_models + + +def create_model_info_response( + model_id: str, + provider: str, + include_metadata: bool = False, + fallback_type: Optional[str] = None, + llm_router: Optional["Router"] = None, +) -> dict: + """ + Create a standardized model info response. + + Args: + model_id: The model ID + provider: The model provider + include_metadata: Whether to include metadata + fallback_type: Type of fallbacks to include + llm_router: LiteLLM router instance + + Returns: + Dictionary containing model information + """ + from litellm.proxy.auth.model_checks import get_all_fallbacks + + model_info = { + "id": model_id, + "object": "model", + "created": DEFAULT_MODEL_CREATED_AT_TIME, + "owned_by": provider, + } + + # Add metadata if requested + if include_metadata: + metadata = {} + + # Default fallback_type to "general" if include_metadata is true + effective_fallback_type = ( + fallback_type if fallback_type is not None else "general" + ) + + # Validate fallback_type + valid_fallback_types = ["general", "context_window", "content_policy"] + if effective_fallback_type not in valid_fallback_types: + raise HTTPException( + status_code=400, + detail=f"Invalid fallback_type. Must be one of: {valid_fallback_types}", + ) + + fallbacks = get_all_fallbacks( + model=model_id, + llm_router=llm_router, + fallback_type=effective_fallback_type, + ) + metadata["fallbacks"] = fallbacks + + model_info["metadata"] = metadata + + return model_info + + +def validate_model_access( + model_id: str, + available_models: List[str], +) -> None: + """ + Validate that a model is accessible to the user. + + Args: + model_id: The model ID to validate + available_models: List of models available to the user + + Raises: + HTTPException: If the model is not accessible + """ + if model_id not in available_models: + raise HTTPException( + status_code=404, + detail="The model `{}` does not exist or is not accessible".format(model_id) + ) diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py new file mode 100644 index 00000000000..28e76ca893f --- /dev/null +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -0,0 +1,301 @@ +import json +from typing import TYPE_CHECKING, Any, List, Optional, Union, cast + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import SpendLogsPayload +from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionResponseMessage, + GenericChatCompletionMessage, + ResponseInputParam, +) +from litellm.types.utils import ChatCompletionMessageToolCall, Message, ModelResponse + +if TYPE_CHECKING: + from litellm.responses.litellm_completion_transformation.transformation import ( + ChatCompletionSession, + ) +else: + ChatCompletionSession = Any + +######################################################## +# Cold Storage Handler +######################################################## +COLD_STORAGE_HANDLER = ColdStorageHandler() +######################################################## + +class ResponsesSessionHandler: + @staticmethod + async def get_chat_completion_message_history_for_previous_response_id( + previous_response_id: str, + ) -> ChatCompletionSession: + """ + Return the chat completion message history for a previous response id + """ + from litellm.responses.litellm_completion_transformation.transformation import ( + ChatCompletionSession, + ) + + verbose_proxy_logger.debug( + "inside get_chat_completion_message_history_for_previous_response_id" + ) + all_spend_logs: List[ + SpendLogsPayload + ] = await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( + previous_response_id + ) + verbose_proxy_logger.debug( + "found %s spend logs for this response id", len(all_spend_logs) + ) + + litellm_session_id: Optional[str] = None + if len(all_spend_logs) > 0: + litellm_session_id = all_spend_logs[0].get("session_id") + + chat_completion_message_history: List[ + Union[ + AllMessageValues, + GenericChatCompletionMessage, + ChatCompletionMessageToolCall, + ChatCompletionResponseMessage, + Message, + ] + ] = [] + for spend_log in all_spend_logs: + chat_completion_message_history = await ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload( + spend_log=spend_log, + chat_completion_message_history=chat_completion_message_history, + ) + + verbose_proxy_logger.debug( + "chat_completion_message_history %s", + json.dumps(chat_completion_message_history, indent=4, default=str), + ) + return ChatCompletionSession( + messages=chat_completion_message_history, + litellm_session_id=litellm_session_id, + ) + + @staticmethod + async def extend_chat_completion_message_with_spend_log_payload( + spend_log: SpendLogsPayload, + chat_completion_message_history: List[ + Union[ + AllMessageValues, + GenericChatCompletionMessage, + ChatCompletionMessageToolCall, + ChatCompletionResponseMessage, + Message, + ] + ] + ): + """ + Extend the chat completion message history with the spend log payload + """ + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + proxy_server_request_dict = await ResponsesSessionHandler.get_proxy_server_request_from_spend_log( + spend_log=spend_log, + ) + response_input_param: Optional[Union[str, ResponseInputParam]] = None + _messages: Optional[Union[str, ResponseInputParam]] = None + + ############################################################ + # Add Input messages for this Spend Log + ############################################################ + if proxy_server_request_dict: + _response_input_param = proxy_server_request_dict.get("input", None) + _messages = proxy_server_request_dict.get("messages", None) + if isinstance(_response_input_param, str): + response_input_param = _response_input_param + elif isinstance(_response_input_param, dict): + response_input_param = cast( + ResponseInputParam, _response_input_param + ) + + if response_input_param: + chat_completion_messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=response_input_param, + responses_api_request=proxy_server_request_dict or {}, + ) + chat_completion_message_history.extend(chat_completion_messages) + + ############################################################ + # Check if `messages` field is present in the proxy server request dict + ############################################################ + elif _messages: + # ensure all messages are /chat/completions/messages + # certain requests can be stored as Responses API format - this ensures they are transformed to /chat/completions/messages + chat_completion_messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=_messages, + responses_api_request=proxy_server_request_dict or {}, + ) + chat_completion_message_history.extend(chat_completion_messages) + + ############################################################ + # Add Output messages for this Spend Log + ############################################################ + _response_output = spend_log.get("response", "{}") + if isinstance(_response_output, dict): + # transform `ChatCompletion Response` to `ResponsesAPIResponse` + model_response = ModelResponse(**_response_output) + for choice in model_response.choices: + if hasattr(choice, "message"): + chat_completion_message_history.append( + getattr(choice, "message") + ) + return chat_completion_message_history + + @staticmethod + async def get_proxy_server_request_from_spend_log( + spend_log: SpendLogsPayload, + ) -> Optional[dict]: + """ + Get the parsed proxy server request from the spend log + """ + proxy_server_request: Union[str, dict] = ( + spend_log.get("proxy_server_request") or "{}" + ) + proxy_server_request_dict: Optional[dict] = None + if isinstance(proxy_server_request, dict): + proxy_server_request_dict = proxy_server_request + else: + proxy_server_request_dict = json.loads(proxy_server_request) + + + ############################################################ + # Check if user has setup cold storage for session handling + ############################################################ + if ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_server_request_dict): + # Try to get cold storage object key from spend log metadata + _proxy_server_request_dict: Optional[dict] = None + cold_storage_object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log) + if cold_storage_object_key: + # Use the object key directly from metadata + _proxy_server_request_dict = await ResponsesSessionHandler.get_proxy_server_request_from_cold_storage_with_object_key( + object_key=cold_storage_object_key, + ) + if _proxy_server_request_dict: + proxy_server_request_dict = _proxy_server_request_dict + + return proxy_server_request_dict + + @staticmethod + def _get_cold_storage_object_key_from_spend_log(spend_log: SpendLogsPayload) -> Optional[str]: + """ + Extract the cold storage object key from spend log metadata. + + Args: + spend_log: The spend log payload containing metadata + + Returns: + Optional[str]: The cold storage object key if found, None otherwise + """ + try: + metadata_str = spend_log.get("metadata", "{}") + if isinstance(metadata_str, str): + metadata_dict = json.loads(metadata_str) + return metadata_dict.get("cold_storage_object_key") + elif isinstance(metadata_str, dict): + return metadata_str.get("cold_storage_object_key") + return None + except (json.JSONDecodeError, TypeError, AttributeError): + verbose_proxy_logger.debug("Failed to parse metadata from spend log to extract cold storage object key") + return None + + @staticmethod + async def get_proxy_server_request_from_cold_storage_with_object_key( + object_key: str, + ) -> Optional[dict]: + """ + Get the proxy server request from cold storage using the object key directly. + + Args: + object_key: The S3/GCS object key to retrieve + + Returns: + Optional[dict]: The proxy server request dict or None if not found + """ + verbose_proxy_logger.debug("inside get_proxy_server_request_from_cold_storage_with_object_key...") + + proxy_server_request_dict = await COLD_STORAGE_HANDLER.get_proxy_server_request_from_cold_storage_with_object_key( + object_key=object_key, + ) + + return proxy_server_request_dict + + @staticmethod + def _should_check_cold_storage_for_full_payload( + proxy_server_request_dict: Optional[dict], + ) -> bool: + """ + Only check cold storage when both are true + 1. `LITELLM_TRUNCATED_PAYLOAD_FIELD` is in the proxy server request dict + 2. `ColdStorageHandler._get_configured_cold_storage_custom_logger()` is not None + """ + from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD + configured_cold_storage_custom_logger = ColdStorageHandler._get_configured_cold_storage_custom_logger() + if configured_cold_storage_custom_logger is None: + return False + if proxy_server_request_dict is None: + return True + if len(proxy_server_request_dict) == 0: + return True + if LITELLM_TRUNCATED_PAYLOAD_FIELD in proxy_server_request_dict: + return True + return False + + + + @staticmethod + async def get_all_spend_logs_for_previous_response_id( + previous_response_id: str, + ) -> List[SpendLogsPayload]: + """ + Get all spend logs for a previous response id + + + SQL query + + SELECT session_id FROM spend_logs WHERE response_id = previous_response_id, SELECT * FROM spend_logs WHERE session_id = session_id + """ + from litellm.proxy.proxy_server import prisma_client + + verbose_proxy_logger.debug("decoding response id=%s", previous_response_id) + + decoded_response_id = ( + ResponsesAPIRequestUtils._decode_responses_api_response_id( + previous_response_id + ) + ) + previous_response_id = decoded_response_id.get( + "response_id", previous_response_id + ) + if prisma_client is None: + return [] + + query = """ + WITH matching_session AS ( + SELECT session_id + FROM "LiteLLM_SpendLogs" + WHERE request_id = $1 + ) + SELECT * + FROM "LiteLLM_SpendLogs" + WHERE session_id IN (SELECT session_id FROM matching_session) + ORDER BY "endTime" ASC; + """ + + spend_logs = await prisma_client.db.query_raw(query, previous_response_id) + + verbose_proxy_logger.debug( + "Found the following spend logs for previous response id %s: %s", + previous_response_id, + json.dumps(spend_logs, indent=4, default=str), + ) + + return spend_logs diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b8cde45c611..3706b2f7fc9 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -7,21 +7,15 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Union, cast from openai.types.responses.tool_param import FunctionToolParam from typing_extensions import TypedDict -from litellm._logging import verbose_logger - -try: - from litellm_enterprise.enterprise_callbacks.session_handler import ( - _ENTERPRISE_ResponsesSessionHandler, - ) -except Exception as e: - verbose_logger.debug( - f"[Non-Blocking] Unable to import _ENTERPRISE_ResponsesSessionHandler - LiteLLM Enterprise Feature - {str(e)}" - ) - _ENTERPRISE_ResponsesSessionHandler = None from litellm.caching import InMemoryCache from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.responses.litellm_completion_transformation.session_handler import ( + ResponsesSessionHandler, +) from litellm.types.llms.openai import ( AllMessageValues, + ChatCompletionImageObject, + ChatCompletionImageUrlObject, ChatCompletionResponseMessage, ChatCompletionSystemMessage, ChatCompletionToolCallChunk, @@ -40,7 +34,6 @@ from litellm.types.llms.openai import ( ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ResponseTextConfig, - ChatCompletionImageUrlObject, ) from litellm.types.responses.main import ( GenericResponseOutputItem, @@ -75,13 +68,6 @@ class ChatCompletionSession(TypedDict, total=False): litellm_session_id: Optional[str] -class ChatCompletionImageItem(TypedDict): - """TypedDict for image items in chat completion content""" - - type: Literal["image"] - image_url: ChatCompletionImageUrlObject - - ########### End of Initialize Classes used for Responses API ########### @@ -210,20 +196,19 @@ class LiteLLMCompletionResponsesConfig: """ Async hook to get the chain of previous input and output pairs and return a list of Chat Completion messages """ - if _ENTERPRISE_ResponsesSessionHandler is not None: - chat_completion_session = ChatCompletionSession( - messages=[], litellm_session_id=None + chat_completion_session = ChatCompletionSession( + messages=[], litellm_session_id=None + ) + if previous_response_id: + chat_completion_session = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + previous_response_id=previous_response_id ) - if previous_response_id: - chat_completion_session = await _ENTERPRISE_ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( - previous_response_id=previous_response_id - ) - _messages = litellm_completion_request.get("messages") or [] - session_messages = chat_completion_session.get("messages") or [] - litellm_completion_request["messages"] = session_messages + _messages - litellm_completion_request[ - "litellm_trace_id" - ] = chat_completion_session.get("litellm_session_id") + _messages = litellm_completion_request.get("messages") or [] + session_messages = chat_completion_session.get("messages") or [] + litellm_completion_request["messages"] = session_messages + _messages + litellm_completion_request[ + "litellm_trace_id" + ] = chat_completion_session.get("litellm_session_id") return litellm_completion_request @staticmethod @@ -264,6 +249,10 @@ class LiteLLMCompletionResponsesConfig: chat_completion_messages = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( input_item=_input ) + + ######################################################### + # If Input Item is a Tool Call Output, add it to the tool_call_output_messages list + ######################################################### if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output( input_item=_input ): @@ -316,6 +305,11 @@ class LiteLLMCompletionResponsesConfig: return LiteLLMCompletionResponsesConfig._transform_responses_api_tool_call_output_to_chat_completion_message( tool_call_output=input_item ) + elif LiteLLMCompletionResponsesConfig._is_input_item_function_call(input_item): + # handle function call input items + return LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( + function_call=input_item + ) else: return [ GenericChatCompletionMessage( @@ -337,6 +331,13 @@ class LiteLLMCompletionResponsesConfig: "computer_call_output", ] + @staticmethod + def _is_input_item_function_call(input_item: Any) -> bool: + """ + Check if the input item is a function call + """ + return input_item.get("type") == "function_call" + @staticmethod def _transform_responses_api_tool_call_output_to_chat_completion_message( tool_call_output: Dict[str, Any], @@ -402,6 +403,52 @@ class LiteLLMCompletionResponsesConfig: return [tool_output_message] + @staticmethod + def _transform_responses_api_function_call_to_chat_completion_message( + function_call: Dict[str, Any], + ) -> List[ + Union[ + AllMessageValues, + GenericChatCompletionMessage, + ChatCompletionResponseMessage, + ] + ]: + """ + Transform a Responses API function_call into a Chat Completion message with tool calls + + Handles Input items of this type: + function_call: + ```json + { + "type": "function_call", + "arguments":"{\"location\": \"SĆ£o Paulo, Brazil\"}", + "call_id": "call_v2wlBzrlTIFl9FxPeY774GHZ", + "name": "get_weather", + "id": "fc_685c42deefc0819a822b6936faaa30be0c76bc1491ab6619", + "status": "completed" + } + ``` + """ + # Create a tool call for the function call + tool_call = ChatCompletionToolCallChunk( + id=function_call.get("call_id") or function_call.get("id") or "", + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=function_call.get("name") or "", + arguments=function_call.get("arguments") or "", + ), + index=0, + ) + + # Create an assistant message with the tool call + chat_completion_response_message = ChatCompletionResponseMessage( + tool_calls=[tool_call], + role="assistant", + content=None, # Function calls don't have content + ) + + return [chat_completion_response_message] + @staticmethod def _transform_input_file_item_to_file_item(item: Dict[str, Any]) -> Dict[str, Any]: """ @@ -423,7 +470,7 @@ class LiteLLMCompletionResponsesConfig: return new_item @staticmethod - def _transform_input_image_item_to_image_item(item: Dict[str, Any]) -> ChatCompletionImageItem: + def _transform_input_image_item_to_image_item(item: Dict[str, Any]) -> ChatCompletionImageObject: """ Transform a Responses API input_image item to a Chat Completion image item """ @@ -432,8 +479,8 @@ class LiteLLMCompletionResponsesConfig: detail=item.get("detail") or "auto" ) - return ChatCompletionImageItem( - type="image", + return ChatCompletionImageObject( + type="image_url", image_url=image_url_obj ) @@ -444,7 +491,6 @@ class LiteLLMCompletionResponsesConfig: """ Transform a Responses API content into a Chat Completion content """ - if isinstance(content, str): return content elif isinstance(content, list): diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 07459547a0a..b5f18cf9e1a 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -243,6 +243,8 @@ async def aresponses( top_p: Optional[float] = None, truncation: Optional[Literal["auto", "disabled"]] = None, user: Optional[str] = None, + service_tier: Optional[str] = None, + safety_identifier: Optional[str] = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Optional[Dict[str, Any]] = None, @@ -294,6 +296,8 @@ async def aresponses( extra_body=extra_body, timeout=timeout, custom_llm_provider=custom_llm_provider, + service_tier=service_tier, + safety_identifier=safety_identifier, **kwargs, ) @@ -350,6 +354,8 @@ def responses( top_p: Optional[float] = None, truncation: Optional[Literal["auto", "disabled"]] = None, user: Optional[str] = None, + service_tier: Optional[str] = None, + safety_identifier: Optional[str] = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Optional[Dict[str, Any]] = None, diff --git a/litellm/router.py b/litellm/router.py index ca724006fdf..3be88596b11 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -406,6 +406,9 @@ class Router: self.default_max_parallel_requests = default_max_parallel_requests self.provider_default_deployment_ids: List[str] = [] self.pattern_router = PatternMatchRouter() + self.team_pattern_routers: Dict[str, PatternMatchRouter] = ( + {} + ) # {"TEAM_ID": PatternMatchRouter} self.auto_routers: Dict[str, "AutoRouter"] = {} if model_list is not None: @@ -1207,7 +1210,7 @@ class Router: verbose_router_logger.error( f"Fallback also failed: {fallback_error}" ) - raise fallback_error + raise fallback_error return FallbackStreamWrapper(stream_with_fallbacks()) @@ -1403,7 +1406,7 @@ class Router: kwargs.setdefault(metadata_variable_name, {}).update(metadata_defaults) def _handle_clientside_credential( - self, deployment: dict, kwargs: dict + self, deployment: dict, kwargs: dict, function_name: Optional[str] = None ) -> Deployment: """ Handle clientside credential @@ -1413,8 +1416,11 @@ class Router: dynamic_litellm_params = get_dynamic_litellm_params( litellm_params=litellm_params, request_kwargs=kwargs ) - metadata = kwargs.get("metadata", {}) - model_group = cast(str, metadata.get("model_group")) + # Use deployment model_name as model_group for generating model_id + metadata_variable_name = _get_router_metadata_variable_name( + function_name=function_name, + ) + model_group = kwargs.get(metadata_variable_name, {}).get("model_group") _model_id = self._generate_model_id( model_group=model_group, litellm_params=dynamic_litellm_params ) @@ -1448,7 +1454,7 @@ class Router: deployment_model_name = deployment["model_name"] if is_clientside_credential(request_kwargs=kwargs): deployment_pydantic_obj = self._handle_clientside_credential( - deployment=deployment, kwargs=kwargs + deployment=deployment, kwargs=kwargs, function_name=function_name ) model_info = deployment_pydantic_obj.model_info.model_dump() deployment_litellm_model_name = deployment_pydantic_obj.litellm_params.model @@ -5109,6 +5115,19 @@ class Router: if deployment.model_info.id: self.provider_default_deployment_ids.append(deployment.model_info.id) + _team_id = deployment.model_info.get("team_id") + _team_public_model_name = deployment.model_info.get("team_public_model_name") + if ( + _team_id is not None + and _team_public_model_name is not None + and "*" in _team_public_model_name + ): + if _team_id not in self.team_pattern_routers: + self.team_pattern_routers[_team_id] = PatternMatchRouter() + self.team_pattern_routers[_team_id].add_pattern( + _team_public_model_name, deployment.to_json(exclude_none=True) + ) + # Azure GPT-Vision Enhancements, users can pass os.environ/ data_sources = deployment.litellm_params.get("dataSources", []) or [] @@ -5920,19 +5939,17 @@ class Router: Map a team model name to a team-specific model name. Returns: - - team_model_name: str - the team-specific model name + - deployment id: str - the deployment id of the team-specific model - None: if no team-specific model name is found """ - for model in self.model_list: - model_team_id = model["model_info"].get("team_id") - model_team_public_model_name = model["model_info"].get( - "team_public_model_name" - ) - if ( - model_team_id == team_id - and model_team_public_model_name == team_model_name - ): - return model["model_name"] + models = self.get_model_list(model_name=team_model_name, team_id=team_id) + if not models: + return None + for model in models: + if model.get("model_info", {}).get("team_id") == team_id: + return model.get("model_name") + + ## wildcard models return None def should_include_deployment( @@ -6073,6 +6090,7 @@ class Router: if team_id specified, returns matching team-specific models """ + if hasattr(self, "model_list"): returned_models: List[DeploymentTypedDict] = [] @@ -6087,7 +6105,17 @@ class Router: ) if len(returned_models) == 0: # check if wildcard route - potential_wildcard_models = self.pattern_router.route(model_name) + potential_wildcard_models = self.pattern_router.route(model_name) or [] + + ## check for team-specific wildcard models + if team_id is not None and team_id in self.team_pattern_routers: + potential_team_only_wildcard_models = ( + self.team_pattern_routers[team_id].route(model_name) or [] + ) + potential_wildcard_models.extend( + potential_team_only_wildcard_models + ) + if model_name is not None and potential_wildcard_models is not None: for m in potential_wildcard_models: deployment_typed_dict = DeploymentTypedDict(**m) # type: ignore @@ -6519,6 +6547,7 @@ class Router: messages: Optional[List[Dict[str, str]]] = None, input: Optional[Union[str, List]] = None, specific_deployment: Optional[bool] = False, + request_kwargs: Optional[Dict] = None, ) -> Tuple[str, Union[List, Dict]]: """ Common checks for 'get_available_deployment' across sync + async call. @@ -6530,6 +6559,14 @@ class Router: - List, if multiple models chosen - Dict, if specific model chosen """ + + request_team_id: Optional[str] = None + if request_kwargs is not None: + metadata = request_kwargs.get("metadata") or {} + litellm_metadata = request_kwargs.get("litellm_metadata") or {} + request_team_id = metadata.get( + "user_api_key_team_id" + ) or litellm_metadata.get("user_api_key_team_id") # check if aliases set on litellm model alias map if specific_deployment is True: return model, self._get_deployment_by_litellm_model(model=model) @@ -6552,9 +6589,22 @@ class Router: pattern_deployments = self.pattern_router.get_deployments_by_pattern( model=model, ) + if pattern_deployments: return model, pattern_deployments + if ( + request_team_id is not None + and request_team_id in self.team_pattern_routers + ): + pattern_deployments = self.team_pattern_routers[ + request_team_id + ].get_deployments_by_pattern( + model=model, + ) + if pattern_deployments: + return model, pattern_deployments + # check if default deployment is set if self.default_deployment is not None: updated_deployment = copy.deepcopy( @@ -6622,6 +6672,7 @@ class Router: messages=messages, input=input, specific_deployment=specific_deployment, + request_kwargs=request_kwargs, ) # type: ignore # IF TEAM ID SPECIFIED ON MODEL, AND REQUEST CONTAINS USER_API_KEY_TEAM_ID, FILTER OUT MODELS THAT ARE NOT IN THE TEAM diff --git a/litellm/types/caching.py b/litellm/types/caching.py index 2531444ae81..67b0238ce8a 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -12,6 +12,7 @@ class LiteLLMCacheType(str, Enum): DISK = "disk" QDRANT_SEMANTIC = "qdrant-semantic" AZURE_BLOB = "azure-blob" + GCS = "gcs" CachingSupportedCallTypes = Literal[ diff --git a/litellm/types/llms/oci.py b/litellm/types/llms/oci.py new file mode 100644 index 00000000000..52c1b2943f7 --- /dev/null +++ b/litellm/types/llms/oci.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel + +OCIRoles = Literal["SYSTEM", "USER", "ASSISTANT", "TOOL"] + + +class OCIVendors(Enum): + """ + A class to hold the vendor names for OCI models. + This is used to map model names to their respective vendors. + """ + + COHERE = "COHERE" + GENERIC = "GENERIC" + + +# --- Base Models and Content Parts --- + + +class OCIContentPart(BaseModel): + """Base model for content parts in an OCI message.""" + + pass + + +class OCITextContentPart(OCIContentPart): + """Text content part for the OCI API.""" + + type: Literal["TEXT"] = "TEXT" + text: str + + +class OCIImageContentPart(OCIContentPart): + """Image content part for the OCI API.""" + + type: Literal["IMAGE"] = "IMAGE" + imageUrl: str + + +OCIContentPartUnion = Union[OCITextContentPart, OCIImageContentPart] + +# --- Models for Tools and Tool Calls --- + + +class OCIToolCall(BaseModel): + """Represents a tool call made by the model.""" + + id: str + type: Literal["FUNCTION"] = "FUNCTION" + name: str + arguments: str # Arguments should be a JSON-serialized string + + +class OCIToolDefinition(BaseModel): + """Defines a tool that can be used by the model.""" + + type: Literal["FUNCTION"] = "FUNCTION" + name: Optional[str] = None + description: Optional[str] = None + parameters: Optional[dict] = None + + +# --- Message Models (Request and Response) --- + + +class OCIMessage(BaseModel): + """Model for a single message in the request/response payload.""" + + role: OCIRoles + content: Optional[List[OCIContentPartUnion]] = None + toolCalls: Optional[List[OCIToolCall]] = None + toolCallId: Optional[str] = None + + +# --- Request Payload Models --- + + +class OCIChatRequestPayload(BaseModel): + """Internal 'chatRequest' payload for the OCI API.""" + + apiFormat: str + messages: List[OCIMessage] + tools: Optional[List[OCIToolDefinition]] = None + isStream: bool = False + numGenerations: Optional[int] = None + maxTokens: Optional[int] = None + temperature: Optional[float] = None + topP: Optional[float] = None + stop: Optional[List[str]] = None + seed: Optional[int] = None + frequencyPenalty: Optional[float] = None + presencePenalty: Optional[float] = None + + +class OCIServingMode(BaseModel): + """Defines the serving mode and the model to be used.""" + + servingType: str + modelId: str + + +class OCICompletionPayload(BaseModel): + """Pydantic model for the complete OCI chat request body.""" + + compartmentId: str + servingMode: OCIServingMode + chatRequest: OCIChatRequestPayload + + +# --- API Response Models (Non-streaming) --- + + +class OCICompletionTokenDetails(BaseModel): + """Completion token details in the OCI response.""" + + acceptedPredictionTokens: int + reasoningTokens: int + + +class OCIPropmtTokensDetails(BaseModel): + """Prompt token details in the OCI response.""" + + cachedTokens: int + + +class OCIResponseUsage(BaseModel): + """Token usage in the OCI response.""" + + promptTokens: int + completionTokens: int + totalTokens: int + completionTokensDetails: OCICompletionTokenDetails + promptTokensDetails: OCIPropmtTokensDetails + + +class OCIResponseChoice(BaseModel): + """A completion choice in the OCI response.""" + + index: int + message: OCIMessage + finishReason: Optional[str] = None + logprobs: Optional[Dict[str, Any]] = None + + +class OCIChatResponse(BaseModel): + """The 'chatResponse' object in the OCI response.""" + + apiFormat: str + timeCreated: str + choices: List[OCIResponseChoice] + usage: OCIResponseUsage + + +class OCICompletionResponse(BaseModel): + """Model for the complete non-streaming OCI response body.""" + + modelId: str + modelVersion: str + chatResponse: OCIChatResponse + + +# --- API Response Models (Streaming) --- + + +class OCIStreamDelta(BaseModel): + """The content delta in a streaming chunk.""" + + content: Optional[List[OCIContentPartUnion]] = None + role: Optional[str] = None + toolCalls: Optional[List[OCIToolCall]] = None + + +class OCIStreamChunk(BaseModel): + """Model for a single SSE event chunk from OCI.""" + + finishReason: Optional[str] = None + message: Optional[OCIStreamDelta] = None + pad: Optional[str] = None + index: Optional[int] = None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 1955bfac5f8..bc3cfeee6a3 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -965,6 +965,8 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): top_p: Optional[float] truncation: Optional[Literal["auto", "disabled"]] user: Optional[str] + service_tier: Optional[str] + safety_identifier: Optional[str] prompt: Optional[PromptObject] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d1b2f57c725..75c7d28460b 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -887,6 +887,10 @@ class Usage(CompletionUsage): ) # hidden param for prompt caching. Might change, once openai introduces their equivalent. server_tool_use: Optional[ServerToolUse] = None + cost: Optional[float] = None + + completion_tokens_details: Optional[CompletionTokensDetailsWrapper] = None + """Breakdown of tokens used in a completion.""" prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None """Breakdown of tokens used in the prompt.""" @@ -904,6 +908,7 @@ class Usage(CompletionUsage): Union[CompletionTokensDetailsWrapper, dict] ] = None, server_tool_use: Optional[ServerToolUse] = None, + cost: Optional[float] = None, **params, ): # handle reasoning_tokens @@ -975,6 +980,11 @@ class Usage(CompletionUsage): else: # maintain openai compatibility in usage object if possible del self.server_tool_use + if cost is not None: + self.cost = cost + else: + del self.cost + ## ANTHROPIC MAPPING ## if "cache_creation_input_tokens" in params and isinstance( params["cache_creation_input_tokens"], int @@ -1900,6 +1910,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): vector_store_request_metadata: Optional[List[StandardLoggingVectorStoreRequest]] applied_guardrails: Optional[List[str]] usage_object: Optional[dict] + cold_storage_object_key: Optional[str] # S3/GCS object key for cold storage retrieval class StandardLoggingAdditionalHeaders(TypedDict, total=False): @@ -2318,6 +2329,7 @@ class LlmProviders(str, Enum): PG_VECTOR = "pg_vector" HYPERBOLIC = "hyperbolic" RECRAFT = "recraft" + OCI = "oci" AUTO_ROUTER = "auto_router" DOTPROMPT = "dotprompt" diff --git a/litellm/utils.py b/litellm/utils.py index 9e9a112f304..ffd8bee382e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2790,7 +2790,10 @@ def get_optional_params_embeddings( # noqa: PLR0915 ) _check_valid_arg(supported_params=supported_params) optional_params = litellm.JinaAIEmbeddingConfig().map_openai_params( - non_default_params=non_default_params, optional_params={} + non_default_params=non_default_params, + optional_params={}, + model=model, + drop_params=drop_params if drop_params is not None else False, ) elif custom_llm_provider == "voyage": supported_params = get_supported_openai_params( @@ -6604,6 +6607,19 @@ def validate_and_fix_openai_messages(messages: List): new_messages.append(cleaned_message) return validate_chat_completion_user_messages(messages=new_messages) +def validate_and_fix_openai_tools(tools: Optional[List]) -> Optional[List[dict]]: + """ + Ensure tools is List[dict] and not List[BaseModel] + """ + new_tools = [] + if tools is None: + return tools + for tool in tools: + if isinstance(tool, BaseModel): + new_tools.append(tool.model_dump()) + elif isinstance(tool, dict): + new_tools.append(tool) + return new_tools def cleanup_none_field_in_message(message: AllMessageValues): """ @@ -6916,6 +6932,8 @@ class ProviderConfigManager: return litellm.OpenAIGPTConfig() elif litellm.LlmProviders.NSCALE == provider: return litellm.NscaleConfig() + elif litellm.LlmProviders.OCI == provider: + return litellm.OCIChatConfig() elif litellm.LlmProviders.HYPERBOLIC == provider: return litellm.HyperbolicChatConfig() return None @@ -6940,6 +6958,12 @@ class ProviderConfigManager: from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig return CohereEmbeddingConfig() + elif litellm.LlmProviders.JINA_AI == provider: + from litellm.llms.jina_ai.embedding.transformation import ( + JinaAIEmbeddingConfig, + ) + + return JinaAIEmbeddingConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ef4891c2686..b4ff923c1e2 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -296,6 +296,60 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "gpt-5-2025-08-07": { + "max_tokens": 128000, + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.25e-07, + "litellm_provider": "openai", + "mode": "chat", + "supports_pdf_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "gpt-5-mini-2025-08-07": { + "max_tokens": 128000, + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openai", + "mode": "chat", + "supports_pdf_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "gpt-5-nano-2025-08-07": { + "max_tokens": 128000, + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 5e-09, + "litellm_provider": "openai", + "mode": "chat", + "supports_pdf_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "watsonx/ibm/granite-3-8b-instruct": { "max_tokens": 8192, "max_input_tokens": 8192, @@ -607,9 +661,9 @@ "supports_system_messages": true, "supports_tool_choice": true, "search_context_cost_per_query": { - "search_context_size_low": 30.0, - "search_context_size_medium": 35.0, - "search_context_size_high": 50.0 + "search_context_size_low": 0.025, + "search_context_size_medium": 0.0275, + "search_context_size_high": 0.03 } }, "codex-mini-latest": { @@ -3750,7 +3804,7 @@ "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3.3e-06, - "output_cost_per_token": 16.5e-06, + "output_cost_per_token": 1.65e-05, "litellm_provider": "azure_ai", "mode": "chat", "supports_function_calling": true, @@ -3764,7 +3818,7 @@ "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-06, - "output_cost_per_token": 15e-06, + "output_cost_per_token": 1.5e-05, "litellm_provider": "azure_ai", "mode": "chat", "supports_function_calling": true, @@ -3777,7 +3831,7 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.25e-06, + "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.27e-06, "litellm_provider": "azure_ai", "mode": "chat", @@ -3792,7 +3846,7 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, - "input_cost_per_token": 0.275e-06, + "input_cost_per_token": 2.75e-07, "output_cost_per_token": 1.38e-06, "litellm_provider": "azure_ai", "mode": "chat", @@ -5486,6 +5540,36 @@ "litellm_provider": "groq", "mode": "audio_transcription" }, + "groq/openai/gpt-oss-20b": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 5e-07, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_web_search": true + }, + "groq/openai/gpt-oss-120b": { + "max_tokens": 32766, + "max_input_tokens": 131072, + "max_output_tokens": 32766, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 7.5e-07, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_web_search": true + }, "cerebras/llama3.1-8b": { "max_tokens": 128000, "max_input_tokens": 128000, @@ -5741,6 +5825,32 @@ "supports_reasoning": true, "supports_computer_use": true }, + "claude-opus-4-1-20250805": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "anthropic", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "claude-sonnet-4-20250514": { "max_tokens": 64000, "max_input_tokens": 200000, @@ -7337,12 +7447,12 @@ "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, "max_pdf_size_mb": 30, - "input_cost_per_token": 3.5e-07, + "input_cost_per_token": 3.5e-07, "input_cost_per_audio_token": 2.1e-06, "input_cost_per_image": 2.1e-06, "input_cost_per_video_per_second": 2.1e-06, "output_cost_per_token": 1.5e-06, - "output_cost_per_audio_token": 8.5e-06, + "output_cost_per_audio_token": 8.5e-06, "litellm_provider": "gemini", "mode": "chat", "rpm": 10, @@ -8690,6 +8800,40 @@ "source": "https://aistudio.google.com", "supports_tool_choice": true }, + "vertex_ai/claude-opus-4-1": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 15e-06, + "output_cost_per_token": 75e-06, + "input_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_batches": 37.5e-06, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_assistant_prefill": true, + "supports_tool_choice": true + }, + "vertex_ai/claude-opus-4-1@20250805": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 15e-06, + "output_cost_per_token": 75e-06, + "input_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_batches": 37.5e-06, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_assistant_prefill": true, + "supports_tool_choice": true + }, "vertex_ai/claude-3-sonnet": { "max_tokens": 4096, "max_input_tokens": 200000, @@ -9039,9 +9183,9 @@ "supports_tool_choice": true }, "vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas": { - "max_tokens": 10000000.0, - "max_input_tokens": 10000000.0, - "max_output_tokens": 10000000.0, + "max_tokens": 10000000, + "max_input_tokens": 10000000, + "max_output_tokens": 10000000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 7e-07, "litellm_provider": "vertex_ai-llama_models", @@ -9059,9 +9203,9 @@ ] }, "vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": { - "max_tokens": 10000000.0, - "max_input_tokens": 10000000.0, - "max_output_tokens": 10000000.0, + "max_tokens": 10000000, + "max_input_tokens": 10000000, + "max_output_tokens": 10000000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 7e-07, "litellm_provider": "vertex_ai-llama_models", @@ -9079,9 +9223,9 @@ ] }, "vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas": { - "max_tokens": 1000000.0, - "max_input_tokens": 1000000.0, - "max_output_tokens": 1000000.0, + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, "input_cost_per_token": 3.5e-07, "output_cost_per_token": 1.15e-06, "litellm_provider": "vertex_ai-llama_models", @@ -9099,9 +9243,9 @@ ] }, "vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": { - "max_tokens": 1000000.0, - "max_input_tokens": 1000000.0, - "max_output_tokens": 1000000.0, + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, "input_cost_per_token": 3.5e-07, "output_cost_per_token": 1.15e-06, "litellm_provider": "vertex_ai-llama_models", @@ -9174,7 +9318,7 @@ "max_input_tokens": 128000, "max_output_tokens": 2048, "input_cost_per_token": 5e-06, - "output_cost_per_token": 16e-06, + "output_cost_per_token": 1.6e-05, "litellm_provider": "vertex_ai-llama_models", "mode": "chat", "supports_system_messages": true, @@ -10480,7 +10624,7 @@ "supports_tool_choice": true, "supports_prompt_caching": true }, - "openrouter/x-ai/grok-4":{ + "openrouter/x-ai/grok-4": { "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, @@ -10494,12 +10638,12 @@ "source": "https://openrouter.ai/x-ai/grok-4", "supports_web_search": true }, - "openrouter/bytedance/ui-tars-1.5-7b":{ + "openrouter/bytedance/ui-tars-1.5-7b": { "max_tokens": 2048, "max_input_tokens": 131072, "max_output_tokens": 2048, - "input_cost_per_token": 0.1e-06, - "output_cost_per_token": 0.2e-06, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, "litellm_provider": "openrouter", "mode": "chat", "source": "https://openrouter.ai/api/v1/models/bytedance/ui-tars-1.5-7b", @@ -11178,8 +11322,8 @@ "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 2048, - "input_cost_per_token": 0.21e-06, - "output_cost_per_token": 0.63e-06, + "input_cost_per_token": 2.1e-07, + "output_cost_per_token": 6.3e-07, "litellm_provider": "openrouter", "mode": "chat", "supports_tool_choice": true @@ -11891,6 +12035,60 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "openai.gpt-oss-20b-1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "openai.gpt-oss-120b-1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true + }, + "anthropic.claude-opus-4-1-20250805-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "anthropic.claude-opus-4-20250514-v1:0": { "max_tokens": 32000, "max_input_tokens": 200000, @@ -12093,6 +12291,32 @@ "supports_tool_choice": true, "supports_reasoning": true }, + "us.anthropic.claude-opus-4-1-20250805-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "us.anthropic.claude-opus-4-20250514-v1:0": { "max_tokens": 32000, "max_input_tokens": 200000, @@ -12266,6 +12490,32 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "eu.anthropic.claude-opus-4-1-20250805-v1:0": { + "max_tokens": 32000, + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 7.5e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01 + }, + "cache_creation_input_token_cost": 1.875e-05, + "cache_read_input_token_cost": 1.5e-06, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "max_tokens": 32000, "max_input_tokens": 200000, @@ -14763,7 +15013,7 @@ "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 16384, - "input_cost_per_token": 0.6e-06, + "input_cost_per_token": 6e-07, "output_cost_per_token": 2.5e-06, "litellm_provider": "fireworks_ai", "mode": "chat", @@ -14809,6 +15059,58 @@ "source": "https://fireworks.ai/pricing", "supports_tool_choice": false }, + "fireworks_ai/accounts/fireworks/models/glm-4p5": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 96000, + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2.19e-06, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://fireworks.ai/models/fireworks/glm-4p5" + }, + "fireworks_ai/accounts/fireworks/models/glm-4p5-air": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 96000, + "input_cost_per_token": 2.2e-07, + "output_cost_per_token": 8.8e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://artificialanalysis.ai/models/glm-4-5-air" + }, + "fireworks_ai/accounts/fireworks/models/gpt-oss-120b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://fireworks.ai/pricing" + }, + "fireworks_ai/accounts/fireworks/models/gpt-oss-20b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://fireworks.ai/pricing" + }, "fireworks_ai/nomic-ai/nomic-embed-text-v1.5": { "max_tokens": 8192, "max_input_tokens": 8192, @@ -17445,5 +17747,136 @@ "supports_vision": false, "supports_system_messages": true, "supports_tool_choice": false + }, + "oci/meta.llama-4-maverick-17b-128e-instruct-fp8": { + "max_tokens": 512000, + "max_input_tokens": 512000, + "max_output_tokens": 4000, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/meta.llama-4-scout-17b-16e-instruct": { + "max_tokens": 192000, + "max_input_tokens": 192000, + "max_output_tokens": 4000, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/meta.llama-3.3-70b-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "input_cost_per_token": 7.2e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/meta.llama-3.2-90b-vision-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "input_cost_per_token": 2.0e-06, + "output_cost_per_token": 2.0e-06, + "litellm_provider": "oci", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/meta.llama-3.1-405b-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "input_cost_per_token": 1.068e-05, + "output_cost_per_token": 1.068e-05, + "litellm_provider": "oci", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + + "oci/xai.grok-4": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 3.0e-06, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/xai.grok-3": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 3.0e-06, + "output_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/xai.grok-3-mini": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 3.0e-07, + "output_cost_per_token": 5.0e-07, + "litellm_provider": "oci", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/xai.grok-3-fast": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 5.0e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "oci", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" + }, + "oci/xai.grok-3-mini-fast": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6.0e-07, + "output_cost_per_token": 4.0e-06, + "litellm_provider": "oci", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" } } diff --git a/poetry.lock b/poetry.lock index 8c703347518..2a1a664ee66 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -6,6 +6,7 @@ version = "2.4.4" description = "Happy Eyeballs for asyncio" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "aiohappyeyeballs-2.4.4-py3-none-any.whl", hash = "sha256:a980909d50efcd44795c4afeca523296716d50cd756ddca6af8c65b996e27de8"}, {file = "aiohappyeyeballs-2.4.4.tar.gz", hash = "sha256:5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745"}, @@ -17,6 +18,7 @@ version = "3.10.11" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "aiohttp-3.10.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5077b1a5f40ffa3ba1f40d537d3bec4383988ee51fbba6b74aa8fb1bc466599e"}, {file = "aiohttp-3.10.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8d6a14a4d93b5b3c2891fca94fa9d41b2322a68194422bef0dd5ec1e57d7d298"}, @@ -121,7 +123,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.12.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] +speedups = ["Brotli ; platform_python_implementation == \"CPython\"", "aiodns (>=3.2.0) ; sys_platform == \"linux\" or sys_platform == \"darwin\"", "brotlicffi ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" @@ -129,6 +131,7 @@ version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, @@ -143,6 +146,8 @@ version = "0.7.13" description = "A configurable sidebar-enabled Sphinx theme" optional = true python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3"}, {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"}, @@ -154,6 +159,8 @@ version = "1.16.4" description = "A database migration tool for SQLAlchemy." optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "alembic-1.16.4-py3-none-any.whl", hash = "sha256:b05e51e8e82efc1abd14ba2af6392897e145930c3e0a2faf2b0da2f7f7fd660d"}, {file = "alembic-1.16.4.tar.gz", hash = "sha256:efab6ada0dd0fae2c92060800e0bf5c1dc26af15a10e02fb4babff164b4725e2"}, @@ -174,6 +181,7 @@ version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -188,6 +196,7 @@ version = "4.5.2" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "anyio-4.5.2-py3-none-any.whl", hash = "sha256:c011ee36bc1e8ba40e5a81cb9df91925c218fe9b778554e0b56a21e1b5d4716f"}, {file = "anyio-4.5.2.tar.gz", hash = "sha256:23009af4ed04ce05991845451e11ef02fc7c5ed29179ac9a420e5ad0ac7ddc5b"}, @@ -201,7 +210,7 @@ typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} [package.extras] doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21.0b1) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\""] trio = ["trio (>=0.26.1)"] [[package]] @@ -210,6 +219,8 @@ version = "3.11.0" description = "In-process task scheduler with Cron-like capabilities" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "APScheduler-3.11.0-py3-none-any.whl", hash = "sha256:fc134ca32e50f5eadcc4938e3a4545ab19131435e851abb40b34d63d5141c6da"}, {file = "apscheduler-3.11.0.tar.gz", hash = "sha256:4c622d250b0955a65d5d0eb91c33e6d43fd879834bf541e0a18661ae60460133"}, @@ -227,7 +238,7 @@ mongodb = ["pymongo (>=3.0)"] redis = ["redis (>=3.0)"] rethinkdb = ["rethinkdb (>=2.4.0)"] sqlalchemy = ["sqlalchemy (>=1.4)"] -test = ["APScheduler[etcd,mongodb,redis,rethinkdb,sqlalchemy,tornado,zookeeper]", "PySide6", "anyio (>=4.5.2)", "gevent", "pytest", "pytz", "twisted"] +test = ["APScheduler[etcd,mongodb,redis,rethinkdb,sqlalchemy,tornado,zookeeper]", "PySide6 ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "anyio (>=4.5.2)", "gevent ; python_version < \"3.14\"", "pytest", "pytz", "twisted ; python_version < \"3.14\""] tornado = ["tornado (>=4.3)"] twisted = ["twisted"] zookeeper = ["kazoo"] @@ -236,8 +247,10 @@ zookeeper = ["kazoo"] name = "async-timeout" version = "5.0.1" description = "Timeout context manager for asyncio programs" -optional = false +optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_full_version < \"3.11.3\" and (extra == \"extra-proxy\" or extra == \"proxy\") or python_version <= \"3.10\"" files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, @@ -249,18 +262,19 @@ version = "25.3.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3"}, {file = "attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b"}, ] [package.extras] -benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] -tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""] [[package]] name = "azure-core" @@ -268,6 +282,7 @@ version = "1.33.0" description = "Microsoft Azure Core Library for Python" optional = false python-versions = ">=3.8" +groups = ["main", "proxy-dev"] files = [ {file = "azure_core-1.33.0-py3-none-any.whl", hash = "sha256:9b5b6d0223a1d38c37500e6971118c1e0f13f54951e6893968b38910bc9cda8f"}, {file = "azure_core-1.33.0.tar.gz", hash = "sha256:f367aa07b5e3005fec2c1e184b882b0b039910733907d001c20fb08ebb8c0eb9"}, @@ -288,6 +303,7 @@ version = "1.21.0" description = "Microsoft Azure Identity Library for Python" optional = false python-versions = ">=3.8" +groups = ["main", "proxy-dev"] files = [ {file = "azure_identity-1.21.0-py3-none-any.whl", hash = "sha256:258ea6325537352440f71b35c3dffe9d240eae4a5126c1b7ce5efd5766bd9fd9"}, {file = "azure_identity-1.21.0.tar.gz", hash = "sha256:ea22ce6e6b0f429bc1b8d9212d5b9f9877bd4c82f1724bfa910760612c07a9a6"}, @@ -306,6 +322,8 @@ version = "4.9.0" description = "Microsoft Azure Key Vault Secrets Client Library for Python" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"extra-proxy\"" files = [ {file = "azure_keyvault_secrets-4.9.0-py3-none-any.whl", hash = "sha256:33c7e2aca2cc2092cebc8c6e96eca36a5cc30c767e16ea429c5fa21270e9fba6"}, {file = "azure_keyvault_secrets-4.9.0.tar.gz", hash = "sha256:2a03bb2ffd9a0d6c8ad1c330d9d0310113985a9de06607ece378fd72a5889fe1"}, @@ -322,6 +340,8 @@ version = "12.26.0" description = "Microsoft Azure Blob Storage Client Library for Python" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "azure_storage_blob-12.26.0-py3-none-any.whl", hash = "sha256:8c5631b8b22b4f53ec5fff2f3bededf34cfef111e2af613ad42c9e6de00a77fe"}, {file = "azure_storage_blob-12.26.0.tar.gz", hash = "sha256:5dd7d7824224f7de00bfeb032753601c982655173061e242f13be6e26d78d71f"}, @@ -342,6 +362,8 @@ version = "2.17.0" description = "Internationalization utilities" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"}, {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"}, @@ -351,7 +373,7 @@ files = [ pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} [package.extras] -dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"] +dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] [[package]] name = "backoff" @@ -359,10 +381,12 @@ version = "2.2.1" description = "Function decoration for backoff and retry" optional = false python-versions = ">=3.7,<4.0" +groups = ["main", "dev"] files = [ {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, ] +markers = {main = "python_version >= \"3.9\" and (extra == \"semantic-router\" or extra == \"proxy\") or extra == \"proxy\""} [[package]] name = "backports-zoneinfo" @@ -370,6 +394,8 @@ version = "0.2.1" description = "Backport of the standard library zoneinfo module" optional = true python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"proxy\" and python_version < \"3.9\"" files = [ {file = "backports.zoneinfo-0.2.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc"}, {file = "backports.zoneinfo-0.2.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722"}, @@ -398,6 +424,7 @@ version = "23.12.1" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "black-23.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0aaf6041986767a5e0ce663c7a2f0e9eaf21e6ff87a5f95cbf3675bfd4c41d2"}, {file = "black-23.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c88b3711d12905b74206227109272673edce0cb29f27e1385f33b0163c414bba"}, @@ -434,7 +461,7 @@ typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +d = ["aiohttp (>=3.7.4) ; sys_platform != \"win32\" or implementation_name != \"pypy\"", "aiohttp (>=3.7.4,!=3.9.0) ; sys_platform == \"win32\" and implementation_name == \"pypy\""] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] @@ -444,6 +471,8 @@ version = "1.9.0" description = "Fast, simple object-to-object and broadcast signaling" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc"}, {file = "blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf"}, @@ -455,6 +484,8 @@ version = "1.34.34" description = "The AWS SDK for Python" optional = true python-versions = ">= 3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "boto3-1.34.34-py3-none-any.whl", hash = "sha256:33a8b6d9136fa7427160edb92d2e50f2035f04e9d63a2d1027349053e12626aa"}, {file = "boto3-1.34.34.tar.gz", hash = "sha256:b2f321e20966f021ec800b7f2c01287a3dd04fc5965acdfbaa9c505a24ca45d1"}, @@ -474,6 +505,8 @@ version = "1.34.162" description = "Low-level, data-driven core of boto 3." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "botocore-1.34.162-py3-none-any.whl", hash = "sha256:2d918b02db88d27a75b48275e6fb2506e9adaaddbec1ffa6a8a0898b34e769be"}, {file = "botocore-1.34.162.tar.gz", hash = "sha256:adc23be4fb99ad31961236342b7cbf3c0bfc62532cd02852196032e8c0d682f3"}, @@ -483,8 +516,8 @@ files = [ jmespath = ">=0.7.1,<2.0.0" python-dateutil = ">=2.1,<3.0.0" urllib3 = [ - {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}, + {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, ] [package.extras] @@ -496,6 +529,8 @@ version = "5.5.2" description = "Extensible memoizing collections and decorators" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "python_version >= \"3.10\" and (extra == \"mlflow\" or extra == \"extra-proxy\") or extra == \"extra-proxy\"" files = [ {file = "cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a"}, {file = "cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4"}, @@ -507,6 +542,7 @@ version = "2025.8.3" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5"}, {file = "certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407"}, @@ -518,6 +554,7 @@ version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, @@ -587,6 +624,7 @@ files = [ {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, ] +markers = {main = "platform_python_implementation != \"PyPy\" or extra == \"proxy\"", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""} [package.dependencies] pycparser = "*" @@ -597,6 +635,7 @@ version = "3.4.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "charset_normalizer-3.4.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941"}, {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd"}, @@ -698,6 +737,7 @@ version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, @@ -712,6 +752,8 @@ version = "3.1.1" description = "Pickler class to extend the standard pickle.Pickler functionality" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "cloudpickle-3.1.1-py3-none-any.whl", hash = "sha256:c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e"}, {file = "cloudpickle-3.1.1.tar.gz", hash = "sha256:b216fa8ae4019d5482a8ac3c95d8f6346115d8835911fd4aefd1a445e4242c64"}, @@ -723,6 +765,8 @@ version = "4.57" description = "Python SDK for the Cohere API" optional = true python-versions = ">=3.8,<4.0" +groups = ["main"] +markers = "python_version >= \"3.9\" and extra == \"semantic-router\"" files = [ {file = "cohere-4.57-py3-none-any.whl", hash = "sha256:479bdea81ae119e53f671f1ae808fcff9df88211780525d7ef2f7b99dfb32e59"}, {file = "cohere-4.57.tar.gz", hash = "sha256:71ace0204a92d1a2a8d4b949b88b353b4f22fc645486851924284cc5a0eb700d"}, @@ -742,10 +786,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "sys_platform == \"win32\" and (extra == \"utils\" or extra == \"semantic-router\") and python_version >= \"3.9\" or platform_system == \"Windows\" or sys_platform == \"win32\" and extra == \"utils\"", dev = "platform_system == \"Windows\" or sys_platform == \"win32\"", proxy-dev = "platform_system == \"Windows\""} [[package]] name = "coloredlogs" @@ -753,6 +799,8 @@ version = "15.0.1" description = "Colored terminal output for Python's logging module" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["main"] +markers = "python_version >= \"3.9\" and extra == \"extra-proxy\" and python_version < \"3.14\"" files = [ {file = "coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934"}, {file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"}, @@ -770,6 +818,8 @@ version = "6.9.0" description = "Add colours to the output of Python's logging module." optional = true python-versions = ">=3.6" +groups = ["main"] +markers = "python_version >= \"3.9\" and extra == \"semantic-router\"" files = [ {file = "colorlog-6.9.0-py3-none-any.whl", hash = "sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff"}, {file = "colorlog-6.9.0.tar.gz", hash = "sha256:bfba54a1b93b94f54e1f4fe48395725a3d92fd2a4af702f6bd70946bdc0c6ac2"}, @@ -787,6 +837,8 @@ version = "1.3.2" description = "Python library for calculating contours of 2D quadrilateral grids" optional = true python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "contourpy-1.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ba38e3f9f330af820c4b27ceb4b9c7feee5fe0493ea53a8720f4792667465934"}, {file = "contourpy-1.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dc41ba0714aa2968d1f8674ec97504a8f7e334f48eeacebcaa6256213acb0989"}, @@ -863,6 +915,7 @@ version = "43.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, @@ -912,6 +965,8 @@ version = "0.12.1" description = "Composable style cycles" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30"}, {file = "cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"}, @@ -927,6 +982,8 @@ version = "0.61.0" description = "Databricks SDK for Python (Beta)" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "databricks_sdk-0.61.0-py3-none-any.whl", hash = "sha256:709ac7c709f843567b04fba6cea8a53ee644b79314e9a9ac4db0c1b3c1d2d5fe"}, {file = "databricks_sdk-0.61.0.tar.gz", hash = "sha256:06e50663c2c87e94f5e505390b74bc5c7c5330f4b4be35616b7aed06cf940af0"}, @@ -937,9 +994,9 @@ google-auth = ">=2.0,<3.0" requests = ">=2.28.1,<3" [package.extras] -dev = ["autoflake", "black", "build", "databricks-connect", "httpx", "ipython", "ipywidgets", "isort", "langchain-openai", "openai", "pycodestyle", "pyfakefs", "pytest", "pytest-cov", "pytest-mock", "pytest-rerunfailures", "pytest-xdist", "requests-mock", "wheel"] +dev = ["autoflake", "black", "build", "databricks-connect", "httpx", "ipython", "ipywidgets", "isort", "langchain-openai ; python_version > \"3.7\"", "openai", "pycodestyle", "pyfakefs", "pytest", "pytest-cov", "pytest-mock", "pytest-rerunfailures", "pytest-xdist", "requests-mock", "wheel"] notebook = ["ipython (>=8,<10)", "ipywidgets (>=8,<9)"] -openai = ["httpx", "langchain-openai", "openai"] +openai = ["httpx", "langchain-openai ; python_version > \"3.7\"", "openai"] [[package]] name = "deprecated" @@ -947,16 +1004,18 @@ version = "1.2.18" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec"}, {file = "deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d"}, ] +markers = {main = "python_version >= \"3.10\""} [package.dependencies] wrapt = ">=1.10,<2" [package.extras] -dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools", "tox"] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools ; python_version >= \"3.12\"", "tox"] [[package]] name = "diskcache" @@ -964,6 +1023,8 @@ version = "5.6.3" description = "Disk Cache -- Disk and file backed persistent cache." optional = true python-versions = ">=3" +groups = ["main"] +markers = "extra == \"caching\"" files = [ {file = "diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19"}, {file = "diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc"}, @@ -975,6 +1036,7 @@ version = "1.9.0" description = "Distro - an OS platform information API" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, @@ -986,6 +1048,8 @@ version = "2.6.1" description = "DNS toolkit" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "dnspython-2.6.1-py3-none-any.whl", hash = "sha256:5ef3b9680161f6fa89daf8ad451b5f1a33b18ae8a1c6778cdf4b43f08c0a6e50"}, {file = "dnspython-2.6.1.tar.gz", hash = "sha256:e8f0f9c23a7b7cb99ded64e6c3a6f3e701d78f50c55e002b839dea7225cff7cc"}, @@ -1006,6 +1070,8 @@ version = "7.1.0" description = "A Python library for the Docker Engine API." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"}, {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"}, @@ -1028,6 +1094,8 @@ version = "0.20.1" description = "Docutils -- Python Documentation Utilities" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6"}, {file = "docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b"}, @@ -1039,6 +1107,8 @@ version = "2.2.0" description = "A robust email address syntax and deliverability validation library." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "email_validator-2.2.0-py3-none-any.whl", hash = "sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631"}, {file = "email_validator-2.2.0.tar.gz", hash = "sha256:cb690f344c617a714f22e66ae771445a1ceb46821152df8e165c5f9a364582b7"}, @@ -1054,6 +1124,8 @@ version = "1.3.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["main", "dev", "proxy-dev"] +markers = "python_version <= \"3.10\"" files = [ {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, @@ -1071,6 +1143,8 @@ version = "0.115.14" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and (extra == \"mlflow\" or extra == \"proxy\") or extra == \"proxy\"" files = [ {file = "fastapi-0.115.14-py3-none-any.whl", hash = "sha256:6c0c8bf9420bd58f565e585036d971872472b4f7d3f6c73b698e10cffdefb3ca"}, {file = "fastapi-0.115.14.tar.gz", hash = "sha256:b1de15cdc1c499a4da47914db35d0e4ef8f1ce62b624e94e0e5824421df99739"}, @@ -1091,6 +1165,8 @@ version = "0.16.0" description = "FastAPI plugin to enable SSO to most common providers (such as Facebook login, Google login and login via Microsoft Office 365 Account)" optional = true python-versions = "<4.0,>=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "fastapi_sso-0.16.0-py3-none-any.whl", hash = "sha256:3a66a942474ef9756d3a9d8b945d55bd9faf99781facdb9b87a40b73d6d6b0c3"}, {file = "fastapi_sso-0.16.0.tar.gz", hash = "sha256:f3941f986347566b7d3747c710cf474a907f581bfb6697ff3bb3e44eb76b438c"}, @@ -1109,6 +1185,8 @@ version = "1.12.0" description = "Fast read/write of AVRO files" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.9\" and extra == \"semantic-router\"" files = [ {file = "fastavro-1.12.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e38497bd24136aad2c47376ee958be4f5b775d6f03c11893fc636eea8c1c3b40"}, {file = "fastavro-1.12.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8d8401b021f4b3dfc05e6f82365f14de8d170a041fbe3345f992c9c13d4f0ff"}, @@ -1160,6 +1238,7 @@ version = "3.16.1" description = "A platform independent file lock." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, @@ -1168,7 +1247,7 @@ files = [ [package.extras] docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"] -typing = ["typing-extensions (>=4.12.2)"] +typing = ["typing-extensions (>=4.12.2) ; python_version < \"3.11\""] [[package]] name = "flake8" @@ -1176,6 +1255,7 @@ version = "6.1.0" description = "the modular source code checker: pep8 pyflakes and co" optional = false python-versions = ">=3.8.1" +groups = ["dev"] files = [ {file = "flake8-6.1.0-py2.py3-none-any.whl", hash = "sha256:ffdfce58ea94c6580c77888a86506937f9a1a227dfcd15f245d694ae20a6b6e5"}, {file = "flake8-6.1.0.tar.gz", hash = "sha256:d5b3857f07c030bdb5bf41c7f53799571d75c4491748a3adcd47de929e34cd23"}, @@ -1192,6 +1272,8 @@ version = "3.1.1" description = "A simple framework for building complex web applications." optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "flask-3.1.1-py3-none-any.whl", hash = "sha256:07aae2bb5eaf77993ef57e357491839f5fd9f4dc281593a81a9e4d79a24f295c"}, {file = "flask-3.1.1.tar.gz", hash = "sha256:284c7b8f2f58cb737f0cf1c30fd7eaf0ccfcde196099d24ecede3fc2005aa59e"}, @@ -1215,6 +1297,8 @@ version = "4.59.0" description = "Tools to manipulate font files" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "fonttools-4.59.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:524133c1be38445c5c0575eacea42dbd44374b310b1ffc4b60ff01d881fabb96"}, {file = "fonttools-4.59.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21e606b2d38fed938dde871c5736822dd6bda7a4631b92e509a1f5cd1b90c5df"}, @@ -1261,17 +1345,17 @@ files = [ ] [package.extras] -all = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres", "pycairo", "scipy", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0)", "xattr", "zopfli (>=0.1.4)"] +all = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\"", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0) ; python_version <= \"3.12\"", "xattr ; sys_platform == \"darwin\"", "zopfli (>=0.1.4)"] graphite = ["lz4 (>=1.7.4.2)"] -interpolatable = ["munkres", "pycairo", "scipy"] +interpolatable = ["munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\""] lxml = ["lxml (>=4.0)"] pathops = ["skia-pathops (>=0.5.0)"] plot = ["matplotlib"] repacker = ["uharfbuzz (>=0.23.0)"] symfont = ["sympy"] -type1 = ["xattr"] -unicode = ["unicodedata2 (>=15.1.0)"] -woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] +type1 = ["xattr ; sys_platform == \"darwin\""] +unicode = ["unicodedata2 (>=15.1.0) ; python_version <= \"3.12\""] +woff = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "zopfli (>=0.1.4)"] [[package]] name = "frozenlist" @@ -1279,6 +1363,7 @@ version = "1.5.0" description = "A list-like structure which implements collections.abc.MutableSequence" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5b6a66c18b5b9dd261ca98dffcb826a525334b2f29e7caa54e182255c5f6a65a"}, {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1b3eb7b05ea246510b43a7e53ed1653e55c2121019a97e60cad7efb881a97bb"}, @@ -1380,6 +1465,7 @@ version = "2025.3.0" description = "File-system specification" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "fsspec-2025.3.0-py3-none-any.whl", hash = "sha256:efb87af3efa9103f94ca91a7f8cb7a4df91af9f74fc106c9c7ea0efd7277c1b3"}, {file = "fsspec-2025.3.0.tar.gz", hash = "sha256:a935fd1ea872591f2b5148907d103488fc523295e6c64b835cfad8c3eca44972"}, @@ -1419,6 +1505,8 @@ version = "4.0.12" description = "Git Object Database" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, @@ -1433,6 +1521,8 @@ version = "3.1.45" description = "GitPython is a Python library used to interact with Git repositories" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77"}, {file = "gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c"}, @@ -1443,7 +1533,7 @@ gitdb = ">=4.0.1,<5" [package.extras] doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] [[package]] name = "google-api-core" @@ -1451,6 +1541,8 @@ version = "2.25.1" description = "Google API client core library" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"extra-proxy\"" files = [ {file = "google_api_core-2.25.1-py3-none-any.whl", hash = "sha256:8a2a56c1fef82987a524371f99f3bd0143702fecc670c72e600c1cda6bf8dbb7"}, {file = "google_api_core-2.25.1.tar.gz", hash = "sha256:d2aaa0b13c78c61cb3f4282c464c046e45fbd75755683c9c525e6e8f7ed0a5e8"}, @@ -1460,15 +1552,15 @@ files = [ google-auth = ">=2.14.1,<3.0.0" googleapis-common-protos = ">=1.56.2,<2.0.0" grpcio = [ - {version = ">=1.33.2,<2.0.0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0.0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, ] grpcio-status = [ - {version = ">=1.33.2,<2.0.0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0.0", optional = true, markers = "extra == \"grpc\""}, ] proto-plus = [ - {version = ">=1.22.3,<2.0.0", markers = "python_version < \"3.13\""}, + {version = ">=1.22.3,<2.0.0"}, {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" @@ -1476,7 +1568,7 @@ requests = ">=2.18.0,<3.0.0" [package.extras] async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.0)"] -grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0)", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0)"] +grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0) ; python_version >= \"3.11\""] grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] @@ -1486,6 +1578,8 @@ version = "2.40.3" description = "Google Authentication Library" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "python_version >= \"3.10\" and (extra == \"mlflow\" or extra == \"extra-proxy\") or extra == \"extra-proxy\"" files = [ {file = "google_auth-2.40.3-py2.py3-none-any.whl", hash = "sha256:1370d4593e86213563547f97a92752fc658456fe4514c809544f330fed45a7ca"}, {file = "google_auth-2.40.3.tar.gz", hash = "sha256:500c3a29adedeb36ea9cf24b8d10858e152f2412e3ca37829b3fa18e33d63b77"}, @@ -1499,19 +1593,44 @@ rsa = ">=3.1.4,<5" [package.extras] aiohttp = ["aiohttp (>=3.6.2,<4.0.0)", "requests (>=2.20.0,<3.0.0)"] enterprise-cert = ["cryptography", "pyopenssl"] -pyjwt = ["cryptography (<39.0.0)", "cryptography (>=38.0.3)", "pyjwt (>=2.0)"] -pyopenssl = ["cryptography (<39.0.0)", "cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +pyjwt = ["cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "pyjwt (>=2.0)"] +pyopenssl = ["cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] requests = ["requests (>=2.20.0,<3.0.0)"] -testing = ["aiohttp (<3.10.0)", "aiohttp (>=3.6.2,<4.0.0)", "aioresponses", "cryptography (<39.0.0)", "cryptography (>=38.0.3)", "flask", "freezegun", "grpcio", "mock", "oauth2client", "packaging", "pyjwt (>=2.0)", "pyopenssl (<24.3.0)", "pyopenssl (>=20.0.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.20.0,<3.0.0)", "responses", "urllib3"] +testing = ["aiohttp (<3.10.0)", "aiohttp (>=3.6.2,<4.0.0)", "aioresponses", "cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "flask", "freezegun", "grpcio", "mock", "oauth2client", "packaging", "pyjwt (>=2.0)", "pyopenssl (<24.3.0)", "pyopenssl (>=20.0.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.20.0,<3.0.0)", "responses", "urllib3"] urllib3 = ["packaging", "urllib3"] +[[package]] +name = "google-cloud-iam" +version = "2.19.1" +description = "Google Cloud Iam API client library" +optional = true +python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"extra-proxy\"" +files = [ + {file = "google_cloud_iam-2.19.1-py3-none-any.whl", hash = "sha256:11b08b86d82510021f9dd9f0beb5a08219e070deab09e28d4c0ce49f8c70997d"}, + {file = "google_cloud_iam-2.19.1.tar.gz", hash = "sha256:f059c369ad98af6be3401f0f5d087775d775fb96833be1e9ab8048c422fb1bf4"}, +] + +[package.dependencies] +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpc-google-iam-v1 = ">=0.12.4,<1.0.0" +proto-plus = [ + {version = ">=1.22.3,<2.0.0"}, + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, +] +protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" + [[package]] name = "google-cloud-kms" version = "2.24.2" description = "Google Cloud Kms API client library" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"extra-proxy\"" files = [ {file = "google_cloud_kms-2.24.2-py2.py3-none-any.whl", hash = "sha256:368209b035dfac691a467c1cf50986d8b1b26cac1166bdfbaa25d738df91ff7b"}, {file = "google_cloud_kms-2.24.2.tar.gz", hash = "sha256:e9e18bbfafd1a4035c76c03fb5ff03f4f57f596d08e1a9ede7e69ec0151b27a1"}, @@ -1530,10 +1649,12 @@ version = "1.70.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8"}, {file = "googleapis_common_protos-1.70.0.tar.gz", hash = "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257"}, ] +markers = {main = "extra == \"extra-proxy\""} [package.dependencies] grpcio = {version = ">=1.44.0,<2.0.0", optional = true, markers = "extra == \"grpc\""} @@ -1548,6 +1669,8 @@ version = "3.4.3" description = "GraphQL Framework for Python" optional = true python-versions = "*" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "graphene-3.4.3-py2.py3-none-any.whl", hash = "sha256:820db6289754c181007a150db1f7fff544b94142b556d12e3ebc777a7bf36c71"}, {file = "graphene-3.4.3.tar.gz", hash = "sha256:2a3786948ce75fe7e078443d37f609cbe5bb36ad8d6b828740ad3b95ed1a0aaa"}, @@ -1569,6 +1692,8 @@ version = "3.2.6" description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." optional = true python-versions = "<4,>=3.6" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "graphql_core-3.2.6-py3-none-any.whl", hash = "sha256:78b016718c161a6fb20a7d97bbf107f331cd1afe53e45566c59f776ed7f0b45f"}, {file = "graphql_core-3.2.6.tar.gz", hash = "sha256:c08eec22f9e40f0bd61d805907e3b3b1b9a320bc606e23dc145eebca07c8fbab"}, @@ -1580,6 +1705,8 @@ version = "3.2.0" description = "Relay library for graphql-core" optional = true python-versions = ">=3.6,<4" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "graphql-relay-3.2.0.tar.gz", hash = "sha256:1ff1c51298356e481a0be009ccdff249832ce53f30559c1338f22a0e0d17250c"}, {file = "graphql_relay-3.2.0-py3-none-any.whl", hash = "sha256:c9b22bd28b170ba1fe674c74384a8ff30a76c8e26f88ac3aa1584dd3179953e5"}, @@ -1594,6 +1721,8 @@ version = "3.2.3" description = "Lightweight in-process concurrent programming" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\") and extra == \"mlflow\" and python_version < \"3.14\"" files = [ {file = "greenlet-3.2.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:1afd685acd5597349ee6d7a88a8bec83ce13c106ac78c196ee9dde7c04fe87be"}, {file = "greenlet-3.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:761917cac215c61e9dc7324b2606107b3b292a8349bdebb31503ab4de3f559ac"}, @@ -1661,6 +1790,8 @@ version = "0.14.2" description = "IAM API client library" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"extra-proxy\"" files = [ {file = "grpc_google_iam_v1-0.14.2-py3-none-any.whl", hash = "sha256:a3171468459770907926d56a440b2bb643eec1d7ba215f48f3ecece42b4d8351"}, {file = "grpc_google_iam_v1-0.14.2.tar.gz", hash = "sha256:b3e1fc387a1a329e41672197d0ace9de22c78dd7d215048c4c78712073f7bd20"}, @@ -1677,6 +1808,7 @@ version = "1.70.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "grpcio-1.70.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:95469d1977429f45fe7df441f586521361e235982a0b39e33841549143ae2851"}, {file = "grpcio-1.70.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:ed9718f17fbdb472e33b869c77a16d0b55e166b100ec57b016dc7de9c8d236bf"}, @@ -1734,6 +1866,7 @@ files = [ {file = "grpcio-1.70.0-cp39-cp39-win_amd64.whl", hash = "sha256:a31d7e3b529c94e930a117b2175b2efd179d96eb3c7a21ccb0289a8ab05b645c"}, {file = "grpcio-1.70.0.tar.gz", hash = "sha256:8d1584a68d5922330025881e63a6c1b54cc8117291d382e4fa69339b6d914c56"}, ] +markers = {main = "extra == \"extra-proxy\""} [package.extras] protobuf = ["grpcio-tools (>=1.70.0)"] @@ -1744,6 +1877,8 @@ version = "1.62.3" description = "Status proto mapping for gRPC" optional = true python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"extra-proxy\"" files = [ {file = "grpcio-status-1.62.3.tar.gz", hash = "sha256:289bdd7b2459794a12cf95dc0cb727bd4a1742c37bd823f760236c937e53a485"}, {file = "grpcio_status-1.62.3-py3-none-any.whl", hash = "sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8"}, @@ -1760,6 +1895,8 @@ version = "23.0.0" description = "WSGI HTTP Server for UNIX" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "platform_system != \"Windows\" and (extra == \"mlflow\" or extra == \"proxy\") and python_version >= \"3.10\" or extra == \"proxy\"" files = [ {file = "gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d"}, {file = "gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec"}, @@ -1781,6 +1918,7 @@ version = "0.16.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, @@ -1792,6 +1930,7 @@ version = "4.1.0" description = "HTTP/2 State-Machine based protocol implementation" optional = false python-versions = ">=3.6.1" +groups = ["proxy-dev"] files = [ {file = "h2-4.1.0-py3-none-any.whl", hash = "sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d"}, {file = "h2-4.1.0.tar.gz", hash = "sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb"}, @@ -1807,6 +1946,8 @@ version = "1.1.5" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ {file = "hf_xet-1.1.5-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23"}, {file = "hf_xet-1.1.5-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:9fa6e3ee5d61912c4a113e0708eaaef987047616465ac7aa30f7121a48fc1af8"}, @@ -1827,6 +1968,7 @@ version = "4.0.0" description = "Pure-Python HPACK header compression" optional = false python-versions = ">=3.6.1" +groups = ["proxy-dev"] files = [ {file = "hpack-4.0.0-py3-none-any.whl", hash = "sha256:84a076fad3dc9a9f8063ccb8041ef100867b1878b25ef0ee63847a5d53818a6c"}, {file = "hpack-4.0.0.tar.gz", hash = "sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095"}, @@ -1838,6 +1980,7 @@ version = "1.0.9" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, @@ -1859,6 +2002,7 @@ version = "0.28.1" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, @@ -1871,7 +2015,7 @@ httpcore = "==1.*" idna = "*" [package.extras] -brotli = ["brotli", "brotlicffi"] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] @@ -1883,6 +2027,8 @@ version = "0.4.1" description = "Consume Server-Sent Event (SSE) messages with HTTPX." optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"proxy\"" files = [ {file = "httpx_sse-0.4.1-py3-none-any.whl", hash = "sha256:cba42174344c3a5b06f255ce65b350880f962d99ead85e776f23c6618a377a37"}, {file = "httpx_sse-0.4.1.tar.gz", hash = "sha256:8f44d34414bc7b21bf3602713005c5df4917884f76072479b21f68befa4ea26e"}, @@ -1894,6 +2040,7 @@ version = "0.34.3" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.8.0" +groups = ["main"] files = [ {file = "huggingface_hub-0.34.3-py3-none-any.whl", hash = "sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492"}, {file = "huggingface_hub-0.34.3.tar.gz", hash = "sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853"}, @@ -1910,16 +2057,16 @@ tqdm = ">=4.42.1" typing-extensions = ">=3.7.4.3" [package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1.3.2)", "fastapi", "gradio (>=4.0.0)", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "mypy (>=1.14.1,<1.15.0)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1.3.2)", "fastapi", "gradio (>=4.0.0)", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0) ; python_version >= \"3.9\"", "mypy (>=1.14.1,<1.15.0) ; python_version == \"3.8\"", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1.3.2)", "fastapi", "gradio (>=4.0.0)", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "mypy (>=1.14.1,<1.15.0)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1.3.2)", "fastapi", "gradio (>=4.0.0)", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0) ; python_version >= \"3.9\"", "mypy (>=1.14.1,<1.15.0) ; python_version == \"3.8\"", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] hf-transfer = ["hf-transfer (>=0.1.4)"] hf-xet = ["hf-xet (>=1.1.2,<2.0.0)"] inference = ["aiohttp"] mcp = ["aiohttp", "mcp (>=1.8.0)", "typer"] oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "mypy (>=1.14.1,<1.15.0)", "ruff (>=0.9.0)"] +quality = ["libcst (>=1.4.0)", "mypy (==1.15.0) ; python_version >= \"3.9\"", "mypy (>=1.14.1,<1.15.0) ; python_version == \"3.8\"", "ruff (>=0.9.0)"] tensorflow = ["graphviz", "pydot", "tensorflow"] tensorflow-testing = ["keras (<3.0)", "tensorflow"] testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1.3.2)", "fastapi", "gradio (>=4.0.0)", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] @@ -1932,6 +2079,8 @@ version = "10.0" description = "Human friendly output for text interfaces using Python" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["main"] +markers = "python_version >= \"3.9\" and extra == \"extra-proxy\" and python_version < \"3.14\"" files = [ {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, @@ -1946,6 +2095,7 @@ version = "0.15.0" description = "A ASGI Server based on Hyper libraries and inspired by Gunicorn" optional = false python-versions = ">=3.7" +groups = ["proxy-dev"] files = [ {file = "hypercorn-0.15.0-py3-none-any.whl", hash = "sha256:5008944999612fd188d7a1ca02e89d20065642b89503020ac392dfed11840730"}, {file = "hypercorn-0.15.0.tar.gz", hash = "sha256:d517f68d5dc7afa9a9d50ecefb0f769f466ebe8c1c18d2c2f447a24e763c9a63"}, @@ -1963,7 +2113,7 @@ wsproto = ">=0.14.0" docs = ["pydata_sphinx_theme", "sphinxcontrib_mermaid"] h3 = ["aioquic (>=0.9.0,<1.0)"] trio = ["exceptiongroup (>=1.1.0)", "trio (>=0.22.0)"] -uvloop = ["uvloop"] +uvloop = ["uvloop ; platform_system != \"Windows\""] [[package]] name = "hyperframe" @@ -1971,6 +2121,7 @@ version = "6.0.1" description = "HTTP/2 framing layer for Python" optional = false python-versions = ">=3.6.1" +groups = ["proxy-dev"] files = [ {file = "hyperframe-6.0.1-py3-none-any.whl", hash = "sha256:0ec6bafd80d8ad2195c4f03aacba3a8265e57bc4cff261e802bf39970ed02a15"}, {file = "hyperframe-6.0.1.tar.gz", hash = "sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914"}, @@ -1982,6 +2133,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -1996,6 +2148,8 @@ version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, @@ -2007,6 +2161,7 @@ version = "6.11.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "importlib_metadata-6.11.0-py3-none-any.whl", hash = "sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b"}, {file = "importlib_metadata-6.11.0.tar.gz", hash = "sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443"}, @@ -2018,7 +2173,7 @@ zipp = ">=0.5" [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] +testing = ["flufl.flake8", "importlib-resources (>=1.3) ; python_version < \"3.9\"", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\"", "pytest-perf (>=0.9.2)", "pytest-ruff"] [[package]] name = "importlib-resources" @@ -2026,6 +2181,8 @@ version = "6.4.5" description = "Read resources from Python packages" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "python_version < \"3.9\"" files = [ {file = "importlib_resources-6.4.5-py3-none-any.whl", hash = "sha256:ac29d5f956f01d5e4bb63102a5a19957f1b9175e45649977264a1416783bb717"}, {file = "importlib_resources-6.4.5.tar.gz", hash = "sha256:980862a1d16c9e147a59603677fa2aa5fd82b87f223b6cb870695bcfce830065"}, @@ -2035,7 +2192,7 @@ files = [ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] @@ -2048,6 +2205,7 @@ version = "2.1.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, @@ -2059,6 +2217,8 @@ version = "0.7.2" description = "An ISO 8601 date/time/duration parser and formatter" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"extra-proxy\" or extra == \"proxy\"" files = [ {file = "isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15"}, {file = "isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6"}, @@ -2070,6 +2230,8 @@ version = "2.2.0" description = "Safely pass data to untrusted environments and back." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef"}, {file = "itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173"}, @@ -2081,6 +2243,7 @@ version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" +groups = ["main", "proxy-dev"] files = [ {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, @@ -2098,6 +2261,7 @@ version = "0.9.1" description = "Fast iterable JSON parser." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "jiter-0.9.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c0163baa7ee85860fdc14cc39263014500df901eeffdf94c1eab9a2d713b2a9d"}, {file = "jiter-0.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:514d4dd845e0af4da15112502e6fcb952f0721f27f17e530454e379472b90c14"}, @@ -2183,6 +2347,8 @@ version = "1.0.1" description = "JSON Matching Expressions" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, @@ -2194,6 +2360,8 @@ version = "1.5.1" description = "Lightweight pipelining with Python functions" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "joblib-1.5.1-py3-none-any.whl", hash = "sha256:4719a31f054c7d766948dcd83e9613686b27114f190f717cec7eaa2084f8a74a"}, {file = "joblib-1.5.1.tar.gz", hash = "sha256:f4f86e351f39fe3d0d32a9f2c3d8af1ee4cec285aafcb27003dda5205576b444"}, @@ -2205,6 +2373,7 @@ version = "4.23.0" description = "An implementation of JSON Schema validation for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, @@ -2228,6 +2397,7 @@ version = "2023.12.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"}, {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"}, @@ -2243,6 +2413,8 @@ version = "1.4.8" description = "A fast implementation of the Cassowary constraint solver" optional = true python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "kiwisolver-1.4.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88c6f252f6816a73b1f8c904f7bbe02fd67c09a69f7cb8a0eecdbf5ce78e63db"}, {file = "kiwisolver-1.4.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c72941acb7b67138f35b879bbe85be0f6c6a70cab78fe3ef6db9c024d9223e5b"}, @@ -2332,6 +2504,7 @@ version = "2.54.1" description = "A client library for accessing langfuse" optional = false python-versions = "<4.0,>=3.8.1" +groups = ["dev"] files = [ {file = "langfuse-2.54.1-py3-none-any.whl", hash = "sha256:1f1261cf763886758c70e192133340ff296169cc0930cde725eee52d467eb661"}, {file = "langfuse-2.54.1.tar.gz", hash = "sha256:7efc70799740ffa0ac7e04066e0596fb6433e8e501fc850c6a4e7967de6de8a7"}, @@ -2353,13 +2526,14 @@ openai = ["openai (>=0.27.8)"] [[package]] name = "litellm-enterprise" -version = "0.1.16" +version = "0.1.19" description = "Package for LiteLLM Enterprise features" optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ - {file = "litellm_enterprise-0.1.16-py3-none-any.whl", hash = "sha256:ceccc8cb579e06fb12c1d209065064188336305be6d024cb050d44e0b5ad9cf3"}, - {file = "litellm_enterprise-0.1.16.tar.gz", hash = "sha256:726194d3c3e8b154912ef021253a4a1dd6cb9ffa7f5249cd32c59c7c1235b3a8"}, + {file = "litellm_enterprise-0.1.19.tar.gz", hash = "sha256:a70794a9c66f069f6eb73b283639f783ac4138ec2684058a696e8d6210cdc4fa"}, ] [[package]] @@ -2368,6 +2542,8 @@ version = "0.2.15" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "litellm_proxy_extras-0.2.15-py3-none-any.whl", hash = "sha256:25e7d7cabe3f10233e2802d4521576bef438eea8b0800b1dbf38ba83ba751bb8"}, {file = "litellm_proxy_extras-0.2.15.tar.gz", hash = "sha256:62a9fdcb77d25aa7bfdfa04ac878e452f185dfb0b9538d4d8988a1b512360649"}, @@ -2379,6 +2555,8 @@ version = "1.3.10" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59"}, {file = "mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28"}, @@ -2398,6 +2576,8 @@ version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, @@ -2422,6 +2602,7 @@ version = "2.1.5" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.7" +groups = ["main", "proxy-dev"] files = [ {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, @@ -2491,6 +2672,8 @@ version = "3.10.5" description = "Python plotting package" optional = true python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "matplotlib-3.10.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:5d4773a6d1c106ca05cb5a5515d277a6bb96ed09e5c8fab6b7741b8fcaa62c8f"}, {file = "matplotlib-3.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dc88af74e7ba27de6cbe6faee916024ea35d895ed3d61ef6f58c4ce97da7185a"}, @@ -2569,6 +2752,7 @@ version = "0.7.0" description = "McCabe checker, plugin for flake8" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, @@ -2580,6 +2764,8 @@ version = "1.12.3" description = "Model Context Protocol SDK" optional = true python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"proxy\"" files = [ {file = "mcp-1.12.3-py3-none-any.whl", hash = "sha256:5483345bf39033b858920a5b6348a303acacf45b23936972160ff152107b850e"}, {file = "mcp-1.12.3.tar.gz", hash = "sha256:ab2e05f5e5c13e1dc90a4a9ef23ac500a6121362a564447855ef0ab643a99fed"}, @@ -2609,6 +2795,8 @@ version = "0.1.2" description = "Markdown URL utilities" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -2620,6 +2808,8 @@ version = "0.4.1" description = "" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.9\" and extra == \"extra-proxy\" and python_version < \"3.14\"" files = [ {file = "ml_dtypes-0.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1fe8b5b5e70cd67211db94b05cfd58dace592f24489b038dc6f9fe347d2e07d5"}, {file = "ml_dtypes-0.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c09a6d11d8475c2a9fd2bc0695628aec105f97cab3b3a3fb7c9660348ff7d24"}, @@ -2642,10 +2832,10 @@ files = [ [package.dependencies] numpy = [ - {version = ">1.20", markers = "python_version < \"3.10\""}, + {version = ">=1.23.3", markers = "python_version >= \"3.11\""}, + {version = ">1.20"}, + {version = ">=1.21.2", markers = "python_version >= \"3.10\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, - {version = ">=1.23.3", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, - {version = ">=1.21.2", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, ] [package.extras] @@ -2657,6 +2847,8 @@ version = "3.2.0rc0" description = "MLflow is an open source platform for the complete machine learning lifecycle" optional = true python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "mlflow-3.2.0rc0-py3-none-any.whl", hash = "sha256:c6b8bf6cf03ff292885cbb01f0c30cb8e80a8f21af96eb9ba0030ed8a8b7ee59"}, {file = "mlflow-3.2.0rc0.tar.gz", hash = "sha256:5e6b3499a8e1b331c7806c64c5e1e44cdb8640e4d438be450cf8f82baca48c42"}, @@ -2698,6 +2890,8 @@ version = "3.2.0rc0" description = "MLflow is an open source platform for the complete machine learning lifecycle" optional = true python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "mlflow_skinny-3.2.0rc0-py3-none-any.whl", hash = "sha256:eb832fbc5232c9917f61c4d51ba6f6294b475e48e7dc747b98d0f55b701cc198"}, {file = "mlflow_skinny-3.2.0rc0.tar.gz", hash = "sha256:1321228d91a01a4fbdfb17083f51b7d475934da4df7fe3637523b6d7949a81e5"}, @@ -2741,6 +2935,8 @@ version = "3.2.0rc0" description = "MLflow Tracing SDK is an open-source, lightweight Python package that only includes the minimum set of dependencies and functionality to instrument your code/models/agents with MLflow Tracing." optional = true python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "mlflow_tracing-3.2.0rc0-py3-none-any.whl", hash = "sha256:4bb5e24e1bee4d16037c985710bf2b5b0589e6e77aef36ca7f96279f4cb6ed46"}, {file = "mlflow_tracing-3.2.0rc0.tar.gz", hash = "sha256:30b487ef6443bffa2f24a0c53e81283d05347c587742efa89c499bbbe049d2d4"}, @@ -2761,6 +2957,7 @@ version = "1.33.0" description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." optional = false python-versions = ">=3.7" +groups = ["main", "proxy-dev"] files = [ {file = "msal-1.33.0-py3-none-any.whl", hash = "sha256:c0cd41cecf8eaed733ee7e3be9e040291eba53b0f262d3ae9c58f38b04244273"}, {file = "msal-1.33.0.tar.gz", hash = "sha256:836ad80faa3e25a7d71015c990ce61f704a87328b1e73bcbb0623a18cbf17510"}, @@ -2772,7 +2969,7 @@ PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} requests = ">=2.0.0,<3" [package.extras] -broker = ["pymsalruntime (>=0.14,<0.19)", "pymsalruntime (>=0.17,<0.19)", "pymsalruntime (>=0.18,<0.19)"] +broker = ["pymsalruntime (>=0.14,<0.19) ; python_version >= \"3.6\" and platform_system == \"Windows\"", "pymsalruntime (>=0.17,<0.19) ; python_version >= \"3.8\" and platform_system == \"Darwin\"", "pymsalruntime (>=0.18,<0.19) ; python_version >= \"3.8\" and platform_system == \"Linux\""] [[package]] name = "msal-extensions" @@ -2780,6 +2977,7 @@ version = "1.3.0" description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." optional = false python-versions = ">=3.7" +groups = ["main", "proxy-dev"] files = [ {file = "msal_extensions-1.3.0-py3-none-any.whl", hash = "sha256:105328ddcbdd342016c9949d8f89e3917554740c8ab26669c0fa0e069e730a0e"}, {file = "msal_extensions-1.3.0.tar.gz", hash = "sha256:96918996642b38c78cd59b55efa0f06fd1373c90e0949be8615697c048fba62c"}, @@ -2797,6 +2995,7 @@ version = "6.1.0" description = "multidict implementation" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"}, {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"}, @@ -2901,6 +3100,7 @@ version = "1.14.1" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mypy-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:52686e37cf13d559f668aa398dd7ddf1f92c5d613e4f8cb262be2fb4fedb0fcb"}, {file = "mypy-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1fb545ca340537d4b45d3eecdb3def05e913299ca72c290326be19b3804b39c0"}, @@ -2960,6 +3160,7 @@ version = "1.1.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, @@ -2971,6 +3172,7 @@ version = "1.9.1" description = "Node.js virtual environment builder" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "proxy-dev"] files = [ {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, @@ -2982,6 +3184,8 @@ version = "1.26.4" description = "Fundamental package for array computing in Python" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.9\" and (python_version >= \"3.10\" or extra == \"extra-proxy\" or extra == \"semantic-router\") and (python_version < \"3.14\" or extra == \"semantic-router\" or extra == \"mlflow\") and (extra == \"extra-proxy\" or extra == \"semantic-router\" or extra == \"mlflow\")" files = [ {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, @@ -3027,6 +3231,8 @@ version = "1.7.0" description = "Sphinx extension to support docstrings in Numpy format" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "numpydoc-1.7.0-py3-none-any.whl", hash = "sha256:5a56419d931310d79a06cfc2a126d1558700feeb9b4f3d8dcae1a8134be829c9"}, {file = "numpydoc-1.7.0.tar.gz", hash = "sha256:866e5ae5b6509dcf873fc6381120f5c31acf13b135636c1a81d68c166a95f921"}, @@ -3038,7 +3244,7 @@ tabulate = ">=0.8.10" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [package.extras] -developer = ["pre-commit (>=3.3)", "tomli"] +developer = ["pre-commit (>=3.3)", "tomli ; python_version < \"3.11\""] doc = ["matplotlib (>=3.5)", "numpy (>=1.22)", "pydata-sphinx-theme (>=0.13.3)", "sphinx (>=7)"] test = ["matplotlib", "pytest", "pytest-cov"] @@ -3048,6 +3254,8 @@ version = "3.3.1" description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1"}, {file = "oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9"}, @@ -3064,6 +3272,7 @@ version = "1.98.0" description = "The official Python library for the openai API" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "openai-1.98.0-py3-none-any.whl", hash = "sha256:b99b794ef92196829120e2df37647722104772d2a74d08305df9ced5f26eae34"}, {file = "openai-1.98.0.tar.gz", hash = "sha256:3ee0fcc50ae95267fd22bd1ad095ba5402098f3df2162592e68109999f685427"}, @@ -3091,10 +3300,12 @@ version = "1.25.0" description = "OpenTelemetry Python API" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "opentelemetry_api-1.25.0-py3-none-any.whl", hash = "sha256:757fa1aa020a0f8fa139f8959e53dec2051cc26b832e76fa839a6d76ecefd737"}, {file = "opentelemetry_api-1.25.0.tar.gz", hash = "sha256:77c4985f62f2614e42ce77ee4c9da5fa5f0bc1e1821085e9a47533a9323ae869"}, ] +markers = {main = "python_version >= \"3.10\""} [package.dependencies] deprecated = ">=1.2.6" @@ -3106,6 +3317,7 @@ version = "1.25.0" description = "OpenTelemetry Collector Exporters" optional = false python-versions = ">=3.8" +groups = ["dev", "proxy-dev"] files = [ {file = "opentelemetry_exporter_otlp-1.25.0-py3-none-any.whl", hash = "sha256:d67a831757014a3bc3174e4cd629ae1493b7ba8d189e8a007003cacb9f1a6b60"}, {file = "opentelemetry_exporter_otlp-1.25.0.tar.gz", hash = "sha256:ce03199c1680a845f82e12c0a6a8f61036048c07ec7a0bd943142aca8fa6ced0"}, @@ -3121,6 +3333,7 @@ version = "1.25.0" description = "OpenTelemetry Protobuf encoding" optional = false python-versions = ">=3.8" +groups = ["dev", "proxy-dev"] files = [ {file = "opentelemetry_exporter_otlp_proto_common-1.25.0-py3-none-any.whl", hash = "sha256:15637b7d580c2675f70246563363775b4e6de947871e01d0f4e3881d1848d693"}, {file = "opentelemetry_exporter_otlp_proto_common-1.25.0.tar.gz", hash = "sha256:c93f4e30da4eee02bacd1e004eb82ce4da143a2f8e15b987a9f603e0a85407d3"}, @@ -3135,6 +3348,7 @@ version = "1.25.0" description = "OpenTelemetry Collector Protobuf over gRPC Exporter" optional = false python-versions = ">=3.8" +groups = ["dev", "proxy-dev"] files = [ {file = "opentelemetry_exporter_otlp_proto_grpc-1.25.0-py3-none-any.whl", hash = "sha256:3131028f0c0a155a64c430ca600fd658e8e37043cb13209f0109db5c1a3e4eb4"}, {file = "opentelemetry_exporter_otlp_proto_grpc-1.25.0.tar.gz", hash = "sha256:c0b1661415acec5af87625587efa1ccab68b873745ca0ee96b69bb1042087eac"}, @@ -3155,6 +3369,7 @@ version = "1.25.0" description = "OpenTelemetry Collector Protobuf over HTTP Exporter" optional = false python-versions = ">=3.8" +groups = ["dev", "proxy-dev"] files = [ {file = "opentelemetry_exporter_otlp_proto_http-1.25.0-py3-none-any.whl", hash = "sha256:2eca686ee11b27acd28198b3ea5e5863a53d1266b91cda47c839d95d5e0541a6"}, {file = "opentelemetry_exporter_otlp_proto_http-1.25.0.tar.gz", hash = "sha256:9f8723859e37c75183ea7afa73a3542f01d0fd274a5b97487ea24cb683d7d684"}, @@ -3175,6 +3390,7 @@ version = "1.25.0" description = "OpenTelemetry Python Proto" optional = false python-versions = ">=3.8" +groups = ["dev", "proxy-dev"] files = [ {file = "opentelemetry_proto-1.25.0-py3-none-any.whl", hash = "sha256:f07e3341c78d835d9b86665903b199893befa5e98866f63d22b00d0b7ca4972f"}, {file = "opentelemetry_proto-1.25.0.tar.gz", hash = "sha256:35b6ef9dc4a9f7853ecc5006738ad40443701e52c26099e197895cbda8b815a3"}, @@ -3189,10 +3405,12 @@ version = "1.25.0" description = "OpenTelemetry Python SDK" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "opentelemetry_sdk-1.25.0-py3-none-any.whl", hash = "sha256:d97ff7ec4b351692e9d5a15af570c693b8715ad78b8aafbec5c7100fe966b4c9"}, {file = "opentelemetry_sdk-1.25.0.tar.gz", hash = "sha256:ce7fc319c57707ef5bf8b74fb9f8ebdb8bfafbe11898410e0d2a761d08a98ec7"}, ] +markers = {main = "python_version >= \"3.10\""} [package.dependencies] opentelemetry-api = "1.25.0" @@ -3205,10 +3423,12 @@ version = "0.46b0" description = "OpenTelemetry Semantic Conventions" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "opentelemetry_semantic_conventions-0.46b0-py3-none-any.whl", hash = "sha256:6daef4ef9fa51d51855d9f8e0ccd3a1bd59e0e545abe99ac6203804e36ab3e07"}, {file = "opentelemetry_semantic_conventions-0.46b0.tar.gz", hash = "sha256:fbc982ecbb6a6e90869b15c1673be90bd18c8a56ff1cffc0864e38e2edffaefa"}, ] +markers = {main = "python_version >= \"3.10\""} [package.dependencies] opentelemetry-api = "1.25.0" @@ -3219,6 +3439,8 @@ version = "3.10.15" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "orjson-3.10.15-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:552c883d03ad185f720d0c09583ebde257e41b9521b74ff40e08b7dec4559c04"}, {file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e3e8d438d02e4854f70bfdc03a6bcdb697358dbaa6bcd19cbe24d24ece1f8"}, @@ -3307,6 +3529,7 @@ version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, @@ -3318,6 +3541,8 @@ version = "2.3.1" description = "Powerful data structures for data analysis, time series, and statistics" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "pandas-2.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:22c2e866f7209ebc3a8f08d75766566aae02bcc91d196935a1d9e59c7b990ac9"}, {file = "pandas-2.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3583d348546201aff730c8c47e49bc159833f971c2899d6097bce68b9112a4f1"}, @@ -3365,9 +3590,9 @@ files = [ [package.dependencies] numpy = [ + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.22.4", markers = "python_version < \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -3404,6 +3629,7 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -3415,6 +3641,8 @@ version = "11.3.0" description = "Python Imaging Library (Fork)" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860"}, {file = "pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad"}, @@ -3530,7 +3758,7 @@ fpx = ["olefile"] mic = ["olefile"] test-arrow = ["pyarrow"] tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "trove-classifiers (>=2024.10.12)"] -typing = ["typing-extensions"] +typing = ["typing-extensions ; python_version < \"3.10\""] xmp = ["defusedxml"] [[package]] @@ -3539,6 +3767,8 @@ version = "1.3.10" description = "Resolve a name to an object." optional = false python-versions = ">=3.6" +groups = ["main"] +markers = "python_version < \"3.9\"" files = [ {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"}, {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, @@ -3550,6 +3780,7 @@ version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, @@ -3566,6 +3797,7 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -3581,6 +3813,8 @@ version = "1.32.0" description = "Blazingly fast DataFrame library" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"proxy\"" files = [ {file = "polars-1.32.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:94f7c6a3b30bc99bc6b682ea42bb1ae983e33a302ca21aacbac50ae19e34fcf2"}, {file = "polars-1.32.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:8bf14c16164839e62c741a863942a94a9a463db21e797452fca996c8afaf8827"}, @@ -3614,7 +3848,7 @@ pyarrow = ["pyarrow (>=7.0.0)"] pydantic = ["pydantic"] sqlalchemy = ["polars[pandas]", "sqlalchemy"] style = ["great-tables (>=0.8.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; platform_system == \"Windows\""] xlsx2csv = ["xlsx2csv (>=0.8.0)"] xlsxwriter = ["xlsxwriter"] @@ -3624,6 +3858,7 @@ version = "2.0.0" description = "A pure-Python implementation of the HTTP/2 priority tree" optional = false python-versions = ">=3.6.1" +groups = ["proxy-dev"] files = [ {file = "priority-2.0.0-py3-none-any.whl", hash = "sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa"}, {file = "priority-2.0.0.tar.gz", hash = "sha256:c965d54f1b8d0d0b19479db3924c7c36cf672dbf2aec92d43fbdaf4492ba18c0"}, @@ -3635,6 +3870,7 @@ version = "0.11.0" description = "Prisma Client Python is an auto-generated and fully type-safe database client" optional = false python-versions = ">=3.7.0" +groups = ["main", "proxy-dev"] files = [ {file = "prisma-0.11.0-py3-none-any.whl", hash = "sha256:22bb869e59a2968b99f3483bb417717273ffbc569fd1e9ceed95e5614cbaf53a"}, {file = "prisma-0.11.0.tar.gz", hash = "sha256:3f2f2fd2361e1ec5ff655f2a04c7860c2f2a5bc4c91f78ca9c5c6349735bf693"}, @@ -3660,6 +3896,7 @@ version = "0.20.0" description = "Python client for the Prometheus monitoring system." optional = false python-versions = ">=3.8" +groups = ["proxy-dev"] files = [ {file = "prometheus_client-0.20.0-py3-none-any.whl", hash = "sha256:cde524a85bce83ca359cc837f28b8c0db5cac7aa653a588fd7e84ba061c329e7"}, {file = "prometheus_client-0.20.0.tar.gz", hash = "sha256:287629d00b147a32dcb2be0b9df905da599b2d82f80377083ec8463309a4bb89"}, @@ -3674,6 +3911,7 @@ version = "0.2.0" description = "Accelerated property cache" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5869b8fd70b81835a6f187c5fdbe67917a04d7e52b6e7cc4e5fe39d55c39d58"}, {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:952e0d9d07609d9c5be361f33b0d6d650cd2bae393aabb11d9b719364521984b"}, @@ -3781,6 +4019,8 @@ version = "1.26.1" description = "Beautiful, Pythonic protocol buffers" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"extra-proxy\"" files = [ {file = "proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66"}, {file = "proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012"}, @@ -3798,6 +4038,7 @@ version = "4.25.8" description = "" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "protobuf-4.25.8-cp310-abi3-win32.whl", hash = "sha256:504435d831565f7cfac9f0714440028907f1975e4bed228e58e72ecfff58a1e0"}, {file = "protobuf-4.25.8-cp310-abi3-win_amd64.whl", hash = "sha256:bd551eb1fe1d7e92c1af1d75bdfa572eff1ab0e5bf1736716814cdccdb2360f9"}, @@ -3811,6 +4052,7 @@ files = [ {file = "protobuf-4.25.8-py3-none-any.whl", hash = "sha256:15a0af558aa3b13efef102ae6e4f3efac06f1eea11afb3a57db2901447d9fb59"}, {file = "protobuf-4.25.8.tar.gz", hash = "sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd"}, ] +markers = {main = "python_version >= \"3.10\" and (extra == \"mlflow\" or extra == \"extra-proxy\") or extra == \"extra-proxy\""} [[package]] name = "pyarrow" @@ -3818,6 +4060,8 @@ version = "21.0.0" description = "Python library for Apache Arrow" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e563271e2c5ff4d4a4cbeb2c83d5cf0d4938b891518e676025f7268c6fe5fe26"}, {file = "pyarrow-21.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:fee33b0ca46f4c85443d6c450357101e47d53e6c3f008d658c27a2d020d44c79"}, @@ -3873,6 +4117,8 @@ version = "0.6.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and (extra == \"mlflow\" or extra == \"extra-proxy\") or extra == \"extra-proxy\"" files = [ {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, @@ -3884,6 +4130,8 @@ version = "0.4.2" description = "A collection of ASN.1-based protocols modules" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and (extra == \"mlflow\" or extra == \"extra-proxy\") or extra == \"extra-proxy\"" files = [ {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, @@ -3898,6 +4146,7 @@ version = "2.11.1" description = "Python style guide checker" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, @@ -3909,10 +4158,12 @@ version = "2.22" description = "C parser in Python" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] +markers = {main = "platform_python_implementation != \"PyPy\" or extra == \"proxy\"", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""} [[package]] name = "pydantic" @@ -3920,6 +4171,7 @@ version = "2.10.6" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "pydantic-2.10.6-py3-none-any.whl", hash = "sha256:427d664bf0b8a2b34ff5dd0f5a18df00591adcee7198fbd71981054cef37b584"}, {file = "pydantic-2.10.6.tar.gz", hash = "sha256:ca5daa827cce33de7a42be142548b0096bf05a7e7b365aebfa5f8eeec7128236"}, @@ -3933,7 +4185,7 @@ typing-extensions = ">=4.12.2" [package.extras] email = ["email-validator (>=2.0.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""] [[package]] name = "pydantic-core" @@ -3941,6 +4193,7 @@ version = "2.27.2" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "pydantic_core-2.27.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2d367ca20b2f14095a8f4fa1210f5a7b78b8a20009ecced6b12818f455b1e9fa"}, {file = "pydantic_core-2.27.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:491a2b73db93fab69731eaee494f320faa4e093dbed776be1a829c2eb222c34c"}, @@ -4053,6 +4306,8 @@ version = "2.10.1" description = "Settings management using Pydantic" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"proxy\"" files = [ {file = "pydantic_settings-2.10.1-py3-none-any.whl", hash = "sha256:a60952460b99cf661dc25c29c0ef171721f98bfcb52ef8d9ea4c943d7c8cc796"}, {file = "pydantic_settings-2.10.1.tar.gz", hash = "sha256:06f0062169818d0f5524420a360d632d5857b83cffd4d42fe29597807a1614ee"}, @@ -4076,6 +4331,7 @@ version = "3.1.0" description = "passive checker of Python programs" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pyflakes-3.1.0-py2.py3-none-any.whl", hash = "sha256:4132f6d49cb4dae6819e5379898f2b8cce3c5f23994194c24b77d5da2e36f774"}, {file = "pyflakes-3.1.0.tar.gz", hash = "sha256:a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc"}, @@ -4087,6 +4343,8 @@ version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"utils\" or extra == \"proxy\"" files = [ {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, @@ -4101,6 +4359,7 @@ version = "2.9.0" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.8" +groups = ["main", "proxy-dev"] files = [ {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, @@ -4121,6 +4380,8 @@ version = "1.5.0" description = "Python binding to the Networking and Cryptography (NaCl) library" optional = true python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"}, {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"}, @@ -4147,6 +4408,8 @@ version = "3.2.3" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf"}, {file = "pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be"}, @@ -4161,6 +4424,8 @@ version = "3.5.4" description = "A python implementation of GNU readline." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.9\" and sys_platform == \"win32\" and extra == \"extra-proxy\" and python_version < \"3.14\"" files = [ {file = "pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6"}, {file = "pyreadline3-3.5.4.tar.gz", hash = "sha256:8d57d53039a1c75adba8e50dd3d992b28143480816187ea5efbd5c78e6c885b7"}, @@ -4175,6 +4440,7 @@ version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, @@ -4197,6 +4463,7 @@ version = "0.21.2" description = "Pytest support for asyncio" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "pytest_asyncio-0.21.2-py3-none-any.whl", hash = "sha256:ab664c88bb7998f711d8039cacd4884da6430886ae8bbd4eded552ed2004f16b"}, {file = "pytest_asyncio-0.21.2.tar.gz", hash = "sha256:d67738fc232b94b326b9d060750beb16e0074210b98dd8b58a5239fa2a154f45"}, @@ -4215,6 +4482,7 @@ version = "3.14.1" description = "Thin-wrapper around the mock package for easier use with pytest" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pytest_mock-3.14.1-py3-none-any.whl", hash = "sha256:178aefcd11307d874b4cd3100344e7e2d888d9791a6a1d9bfe90fbc1b74fd1d0"}, {file = "pytest_mock-3.14.1.tar.gz", hash = "sha256:159e9edac4c451ce77a5cdb9fc5d1100708d2dd4ba3c3df572f14097351af80e"}, @@ -4232,6 +4500,8 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +markers = "python_version >= \"3.10\" and (extra == \"mlflow\" or extra == \"proxy\") or extra == \"proxy\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -4246,6 +4516,7 @@ version = "1.0.1" description = "Read key-value pairs from a .env file and set them as environment variables" optional = false python-versions = ">=3.8" +groups = ["main", "proxy-dev"] files = [ {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, @@ -4260,6 +4531,8 @@ version = "0.0.18" description = "A streaming multipart parser for Python" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "python_multipart-0.0.18-py3-none-any.whl", hash = "sha256:efe91480f485f6a361427a541db4796f9e1591afc0fb8e7a4ba06bfbc6708996"}, {file = "python_multipart-0.0.18.tar.gz", hash = "sha256:7a68db60c8bfb82e460637fa4750727b45af1d5e2ed215593f917f64694d34fe"}, @@ -4271,6 +4544,8 @@ version = "3.0.0" description = "Universally unique lexicographically sortable identifier" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.9\" and extra == \"extra-proxy\" and python_version < \"3.14\"" files = [ {file = "python_ulid-3.0.0-py3-none-any.whl", hash = "sha256:e4c4942ff50dbd79167ad01ac725ec58f924b4018025ce22c858bfcff99a5e31"}, {file = "python_ulid-3.0.0.tar.gz", hash = "sha256:e50296a47dc8209d28629a22fc81ca26c00982c78934bd7766377ba37ea49a9f"}, @@ -4285,6 +4560,8 @@ version = "2025.2" description = "World timezone definitions, modern and historical" optional = true python-versions = "*" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\" or python_version < \"3.9\" and extra == \"utils\"" files = [ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, @@ -4296,6 +4573,8 @@ version = "311" description = "Python for Window Extensions" optional = true python-versions = "*" +groups = ["main"] +markers = "python_version >= \"3.10\" and sys_platform == \"win32\" and (extra == \"proxy\" or extra == \"mlflow\")" files = [ {file = "pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3"}, {file = "pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b"}, @@ -4325,6 +4604,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -4387,6 +4667,8 @@ version = "5.3.1" description = "Python client for Redis database and key-value store" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.9\" and (extra == \"extra-proxy\" or extra == \"proxy\") and python_version < \"3.14\" or extra == \"proxy\"" files = [ {file = "redis-5.3.1-py3-none-any.whl", hash = "sha256:dc1909bd24669cc31b5f67a039700b16ec30571096c5f1f0d9d2324bff31af97"}, {file = "redis-5.3.1.tar.gz", hash = "sha256:ca49577a531ea64039b5a36db3d6cd1a0c7a60c34124d46924a45b956e8cf14c"}, @@ -4406,6 +4688,8 @@ version = "0.4.1" description = "Python client library and CLI for using Redis as a vector database" optional = true python-versions = "<3.14,>=3.9" +groups = ["main"] +markers = "python_version >= \"3.9\" and extra == \"extra-proxy\" and python_version < \"3.14\"" files = [ {file = "redisvl-0.4.1-py3-none-any.whl", hash = "sha256:6db5d5bc95b1fe8032a1cdae74ce1c65bc7fe9054e5429b5d34d5a91d28bae5f"}, {file = "redisvl-0.4.1.tar.gz", hash = "sha256:fd6a36426ba94792c0efca20915c31232d4ee3cc58eb23794a62c142696401e6"}, @@ -4430,7 +4714,7 @@ bedrock = ["boto3[bedrock] (>=1.36.0,<2.0.0)"] cohere = ["cohere (>=4.44)"] mistralai = ["mistralai (>=1.0.0)"] openai = ["openai (>=1.13.0,<2.0.0)"] -sentence-transformers = ["scipy (<1.15)", "scipy (>=1.15,<2.0)", "sentence-transformers (>=3.4.0,<4.0.0)"] +sentence-transformers = ["scipy (<1.15) ; python_version < \"3.10\"", "scipy (>=1.15,<2.0) ; python_version >= \"3.10\"", "sentence-transformers (>=3.4.0,<4.0.0)"] vertexai = ["google-cloud-aiplatform (>=1.26,<2.0)", "protobuf (>=5.29.1,<6.0.0)"] voyageai = ["voyageai (>=0.2.2)"] @@ -4440,6 +4724,7 @@ version = "0.35.1" description = "JSON Referencing + Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"}, {file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"}, @@ -4455,6 +4740,7 @@ version = "2024.11.6" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91"}, {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:658f90550f38270639e83ce492f27d2c8d2cd63805c65a13a14d36ca126753f0"}, @@ -4558,6 +4844,7 @@ version = "2.31.0" description = "Python HTTP for Humans." optional = false python-versions = ">=3.7" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, @@ -4579,6 +4866,7 @@ version = "1.12.1" description = "Mock out responses from the requests package" optional = false python-versions = ">=3.5" +groups = ["dev"] files = [ {file = "requests-mock-1.12.1.tar.gz", hash = "sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401"}, {file = "requests_mock-1.12.1-py2.py3-none-any.whl", hash = "sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563"}, @@ -4596,6 +4884,8 @@ version = "0.8.0" description = "Resend Python SDK" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"extra-proxy\"" files = [ {file = "resend-0.8.0-py2.py3-none-any.whl", hash = "sha256:adc1515dadf4f4fc6b90db55a237f0f37fc56fd74287a986519a8a187fdb661d"}, {file = "resend-0.8.0.tar.gz", hash = "sha256:94142394701724dbcfcd8f760f675c662a1025013e741dd7cc773ca885526257"}, @@ -4610,6 +4900,7 @@ version = "0.25.7" description = "A utility library for mocking out the `requests` Python library." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "responses-0.25.7-py3-none-any.whl", hash = "sha256:92ca17416c90fe6b35921f52179bff29332076bb32694c0df02dcac2c6bc043c"}, {file = "responses-0.25.7.tar.gz", hash = "sha256:8ebae11405d7a5df79ab6fd54277f6f2bc29b2d002d0dd2d5c632594d1ddcedb"}, @@ -4621,7 +4912,7 @@ requests = ">=2.30.0,<3.0" urllib3 = ">=1.25.10,<3.0" [package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-PyYAML", "types-requests"] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli ; python_version < \"3.11\"", "tomli-w", "types-PyYAML", "types-requests"] [[package]] name = "respx" @@ -4629,6 +4920,7 @@ version = "0.22.0" description = "A utility for mocking out the Python HTTPX and HTTP Core libraries." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "respx-0.22.0-py2.py3-none-any.whl", hash = "sha256:631128d4c9aba15e56903fb5f66fb1eff412ce28dd387ca3a81339e52dbd3ad0"}, {file = "respx-0.22.0.tar.gz", hash = "sha256:3c8924caa2a50bd71aefc07aa812f2466ff489f1848c96e954a5362d17095d91"}, @@ -4643,6 +4935,8 @@ version = "13.7.1" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = true python-versions = ">=3.7.0" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, @@ -4662,6 +4956,7 @@ version = "0.20.1" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "rpds_py-0.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a649dfd735fff086e8a9d0503a9f0c7d01b7912a333c7ae77e1515c08c146dad"}, {file = "rpds_py-0.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f16bc1334853e91ddaaa1217045dd7be166170beec337576818461268a3de67f"}, @@ -4774,6 +5069,8 @@ version = "2.3.3" description = "RQ is a simple, lightweight, library for creating background jobs, and processing them." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "rq-2.3.3-py3-none-any.whl", hash = "sha256:2202c4409c4c527ac4bee409867d6c02515dd110030499eb0de54c7374aee0ce"}, {file = "rq-2.3.3.tar.gz", hash = "sha256:20c41c977b6f27c852a41bd855893717402bae7b8d9607dca21fe9dd55453e22"}, @@ -4789,6 +5086,8 @@ version = "4.9.1" description = "Pure-Python RSA implementation" optional = true python-versions = "<4,>=3.6" +groups = ["main"] +markers = "python_version >= \"3.10\" and (extra == \"mlflow\" or extra == \"extra-proxy\") or extra == \"extra-proxy\"" files = [ {file = "rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762"}, {file = "rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75"}, @@ -4803,6 +5102,7 @@ version = "0.1.15" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "ruff-0.1.15-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5fe8d54df166ecc24106db7dd6a68d44852d14eb0729ea4672bb4d96c320b7df"}, {file = "ruff-0.1.15-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6f0bfbb53c4b4de117ac4d6ddfd33aa5fc31beeaa21d23c45c6dd249faf9126f"}, @@ -4829,6 +5129,8 @@ version = "0.10.4" description = "An Amazon S3 Transfer Manager" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e"}, {file = "s3transfer-0.10.4.tar.gz", hash = "sha256:29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7"}, @@ -4846,6 +5148,8 @@ version = "1.7.1" description = "A set of python modules for machine learning and data mining" optional = true python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "scikit_learn-1.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:406204dd4004f0517f0b23cf4b28c6245cbd51ab1b6b78153bc784def214946d"}, {file = "scikit_learn-1.7.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:16af2e44164f05d04337fd1fc3ae7c4ea61fd9b0d527e22665346336920fe0e1"}, @@ -4896,6 +5200,8 @@ version = "1.15.3" description = "Fundamental algorithms for scientific computing in Python" optional = true python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "scipy-1.15.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a345928c86d535060c9c2b25e71e87c39ab2f22fc96e9636bd74d1dbf9de448c"}, {file = "scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ad3432cb0f9ed87477a8d97f03b763fd1d57709f1bbde3c9369b1dff5503b253"}, @@ -4951,7 +5257,7 @@ numpy = ">=1.23.5,<2.5" [package.extras] dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.19.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.0.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)"] -test = ["Cython", "array-api-strict (>=2.0,<2.1.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +test = ["Cython", "array-api-strict (>=2.0,<2.1.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja ; sys_platform != \"emscripten\"", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] [[package]] name = "semantic-router" @@ -4959,6 +5265,8 @@ version = "0.0.20" description = "Super fast semantic router for AI decision making" optional = true python-versions = ">=3.9,<4.0" +groups = ["main"] +markers = "python_version >= \"3.9\" and extra == \"semantic-router\"" files = [ {file = "semantic_router-0.0.20-py3-none-any.whl", hash = "sha256:7a713401564fb6cf22b566046ad32a4224e4f357be8de6583ca3b9ee328c8f95"}, {file = "semantic_router-0.0.20.tar.gz", hash = "sha256:26119a4628ca72b2fa9eacd446ea763b6f1925a661a34e26945433d2601efac7"}, @@ -4974,7 +5282,7 @@ pydantic = ">=2.5.3,<3.0.0" pyyaml = ">=6.0.1,<7.0.0" [package.extras] -fastembed = ["fastembed (>=0.1.3,<0.2.0)"] +fastembed = ["fastembed (>=0.1.3,<0.2.0) ; python_version < \"3.12\""] hybrid = ["pinecone-text (>=0.7.1,<0.8.0)"] local = ["llama-cpp-python (>=0.2.28,<0.3.0)", "torch (>=2.1.0,<3.0.0)", "transformers (>=4.36.2,<5.0.0)"] @@ -4984,6 +5292,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main", "proxy-dev"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -4995,6 +5304,8 @@ version = "5.0.2" description = "A pure Python implementation of a sliding window memory map manager" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e"}, {file = "smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5"}, @@ -5006,6 +5317,7 @@ version = "1.3.1" description = "Sniff out which async library your code is running under" optional = false python-versions = ">=3.7" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, @@ -5017,6 +5329,8 @@ version = "3.0.1" description = "This package provides 32 stemmers for 30 languages generated from Snowball algorithms." optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064"}, {file = "snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895"}, @@ -5028,6 +5342,8 @@ version = "7.1.2" description = "Python documentation generator" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "sphinx-7.1.2-py3-none-any.whl", hash = "sha256:d170a81825b2fcacb6dfd5a0d7f578a053e45d3f2b153fecc948c37344eb4cbe"}, {file = "sphinx-7.1.2.tar.gz", hash = "sha256:780f4d32f1d7d1126576e0e5ecc19dc32ab76cd24e950228dcf7b1f6d3d9e22f"}, @@ -5063,6 +5379,8 @@ version = "1.0.4" description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e"}, {file = "sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228"}, @@ -5078,6 +5396,8 @@ version = "1.0.2" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." optional = true python-versions = ">=3.5" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"}, {file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"}, @@ -5093,6 +5413,8 @@ version = "2.0.1" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff"}, {file = "sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903"}, @@ -5108,6 +5430,8 @@ version = "1.0.1" description = "A sphinx extension which renders display math in HTML via JavaScript" optional = true python-versions = ">=3.5" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, @@ -5122,6 +5446,8 @@ version = "1.0.3" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." optional = true python-versions = ">=3.5" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"}, {file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"}, @@ -5137,6 +5463,8 @@ version = "1.1.5" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." optional = true python-versions = ">=3.5" +groups = ["main"] +markers = "extra == \"utils\"" files = [ {file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"}, {file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"}, @@ -5152,6 +5480,8 @@ version = "2.0.42" description = "Database Abstraction Library" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "SQLAlchemy-2.0.42-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7ee065898359fdee83961aed5cf1fb4cfa913ba71b58b41e036001d90bebbf7a"}, {file = "SQLAlchemy-2.0.42-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56bc76d86216443daa2e27e6b04a9b96423f0b69b5d0c40c7f4b9a4cdf7d8d90"}, @@ -5247,6 +5577,8 @@ version = "0.5.3" description = "A non-validating SQL parser." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "sqlparse-0.5.3-py3-none-any.whl", hash = "sha256:cf2196ed3418f3ba5de6af7e82c694a9fbdbfecccdfc72e281548517081f16ca"}, {file = "sqlparse-0.5.3.tar.gz", hash = "sha256:09f67787f56a0b16ecdbde1bfc7f5d9c3371ca683cfeaa8e6ff60b4807ec9272"}, @@ -5262,6 +5594,8 @@ version = "2.1.3" description = "SSE plugin for Starlette" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"proxy\"" files = [ {file = "sse_starlette-2.1.3-py3-none-any.whl", hash = "sha256:8ec846438b4665b9e8c560fcdea6bc8081a3abf7942faa95e5a744999d219772"}, {file = "sse_starlette-2.1.3.tar.gz", hash = "sha256:9cd27eb35319e1414e3d2558ee7414487f9529ce3b3cf9b21434fd110e017169"}, @@ -5281,6 +5615,8 @@ version = "0.44.0" description = "The little ASGI library that shines." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and (extra == \"mlflow\" or extra == \"proxy\") or extra == \"proxy\"" files = [ {file = "starlette-0.44.0-py3-none-any.whl", hash = "sha256:19edeb75844c16dcd4f9dd72f22f9108c1539f3fc9c4c88885654fef64f85aea"}, {file = "starlette-0.44.0.tar.gz", hash = "sha256:e35166950a3ccccc701962fe0711db0bc14f2ecd37c6f9fe5e3eae0cbaea8715"}, @@ -5288,7 +5624,6 @@ files = [ [package.dependencies] anyio = ">=3.4.0,<5" -typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} [package.extras] full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] @@ -5299,6 +5634,8 @@ version = "0.9.0" description = "Pretty-print tabular data" optional = true python-versions = ">=3.7" +groups = ["main"] +markers = "python_version >= \"3.9\" and (extra == \"extra-proxy\" or extra == \"utils\") and python_version < \"3.14\" or extra == \"utils\"" files = [ {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, @@ -5313,6 +5650,8 @@ version = "0.2.2" description = "backport of asyncio.TaskGroup, asyncio.Runner and asyncio.timeout" optional = false python-versions = "*" +groups = ["proxy-dev"] +markers = "python_version <= \"3.10\"" files = [ {file = "taskgroup-0.2.2-py2.py3-none-any.whl", hash = "sha256:e2c53121609f4ae97303e9ea1524304b4de6faf9eb2c9280c7f87976479a52fb"}, {file = "taskgroup-0.2.2.tar.gz", hash = "sha256:078483ac3e78f2e3f973e2edbf6941374fbea81b9c5d0a96f51d297717f4752d"}, @@ -5328,6 +5667,8 @@ version = "9.1.2" description = "Retry code until it succeeds" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.9\" and extra == \"extra-proxy\" and python_version < \"3.14\"" files = [ {file = "tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138"}, {file = "tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb"}, @@ -5343,6 +5684,8 @@ version = "3.6.0" description = "threadpoolctl" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb"}, {file = "threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e"}, @@ -5354,6 +5697,7 @@ version = "0.7.0" description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "tiktoken-0.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485f3cc6aba7c6b6ce388ba634fbba656d9ee27f766216f45146beb4ac18b25f"}, {file = "tiktoken-0.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e54be9a2cd2f6d6ffa3517b064983fb695c9a9d8aa7d574d1ef3c3f931a99225"}, @@ -5406,6 +5750,7 @@ version = "0.21.0" description = "" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tokenizers-0.21.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3c4c93eae637e7d2aaae3d376f06085164e1660f89304c0ab2b1d08a406636b2"}, {file = "tokenizers-0.21.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:f53ea537c925422a2e0e92a24cce96f6bc5046bbef24a1652a5edc8ba975f62e"}, @@ -5438,6 +5783,8 @@ version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] +markers = "python_version <= \"3.10\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -5479,6 +5826,7 @@ version = "0.13.3" description = "Style preserving TOML library" optional = false python-versions = ">=3.8" +groups = ["main", "proxy-dev"] files = [ {file = "tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0"}, {file = "tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1"}, @@ -5490,6 +5838,7 @@ version = "4.67.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, @@ -5511,6 +5860,7 @@ version = "1.16.0.20241221" description = "Typing stubs for cffi" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types_cffi-1.16.0.20241221-py3-none-any.whl", hash = "sha256:e5b76b4211d7a9185f6ab8d06a106d56c7eb80af7cdb8bfcb4186ade10fb112f"}, {file = "types_cffi-1.16.0.20241221.tar.gz", hash = "sha256:1c96649618f4b6145f58231acb976e0b448be6b847f7ab733dabe62dfbff6591"}, @@ -5525,6 +5875,7 @@ version = "24.1.0.20240722" description = "Typing stubs for pyOpenSSL" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-pyOpenSSL-24.1.0.20240722.tar.gz", hash = "sha256:47913b4678a01d879f503a12044468221ed8576263c1540dcb0484ca21b08c39"}, {file = "types_pyOpenSSL-24.1.0.20240722-py3-none-any.whl", hash = "sha256:6a7a5d2ec042537934cfb4c9d4deb0e16c4c6250b09358df1f083682fe6fda54"}, @@ -5540,6 +5891,7 @@ version = "6.0.12.20241230" description = "Typing stubs for PyYAML" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types_PyYAML-6.0.12.20241230-py3-none-any.whl", hash = "sha256:fa4d32565219b68e6dee5f67534c722e53c00d1cfc09c435ef04d7353e1e96e6"}, {file = "types_pyyaml-6.0.12.20241230.tar.gz", hash = "sha256:7f07622dbd34bb9c8b264fe860a17e0efcad00d50b5f27e93984909d9363498c"}, @@ -5551,6 +5903,7 @@ version = "4.6.0.20241004" description = "Typing stubs for redis" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-redis-4.6.0.20241004.tar.gz", hash = "sha256:5f17d2b3f9091ab75384153bfa276619ffa1cf6a38da60e10d5e6749cc5b902e"}, {file = "types_redis-4.6.0.20241004-py3-none-any.whl", hash = "sha256:ef5da68cb827e5f606c8f9c0b49eeee4c2669d6d97122f301d3a55dc6a63f6ed"}, @@ -5566,6 +5919,8 @@ version = "2.31.0.6" description = "Typing stubs for requests" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "python_version < \"3.10\"" files = [ {file = "types-requests-2.31.0.6.tar.gz", hash = "sha256:cd74ce3b53c461f1228a9b783929ac73a666658f223e28ed29753771477b3bd0"}, {file = "types_requests-2.31.0.6-py3-none-any.whl", hash = "sha256:a2db9cb228a81da8348b49ad6db3f5519452dd20a9c1e1a868c83c5fe88fd1a9"}, @@ -5580,6 +5935,8 @@ version = "2.32.0.20241016" description = "Typing stubs for requests" optional = false python-versions = ">=3.8" +groups = ["dev"] +markers = "python_version >= \"3.10\"" files = [ {file = "types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95"}, {file = "types_requests-2.32.0.20241016-py3-none-any.whl", hash = "sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747"}, @@ -5594,6 +5951,7 @@ version = "75.8.0.20250110" description = "Typing stubs for setuptools" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types_setuptools-75.8.0.20250110-py3-none-any.whl", hash = "sha256:a9f12980bbf9bcdc23ecd80755789085bad6bfce4060c2275bc2b4ca9f2bc480"}, {file = "types_setuptools-75.8.0.20250110.tar.gz", hash = "sha256:96f7ec8bbd6e0a54ea180d66ad68ad7a1d7954e7281a710ea2de75e355545271"}, @@ -5605,6 +5963,8 @@ version = "1.26.25.14" description = "Typing stubs for urllib3" optional = false python-versions = "*" +groups = ["dev"] +markers = "python_version < \"3.10\"" files = [ {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, @@ -5616,6 +5976,7 @@ version = "4.13.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c"}, {file = "typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"}, @@ -5627,6 +5988,8 @@ version = "0.4.1" description = "Runtime typing introspection tools" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"proxy\"" files = [ {file = "typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51"}, {file = "typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28"}, @@ -5641,6 +6004,8 @@ version = "2025.2" description = "Provider of IANA time zone data" optional = true python-versions = ">=2" +groups = ["main"] +markers = "python_version >= \"3.10\" and platform_system == \"Windows\" and (extra == \"proxy\" or extra == \"mlflow\") or python_version >= \"3.10\" and extra == \"mlflow\" or platform_system == \"Windows\" and extra == \"proxy\"" files = [ {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"}, {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, @@ -5652,6 +6017,8 @@ version = "5.2" description = "tzinfo object for the local timezone" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "tzlocal-5.2-py3-none-any.whl", hash = "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8"}, {file = "tzlocal-5.2.tar.gz", hash = "sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e"}, @@ -5670,14 +6037,16 @@ version = "1.26.20" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +groups = ["main", "dev", "proxy-dev"] +markers = "python_version < \"3.10\"" files = [ {file = "urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e"}, {file = "urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32"}, ] [package.extras] -brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +brotli = ["brotli (==1.0.9) ; os_name != \"nt\" and python_version < \"3\" and platform_python_implementation == \"CPython\"", "brotli (>=1.0.9) ; python_version >= \"3\" and platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; (os_name != \"nt\" or python_version >= \"3\") and platform_python_implementation != \"CPython\"", "brotlipy (>=0.6.0) ; os_name == \"nt\" and python_version < \"3\""] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress ; python_version == \"2.7\"", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] @@ -5686,13 +6055,15 @@ version = "2.2.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] +markers = "python_version >= \"3.10\"" files = [ {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -5703,6 +6074,8 @@ version = "0.29.0" description = "The lightning-fast ASGI server." optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\" and (extra == \"mlflow\" or extra == \"proxy\") or extra == \"proxy\"" files = [ {file = "uvicorn-0.29.0-py3-none-any.whl", hash = "sha256:2c2aac7ff4f4365c206fd773a39bf4ebd1047c238f8b8268ad996829323473de"}, {file = "uvicorn-0.29.0.tar.gz", hash = "sha256:6a69214c0b6a087462412670b3ef21224fa48cae0e452b5883e8e8bdfdd11dd0"}, @@ -5714,7 +6087,7 @@ h11 = ">=0.8" typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} [package.extras] -standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] +standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"] [[package]] name = "uvloop" @@ -5722,6 +6095,8 @@ version = "0.21.0" description = "Fast implementation of asyncio event loop on top of libuv" optional = true python-versions = ">=3.8.0" +groups = ["main"] +markers = "sys_platform != \"win32\" and extra == \"proxy\"" files = [ {file = "uvloop-0.21.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ec7e6b09a6fdded42403182ab6b832b71f4edaf7f37a9a0e371a01db5f0cb45f"}, {file = "uvloop-0.21.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:196274f2adb9689a289ad7d65700d37df0c0930fd8e4e743fa4834e850d7719d"}, @@ -5773,6 +6148,8 @@ version = "3.0.2" description = "Waitress WSGI server" optional = true python-versions = ">=3.9.0" +groups = ["main"] +markers = "python_version >= \"3.10\" and platform_system == \"Windows\" and extra == \"mlflow\"" files = [ {file = "waitress-3.0.2-py3-none-any.whl", hash = "sha256:c56d67fd6e87c2ee598b76abdd4e96cfad1f24cacdea5078d382b1f9d7b5ed2e"}, {file = "waitress-3.0.2.tar.gz", hash = "sha256:682aaaf2af0c44ada4abfb70ded36393f0e307f4ab9456a215ce0020baefc31f"}, @@ -5788,6 +6165,8 @@ version = "13.1" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" optional = true python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" files = [ {file = "websockets-13.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f48c749857f8fb598fb890a75f540e3221d0976ed0bf879cf3c7eef34151acee"}, {file = "websockets-13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c7e72ce6bda6fb9409cc1e8164dd41d7c91466fb599eb047cfda72fe758a34a7"}, @@ -5883,6 +6262,8 @@ version = "3.1.3" description = "The comprehensive WSGI web application library." optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"mlflow\"" files = [ {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"}, {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"}, @@ -5900,6 +6281,7 @@ version = "1.17.2" description = "Module for decorators, wrappers and monkey patching." optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "wrapt-1.17.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3d57c572081fed831ad2d26fd430d565b76aa277ed1d30ff4d40670b1c0dd984"}, {file = "wrapt-1.17.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5e251054542ae57ac7f3fba5d10bfff615b6c2fb09abeb37d2f1463f841ae22"}, @@ -5981,6 +6363,7 @@ files = [ {file = "wrapt-1.17.2-py3-none-any.whl", hash = "sha256:b18f2d1533a71f069c7f82d524a52599053d4c7166e9dd374ae2136b7f40f7c8"}, {file = "wrapt-1.17.2.tar.gz", hash = "sha256:41388e9d4d1522446fe79d3213196bd9e3b301a336965b9e27ca2788ebd122f3"}, ] +markers = {main = "python_version >= \"3.10\""} [[package]] name = "wsproto" @@ -5988,6 +6371,7 @@ version = "1.2.0" description = "WebSockets state-machine based protocol implementation" optional = false python-versions = ">=3.7.0" +groups = ["proxy-dev"] files = [ {file = "wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736"}, {file = "wsproto-1.2.0.tar.gz", hash = "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065"}, @@ -6002,6 +6386,7 @@ version = "1.15.2" description = "Yet another URL library" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "yarl-1.15.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e4ee8b8639070ff246ad3649294336b06db37a94bdea0d09ea491603e0be73b8"}, {file = "yarl-1.15.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a7cf963a357c5f00cb55b1955df8bbe68d2f2f65de065160a1c26b85a1e44172"}, @@ -6114,28 +6499,29 @@ version = "3.20.2" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "proxy-dev"] files = [ {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] [extras] caching = ["diskcache"] -extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-kms", "prisma", "redisvl", "resend"] +extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-iam", "google-cloud-kms", "prisma", "redisvl", "resend"] mlflow = ["mlflow"] proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "python-multipart", "pyyaml", "rich", "rq", "uvicorn", "uvloop", "websockets"] semantic-router = ["semantic-router"] utils = ["numpydoc"] [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "3b9cdf44dbcb105f490450cd4714bf5ced62825cb2b319a423afeac0219b5575" +content-hash = "8caa7dd3ee7d56562ccb799b97de7f2d206010d8300139bdc3614853c9ea1d31" diff --git a/pyproject.toml b/pyproject.toml index 27bded35275..c4a0ed0477a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.74.15" +version = "1.75.2" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -51,6 +51,7 @@ azure-identity = {version = "^1.15.0", optional = true} azure-keyvault-secrets = {version = "^4.8.0", optional = true} azure-storage-blob = {version="^12.25.1", optional=true} google-cloud-kms = {version = "^2.21.3", optional = true} +google-cloud-iam = {version = "^2.19.1", optional = true} resend = {version = "^0.8.0", optional = true} pynacl = {version = "^1.5.0", optional = true} websockets = {version = "^13.1.0", optional = true} @@ -59,7 +60,7 @@ redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3. mcp = {version = "^1.10.0", optional = true, python = ">=3.10"} litellm-proxy-extras = {version = "0.2.15", optional = true} rich = {version = "13.7.1", optional = true} -litellm-enterprise = {version = "0.1.16", optional = true} +litellm-enterprise = {version = "0.1.19", optional = true} diskcache = {version = "^5.6.1", optional = true} polars = {version = "^1.31.0", optional = true, python = ">=3.10"} semantic-router = {version = "*", optional = true, python = ">=3.9"} @@ -97,6 +98,7 @@ extra_proxy = [ "azure-identity", "azure-keyvault-secrets", "google-cloud-kms", + "google-cloud-iam", "resend", "redisvl" ] @@ -152,7 +154,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.74.15" +version = "1.75.2" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index 364a4368827..84eb146aee2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ prisma==0.11.0 # for db mangum==0.17.0 # for aws lambda functions pynacl==1.5.0 # for encrypting keys google-cloud-aiplatform==1.47.0 # for vertex ai calls +google-cloud-iam==2.19.1 # for GCP IAM Redis authentication google-genai==1.22.0 anthropic[vertex]==0.54.0 mcp==1.10.1 # for MCP server @@ -59,4 +60,4 @@ websockets==13.1.0 # for realtime API ######################## # LITELLM ENTERPRISE DEPENDENCIES ######################## -litellm-enterprise==0.1.16 +litellm-enterprise==0.1.19 diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index 31c849fea52..936d93c4417 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -7,6 +7,8 @@ sys.path.insert(0, os.path.abspath("../..")) import asyncio import logging import uuid +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock, call, patch import pytest from prometheus_client import REGISTRY, CollectorRegistry @@ -16,16 +18,18 @@ from litellm import completion from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.utils import ( - StandardLoggingPayload, - StandardLoggingMetadata, StandardLoggingHiddenParams, + StandardLoggingMetadata, StandardLoggingModelInformation, + StandardLoggingPayload, ) -import pytest -from unittest.mock import MagicMock, patch, call -from datetime import datetime, timedelta, timezone + try: - from litellm_enterprise.integrations.prometheus import PrometheusLogger, UserAPIKeyLabelValues, get_custom_labels_from_metadata + from litellm_enterprise.integrations.prometheus import ( + PrometheusLogger, + UserAPIKeyLabelValues, + get_custom_labels_from_metadata, + ) except Exception: PrometheusLogger = None from litellm.proxy._types import UserAPIKeyAuth @@ -1054,6 +1058,7 @@ def test_increment_deployment_cooled_down(prometheus_logger): @pytest.mark.parametrize("enable_end_user_cost_tracking_prometheus_only", [True, False]) def test_prometheus_factory(monkeypatch, enable_end_user_cost_tracking_prometheus_only): from litellm_enterprise.integrations.prometheus import prometheus_label_factory + from litellm.types.integrations.prometheus import UserAPIKeyLabelValues monkeypatch.setattr( @@ -1130,6 +1135,119 @@ def test_get_custom_labels_from_tags_no_tags(monkeypatch): } +def test_get_custom_labels_from_tags_wildcard_patterns(monkeypatch): + """Test wildcard pattern matching for custom labels from tags.""" + from litellm_enterprise.integrations.prometheus import get_custom_labels_from_tags + + # Configure tags with wildcard patterns + monkeypatch.setattr( + "litellm.custom_prometheus_tags", + ["User-Agent: curl/*", "User-Agent: python-requests/*", "Environment: prod*", "Service: api-gateway*", "exact-match"] + ) + + # Test tags that should match the wildcard patterns + tags = [ + "User-Agent: curl/7.68.0", + "User-Agent: python-requests/2.28.1", + "Environment: production", + "Service: api-gateway-v2", + "exact-match", + "other-tag" + ] + + result = get_custom_labels_from_tags(tags) + + expected = { + "tag_User_Agent__curl__": "true", # matches "User-Agent: curl/*" + "tag_User_Agent__python_requests__": "true", # matches "User-Agent: python-requests/*" + "tag_Environment__prod_": "true", # matches "Environment: prod*" + "tag_Service__api_gateway_": "true", # matches "Service: api-gateway*" + "tag_exact_match": "true", # exact match + } + + assert result == expected + + +def test_get_custom_labels_from_tags_wildcard_no_matches(monkeypatch): + """Test wildcard patterns that don't match any tags.""" + from litellm_enterprise.integrations.prometheus import get_custom_labels_from_tags + + # Configure tags with wildcard patterns + monkeypatch.setattr( + "litellm.custom_prometheus_tags", + ["User-Agent: firefox/*", "Environment: dev*", "Service: web-app*"] + ) + + # Test tags that should NOT match the wildcard patterns + tags = [ + "User-Agent: curl/7.68.0", # doesn't match "User-Agent: firefox/*" + "Environment: production", # doesn't match "Environment: dev*" + "Service: api-gateway-v2", # doesn't match "Service: web-app*" + "other-tag" + ] + + result = get_custom_labels_from_tags(tags) + + expected = { + "tag_User_Agent__firefox__": "false", # no match for "User-Agent: firefox/*" + "tag_Environment__dev_": "false", # no match for "Environment: dev*" + "tag_Service__web_app_": "false", # no match for "Service: web-app*" + } + + assert result == expected + + +def test_tag_matches_wildcard_configured_pattern(): + """Test the helper function for wildcard pattern matching.""" + from litellm_enterprise.integrations.prometheus import ( + _tag_matches_wildcard_configured_pattern, + ) + + # Test cases that should match + assert _tag_matches_wildcard_configured_pattern( + tags=["User-Agent: curl/7.68.0", "prod", "other"], + configured_tag="User-Agent: curl/*" + ) is True + + assert _tag_matches_wildcard_configured_pattern( + tags=["User-Agent: python-requests/2.28.1", "test"], + configured_tag="User-Agent: python-requests/*" + ) is True + + assert _tag_matches_wildcard_configured_pattern( + tags=["Environment: production", "debug"], + configured_tag="Environment: prod*" + ) is True + + # Test exact match (no wildcard) + assert _tag_matches_wildcard_configured_pattern( + tags=["prod", "test"], + configured_tag="prod" + ) is True + + # Test cases that should NOT match + assert _tag_matches_wildcard_configured_pattern( + tags=["User-Agent: firefox/98.0", "prod"], + configured_tag="User-Agent: curl/*" + ) is False + + assert _tag_matches_wildcard_configured_pattern( + tags=["Environment: development", "test"], + configured_tag="Environment: prod*" + ) is False + + assert _tag_matches_wildcard_configured_pattern( + tags=["staging", "test"], + configured_tag="prod" + ) is False + + # Test with empty tags + assert _tag_matches_wildcard_configured_pattern( + tags=[], + configured_tag="User-Agent: curl/*" + ) is False + + @pytest.mark.asyncio(scope="session") async def test_initialize_remaining_budget_metrics(prometheus_logger): """ @@ -1532,7 +1650,11 @@ def test_prometheus_label_factory_with_custom_tags(monkeypatch): """ Test that prometheus_label_factory correctly handles custom tags """ - from litellm_enterprise.integrations.prometheus import get_custom_labels_from_tags, prometheus_label_factory + from litellm_enterprise.integrations.prometheus import ( + get_custom_labels_from_tags, + prometheus_label_factory, + ) + from litellm.types.integrations.prometheus import UserAPIKeyLabelValues # Set custom tags configuration @@ -1567,7 +1689,11 @@ def test_prometheus_label_factory_with_no_custom_tags(monkeypatch): """ Test that prometheus_label_factory works when no custom tags are configured """ - from litellm_enterprise.integrations.prometheus import get_custom_labels_from_tags, prometheus_label_factory + from litellm_enterprise.integrations.prometheus import ( + get_custom_labels_from_tags, + prometheus_label_factory, + ) + from litellm.types.integrations.prometheus import UserAPIKeyLabelValues # Set empty custom tags configuration @@ -1776,3 +1902,154 @@ def test_set_llm_deployment_success_metrics_with_label_filtering(): ) prometheus_logger.litellm_deployment_success_responses.labels().inc.assert_called_once() prometheus_logger.litellm_deployment_total_requests.labels().inc.assert_called_once() + + +@pytest.mark.asyncio +async def test_prometheus_token_metrics_with_prometheus_config(): + """ + Test that validates the renamed token metrics are incremented correctly with a prometheus config. + + This test ensures that after the metric renaming (git diff): + - litellm_total_tokens -> litellm_total_tokens_metric + - litellm_input_tokens -> litellm_input_tokens_metric + - litellm_output_tokens -> litellm_output_tokens_metric + + All three metrics should be properly incremented when making a successful completion request. + """ + from prometheus_client import CollectorRegistry, Counter + + import litellm + from litellm.types.integrations.prometheus import PrometheusMetricsConfig + + # Clear registry before test + collectors = list(REGISTRY._collector_to_names.keys()) + for collector in collectors: + REGISTRY.unregister(collector) + + # Set up prometheus configuration that includes the token metrics + config = [ + PrometheusMetricsConfig( + group="token_metrics_test", + metrics=[ + "litellm_total_tokens_metric", + "litellm_input_tokens_metric", + "litellm_output_tokens_metric", + "litellm_requests_metric" + ], + include_labels=[ + "model", + "hashed_api_key", + "api_key_alias", + "team", + "team_alias" + ], + ) + ] + + # Mock litellm.prometheus_metrics_config + with patch("litellm.prometheus_metrics_config", config): + # Create PrometheusLogger with the configuration + prometheus_logger = PrometheusLogger() + + # Test data with specific token counts + standard_logging_payload = create_standard_logging_payload() + standard_logging_payload["total_tokens"] = 1500 + standard_logging_payload["prompt_tokens"] = 900 + standard_logging_payload["completion_tokens"] = 600 + standard_logging_payload["response_cost"] = 0.075 + + kwargs = { + "model": "gpt-3.5-turbo", + "stream": False, + "litellm_params": { + "metadata": { + "user_api_key": "test_key_hash", + "user_api_key_user_id": "test_user", + "user_api_key_team_id": "test_team", + "user_api_key_alias": "test_alias", + "user_api_key_team_alias": "test_team_alias", + } + }, + "start_time": datetime.now() - timedelta(seconds=2), + "completion_start_time": datetime.now() - timedelta(seconds=1), + "api_call_start_time": datetime.now() - timedelta(seconds=1.5), + "end_time": datetime.now(), + "standard_logging_object": standard_logging_payload, + } + response_obj = MagicMock() + + # Make the completion call through the logger + await prometheus_logger.async_log_success_event( + kwargs, response_obj, kwargs["start_time"], kwargs["end_time"] + ) + + await asyncio.sleep(2) + + print("final registry values", REGISTRY._collector_to_names) + + # Get metric collectors directly from registry + metric_collectors = {} + for collector, names in REGISTRY._collector_to_names.items(): + metric_name = names[0] # First name is the base metric name + metric_collectors[metric_name] = collector + + print("=== Final Metric Values (Direct Access) ===") + + # Expected values + expected_values = { + "litellm_total_tokens_metric": 1500.0, + "litellm_input_tokens_metric": 900.0, + "litellm_output_tokens_metric": 600.0, + "litellm_requests_metric": 1.0 + } + + expected_label_values = { + 'api_key_alias': 'test_alias', + 'hashed_api_key': 'test_hash', + 'model': 'gpt-3.5-turbo', + 'team': 'test_team', + 'team_alias': 'test_team_alias' + } + + # Validate each metric directly + for metric_name, expected_value in expected_values.items(): + if metric_name in metric_collectors: + collector = metric_collectors[metric_name] + + # Get all samples for this metric + samples = list(collector.collect())[0].samples + + # Find the _total sample (the actual counter value) + total_sample = None + for sample in samples: + if sample.name.endswith('_total'): + total_sample = sample + break + + if total_sample: + actual_value = total_sample.value + actual_labels = total_sample.labels + + print(f"āœ“ {metric_name}: expected={expected_value}, actual={actual_value}") + print(f" Labels: {actual_labels}") + + # Validate the value + assert actual_value == expected_value, f"Expected {expected_value}, got {actual_value} for {metric_name}" + + # Validate the labels + for label_key, expected_label_value in expected_label_values.items(): + actual_label_value = actual_labels.get(label_key) + assert actual_label_value == expected_label_value, f"Expected label {label_key}={expected_label_value}, got {actual_label_value}" + + print(f" āœ“ {metric_name} VALIDATED") + else: + raise AssertionError(f"No _total sample found for {metric_name}") + else: + raise AssertionError(f"Metric {metric_name} not found in registry") + + print("āœ“ All token metrics validated successfully!") + + # check final value of metrics in registry + + + diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 8087e0ead3c..a4ee9a8835a 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -25,6 +25,9 @@ from litellm.types.llms.openai import ( ResponseAPIUsage, IncompleteDetails, ) +from openai.types.responses.response_create_params import ( + ResponseInputParam, +) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -184,8 +187,8 @@ class BaseResponsesAPITest(ABC): # basic test assert the usage seems reasonable print("response_completed_event.response.usage=", response_completed_event.response.usage) assert response_completed_event.response.usage.input_tokens > 0 and response_completed_event.response.usage.input_tokens < 100 - assert response_completed_event.response.usage.output_tokens > 0 and response_completed_event.response.usage.output_tokens < 1000 - assert response_completed_event.response.usage.total_tokens > 0 and response_completed_event.response.usage.total_tokens < 1000 + assert response_completed_event.response.usage.output_tokens > 0 and response_completed_event.response.usage.output_tokens < 2000 + assert response_completed_event.response.usage.total_tokens > 0 and response_completed_event.response.usage.total_tokens < 2000 # total tokens should be the sum of input and output tokens assert response_completed_event.response.usage.total_tokens == response_completed_event.response.usage.input_tokens + response_completed_event.response.usage.output_tokens @@ -229,6 +232,7 @@ class BaseResponsesAPITest(ABC): @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.flaky(retries=3, delay=2) @pytest.mark.asyncio async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode): #litellm._turn_on_debug() @@ -278,6 +282,7 @@ class BaseResponsesAPITest(ABC): ) @pytest.mark.parametrize("sync_mode", [False, True]) + @pytest.mark.flaky(retries=3, delay=2) @pytest.mark.asyncio async def test_basic_openai_responses_get_endpoint(self, sync_mode): litellm._turn_on_debug() @@ -318,6 +323,7 @@ class BaseResponsesAPITest(ABC): raise ValueError("response is not a ResponsesAPIResponse") @pytest.mark.asyncio + @pytest.mark.flaky(retries=3, delay=2) async def test_basic_openai_list_input_items_endpoint(self): """Test that calls the OpenAI List Input Items endpoint""" litellm._turn_on_debug() @@ -364,3 +370,73 @@ class BaseResponsesAPITest(ABC): # assert the response is not None assert response_1 is not None assert response_2 is not None + + @pytest.mark.asyncio + async def test_responses_api_with_tool_calls(self): + """Test that calls the Responses API with tool calls including function call and output""" + litellm._turn_on_debug() + litellm.set_verbose = True + base_completion_call_args = self.get_base_completion_call_args() + + # Define the input with message, function call, and function call output + input_data: ResponseInputParam = [ + { + "type": "message", + "role": "user", + "content": "How is the weather in SĆ£o Paulo today ?" + }, + { + "type": "function_call", + "arguments": "{\"location\": \"SĆ£o Paulo, Brazil\"}", + "call_id": "fc_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "name": "get_weather", + "id": "fc_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "status": "completed" + }, + { + "type": "function_call_output", + "call_id": "fc_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "output": "Rainy" + } + ] + + # Define the tools + tools = [ + { + "type": "function", + "name": "get_weather", + "description": "Get current temperature for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and country e.g. BogotĆ”, Colombia" + } + }, + "required": ["location"], + "additionalProperties": False + } + } + ] + + try: + # Make the responses API call + response = await litellm.aresponses( + input=input_data, + store=False, + tools=tools, + **base_completion_call_args + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + + print("litellm response=", json.dumps(response, indent=4, default=str)) + + # Validate the response structure + validate_responses_api_response(response, final_chunk=True) + + # Additional assertions specific to tool calls + assert response is not None + assert "output" in response + assert len(response["output"]) > 0 diff --git a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py b/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py index 0357f8c6496..81daaea238d 100644 --- a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py +++ b/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py @@ -5,7 +5,7 @@ from unittest.mock import patch, AsyncMock sys.path.insert(0, os.path.abspath("../..")) import litellm import json - +from base_responses_api import BaseResponsesAPITest @pytest.mark.asyncio async def test_basic_google_ai_studio_responses_api_with_tools(): litellm._turn_on_debug() @@ -85,10 +85,22 @@ async def test_mock_basic_google_ai_studio_responses_api_with_tools(): assert call_kwargs["messages"][0]["content"] == "what is the latest version of supabase python package and when was it released?" assert call_kwargs["tools"] == [] # web search tools are converted to web_search_options, not kept as tools +class TestGoogleAIStudioResponsesAPITest(BaseResponsesAPITest): + def get_base_completion_call_args(self): + #litellm._turn_on_debug() + return { + "model": "gemini/gemini-2.5-flash-lite" + } + + async def test_basic_openai_responses_delete_endpoint(self, sync_mode=False): + pass + + async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode=False): + pass - - - + async def test_basic_openai_responses_get_endpoint(self, sync_mode=False): + pass + diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 87488f67e7a..5cd515be232 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -1308,3 +1308,90 @@ async def test_store_field_transformation(): assert response.created_at == 1751443898, "created_at should maintain the same value after conversion" +@pytest.mark.asyncio +async def test_aresponses_service_tier_and_safety_identifier(): + """ + Test that service_tier and safety_identifier parameters are correctly sent in the request body + when using litellm.aresponses. + """ + mock_response = { + "id": "resp_01234567890abcdef", + "object": "response", + "created_at": 1753060947, + "status": "completed", + "error": None, + "incomplete_details": None, + "instructions": None, + "max_output_tokens": None, + "model": "gpt-4o-2024-05-13", + "output": [ + { + "type": "text", + "id": "out_01234567890abcdef", + "text": "This is a test response with service tier and safety identifier.", + } + ], + "parallel_tool_calls": True, + "previous_response_id": None, + "reasoning": None, + "store": True, + "temperature": 1.0, + "text": {"format": {"type": "text"}}, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 15, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 25, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 40, + }, + "user": None, + "metadata": {}, + } + + class MockResponse: + def __init__(self, json_data, status_code): + self._json_data = json_data + self.status_code = status_code + self.text = json.dumps(json_data) + + def json(self): + return self._json_data + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + # Configure the mock to return our response + mock_post.return_value = MockResponse(mock_response, 200) + + litellm._turn_on_debug() + litellm.set_verbose = True + + # Call aresponses with service_tier and safety_identifier + response = await litellm.aresponses( + model="openai/gpt-4o", + input="Test with service tier and safety identifier", + service_tier="flex", + safety_identifier="123", + ) + + # Verify the request was made correctly + mock_post.assert_called_once() + request_body = mock_post.call_args.kwargs["json"] + print("request_body=", json.dumps(request_body, indent=4, default=str)) + + # Validate that both parameters are present in the request body + assert request_body["service_tier"] == "flex", "service_tier should be 'flex' in request body" + assert request_body["safety_identifier"] == "123", "safety_identifier should be '123' in request body" + assert request_body["model"] == "gpt-4o" + assert request_body["input"] == "Test with service tier and safety identifier" + + # Validate the response + print("Response:", json.dumps(response, indent=4, default=str)) + + + diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 04953e32577..4c24d1fcd5b 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -1200,96 +1200,99 @@ class BaseLLMChatTest(ABC): from litellm.utils import supports_function_calling from litellm import completion litellm._turn_on_debug() + try: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") - base_completion_call_args = self.get_base_completion_call_args() - if not supports_function_calling(base_completion_call_args["model"], None): - print("Model does not support function calling") - pytest.skip("Model does not support function calling") - - def get_weather(city: str): - return f"City: {city}, Weather: Sunny with 34 degree Celcius" + base_completion_call_args = self.get_base_completion_call_args() + if not supports_function_calling(base_completion_call_args["model"], None): + print("Model does not support function calling") + pytest.skip("Model does not support function calling") + + def get_weather(city: str): + return f"City: {city}, Weather: Sunny with 34 degree Celcius" - TOOLS = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather in a city", - "parameters": { - "$id": "https://some/internal/name", - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The city to get the weather for", - } + TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather in a city", + "parameters": { + "$id": "https://some/internal/name", + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to get the weather for", + } + }, + "required": ["city"], + "additionalProperties": False, }, - "required": ["city"], - "additionalProperties": False, + "strict": True, }, - "strict": True, - }, - } - ] + } + ] - messages = [{ "content": "How is the weather in Mumbai?","role": "user"}] - response, iteration = "", 0 - while True: - if response: - break - # Create a streaming response with tool calling enabled - stream = completion( - **base_completion_call_args, - messages=messages, - tools=TOOLS, - stream=True, - ) + messages = [{ "content": "How is the weather in Mumbai?","role": "user"}] + response, iteration = "", 0 + while True: + if response: + break + # Create a streaming response with tool calling enabled + stream = completion( + **base_completion_call_args, + messages=messages, + tools=TOOLS, + stream=True, + ) - final_tool_calls = {} - for chunk in stream: - delta = chunk.choices[0].delta - print(delta) - if delta.content: - response += delta.content - elif delta.tool_calls: - for tool_call in chunk.choices[0].delta.tool_calls or []: - index = tool_call.index - if index not in final_tool_calls: - final_tool_calls[index] = tool_call - else: - final_tool_calls[ - index - ].function.arguments += tool_call.function.arguments - if final_tool_calls: - for tool_call in final_tool_calls.values(): - if tool_call.function.name == "get_weather": - city = json.loads(tool_call.function.arguments)["city"] - tool_response = get_weather(city) - messages.append( - { - "role": "assistant", - "tool_calls": [tool_call], - "content": None, - } - ) - messages.append( - { - "role": "tool", - "tool_call_id": tool_call.id, - "content": tool_response, - } - ) - iteration += 1 - if iteration > 2: - print("Something went wrong!") - break + final_tool_calls = {} + for chunk in stream: + delta = chunk.choices[0].delta + print(delta) + if delta.content: + response += delta.content + elif delta.tool_calls: + for tool_call in chunk.choices[0].delta.tool_calls or []: + index = tool_call.index + if index not in final_tool_calls: + final_tool_calls[index] = tool_call + else: + final_tool_calls[ + index + ].function.arguments += tool_call.function.arguments + if final_tool_calls: + for tool_call in final_tool_calls.values(): + if tool_call.function.name == "get_weather": + city = json.loads(tool_call.function.arguments)["city"] + tool_response = get_weather(city) + messages.append( + { + "role": "assistant", + "tool_calls": [tool_call], + "content": None, + } + ) + messages.append( + { + "role": "tool", + "tool_call_id": tool_call.id, + "content": tool_response, + } + ) + iteration += 1 + if iteration > 2: + print("Something went wrong!") + break - print(response) + print(response) + except litellm.ServiceUnavailableError: + pass def test_reasoning_effort(self): """Test that reasoning_effort is passed correctly to the model""" diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 98707cbc1ee..f5de082ded2 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -603,3 +603,54 @@ def test_openai_deepresearch_model_bridge(): ) print("response: ", response) + + +def test_openai_tool_calling(): + from pydantic import BaseModel + from typing import Any, Literal + + class OpenAIFunction(BaseModel): + description: Optional[str] = None + name: str + parameters: Optional[dict[str, Any]] = None + + class OpenAITool(BaseModel): + type: Literal["function"] + function: OpenAIFunction + + completion_params = { + "model": "openai/gpt-4.1", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is TSLA stock price at today?"} + ], + } + ], + "stream": False, + "temperature": 0.5, + "stop": None, + "max_tokens": 1600, + "tools": [ + OpenAITool( + type="function", + function=OpenAIFunction( + description="Get the current stock price for a given ticker symbol.", + name="get_stock_price", + parameters={ + "type": "object", + "properties": { + "ticker": { + "type": "string", + "description": "The stock ticker symbol, e.g. AAPL for Apple Inc.", + } + }, + "required": ["ticker"], + }, + ), + ) + ], + } + + response = litellm.completion(**completion_params) diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index 7095285eb60..d1c0fb4e01c 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -58,10 +58,10 @@ VERTEX_MODELS_TO_NOT_TEST = [ "gemini-1.5-pro-preview-0215", "gemini-pro-experimental", "gemini-flash-experimental", - "gemini-1.5-flash-exp-0827", + "gemini-2.5-flash-lite-exp-0827", "gemini-2.0-pro-exp-02-05", "gemini-pro-flash", - "gemini-1.5-flash-exp-0827", + "gemini-2.5-flash-lite-exp-0827", "gemini-2.0-flash-exp", "gemini-2.0-flash-thinking-exp", "gemini-2.0-flash-thinking-exp-01-21", @@ -149,7 +149,7 @@ async def test_get_response(): prompt = '\ndef count_nums(arr):\n """\n Write a function count_nums which takes an array of integers and returns\n the number of elements which has a sum of digits > 0.\n If a number is negative, then its first signed digit will be negative:\n e.g. -123 has signed digits -1, 2, and 3.\n >>> count_nums([]) == 0\n >>> count_nums([-1, 11, -11]) == 1\n >>> count_nums([1, 1, 2]) == 3\n """\n' try: response = await acompletion( - model="gemini-1.5-flash", + model="gemini-2.5-flash-lite", messages=[ { "role": "system", @@ -518,7 +518,7 @@ async def test_gemini_pro_vision(provider, sync_mode): litellm.num_retries = 3 if sync_mode: resp = litellm.completion( - model="{}/gemini-1.5-flash-preview-0514".format(provider), + model="{}/gemini-2.5-flash-lite".format(provider), messages=[ {"role": "system", "content": "Be a good bot"}, { @@ -537,7 +537,7 @@ async def test_gemini_pro_vision(provider, sync_mode): ) else: resp = await litellm.acompletion( - model="{}/gemini-1.5-flash-preview-0514".format(provider), + model="{}/gemini-2.5-flash-lite".format(provider), messages=[ {"role": "system", "content": "Be a good bot"}, { @@ -605,7 +605,7 @@ def test_completion_function_plus_pdf(load_pdf): image_message = {"role": "user", "content": image_content} response = completion( - model="vertex_ai_beta/gemini-1.5-flash-preview-0514", + model="vertex_ai_beta/gemini-2.5-flash-lite", messages=[image_message], stream=False, ) @@ -1194,7 +1194,7 @@ Using this JSON schema: with patch.object(client, "post", side_effect=_side_effect) as mock_call: response = completion( - model="vertex_ai_beta/gemini-1.5-flash", + model="vertex_ai_beta/gemini-2.5-flash-lite", messages=messages, response_format={"type": "json_object"}, client=client, @@ -1383,7 +1383,7 @@ def vertex_httpx_mock_post_invalid_schema_response_anthropic(*args, **kwargs): [ ("vertex_ai_beta/gemini-1.5-pro-001", "us-central1", True), ("gemini/gemini-1.5-pro", None, True), - ("vertex_ai_beta/gemini-1.5-flash", "us-central1", True), + ("vertex_ai_beta/gemini-2.5-flash-lite", "us-central1", True), ("vertex_ai/claude-3-5-sonnet@20240620", "us-east5", False), ], ) @@ -1572,7 +1572,7 @@ async def test_anthropic_message_via_anthropic_messages(): [ ("vertex_ai_beta/gemini-1.5-pro-001", "us-central1", True), ("gemini/gemini-1.5-pro", None, True), - ("vertex_ai_beta/gemini-1.5-flash", "us-central1", True), + ("vertex_ai_beta/gemini-2.5-flash-lite", "us-central1", True), ("vertex_ai/claude-3-5-sonnet@20240620", "us-east5", False), ], ) @@ -1680,7 +1680,7 @@ async def test_gemini_pro_json_schema_args_sent_httpx_openai_schema( @pytest.mark.parametrize( - "model", ["gemini-1.5-flash", "claude-3-5-sonnet@20240620"] + "model", ["gemini-2.5-flash-lite", "claude-3-5-sonnet@20240620"] ) # "vertex_ai", @pytest.mark.asyncio async def test_gemini_pro_httpx_custom_api_base(model): @@ -1820,7 +1820,7 @@ async def test_gemini_pro_function_calling_streaming(sync_mode): load_vertex_ai_credentials() litellm.set_verbose = True data = { - "model": "vertex_ai/gemini-1.5-flash", + "model": "vertex_ai/gemini-2.5-flash-lite", "messages": [ { "role": "user", @@ -2541,7 +2541,7 @@ def mock_gemini_request(*args, **kwargs): if "cachedContents" in kwargs["url"]: mock_response.json.return_value = { "name": "cachedContents/4d2kd477o3pg", - "model": "models/gemini-1.5-flash-001", + "model": "models/gemini-2.5-flash-lite-001", "createTime": "2024-08-26T22:31:16.147190Z", "updateTime": "2024-08-26T22:31:16.147190Z", "expireTime": "2024-08-26T22:36:15.548934784Z", @@ -2671,7 +2671,7 @@ async def test_gemini_context_caching_anthropic_format(sync_mode): try: if sync_mode: response = litellm.completion( - model="gemini/gemini-1.5-flash-001", + model="gemini/gemini-2.5-flash-lite-001", messages=gemini_context_caching_messages, temperature=0.2, max_tokens=10, @@ -2679,7 +2679,7 @@ async def test_gemini_context_caching_anthropic_format(sync_mode): ) else: response = await litellm.acompletion( - model="gemini/gemini-1.5-flash-001", + model="gemini/gemini-2.5-flash-lite-001", messages=gemini_context_caching_messages, temperature=0.2, max_tokens=10, diff --git a/tests/local_testing/test_batch_completions.py b/tests/local_testing/test_batch_completions.py index 0883fd36d77..2125a998f84 100644 --- a/tests/local_testing/test_batch_completions.py +++ b/tests/local_testing/test_batch_completions.py @@ -72,7 +72,7 @@ def test_batch_completions_models(): def test_batch_completion_models_all_responses(): try: responses = batch_completion_models_all_responses( - models=["gemini/gemini-1.5-flash", "claude-3-haiku-20240307"], + models=["gemini/gemini-2.5-flash-lite", "claude-3-haiku-20240307"], messages=[{"role": "user", "content": "write a poem"}], max_tokens=10, ) diff --git a/tests/local_testing/test_caching.py b/tests/local_testing/test_caching.py index 26df08ecc64..1f68e8a43ad 100644 --- a/tests/local_testing/test_caching.py +++ b/tests/local_testing/test_caching.py @@ -2155,8 +2155,8 @@ async def test_caching_kwargs_input(sync_mode): Message, ModelResponse, Usage, - CompletionTokensDetails, - PromptTokensDetails, + CompletionTokensDetailsWrapper, + PromptTokensDetailsWrapper, ) from datetime import datetime @@ -2187,10 +2187,10 @@ async def test_caching_kwargs_input(sync_mode): completion_tokens=31, prompt_tokens=16, total_tokens=47, - completion_tokens_details=CompletionTokensDetails( + completion_tokens_details=CompletionTokensDetailsWrapper( audio_tokens=None, reasoning_tokens=0 ), - prompt_tokens_details=PromptTokensDetails( + prompt_tokens_details=PromptTokensDetailsWrapper( audio_tokens=None, cached_tokens=0 ), ), diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 494fe231daf..535f5cb00af 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -3696,7 +3696,7 @@ def test_completion_volcengine(): [ # "gemini-1.0-pro", "gemini-1.5-pro", - # "gemini-1.5-flash", + # "gemini-2.5-flash-lite", ], ) @pytest.mark.flaky(retries=3, delay=1) @@ -3750,7 +3750,7 @@ def test_completion_gemini(model): @pytest.mark.asyncio async def test_acompletion_gemini(): litellm.set_verbose = True - model_name = "gemini/gemini-1.5-flash" + model_name = "gemini/gemini-2.5-flash-lite" messages = [{"role": "user", "content": "Hey, how's it going?"}] try: response = await litellm.acompletion(model=model_name, messages=messages) diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index 72f057bc60d..06f8346994f 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -1187,3 +1187,93 @@ async def test_embedding_with_extra_headers(sync_mode): mock_post.assert_called_once() assert "my-test-param" in mock_post.call_args.kwargs["headers"] + + +@pytest.mark.parametrize( + "input_data, expected_payload_input", + [ + # Case 1: Input with only text strings + ( + ["hello world", "foo bar"], + ["hello world", "foo bar"], + ), + # Case 2: Input with a mix of text and a base64 encoded image + ( + [ + "A picture of a cat", + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=", + ], + [ + {"text": "A picture of a cat"}, + { + "image": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + }, + ], + ), + # Case 3: Input with only a base64 encoded image + ( + [ + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + ], + [ + { + "image": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + } + ], + ), + ], +) +def test_jina_ai_img_embeddings(input_data, expected_payload_input): + """ + Tests the input transformation logic for Jina AI embeddings using mocks. + + This test verifies that when litellm.embedding is called with a jina_ai model, + the 'input' field in the request payload is formatted correctly based on whether + the input contains text or base64 encoded images. + """ + # We patch the `post` method of the HTTPHandler. This intercepts the network + # request before it's actually sent. + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: + # Configure the mock to return a successful, minimal valid response. + # This prevents litellm from raising an error when processing the response. + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [0.1] * 768, # Dummy embedding vector + } + ], + "model": "jina-embeddings-v4", + } + mock_post.return_value = mock_response + + # Call the function we want to test + try: + litellm.embedding( + model="jina_ai/jina-embeddings-v4", input=input_data + ) + except Exception as e: + pytest.fail( + f"litellm.embedding call failed with an unexpected exception: {e}" + ) + + # --- Assertions --- + # 1. Check that our mock `post` method was called exactly once. + mock_post.assert_called_once() + + # 2. Extract the keyword arguments passed to the mock call. + # The request payload is in the 'data' keyword argument. + kwargs = mock_post.call_args.kwargs + assert "data" in kwargs + + # 3. Parse the JSON payload string into a Python dictionary. + sent_data = json.loads(kwargs["data"]) + + # 4. This is the core of our test: + # Assert that the 'input' field in the payload matches our expectation. + assert "input" in sent_data + assert sent_data["input"] == expected_payload_input diff --git a/tests/local_testing/test_gcs_cache_unit_tests.py b/tests/local_testing/test_gcs_cache_unit_tests.py new file mode 100644 index 00000000000..305dfd95d7d --- /dev/null +++ b/tests/local_testing/test_gcs_cache_unit_tests.py @@ -0,0 +1,6 @@ +from cache_unit_tests import LLMCachingUnitTests +from litellm.caching import LiteLLMCacheType + +class TestGCSCacheUnitTests(LLMCachingUnitTests): + def get_cache_type(self) -> LiteLLMCacheType: + return LiteLLMCacheType.GCS diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index 3445dd5a519..73c31f0ba44 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -2133,7 +2133,7 @@ def test_router_correctly_reraise_error(): """ User feedback: There is a problem with my messages array, but the error exception thrown is a Rate Limit error. ``` - Rate Limit: Error code: 429 - {'error': {'message': 'No deployments available for selected model, Try again in 60 seconds. Passed model=gemini-1.5-flash.. + Rate Limit: Error code: 429 - {'error': {'message': 'No deployments available for selected model, Try again in 60 seconds. Passed model=gemini-2.5-flash-lite.. ``` What they want? Propagation of the real error. """ diff --git a/tests/local_testing/test_router_utils.py b/tests/local_testing/test_router_utils.py index cd26f8ad603..f3b0238be51 100644 --- a/tests/local_testing/test_router_utils.py +++ b/tests/local_testing/test_router_utils.py @@ -414,6 +414,7 @@ def test_router_handle_clientside_credential(): "api_key": "123", "metadata": {"model_group": "gemini/gemini-1.5-flash"}, }, + function_name="acompletion", ) assert new_deployment.litellm_params.api_key == "123" diff --git a/tests/local_testing/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py index 63907eb7d5e..85df25e8f21 100644 --- a/tests/local_testing/test_stream_chunk_builder.py +++ b/tests/local_testing/test_stream_chunk_builder.py @@ -198,7 +198,7 @@ def test_stream_chunk_builder_litellm_usage_chunks(): # make a streaming gemini call try: response = completion( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash-lite", messages=messages, stream=True, complete_response=True, diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index 823b6350285..184ede2222f 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -709,7 +709,7 @@ async def test_completion_gemini_stream(sync_mode): chunks = [] if sync_mode: response = completion( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash-lite", messages=messages, stream=True, functions=function1, @@ -726,7 +726,7 @@ async def test_completion_gemini_stream(sync_mode): complete_response += chunk else: response = await litellm.acompletion( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash-lite", messages=messages, stream=True, functions=function1, @@ -829,7 +829,7 @@ async def test_completion_gemini_stream_accumulated_json(sync_mode): client, "post", side_effect=gemini_mock_post_streaming ) as mock_client: response = completion( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash-lite", messages=messages, stream=True, functions=function1, @@ -854,7 +854,7 @@ async def test_completion_gemini_stream_accumulated_json(sync_mode): client, "post", side_effect=gemini_mock_post_streaming ) as mock_client: response = await litellm.acompletion( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash-lite", messages=messages, stream=True, functions=function1, @@ -2982,7 +2982,7 @@ def test_completion_claude_3_function_call_with_streaming(): @pytest.mark.parametrize( "model", [ - "gemini/gemini-1.5-flash", + "gemini/gemini-2.5-flash-lite", ], # "claude-3-opus-20240229" ) # @pytest.mark.asyncio @@ -3669,7 +3669,7 @@ def test_unit_test_custom_stream_wrapper_function_call(): ) ], created=1720755257, - model="gemini-1.5-flash", + model="gemini-2.5-flash-lite", object="chat.completion.chunk", system_fingerprint=None, usage=Usage(prompt_tokens=67, completion_tokens=55, total_tokens=122), @@ -3948,3 +3948,45 @@ def test_is_delta_empty(): audio=None, ) ) + + +def test_streaming_with_cost_calculation(): + from litellm.types.utils import Usage + from typing import Optional + + litellm.include_cost_in_streaming_usage = True + + ## Test 1: check if usage object can handle 'cost' field + usage_object = Usage( + prompt_tokens=100, + completion_tokens=100, + total_tokens=200, + cost=1.0, + ) + assert usage_object.cost is not None + + print(f"usage_object: {usage_object}") + + ## Test 2: check if usage object has 'cost' field when streaming + + response = litellm.completion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "What is the capital of France?"}], + stream=True, + stream_options={"include_usage": True}, + ) + + usage_object: Optional[Usage] = None + for chunk in response: + _usage_obj = getattr(chunk, "usage", None) + if _usage_obj is not None: + usage_object = _usage_obj + break + + assert usage_object is not None + assert usage_object.total_tokens is not None + assert usage_object.total_tokens > 0 + assert usage_object.prompt_tokens is not None + assert usage_object.prompt_tokens > 0 + assert usage_object.cost is not None + assert usage_object.cost > 0 diff --git a/tests/local_testing/test_text_completion.py b/tests/local_testing/test_text_completion.py index 89527354f3c..26aca81adf9 100644 --- a/tests/local_testing/test_text_completion.py +++ b/tests/local_testing/test_text_completion.py @@ -4152,7 +4152,7 @@ def test_completion_vllm(provider): client.completions.with_raw_response, "create", side_effect=mock_post ) as mock_call: response = text_completion( - model="{provider}/gemini-1.5-flash".format(provider=provider), + model="{provider}/gemini-2.5-flash-lite".format(provider=provider), prompt="ping", client=client, hello="world", diff --git a/tests/logging_callback_tests/test_amazing_s3_logs.py b/tests/logging_callback_tests/test_amazing_s3_logs.py index 37666d72b79..c9d0987e86f 100644 --- a/tests/logging_callback_tests/test_amazing_s3_logs.py +++ b/tests/logging_callback_tests/test_amazing_s3_logs.py @@ -476,3 +476,18 @@ def test_s3_logging_r2(): # post, close log file and verify # Reset stdout to the original value print("Passed! Testing async s3 logging") + +from litellm.integrations.s3_v2 import S3Logger + +class TestS3Logger(S3Logger): + def __init__(self, *args, **kwargs): + self.recorded_requests = {} + self.logged_standard_logging_payload: Optional[StandardLoggingPayload] = None + super().__init__(*args, **kwargs) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.recorded_requests[response_obj["id"]] = start_time + print("recorded request", self.recorded_requests) + self.logged_standard_logging_payload = kwargs["standard_logging_object"] + return await super().async_log_success_event(kwargs, response_obj, start_time, end_time) + diff --git a/tests/logging_callback_tests/test_gcs_pub_sub.py b/tests/logging_callback_tests/test_gcs_pub_sub.py index f231d01d3fb..4172659e659 100644 --- a/tests/logging_callback_tests/test_gcs_pub_sub.py +++ b/tests/logging_callback_tests/test_gcs_pub_sub.py @@ -38,6 +38,7 @@ ignored_keys = [ "endTime", "metadata.model_map_information", "metadata.usage_object", + "metadata.cold_storage_object_key", ] diff --git a/tests/logging_callback_tests/test_otel_logging.py b/tests/logging_callback_tests/test_otel_logging.py index ff9d8300fef..aeb42bdaf79 100644 --- a/tests/logging_callback_tests/test_otel_logging.py +++ b/tests/logging_callback_tests/test_otel_logging.py @@ -279,6 +279,7 @@ def validate_redacted_message_span_attributes(span): "metadata.mcp_tool_call_metadata", "metadata.vector_store_request_metadata", "metadata.requester_custom_headers", + "metadata.cold_storage_object_key", ] _all_attributes = set( diff --git a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py index 3c9d31890c7..a1760ca6371 100644 --- a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py +++ b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py @@ -33,6 +33,7 @@ expected_env_vars = { "LAGO_API_BASE": "mock_base", "LAGO_API_EVENT_CODE": "mock_event_code", "OPENMETER_API_KEY": "openmeter_api_key", + "BRAINTRUST_API_BASE": "braintrust_api_base", "BRAINTRUST_API_KEY": "braintrust_api_key", "GALILEO_API_KEY": "galileo_api_key", "LITERAL_API_KEY": "literal_api_key", diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 35884972532..2a2d60d96ac 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -583,9 +583,14 @@ async def test_list_tools_rest_api_server_not_found(): # Mock UserAPIKeyAuth mock_user_auth = UserAPIKeyAuth(api_key="test", user_id="test") + + # Mock request + mock_request = MagicMock() + mock_request.headers = {} # Test with non-existent server ID response = await list_tool_rest_api( + request=mock_request, server_id="non_existent_server_id", user_api_key_dict=mock_user_auth ) @@ -644,8 +649,13 @@ async def test_list_tools_rest_api_success(): # Get the server ID server_id = list(global_mcp_server_manager.get_registry().keys())[0] + # Mock request + mock_request = MagicMock() + mock_request.headers = {} + # Test successful case response = await list_tool_rest_api( + request=mock_request, server_id=server_id, user_api_key_dict=mock_user_auth ) @@ -1237,3 +1247,411 @@ async def test_mcp_protocol_version_passed_to_client(): mock_client.list_tools.assert_called() +def test_get_server_auth_header_with_alias(): + """Test _get_server_auth_header function with server alias.""" + from litellm.proxy._experimental.mcp_server.rest_endpoints import _get_server_auth_header + + # Create a mock server with alias + mock_server = MagicMock() + mock_server.alias = "zapier" + mock_server.server_name = "zapier_server" + + # Test with server-specific auth headers + mcp_server_auth_headers = { + "zapier": "Bearer zapier_token", + "slack": "Bearer slack_token" + } + mcp_auth_header = "Bearer default_token" + + result = _get_server_auth_header(mock_server, mcp_server_auth_headers, mcp_auth_header) + assert result == "Bearer zapier_token" + + # Test case-insensitive matching + mcp_server_auth_headers = { + "ZAPIER": "Bearer zapier_token_upper", + "slack": "Bearer slack_token" + } + + result = _get_server_auth_header(mock_server, mcp_server_auth_headers, mcp_auth_header) + assert result == "Bearer zapier_token_upper" + + +def test_get_server_auth_header_with_server_name(): + """Test _get_server_auth_header function with server name (no alias).""" + from litellm.proxy._experimental.mcp_server.rest_endpoints import _get_server_auth_header + + # Create a mock server with server_name but no alias + mock_server = MagicMock() + mock_server.alias = None + mock_server.server_name = "slack_server" + + # Test with server-specific auth headers + mcp_server_auth_headers = { + "slack_server": "Bearer slack_token", + "zapier": "Bearer zapier_token" + } + mcp_auth_header = "Bearer default_token" + + result = _get_server_auth_header(mock_server, mcp_server_auth_headers, mcp_auth_header) + assert result == "Bearer slack_token" + + # Test case-insensitive matching + mcp_server_auth_headers = { + "SLACK_SERVER": "Bearer slack_token_upper", + "zapier": "Bearer zapier_token" + } + + result = _get_server_auth_header(mock_server, mcp_server_auth_headers, mcp_auth_header) + assert result == "Bearer slack_token_upper" + + +def test_get_server_auth_header_fallback_to_default(): + """Test _get_server_auth_header function fallback to default auth header.""" + from litellm.proxy._experimental.mcp_server.rest_endpoints import _get_server_auth_header + + # Create a mock server + mock_server = MagicMock() + mock_server.alias = "unknown_server" + mock_server.server_name = "unknown_server_name" + + # Test with no matching server-specific headers + mcp_server_auth_headers = { + "zapier": "Bearer zapier_token", + "slack": "Bearer slack_token" + } + mcp_auth_header = "Bearer default_token" + + result = _get_server_auth_header(mock_server, mcp_server_auth_headers, mcp_auth_header) + assert result == "Bearer default_token" + + # Test with no server-specific headers at all + result = _get_server_auth_header(mock_server, None, mcp_auth_header) + assert result == "Bearer default_token" + + +def test_get_server_auth_header_no_auth_headers(): + """Test _get_server_auth_header function with no auth headers.""" + from litellm.proxy._experimental.mcp_server.rest_endpoints import _get_server_auth_header + + # Create a mock server + mock_server = MagicMock() + mock_server.alias = "zapier" + mock_server.server_name = "zapier_server" + + # Test with no auth headers + result = _get_server_auth_header(mock_server, None, None) + assert result is None + + result = _get_server_auth_header(mock_server, {}, None) + assert result is None + + +def test_create_tool_response_objects(): + """Test _create_tool_response_objects function.""" + from litellm.proxy._experimental.mcp_server.rest_endpoints import _create_tool_response_objects + from mcp.types import Tool as MCPTool + + # Create mock tools + mock_tools = [ + MCPTool( + name="send_email", + description="Send an email", + inputSchema={"type": "object", "properties": {"to": {"type": "string"}}} + ), + MCPTool( + name="create_event", + description="Create a calendar event", + inputSchema={"type": "object", "properties": {"title": {"type": "string"}}} + ) + ] + + server_mcp_info = { + "server_name": "zapier", + "logo_url": "https://zapier.com/logo.png" + } + + result = _create_tool_response_objects(mock_tools, server_mcp_info) + + assert len(result) == 2 + assert result[0].name == "send_email" + assert result[0].description == "Send an email" + assert result[0].mcp_info == server_mcp_info + assert result[1].name == "create_event" + assert result[1].description == "Create a calendar event" + assert result[1].mcp_info == server_mcp_info + + +@pytest.mark.asyncio +async def test_get_tools_for_single_server(): + """Test _get_tools_for_single_server function.""" + from litellm.proxy._experimental.mcp_server.rest_endpoints import _get_tools_for_single_server + from mcp.types import Tool as MCPTool + + # Create a mock server + mock_server = MagicMock() + mock_server.mcp_info = {"server_name": "zapier"} + + # Create mock tools + mock_tools = [ + MCPTool( + name="send_email", + description="Send an email", + inputSchema={"type": "object", "properties": {"to": {"type": "string"}}} + ) + ] + + # Mock the global_mcp_server_manager + with patch('litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager') as mock_manager: + mock_manager._get_tools_from_server = AsyncMock(return_value=mock_tools) + + result = await _get_tools_for_single_server(mock_server, "Bearer test_token", "2025-03-26") + + # Verify the manager was called with correct parameters + mock_manager._get_tools_from_server.assert_called_once_with( + server=mock_server, + mcp_auth_header="Bearer test_token", + mcp_protocol_version="2025-03-26" + ) + + # Verify the result + assert len(result) == 1 + assert result[0].name == "send_email" + assert result[0].mcp_info == {"server_name": "zapier"} + + +@pytest.mark.asyncio +async def test_list_tool_rest_api_with_server_specific_auth(): + """Test list_tool_rest_api with server-specific auth headers.""" + from litellm.proxy._experimental.mcp_server.rest_endpoints import list_tool_rest_api + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + + # Create mock request with server-specific auth headers + mock_request = MagicMock() + mock_request.headers = { + "authorization": "Bearer user_token", + "x-mcp-zapier-authorization": "Bearer zapier_token", + "x-mcp-slack-authorization": "Bearer slack_token", + "MCP-Protocol-Version": "2025-06-18" + } + + # Create mock user_api_key_dict + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test_user" + + # Mock the MCPRequestHandler methods + with patch.object(MCPRequestHandler, '_get_mcp_auth_header_from_headers') as mock_get_auth: + with patch.object(MCPRequestHandler, '_get_mcp_server_auth_headers_from_headers') as mock_get_server_auth: + mock_get_auth.return_value = "Bearer default_token" + mock_get_server_auth.return_value = { + "zapier": "Bearer zapier_token", + "slack": "Bearer slack_token" + } + + # Mock the global_mcp_server_manager + with patch('litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager') as mock_manager: + # Create a mock server + mock_server = MagicMock() + mock_server.server_id = "test-server-123" + mock_server.alias = "zapier" + mock_server.name = "zapier_server" + mock_server.mcp_info = {"server_name": "zapier"} + + mock_manager.get_mcp_server_by_id.return_value = mock_server + + # Mock the _get_tools_for_single_server function + with patch('litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server') as mock_get_tools: + from litellm.proxy._experimental.mcp_server.server import ListMCPToolsRestAPIResponseObject + + mock_tools = [ + ListMCPToolsRestAPIResponseObject( + name="send_email", + description="Send an email", + inputSchema={"type": "object"}, + mcp_info={"server_name": "zapier"} + ) + ] + mock_get_tools.return_value = mock_tools + + # Call the function + result = await list_tool_rest_api( + request=mock_request, + server_id="test-server-123", + user_api_key_dict=mock_user_api_key_dict + ) + + # Verify the result + assert result["error"] is None + assert len(result["tools"]) == 1 + assert result["tools"][0].name == "send_email" + + # Verify that _get_tools_for_single_server was called with the correct auth header + mock_get_tools.assert_called_once() + call_args = mock_get_tools.call_args + assert call_args[0][0] == mock_server # server + assert call_args[0][1] == "Bearer zapier_token" # server_auth_header + assert call_args[0][2] == "2025-06-18" # mcp_protocol_version + + +@pytest.mark.asyncio +async def test_list_tool_rest_api_with_default_auth(): + """Test list_tool_rest_api with default auth header when no server-specific header is found.""" + from litellm.proxy._experimental.mcp_server.rest_endpoints import list_tool_rest_api + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + + # Create mock request with default auth header only + mock_request = MagicMock() + mock_request.headers = { + "authorization": "Bearer user_token", + "x-mcp-authorization": "Bearer default_token", + "MCP-Protocol-Version": "2025-06-18" + } + + # Create mock user_api_key_dict + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test_user" + + # Mock the MCPRequestHandler methods + with patch.object(MCPRequestHandler, '_get_mcp_auth_header_from_headers') as mock_get_auth: + with patch.object(MCPRequestHandler, '_get_mcp_server_auth_headers_from_headers') as mock_get_server_auth: + mock_get_auth.return_value = "Bearer default_token" + mock_get_server_auth.return_value = {} # No server-specific headers + + # Mock the global_mcp_server_manager + with patch('litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager') as mock_manager: + # Create a mock server + mock_server = MagicMock() + mock_server.server_id = "test-server-123" + mock_server.alias = "unknown_server" + mock_server.name = "unknown_server" + mock_server.mcp_info = {"server_name": "unknown_server"} + + mock_manager.get_mcp_server_by_id.return_value = mock_server + + # Mock the _get_tools_for_single_server function + with patch('litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server') as mock_get_tools: + from litellm.proxy._experimental.mcp_server.server import ListMCPToolsRestAPIResponseObject + + mock_tools = [ + ListMCPToolsRestAPIResponseObject( + name="send_email", + description="Send an email", + inputSchema={"type": "object"}, + mcp_info={"server_name": "unknown_server"} + ) + ] + mock_get_tools.return_value = mock_tools + + # Call the function + result = await list_tool_rest_api( + request=mock_request, + server_id="test-server-123", + user_api_key_dict=mock_user_api_key_dict + ) + + # Verify the result + assert result["error"] is None + assert len(result["tools"]) == 1 + assert result["tools"][0].name == "send_email" + + # Verify that _get_tools_for_single_server was called with the default auth header + mock_get_tools.assert_called_once() + call_args = mock_get_tools.call_args + assert call_args[0][0] == mock_server # server + assert call_args[0][1] == "Bearer default_token" # server_auth_header + assert call_args[0][2] == "2025-06-18" # mcp_protocol_version + + +@pytest.mark.asyncio +async def test_list_tool_rest_api_all_servers_with_auth(): + """Test list_tool_rest_api for all servers with server-specific auth headers.""" + from litellm.proxy._experimental.mcp_server.rest_endpoints import list_tool_rest_api + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + + # Create mock request with server-specific auth headers + mock_request = MagicMock() + mock_request.headers = { + "authorization": "Bearer user_token", + "x-mcp-zapier-authorization": "Bearer zapier_token", + "x-mcp-slack-authorization": "Bearer slack_token", + "MCP-Protocol-Version": "2025-06-18" + } + + # Create mock user_api_key_dict + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test_user" + + # Mock the MCPRequestHandler methods + with patch.object(MCPRequestHandler, '_get_mcp_auth_header_from_headers') as mock_get_auth: + with patch.object(MCPRequestHandler, '_get_mcp_server_auth_headers_from_headers') as mock_get_server_auth: + mock_get_auth.return_value = "Bearer default_token" + mock_get_server_auth.return_value = { + "zapier": "Bearer zapier_token", + "slack": "Bearer slack_token" + } + + # Mock the global_mcp_server_manager + with patch('litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager') as mock_manager: + # Create mock servers + mock_zapier_server = MagicMock() + mock_zapier_server.alias = "zapier" + mock_zapier_server.server_name = "zapier_server" + mock_zapier_server.mcp_info = {"server_name": "zapier"} + + mock_slack_server = MagicMock() + mock_slack_server.alias = "slack" + mock_slack_server.server_name = "slack_server" + mock_slack_server.mcp_info = {"server_name": "slack"} + + mock_manager.get_registry.return_value = { + "zapier": mock_zapier_server, + "slack": mock_slack_server + } + + # Mock the _get_tools_for_single_server function + with patch('litellm.proxy._experimental.mcp_server.rest_endpoints._get_tools_for_single_server') as mock_get_tools: + from litellm.proxy._experimental.mcp_server.server import ListMCPToolsRestAPIResponseObject + + # Mock tools for each server + mock_get_tools.side_effect = [ + [ListMCPToolsRestAPIResponseObject( + name="send_email", + description="Send an email", + inputSchema={"type": "object"}, + mcp_info={"server_name": "zapier"} + )], + [ListMCPToolsRestAPIResponseObject( + name="send_message", + description="Send a message", + inputSchema={"type": "object"}, + mcp_info={"server_name": "slack"} + )] + ] + + # Call the function without server_id (query all servers) + result = await list_tool_rest_api( + request=mock_request, + server_id=None, + user_api_key_dict=mock_user_api_key_dict + ) + + # Verify the result + assert result["error"] is None + assert len(result["tools"]) == 2 + assert result["tools"][0].name == "send_email" + assert result["tools"][1].name == "send_message" + + # Verify that _get_tools_for_single_server was called for both servers with correct auth headers + assert mock_get_tools.call_count == 2 + calls = mock_get_tools.call_args_list + + # First call should be for zapier server with zapier auth + assert calls[0][0][0] == mock_zapier_server # server + assert calls[0][0][1] == "Bearer zapier_token" # server_auth_header + assert calls[0][0][2] == "2025-06-18" # mcp_protocol_version + + # Second call should be for slack server with slack auth + assert calls[1][0][0] == mock_slack_server # server + assert calls[1][0][1] == "Bearer slack_token" # server_auth_header + assert calls[1][0][2] == "2025-06-18" # mcp_protocol_version + + diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index b78543f2c8c..2c829406c01 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -54,6 +54,7 @@ async def test_get_end_user_object(customer_spend, customer_budget): end_user_id=end_user_id, prisma_client="RANDOM VALUE", # type: ignore user_api_key_cache=_cache, + route="/v1/chat/completions", ) if customer_spend > customer_budget: pytest.fail( diff --git a/tests/proxy_unit_tests/test_proxy_token_counter.py b/tests/proxy_unit_tests/test_proxy_token_counter.py index 11dededd6ca..eefe5059b0b 100644 --- a/tests/proxy_unit_tests/test_proxy_token_counter.py +++ b/tests/proxy_unit_tests/test_proxy_token_counter.py @@ -136,3 +136,427 @@ async def test_gpt_token_counting(): response.tokenizer_type == "openai_tokenizer" ) # SHOULD use the OpenAI tokenizer assert response.request_model == "gpt-4" + + +@pytest.mark.asyncio +async def test_anthropic_messages_count_tokens_endpoint(): + """ + Test /v1/messages/count_tokens endpoint with Anthropic model + - Should return response in Anthropic format: {"input_tokens": } + - Should work as wrapper around internal token_counter function + """ + from litellm.proxy.anthropic_endpoints.endpoints import count_tokens + from fastapi import Request + from unittest.mock import AsyncMock, MagicMock + + # Mock request object + mock_request = MagicMock(spec=Request) + mock_request_data = { + "model": "claude-3-sonnet-20240229", + "messages": [{"role": "user", "content": "Hello Claude!"}] + } + + # Mock the _read_request_body function + async def mock_read_request_body(request): + return mock_request_data + + # Mock UserAPIKeyAuth + mock_user_api_key_dict = MagicMock() + + # Patch the _read_request_body function + import litellm.proxy.anthropic_endpoints.endpoints as anthropic_endpoints + original_read_request_body = anthropic_endpoints._read_request_body + anthropic_endpoints._read_request_body = mock_read_request_body + + # Mock the internal token_counter function to return a controlled response + async def mock_token_counter(request, is_direct_request=True): + assert is_direct_request == False, "Should be called with is_direct_request=False for Anthropic endpoint" + assert request.model == "claude-3-sonnet-20240229" + assert request.messages == [{"role": "user", "content": "Hello Claude!"}] + + from litellm.proxy._types import TokenCountResponse + return TokenCountResponse( + total_tokens=15, + request_model="claude-3-sonnet-20240229", + model_used="claude-3-sonnet-20240229", + tokenizer_type="openai_tokenizer" + ) + + # Patch the imported token_counter function from proxy_server + import litellm.proxy.proxy_server as proxy_server + original_token_counter = proxy_server.token_counter + proxy_server.token_counter = mock_token_counter + + try: + # Call the endpoint + response = await count_tokens(mock_request, mock_user_api_key_dict) + + # Verify response format matches Anthropic spec + assert isinstance(response, dict) + assert "input_tokens" in response + assert response["input_tokens"] == 15 + assert len(response) == 1 # Should only contain input_tokens + + print("āœ… Anthropic endpoint test passed!") + + finally: + # Restore original functions + anthropic_endpoints._read_request_body = original_read_request_body + proxy_server.token_counter = original_token_counter + + +@pytest.mark.asyncio +async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): + """ + Test /v1/messages/count_tokens endpoint with non-Anthropic model (GPT-4) + - Should still work and return Anthropic format + - Should call internal token_counter with from_anthropic_endpoint=True + """ + from litellm.proxy.anthropic_endpoints.endpoints import count_tokens + from fastapi import Request + from unittest.mock import AsyncMock, MagicMock + + # Mock request object + mock_request = MagicMock(spec=Request) + mock_request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello GPT!"}] + } + + # Mock the _read_request_body function + async def mock_read_request_body(request): + return mock_request_data + + # Mock UserAPIKeyAuth + mock_user_api_key_dict = MagicMock() + + # Patch the _read_request_body function + import litellm.proxy.anthropic_endpoints.endpoints as anthropic_endpoints + original_read_request_body = anthropic_endpoints._read_request_body + anthropic_endpoints._read_request_body = mock_read_request_body + + # Mock the internal token_counter function to return a controlled response + async def mock_token_counter(request, is_direct_request=True): + assert is_direct_request == False, "Should be called with is_direct_request=False for Anthropic endpoint" + assert request.model == "gpt-4" + assert request.messages == [{"role": "user", "content": "Hello GPT!"}] + + from litellm.proxy._types import TokenCountResponse + return TokenCountResponse( + total_tokens=12, + request_model="gpt-4", + model_used="gpt-4", + tokenizer_type="openai_tokenizer" + ) + + # Patch the imported token_counter function from proxy_server + import litellm.proxy.proxy_server as proxy_server + original_token_counter = proxy_server.token_counter + proxy_server.token_counter = mock_token_counter + + try: + # Call the endpoint + response = await count_tokens(mock_request, mock_user_api_key_dict) + + # Verify response format matches Anthropic spec + assert isinstance(response, dict) + assert "input_tokens" in response + assert response["input_tokens"] == 12 + assert len(response) == 1 # Should only contain input_tokens + + print("āœ… Non-Anthropic model test passed!") + + finally: + # Restore original functions + anthropic_endpoints._read_request_body = original_read_request_body + proxy_server.token_counter = original_token_counter + + +@pytest.mark.asyncio +async def test_internal_token_counter_anthropic_provider_detection(): + """ + Test that the internal token_counter correctly detects Anthropic providers + and handles the from_anthropic_endpoint flag appropriately + """ + + # Test with Anthropic provider + llm_router = Router( + model_list=[ + { + "model_name": "claude-test", + "litellm_params": { + "model": "anthropic/claude-3-sonnet-20240229", + "api_key": "test-key" + }, + } + ] + ) + + setattr(litellm.proxy.proxy_server, "llm_router", llm_router) + + # Test with is_direct_request=False (simulating call from Anthropic endpoint) + response = await token_counter( + request=TokenCountRequest( + model="claude-test", + messages=[{"role": "user", "content": "hello"}], + ), + is_direct_request=False + ) + + print("Anthropic provider test response:", response) + + # Verify response structure + assert response.request_model == "claude-test" + assert response.model_used == "claude-3-sonnet-20240229" + assert response.total_tokens > 0 + + # Test with non-Anthropic provider + llm_router = Router( + model_list=[ + { + "model_name": "gpt-test", + "litellm_params": { + "model": "gpt-4", + }, + } + ] + ) + + setattr(litellm.proxy.proxy_server, "llm_router", llm_router) + + # Test with is_direct_request=False but non-Anthropic provider + response = await token_counter( + request=TokenCountRequest( + model="gpt-test", + messages=[{"role": "user", "content": "hello"}], + ), + is_direct_request=False + ) + + print("Non-Anthropic provider test response:", response) + + # Verify response structure + assert response.request_model == "gpt-test" + assert response.model_used == "gpt-4" + assert response.total_tokens > 0 + assert response.tokenizer_type == "openai_tokenizer" # Should use LiteLLM tokenizer + + +@pytest.mark.asyncio +async def test_anthropic_endpoint_error_handling(): + """ + Test error handling in the /v1/messages/count_tokens endpoint + """ + from litellm.proxy.anthropic_endpoints.endpoints import count_tokens + from fastapi import Request, HTTPException + from unittest.mock import MagicMock + + # Mock request object + mock_request = MagicMock(spec=Request) + mock_user_api_key_dict = MagicMock() + + # Test missing model parameter + mock_request_data = { + "messages": [{"role": "user", "content": "Hello!"}] + # Missing "model" key + } + + async def mock_read_request_body(request): + return mock_request_data + + import litellm.proxy.anthropic_endpoints.endpoints as anthropic_endpoints + original_read_request_body = anthropic_endpoints._read_request_body + anthropic_endpoints._read_request_body = mock_read_request_body + + try: + # Should raise HTTPException for missing model + with pytest.raises(HTTPException) as exc_info: + await count_tokens(mock_request, mock_user_api_key_dict) + + assert exc_info.value.status_code == 400 + assert "model parameter is required" in str(exc_info.value.detail) + + print("āœ… Error handling test passed!") + + finally: + anthropic_endpoints._read_request_body = original_read_request_body + + +@pytest.mark.asyncio +async def test_factory_anthropic_endpoint_calls_anthropic_counter(): + """Test that /v1/messages/count_tokens with Anthropic model uses Anthropic counter.""" + from unittest.mock import patch + from fastapi.testclient import TestClient + from litellm.proxy.proxy_server import app + + # Mock the anthropic token counting function + with patch('litellm.proxy.utils.count_tokens_with_anthropic_api') as mock_anthropic_count: + mock_anthropic_count.return_value = { + "total_tokens": 42, + "tokenizer_used": "anthropic" + } + + # Mock router to return Anthropic deployment + with patch('litellm.proxy.proxy_server.llm_router') as mock_router: + mock_router.model_list = [{ + "model_name": "claude-3-5-sonnet", + "litellm_params": {"model": "anthropic/claude-3-5-sonnet-20241022"}, + "model_info": {} + }] + + client = TestClient(app) + + response = client.post( + "/v1/messages/count_tokens", + json={ + "model": "claude-3-5-sonnet", + "messages": [{"role": "user", "content": "Hello"}] + }, + headers={"Authorization": "Bearer test-key"} + ) + + assert response.status_code == 200 + data = response.json() + assert data["input_tokens"] == 42 + + # Verify that Anthropic API was called + mock_anthropic_count.assert_called_once() + + +@pytest.mark.asyncio +async def test_factory_gpt4_endpoint_does_not_call_anthropic_counter(): + """Test that /v1/messages/count_tokens with GPT-4 does NOT use Anthropic counter.""" + from unittest.mock import patch + from fastapi.testclient import TestClient + from litellm.proxy.proxy_server import app + + # Mock the anthropic token counting function + with patch('litellm.proxy.utils.count_tokens_with_anthropic_api') as mock_anthropic_count: + # Mock litellm token counter + with patch('litellm.token_counter') as mock_litellm_counter: + mock_litellm_counter.return_value = 50 + + # Mock router to return GPT-4 deployment + with patch('litellm.proxy.proxy_server.llm_router') as mock_router: + mock_router.model_list = [{ + "model_name": "gpt-4", + "litellm_params": {"model": "openai/gpt-4"}, + "model_info": {} + }] + + client = TestClient(app) + + response = client.post( + "/v1/messages/count_tokens", + json={ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}] + }, + headers={"Authorization": "Bearer test-key"} + ) + + assert response.status_code == 200 + data = response.json() + assert data["input_tokens"] == 50 + + # Verify that Anthropic API was NOT called + mock_anthropic_count.assert_not_called() + + +@pytest.mark.asyncio +async def test_factory_normal_token_counter_endpoint_does_not_call_anthropic(): + """Test that /utils/token_counter does NOT use Anthropic counter even with Anthropic model.""" + from unittest.mock import patch + from fastapi.testclient import TestClient + from litellm.proxy.proxy_server import app + + # Mock the anthropic token counting function + with patch('litellm.proxy.utils.count_tokens_with_anthropic_api') as mock_anthropic_count: + # Mock litellm token counter + with patch('litellm.token_counter') as mock_litellm_counter: + mock_litellm_counter.return_value = 35 + + # Mock router to return Anthropic deployment + with patch('litellm.proxy.proxy_server.llm_router') as mock_router: + mock_router.model_list = [{ + "model_name": "claude-3-5-sonnet", + "litellm_params": {"model": "anthropic/claude-3-5-sonnet-20241022"}, + "model_info": {} + }] + + client = TestClient(app) + + response = client.post( + "/utils/token_counter", + json={ + "model": "claude-3-5-sonnet", + "messages": [{"role": "user", "content": "Hello"}] + }, + headers={"Authorization": "Bearer test-key"} + ) + + assert response.status_code == 200 + data = response.json() + assert data["total_tokens"] == 35 + + # Verify that Anthropic API was NOT called (since is_direct_request=True) + mock_anthropic_count.assert_not_called() + + +@pytest.mark.asyncio +async def test_factory_registration(): + """Test that the new factory pattern correctly provides counters.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + # Test Anthropic ModelInfo provides token counter + anthropic_model_info = AnthropicModelInfo() + counter = anthropic_model_info.get_token_counter() + assert counter is not None + + # Create test deployments + anthropic_deployment = { + "litellm_params": {"model": "anthropic/claude-3-5-sonnet-20241022"} + } + + non_anthropic_deployment = { + "litellm_params": {"model": "openai/gpt-4"} + } + + # Test Anthropic counter supports provider + assert counter.supports_provider(anthropic_deployment, from_endpoint=True) + assert not counter.supports_provider(anthropic_deployment, from_endpoint=False) + + # Test non-Anthropic provider + assert not counter.supports_provider(non_anthropic_deployment, from_endpoint=True) + assert not counter.supports_provider(non_anthropic_deployment, from_endpoint=False) + + # Test None deployment + assert not counter.supports_provider(None, from_endpoint=True) + assert not counter.supports_provider(None, from_endpoint=False) + + +@pytest.mark.asyncio +async def test_factory_anthropic_counter_supports_provider(): + """Test AnthropicTokenCounter provider detection logic.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + anthropic_model_info = AnthropicModelInfo() + counter = anthropic_model_info.get_token_counter() + + # Test Anthropic provider detection + anthropic_deployment = { + "litellm_params": {"model": "anthropic/claude-3-5-sonnet-20241022"} + } + assert counter.supports_provider(anthropic_deployment, from_endpoint=True) + assert not counter.supports_provider(anthropic_deployment, from_endpoint=False) + + # Test non-Anthropic provider + openai_deployment = { + "litellm_params": {"model": "openai/gpt-4"} + } + assert not counter.supports_provider(openai_deployment, from_endpoint=True) + assert not counter.supports_provider(openai_deployment, from_endpoint=False) + + # Test None deployment + assert not counter.supports_provider(None, from_endpoint=True) + assert not counter.supports_provider(None, from_endpoint=False) diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index ff4a64085f9..07a32530956 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1332,3 +1332,340 @@ def test_init_auto_router_deployment_duplicate_model_name(mock_auto_router, mode with pytest.raises(ValueError, match="Auto-router deployment test-auto-router already exists"): router.init_auto_router_deployment(deployment) + +def test_generate_model_id_with_deployment_model_name(model_list): + """Test that _generate_model_id works correctly with deployment model_name and handles None values properly""" + router = Router(model_list=model_list) + + # Test case 1: Normal case with valid model_group and litellm_params + model_group = "gpt-4.1" + litellm_params = { + "model": "gpt-4.1", + "api_key": "test_key", + "api_base": "https://api.openai.com/v1" + } + + try: + result = router._generate_model_id(model_group=model_group, litellm_params=litellm_params) + assert isinstance(result, str) + assert len(result) > 0 + print(f"āœ“ Success with valid model_group: {result}") + except Exception as e: + pytest.fail(f"Failed with valid model_group: {e}") + + # Test case 2: Edge case with None model_group (this should fail as expected - our fix prevents this from happening) + try: + result = router._generate_model_id(model_group=None, litellm_params=litellm_params) + pytest.fail("Expected TypeError when model_group is None - this confirms our fix is needed") + except TypeError as e: + assert "unsupported operand type(s) for +=" in str(e) + print(f"āœ“ Correctly failed with None model_group (as expected): {e}") + except Exception as e: + pytest.fail(f"Unexpected error with None model_group: {e}") + + # Test case 3: Edge case with None key in litellm_params + litellm_params_with_none_key = { + "model": "gpt-4.1", + "api_key": "test_key", + None: "should_be_skipped" # This should be handled gracefully + } + + try: + result = router._generate_model_id(model_group=model_group, litellm_params=litellm_params_with_none_key) + assert isinstance(result, str) + assert len(result) > 0 + print(f"āœ“ Success with None key in litellm_params: {result}") + except Exception as e: + pytest.fail(f"Failed with None key in litellm_params: {e}") + + # Test case 4: Edge case with empty litellm_params + try: + result = router._generate_model_id(model_group=model_group, litellm_params={}) + assert isinstance(result, str) + assert len(result) > 0 + print(f"āœ“ Success with empty litellm_params: {result}") + except Exception as e: + pytest.fail(f"Failed with empty litellm_params: {e}") + + # Test case 5: Verify that the same inputs produce the same result (deterministic) + result1 = router._generate_model_id(model_group=model_group, litellm_params=litellm_params) + result2 = router._generate_model_id(model_group=model_group, litellm_params=litellm_params) + assert result1 == result2, "Model ID generation should be deterministic" + + print("āœ“ All _generate_model_id tests passed!") + + +def test_handle_clientside_credential_with_deployment_model_name(model_list): + """Test that _handle_clientside_credential uses deployment model_name correctly""" + router = Router(model_list=model_list) + + # Mock deployment with model_name + deployment = { + "model_name": "gpt-4.1", + "litellm_params": { + "model": "gpt-4.1", + "api_key": "test_key" + } + } + + # Mock kwargs with empty metadata (simulating the original issue) + kwargs = { + "metadata": {}, # Empty metadata, no model_group + "litellm_params": { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1" + } + } + + # Mock dynamic_litellm_params that would be returned by get_dynamic_litellm_params + dynamic_litellm_params = { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1" + } + + # Test that the method doesn't fail when metadata is empty + try: + # This would normally call _generate_model_id internally + # We're testing that the fix prevents the TypeError + model_group = deployment["model_name"] # This is what our fix does + assert model_group == "gpt-4.1" + + # Verify that _generate_model_id works with this model_group + result = router._generate_model_id(model_group=model_group, litellm_params=dynamic_litellm_params) + assert isinstance(result, str) + assert len(result) > 0 + + print(f"āœ“ Success with deployment model_name: {result}") + except Exception as e: + pytest.fail(f"Failed with deployment model_name: {e}") + + print("āœ“ _handle_clientside_credential test passed!") + + +@pytest.mark.parametrize("function_name, expected_metadata_key", [ + ("acompletion", "metadata"), + ("_ageneric_api_call_with_fallbacks", "litellm_metadata"), + ("batch", "litellm_metadata"), + ("completion", "metadata"), + ("acreate_file", "litellm_metadata"), + ("aget_file", "litellm_metadata"), +]) +def test_handle_clientside_credential_metadata_loading(model_list, function_name, expected_metadata_key): + """Test that _handle_clientside_credential correctly loads metadata based on function name""" + router = Router(model_list=model_list) + + # Mock deployment + deployment = { + "model_name": "gpt-4.1", + "litellm_params": { + "model": "gpt-4.1", + "api_key": "test_key" + }, + "model_info": { + "id": "original-id-123" + } + } + + # Mock kwargs with clientside credentials and metadata + kwargs = { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1", + expected_metadata_key: { + "model_group": "gpt-4.1", + "custom_field": "test_value" + } + } + + # Call the function + result_deployment = router._handle_clientside_credential( + deployment=deployment, + kwargs=kwargs, + function_name=function_name + ) + + # Verify the result is a Deployment object + assert isinstance(result_deployment, Deployment) + + # Verify the deployment has the correct model_name (should be the model_group from metadata) + assert result_deployment.model_name == "gpt-4.1" + + # Verify the litellm_params contain the clientside credentials + assert result_deployment.litellm_params.api_key == "client_side_key" + assert result_deployment.litellm_params.api_base == "https://api.openai.com/v1" + + # Verify the model_info has been updated with a new ID + assert result_deployment.model_info.id != "original-id-123" + assert result_deployment.model_info.original_model_id == "original-id-123" + + # Verify the deployment was added to the router + assert len(router.model_list) == len(model_list) + 1 + + # Test that the function correctly uses the right metadata key + # For acompletion, it should use "metadata" + # For _ageneric_api_call_with_fallbacks/batch, it should use "litellm_metadata" + if function_name == "acompletion": + assert "metadata" in kwargs + assert "litellm_metadata" not in kwargs + elif function_name in ["_ageneric_api_call_with_fallbacks", "batch", "acreate_file", "aget_file"]: + assert "litellm_metadata" in kwargs + # Note: acompletion would not have litellm_metadata, but other functions might have both + + print(f"āœ“ Success with function_name '{function_name}' using '{expected_metadata_key}' metadata key") + + +@pytest.mark.parametrize("function_name, metadata_key", [ + ("acompletion", "metadata"), + ("_ageneric_api_call_with_fallbacks", "litellm_metadata"), +]) +def test_handle_clientside_credential_metadata_variable_name(model_list, function_name, metadata_key): + """Test that _handle_clientside_credential uses the correct metadata variable name based on function name""" + from litellm.router_utils.batch_utils import _get_router_metadata_variable_name + + router = Router(model_list=model_list) + + # Verify the metadata variable name is correct for each function + expected_metadata_key = _get_router_metadata_variable_name(function_name=function_name) + assert expected_metadata_key == metadata_key + + # Mock deployment + deployment = { + "model_name": "gpt-4.1", + "litellm_params": { + "model": "gpt-4.1", + "api_key": "test_key" + }, + "model_info": { + "id": "original-id-456" + } + } + + # Mock kwargs with clientside credentials and the correct metadata key + kwargs = { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1", + metadata_key: { + "model_group": "gpt-4.1", + "test_field": "test_value" + } + } + + # Call the function + result_deployment = router._handle_clientside_credential( + deployment=deployment, + kwargs=kwargs, + function_name=function_name + ) + + # Verify the function correctly extracted model_group from the right metadata key + assert result_deployment.model_name == "gpt-4.1" + + # Verify the deployment was created with the correct metadata + assert result_deployment.litellm_params.api_key == "client_side_key" + assert result_deployment.litellm_params.api_base == "https://api.openai.com/v1" + + print(f"āœ“ Success with function_name '{function_name}' correctly using '{metadata_key}' for metadata") + + +def test_handle_clientside_credential_no_metadata(model_list): + """Test that _handle_clientside_credential handles cases where no metadata is provided""" + router = Router(model_list=model_list) + + # Mock deployment + deployment = { + "model_name": "gpt-4.1", + "litellm_params": { + "model": "gpt-4.1", + "api_key": "test_key" + }, + "model_info": { + "id": "original-id-789" + } + } + + # Mock kwargs with clientside credentials but NO metadata + kwargs = { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1" + # No metadata key at all + } + + # This should fail because there's no model_group in metadata + # The function expects to find model_group in the metadata + try: + result_deployment = router._handle_clientside_credential( + deployment=deployment, + kwargs=kwargs, + function_name="acompletion" + ) + # If we get here, the function should have used deployment.model_name as fallback + assert result_deployment.model_name == "gpt-4.1" + print("āœ“ Success with no metadata - used deployment.model_name as fallback") + except Exception as e: + # This is expected behavior - the function needs model_group to generate model_id + print(f"āœ“ Correctly handled no metadata case: {e}") + + # Test with empty metadata + kwargs_with_empty_metadata = { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1", + "metadata": {} # Empty metadata + } + + try: + result_deployment = router._handle_clientside_credential( + deployment=deployment, + kwargs=kwargs_with_empty_metadata, + function_name="acompletion" + ) + # Should fail because empty metadata has no model_group + pytest.fail("Expected failure with empty metadata") + except Exception as e: + print(f"āœ“ Correctly handled empty metadata case: {e}") + + +def test_handle_clientside_credential_with_responses_function(model_list): + """Test that _handle_clientside_credential works correctly with responses function name""" + router = Router(model_list=model_list) + + # Mock deployment + deployment = { + "model_name": "gpt-4.1", + "litellm_params": { + "model": "gpt-4.1", + "api_key": "test_key" + }, + "model_info": { + "id": "original-id-responses" + } + } + + # Mock kwargs with clientside credentials and litellm_metadata (for responses function) + kwargs = { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1", + "litellm_metadata": { + "model_group": "gpt-4.1", + "responses_field": "responses_value" + } + } + + # Call the function with _ageneric_api_call_with_fallbacks function name (which handles responses) + result_deployment = router._handle_clientside_credential( + deployment=deployment, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks" + ) + + # Verify the result + assert isinstance(result_deployment, Deployment) + assert result_deployment.model_name == "gpt-4.1" + assert result_deployment.litellm_params.api_key == "client_side_key" + assert result_deployment.litellm_params.api_base == "https://api.openai.com/v1" + assert result_deployment.model_info.id != "original-id-responses" + assert result_deployment.model_info.original_model_id == "original-id-responses" + + # Verify the deployment was added to the router + assert len(router.model_list) == len(model_list) + 1 + + print("āœ“ Success with _ageneric_api_call_with_fallbacks function name and litellm_metadata") + diff --git a/tests/store_model_in_db_tests/test_mcp_servers.py b/tests/store_model_in_db_tests/test_mcp_servers.py index c2f600d8ccd..40273f051e7 100644 --- a/tests/store_model_in_db_tests/test_mcp_servers.py +++ b/tests/store_model_in_db_tests/test_mcp_servers.py @@ -100,8 +100,8 @@ async def test_create_mcp_server_direct(): # Mock the database functions directly with mock.patch("litellm.proxy.management_endpoints.mcp_management_endpoints.MCP_AVAILABLE", True), \ mock.patch("litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw") as mock_get_prisma, \ - mock.patch("litellm.proxy.management_endpoints.mcp_management_endpoints.create_mcp_server") as mock_create, \ - mock.patch("litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server") as mock_get_server, \ + mock.patch("litellm.proxy.management_endpoints.mcp_management_endpoints.create_mcp_server", new_callable=mock.AsyncMock) as mock_create, \ + mock.patch("litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", new_callable=mock.AsyncMock) as mock_get_server, \ mock.patch("litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager") as mock_manager: # Import after mocking @@ -113,6 +113,7 @@ async def test_create_mcp_server_direct(): # Mock server manager mock_manager.add_update_server = mock.Mock() + mock_manager.reload_servers_from_database = mock.AsyncMock() # Set up test data server_id = str(uuid.uuid4()) @@ -138,7 +139,7 @@ async def test_create_mcp_server_direct(): # Mock the database calls mock_get_server.return_value = None # Server doesn't exist yet - # Set up async mock for create_mcp_server + # Set up async mock for create_mcp_server using AsyncMock mock_create.return_value = expected_response # Create mock user auth @@ -174,7 +175,7 @@ async def test_create_duplicate_mcp_server(): # Mock the database functions directly with mock.patch("litellm.proxy.management_endpoints.mcp_management_endpoints.MCP_AVAILABLE", True), \ mock.patch("litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw") as mock_get_prisma, \ - mock.patch("litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server") as mock_get_server: + mock.patch("litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", new_callable=mock.AsyncMock) as mock_get_server: # Import after mocking from litellm.proxy.management_endpoints.mcp_management_endpoints import add_mcp_server diff --git a/tests/test_litellm/caching/test_gcs_cache.py b/tests/test_litellm/caching/test_gcs_cache.py new file mode 100644 index 00000000000..c346570bb05 --- /dev/null +++ b/tests/test_litellm/caching/test_gcs_cache.py @@ -0,0 +1,36 @@ +import os +import sys +from unittest.mock import MagicMock, AsyncMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.caching.gcs_cache import GCSCache + + +@pytest.fixture +def mock_gcs_dependencies(): + """Mock httpx clients and GCS auth""" + mock_sync_client = MagicMock() + mock_async_client = AsyncMock() + + with patch("litellm.caching.gcs_cache._get_httpx_client", return_value=mock_sync_client), \ + patch("litellm.caching.gcs_cache.get_async_httpx_client", return_value=mock_async_client), \ + patch("litellm.caching.gcs_cache.GCSBucketBase.sync_construct_request_headers", return_value={}): + yield { + "sync_client": mock_sync_client, + "async_client": mock_async_client, + } + + +@pytest.mark.asyncio +async def test_gcs_cache_async_set_and_get(mock_gcs_dependencies): + cache = GCSCache(bucket_name="test-bucket") + await cache.async_set_cache("key", {"foo": "bar"}) + mock_gcs_dependencies["async_client"].post.assert_called_once() + + mock_gcs_dependencies["async_client"].get.return_value.status_code = 200 + mock_gcs_dependencies["async_client"].get.return_value.text = "{\"foo\": \"bar\"}" + result = await cache.async_get_cache("key") + assert result == {"foo": "bar"} diff --git a/tests/test_litellm/integrations/test_braintrust_logging.py b/tests/test_litellm/integrations/test_braintrust_logging.py new file mode 100644 index 00000000000..5ae40e82760 --- /dev/null +++ b/tests/test_litellm/integrations/test_braintrust_logging.py @@ -0,0 +1,43 @@ +import os +import unittest +from unittest.mock import patch + +from litellm.integrations.braintrust_logging import BraintrustLogger + +class TestBraintrustLogger(unittest.TestCase): + @patch.dict(os.environ, {"BRAINTRUST_API_KEY": "test-env-api-key"}) + @patch.dict(os.environ, {"BRAINTRUST_API_BASE": "https://test-env-api.com/v1"}) + def test_init_with_env_var(self): + """Test BraintrustLogger initialization with environment variable.""" + logger = BraintrustLogger() + self.assertEqual(logger.api_key, "test-env-api-key") + self.assertEqual(logger.api_base, "https://test-env-api.com/v1") + self.assertEqual(logger.headers["Authorization"], "Bearer test-env-api-key") + self.assertEqual(logger.headers["Content-Type"], "application/json") + + def test_init_with_explicit_params(self): + """Test BraintrustLogger initialization with explicit parameters.""" + logger = BraintrustLogger(api_key="explicit-key", api_base="https://custom-api.com/v1") + self.assertEqual(logger.api_key, "explicit-key") + self.assertEqual(logger.api_base, "https://custom-api.com/v1") + self.assertEqual(logger.headers["Authorization"], "Bearer explicit-key") + + @patch.dict(os.environ, {}, clear=True) + def test_init_missing_api_key(self): + """Test BraintrustLogger initialization fails without API key.""" + with self.assertRaises(Exception) as context: + BraintrustLogger() + self.assertIn("Missing keys=['BRAINTRUST_API_KEY']", str(context.exception)) + + def test_validate_environment_with_api_key(self): + """Test validate_environment method with valid API key.""" + logger = BraintrustLogger(api_key="test-key") + # Should not raise an exception + logger.validate_environment(api_key="test-key") + + def test_validate_environment_missing_api_key(self): + """Test validate_environment method with missing API key.""" + with patch.dict(os.environ, {}, clear=True): + with self.assertRaises(Exception) as context: + BraintrustLogger(api_key=None) + self.assertIn("Missing keys=['BRAINTRUST_API_KEY']", str(context.exception)) \ No newline at end of file diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 5cc30f39181..e3ca7101c6c 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -139,6 +139,70 @@ def test_bedrock_validate_format_image_or_video(): result = BedrockImageProcessor._validate_format(f"video/{format}", format) assert result == format, f"Expected {format}, got {result}" + # Test valid document formats + valid_document_formats = { + "application/pdf": "pdf", + "text/csv": "csv", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx", + } + for mime, expected in valid_document_formats.items(): + print("testing mime", mime, "expected", expected) + result = BedrockImageProcessor._validate_format( + mime, mime.split("/")[1] + ) + assert result == expected, f"Expected {expected}, got {result}" + + +def test_bedrock_get_document_format_fallback_mimes(): + """ + Test the _get_document_format method with fallback MIME types for DOCX and XLSX. + + This tests the fallback mechanism when mimetypes.guess_all_extensions returns empty results, + which can happen in Docker containers where mimetypes depends on OS-installed MIME types. + """ + from unittest.mock import patch + + # Test DOCX fallback + docx_mime = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + supported_formats = ["pdf", "docx", "xlsx", "csv"] + + # Mock mimetypes.guess_all_extensions to return empty list (simulating Docker container scenario) + with patch('mimetypes.guess_all_extensions', return_value=[]): + result = BedrockImageProcessor._get_document_format( + mime_type=docx_mime, + supported_doc_formats=supported_formats + ) + assert result == "docx", f"Expected 'docx', got '{result}'" + + # Test XLSX fallback + xlsx_mime = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + + with patch('mimetypes.guess_all_extensions', return_value=[]): + result = BedrockImageProcessor._get_document_format( + mime_type=xlsx_mime, + supported_doc_formats=supported_formats + ) + assert result == "xlsx", f"Expected 'xlsx', got '{result}'" + + +def test_bedrock_get_document_format_mimetypes_success(): + """ + Test the _get_document_format method when mimetypes.guess_all_extensions works normally. + """ + docx_mime = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + supported_formats = ["pdf", "docx", "xlsx", "csv"] + + # Test normal mimetypes behavior (should not hit fallback) + result = BedrockImageProcessor._get_document_format( + mime_type=docx_mime, + supported_doc_formats=supported_formats + ) + assert result == "docx", f"Expected 'docx', got '{result}'" + + + + # def test_ollama_pt_consecutive_system_messages(): # """Test handling consecutive system messages""" @@ -435,3 +499,75 @@ def test_convert_gemini_messages(): message=message, last_message_with_tool_calls=last_message_with_tool_calls, ) + + +def test_bedrock_tools_unpack_defs(): + """ + Test that the unpack_defs method handles nested $ref inside anyOf items correctly + """ + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt + + circularRefSchema = { + "type": "object", + "properties": { + "type": {"type": "string", "enum": ["doc"]}, + "content": {"type": "array", "items": {"$ref": "#/$defs/node"}}, + }, + "required": ["type", "content"], + "additionalProperties": False, + "$defs": { + "node": { + "type": "object", + "anyOf": [ + { + "type": "object", + "properties": { + "type": {"type": "string", "enum": ["bulletList"]}, + "content": { + "type": "array", + "items": {"$ref": "#/$defs/listItem"}, + }, + }, + "required": ["type"], + "additionalProperties": True, + }, + { + "type": "object", + "properties": { + "type": {"type": "string", "enum": ["orderedList"]}, + "content": { + "type": "array", + "items": {"$ref": "#/$defs/listItem"}, + }, + }, + "required": ["type"], + "additionalProperties": True, + }, + ], + }, + "listItem": { + "type": "object", + "properties": { + "type": {"type": "string", "enum": ["listItem"]}, + "content": {"type": "array", "items": {"$ref": "#/$defs/node"}}, + }, + "required": ["type"], + "additionalProperties": True, + }, + }, + } + + tools = [ + { + "type": "function", + "function": { + "name": "json_schema", + "description": "Process the content using json schema validation", + "parameters": circularRefSchema, + }, + } + ] + + _bedrock_tools_pt(tools=tools) + + diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index e9b962f189f..317491d3172 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -394,3 +394,73 @@ def test_get_masked_values(): sensitive_object, unmasked_length=4, number_of_asterisks=4 ) assert masked_values["presidio_anonymizer_api_base"] is None + + +@pytest.mark.asyncio +async def test_e2e_generate_cold_storage_object_key_successful(): + """ + Test end-to-end generation of cold storage object key when cold storage is properly configured. + """ + from datetime import datetime, timezone + from unittest.mock import patch + + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + # Create test data + start_time = datetime(2025, 1, 15, 10, 30, 45, 123456, timezone.utc) + response_id = "chatcmpl-test-12345" + team_alias = "test-team" + + with patch("litellm.proxy.spend_tracking.cold_storage_handler.ColdStorageHandler._get_configured_cold_storage_custom_logger", return_value="s3"), \ + patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key: + + # Mock the S3 object key generation to return a predictable result + mock_get_s3_key.return_value = "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json" + + # Call the function + result = StandardLoggingPayloadSetup._generate_cold_storage_object_key( + start_time=start_time, + response_id=response_id, + team_alias=team_alias + ) + + # Verify the S3 function was called with correct parameters + mock_get_s3_key.assert_called_once_with( + s3_path="", # Empty path as default + team_alias_prefix="", # No team alias prefix for cold storage + start_time=start_time, + s3_file_name="time-10-30-45-123456_chatcmpl-test-12345" + ) + + # Verify the result + assert result == "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json" + assert result is not None + assert isinstance(result, str) + + +@pytest.mark.asyncio +async def test_e2e_generate_cold_storage_object_key_not_configured(): + """ + Test end-to-end generation of cold storage object key when cold storage is not configured. + """ + from datetime import datetime, timezone + from unittest.mock import patch + + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + # Create test data + start_time = datetime(2025, 1, 15, 10, 30, 45, 123456, timezone.utc) + response_id = "chatcmpl-test-67890" + team_alias = "another-team" + + with patch("litellm.proxy.spend_tracking.cold_storage_handler.ColdStorageHandler._get_configured_cold_storage_custom_logger", return_value=None): + + # Call the function + result = StandardLoggingPayloadSetup._generate_cold_storage_object_key( + start_time=start_time, + response_id=response_id, + team_alias=team_alias + ) + + # Verify the result is None when cold storage is not configured + assert result is None diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index 1026a210098..fc44d44aba9 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -45,3 +45,48 @@ def test_transform_choices(): assert choices[0].message.reasoning_content == "i'm thinking." assert choices[0].message.thinking_blocks is not None assert choices[0].message.tool_calls is None + + +def test_transform_choices_without_signature(): + """ + Test that the transformation works correctly when the signature field is missing + from the summary, which occurs with new Databricks Foundation Models like + databricks-gpt-oss-20b and databricks-gpt-oss-120b. + """ + config = DatabricksConfig() + databricks_choices = [ + { + "message": { + "role": "assistant", + "content": [ + { + "type": "reasoning", + "summary": [ + { + "type": "summary_text", + "text": "i'm thinking without signature.", + # Note: no signature field here + } + ], + }, + {"type": "text", "text": "Response without signature"}, + ], + }, + "index": 0, + "finish_reason": "stop", + } + ] + + # This should not raise a KeyError for missing signature + choices = config._transform_dbrx_choices(choices=databricks_choices) + + assert len(choices) == 1 + assert choices[0].message.content == "Response without signature" + assert choices[0].message.reasoning_content == "i'm thinking without signature." + assert choices[0].message.thinking_blocks is not None + assert len(choices[0].message.thinking_blocks) == 1 + + # Verify the thinking block was created successfully without signature + thinking_block = choices[0].message.thinking_blocks[0] + assert thinking_block["type"] == "thinking" + assert thinking_block["thinking"] == "i'm thinking without signature." diff --git a/tests/test_litellm/llms/jina_ai/embedding/test_jina_embedding_transformation.py b/tests/test_litellm/llms/jina_ai/embedding/test_jina_embedding_transformation.py new file mode 100644 index 00000000000..715d12043d0 --- /dev/null +++ b/tests/test_litellm/llms/jina_ai/embedding/test_jina_embedding_transformation.py @@ -0,0 +1,88 @@ +import os +import sys +from unittest.mock import MagicMock + +sys.path.insert( + 0, os.path.abspath("../../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig + + +class TestJinaAIEmbeddingTransform: + def setup_method(self): + self.config = JinaAIEmbeddingConfig() + self.model = "jina-embeddings-v2-base-en" + self.logging_obj = MagicMock() + + def test_map_openai_params(self): + """Test that 'dimensions' parameter is correctly mapped""" + test_params = {"dimensions": 1024} + result = self.config.map_openai_params( + non_default_params=test_params, + optional_params={}, + model=self.model, + drop_params=False, + ) + assert result == {"dimensions": 1024} + + def test_transform_embedding_request_text_input(self): + """Test transformation of a standard text embedding request""" + input_data = ["hello world", "hello world again"] + result = self.config.transform_embedding_request( + model=self.model, + input=input_data, + optional_params={}, + headers={}, + ) + expected_result = { + "model": self.model, + "input": input_data, + } + assert result == expected_result + + def test_transform_embedding_request_image_input(self): + """Test transformation of an image embedding request""" + # a fake base64 string for testing purposes + input_data = [ + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + ] + result = self.config.transform_embedding_request( + model=self.model, + input=input_data, + optional_params={}, + headers={}, + ) + expected_input = [ + { + "image": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + } + ] + expected_result = { + "model": self.model, + "input": expected_input, + } + assert result == expected_result + + def test_transform_embedding_request_mixed_input(self): + """Test transformation of a mixed text and image embedding request""" + # a fake base64 string for testing purposes + base64_str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + input_data = ["hello world", base64_str] + result = self.config.transform_embedding_request( + model=self.model, + input=input_data, + optional_params={}, + headers={}, + ) + expected_input = [ + {"text": "hello world"}, + { + "image": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + }, + ] + expected_result = { + "model": self.model, + "input": expected_input, + } + assert result == expected_result diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py new file mode 100644 index 00000000000..d5369253503 --- /dev/null +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -0,0 +1,297 @@ +import datetime +import os +import sys +import httpx +import pytest +import json + +import litellm + +# Adds the parent directory to the system path +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm import ModelResponse +from litellm.llms.oci.chat.transformation import OCIChatConfig, version + +TEST_MODEL_NAME = "xai.grok-4" +TEST_MODEL = f"oci/{TEST_MODEL_NAME}" +TEST_MESSAGES = [{"role": "user", "content": "Hello, how are you?"}] +TEST_COMPARTMENT_ID = "ocid1.compartment.oc1..xxxxxx" +TEST_OCI_PARAMS = { + "oci_region": "us-ashburn-1", + "oci_user": "ocid1.user.oc1..xxxxxxEXAMPLExxxxxx", + "oci_fingerprint": "4f:29:77:cc:b1:3e:55:ab:61:2a:de:47:f1:38:4c:90", + "oci_tenancy": "ocid1.tenancy.oc1..xxxxxxEXAMPLExxxxxx", + "oci_compartment_id": TEST_COMPARTMENT_ID, + "oci_key": "" +} + + +class TestOCIChatConfig: + def test_validate_environment_with_oci_region(self): + config = OCIChatConfig() + + headers = {} + + result = config.validate_environment( + headers=headers, + model=TEST_MODEL, + messages=TEST_MESSAGES, # type: ignore + optional_params=TEST_OCI_PARAMS, + litellm_params={}, + ) + + assert result["content-type"] == "application/json" + assert result["user-agent"] == f"litellm/{version}" + + def test_missing_oci_auth_parameters(self): + optional_params = TEST_OCI_PARAMS.copy() + optional_params.pop("oci_region") + + # Remove optional_params one by one and verify that an exception is raised + for key in optional_params.keys(): + modified_params = optional_params.copy() + del modified_params[key] + + with pytest.raises(Exception) as excinfo: + config = OCIChatConfig() + + headers = {} + + config.validate_environment( + headers=headers, + model=TEST_MODEL, + messages=TEST_MESSAGES, # type: ignore + optional_params=modified_params, + api_base="https://api.oci.example.com", + litellm_params={}, + ) + + assert f"Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key, oci_compartment_id" in str(excinfo.value) + + def test_transform_request_simple(self): + """ + Tests if a simple request is transformed correctly. + """ + config = OCIChatConfig() + optional_params = {"oci_compartment_id": TEST_COMPARTMENT_ID} + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + expected_output = { + "compartmentId": TEST_COMPARTMENT_ID, + "servingMode": {"servingType": "ON_DEMAND", "modelId": TEST_MODEL_NAME}, + "chatRequest": { + "apiFormat": "GENERIC", + "isStream": False, + "messages": [ + { + "role": "USER", + "content": [{"type": "TEXT", "text": "Hello, how are you?"}], + } + ], + }, + } + assert transformed_request == expected_output + + def test_transform_request_with_tools(self): + """ + Tests if a request with tools is transformed correctly. + """ + config = OCIChatConfig() + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + "required": ["location"], + }, + }, + } + ] + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "tools": tools, + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert "tools" in transformed_request["chatRequest"] + assert transformed_request["chatRequest"]["tools"][0]["name"] == "get_current_weather" + assert transformed_request["chatRequest"]["tools"][0]["type"] == "FUNCTION" + assert transformed_request["chatRequest"]["tools"][0]["description"] == "Get the current weather in a given location" + assert transformed_request["chatRequest"]["tools"][0]["parameters"] is not None + + def test_transform_response_simple_text(self): + """ + Tests if a simple text response is transformed correctly. + """ + config = OCIChatConfig() + created_time = datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z") + mock_oci_response = { + "modelId": TEST_MODEL_NAME, + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "choices": [ + { + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [{"type": "TEXT", "text": "I am doing well, thank you!"}], + }, + "finishReason": "STOP", + } + ], + "timeCreated": created_time, + "usage": { + "promptTokens": 10, + "completionTokens": 20, + "totalTokens": 30, + "completionTokensDetails": { + "acceptedPredictionTokens": 20, + "reasoningTokens": 20, + }, + "promptTokensDetails": { + "cachedTokens": 10, + }, + }, + }, + } + response = httpx.Response( + status_code=200, json=mock_oci_response, headers={"Content-Type": "application/json"} + ) + result = config.transform_response( + model=TEST_MODEL_NAME, + raw_response=response, + model_response=ModelResponse(), + logging_obj={}, # type: ignore + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding={}, + ) + + assert isinstance(result, ModelResponse) + assert len(result.choices) == 1 + assert isinstance(result.choices[0], litellm.Choices) + assert result.choices[0].message + assert result.choices[0].message.content == "I am doing well, thank you!" + assert result.choices[0].finish_reason == "stop" + assert result.model == TEST_MODEL_NAME + assert hasattr(result, "usage") + assert isinstance(result.usage, litellm.Usage) # type: ignore + assert result.usage.prompt_tokens == 10 # type: ignore + assert result.usage.completion_tokens == 20 # type: ignore + assert result.usage.total_tokens == 30 # type: ignore + + def test_transform_response_with_tool_calls(self): + """ + Tests if a response with tool calls is transformed correctly. + """ + config = OCIChatConfig() + created_time = datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z") + mock_oci_response = { + "modelId": TEST_MODEL_NAME, + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "choices": [ + { + "index": 0, + "message": { + "role": "ASSISTANT", + "content": None, + "toolCalls": [ + { + "id": "call_abc123", + "type": "FUNCTION", + "name": "get_weather", + "arguments": '{"location": "Vila Velha, BR"}', + } + ], + }, + "finishReason": "stop", + } + ], + "timeCreated": created_time, + "usage": { + "promptTokens": 10, + "completionTokens": 20, + "totalTokens": 30, + "completionTokensDetails": { + "acceptedPredictionTokens": 20, + "reasoningTokens": 20, + }, + "promptTokensDetails": { + "cachedTokens": 10, + }, + }, + }, + } + response = httpx.Response(status_code=200, json=mock_oci_response) + model_response = ModelResponse( + choices=[litellm.Choices(index=0, message=litellm.Message())] + ) + + result = config.transform_response( + model=TEST_MODEL_NAME, + raw_response=response, + model_response=model_response, + logging_obj={}, # type: ignore + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding={}, + ) + + # General assertions + assert isinstance(result, ModelResponse) + assert len(result.choices) == 1 + + choice = result.choices[0] + assert isinstance(choice, litellm.Choices) + assert choice.finish_reason == "stop" + + # Message and tool_calls assertions + message = choice.message + assert isinstance(message, litellm.Message) + assert hasattr(message, "tool_calls") + assert isinstance(message.tool_calls, list) + assert len(message.tool_calls) == 1 + + # Specific tool_call assertions + tool_call = message.tool_calls[0] + assert isinstance(tool_call, litellm.utils.ChatCompletionMessageToolCall) + assert tool_call.id == "call_abc123" + assert tool_call.type == "function" + assert tool_call.function["name"] == "get_weather" + assert tool_call.function["arguments"] == '{"location": "Vila Velha, BR"}' + + # Usage assertions + assert hasattr(result, "usage") + usage = result.usage # type: ignore + assert isinstance(usage, litellm.Usage) # type: ignore + assert usage.prompt_tokens == 10 # type: ignore + assert usage.completion_tokens == 20 # type: ignore + assert usage.total_tokens == 30 # type: ignore diff --git a/tests/test_litellm/llms/openai/vector_stores/test_openai_vector_stores_transformation.py b/tests/test_litellm/llms/openai/vector_stores/test_openai_vector_stores_transformation.py new file mode 100644 index 00000000000..17a611f571e --- /dev/null +++ b/tests/test_litellm/llms/openai/vector_stores/test_openai_vector_stores_transformation.py @@ -0,0 +1,67 @@ +import pytest + +from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig +from litellm.types.vector_stores import ( + VectorStoreCreateOptionalRequestParams, +) + + +class TestOpenAIVectorStoreAPIConfig: + + @pytest.mark.parametrize( + "metadata", [{}, None] + ) + def test_transform_create_vector_store_request_with_metadata_empty_or_none(self, metadata): + """ + Test transform_create_vector_store_request when metadata is None or empty dict. + """ + config = OpenAIVectorStoreConfig() + api_base = "https://api.openai.com/v1/vector_stores" + + vector_store_create_params: VectorStoreCreateOptionalRequestParams = { + "name": "test-vector-store", + "file_ids": ["file-123", "file-456"], + "metadata": metadata, + } + + url, request_body = config.transform_create_vector_store_request( + vector_store_create_params, api_base + ) + + assert url == api_base + assert request_body["name"] == "test-vector-store" + assert request_body["file_ids"] == ["file-123", "file-456"] + assert request_body["metadata"] == metadata + + + def test_transform_create_vector_store_request_with_large_metadata(self): + """ + Test transform_create_vector_store_request with metadata exceeding 16 keys. + + OpenAI limits metadata to 16 keys maximum. + """ + config = OpenAIVectorStoreConfig() + api_base = "https://api.openai.com/v1/vector_stores" + + # Create metadata with more than 16 keys + large_metadata = {f"key_{i}": f"value_{i}" for i in range(20)} + + vector_store_create_params: VectorStoreCreateOptionalRequestParams = { + "name": "test-vector-store", + "metadata": large_metadata, + } + + url, request_body = config.transform_create_vector_store_request( + vector_store_create_params, api_base + ) + + assert url == api_base + assert request_body["name"] == "test-vector-store" + + # Should be trimmed to 16 keys + assert len(request_body["metadata"]) == 16 + + # Should contain the first 16 keys (as per add_openai_metadata implementation) + for i in range(16): + assert f"key_{i}" in request_body["metadata"] + assert request_body["metadata"][f"key_{i}"] == f"value_{i}" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py new file mode 100644 index 00000000000..9e44a3eb419 --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -0,0 +1,861 @@ +""" +Unit tests for Bedrock Guardrails +""" + +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../../..")) + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( + BedrockGuardrail, + _redact_pii_matches, +) + + +@pytest.mark.asyncio +async def test__redact_pii_matches_function(): + """Test the _redact_pii_matches function directly""" + + # Test case 1: Response with PII entities + response_with_pii = { + "action": "GUARDRAIL_INTERVENED", + "assessments": [ + { + "sensitiveInformationPolicy": { + "piiEntities": [ + {"type": "NAME", "match": "John Smith", "action": "BLOCKED"}, + { + "type": "US_SOCIAL_SECURITY_NUMBER", + "match": "324-12-3212", + "action": "BLOCKED", + }, + {"type": "PHONE", "match": "607-456-7890", "action": "BLOCKED"}, + ] + } + } + ], + "outputs": [{"text": "Input blocked by PII policy"}], + } + + # Call the redaction function + redacted_response = _redact_pii_matches(response_with_pii) + + # Verify that PII matches are redacted + pii_entities = redacted_response["assessments"][0]["sensitiveInformationPolicy"][ + "piiEntities" + ] + + assert pii_entities[0]["match"] == "[REDACTED]", "Name should be redacted" + assert pii_entities[1]["match"] == "[REDACTED]", "SSN should be redacted" + assert pii_entities[2]["match"] == "[REDACTED]", "Phone should be redacted" + + # Verify other fields remain unchanged + assert pii_entities[0]["type"] == "NAME" + assert pii_entities[1]["type"] == "US_SOCIAL_SECURITY_NUMBER" + assert pii_entities[2]["type"] == "PHONE" + assert redacted_response["action"] == "GUARDRAIL_INTERVENED" + assert redacted_response["outputs"][0]["text"] == "Input blocked by PII policy" + + print("PII redaction function test passed") + + +@pytest.mark.asyncio +async def test__redact_pii_matches_no_pii(): + """Test _redact_pii_matches with response that has no PII""" + + response_no_pii = {"action": "NONE", "assessments": [], "outputs": []} + + # Call the redaction function + redacted_response = _redact_pii_matches(response_no_pii) + + # Should return the same response unchanged + assert redacted_response == response_no_pii + print("No PII redaction test passed") + + +@pytest.mark.asyncio +async def test__redact_pii_matches_empty_assessments(): + """Test _redact_pii_matches with empty assessments""" + + response_empty_assessments = { + "action": "GUARDRAIL_INTERVENED", + "assessments": [{"sensitiveInformationPolicy": {"piiEntities": []}}], + "outputs": [{"text": "Some output"}], + } + + # Call the redaction function + redacted_response = _redact_pii_matches(response_empty_assessments) + + # Should return the same response unchanged + assert redacted_response == response_empty_assessments + print("Empty assessments redaction test passed") + + +@pytest.mark.asyncio +async def test__redact_pii_matches_malformed_response(): + """Test _redact_pii_matches with malformed response (should not crash)""" + + # Test with completely malformed response + malformed_response = { + "action": "GUARDRAIL_INTERVENED", + "assessments": "not_a_list", # This should cause an exception + } + + # Should not crash and return original response + redacted_response = _redact_pii_matches(malformed_response) + assert redacted_response == malformed_response + + # Test with missing keys + missing_keys_response = { + "action": "GUARDRAIL_INTERVENED" + # Missing assessments key + } + + redacted_response = _redact_pii_matches(missing_keys_response) + assert redacted_response == missing_keys_response + + print("Malformed response redaction test passed") + + +@pytest.mark.asyncio +async def test__redact_pii_matches_multiple_assessments(): + """Test _redact_pii_matches with multiple assessments containing PII""" + + response_multiple_assessments = { + "action": "GUARDRAIL_INTERVENED", + "assessments": [ + { + "sensitiveInformationPolicy": { + "piiEntities": [ + { + "type": "EMAIL", + "match": "john@example.com", + "action": "ANONYMIZED", + } + ] + } + }, + { + "sensitiveInformationPolicy": { + "piiEntities": [ + { + "type": "CREDIT_DEBIT_CARD_NUMBER", + "match": "1234-5678-9012-3456", + "action": "BLOCKED", + }, + { + "type": "ADDRESS", + "match": "123 Main St, Anytown USA", + "action": "ANONYMIZED", + }, + ] + } + }, + ], + "outputs": [{"text": "Multiple PII detected"}], + } + + # Call the redaction function + redacted_response = _redact_pii_matches(response_multiple_assessments) + + # Verify all PII in all assessments are redacted + assessment1_pii = redacted_response["assessments"][0]["sensitiveInformationPolicy"][ + "piiEntities" + ] + assessment2_pii = redacted_response["assessments"][1]["sensitiveInformationPolicy"][ + "piiEntities" + ] + + assert assessment1_pii[0]["match"] == "[REDACTED]", "Email should be redacted" + assert assessment2_pii[0]["match"] == "[REDACTED]", "Credit card should be redacted" + assert assessment2_pii[1]["match"] == "[REDACTED]", "Address should be redacted" + + # Verify types remain unchanged + assert assessment1_pii[0]["type"] == "EMAIL" + assert assessment2_pii[0]["type"] == "CREDIT_DEBIT_CARD_NUMBER" + assert assessment2_pii[1]["type"] == "ADDRESS" + + print("Multiple assessments redaction test passed") + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_logging_uses_redacted_response(): + """Test that the Bedrock guardrail uses redacted response for logging""" + + # Create proper mock objects + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = BedrockGuardrail( + guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT" + ) + + # Mock the Bedrock API response with PII + mock_bedrock_response = MagicMock() + mock_bedrock_response.status_code = 200 + mock_bedrock_response.json.return_value = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{"text": "Hello, my phone number is {PHONE}"}], + "assessments": [ + { + "sensitiveInformationPolicy": { + "piiEntities": [ + { + "type": "PHONE", + "match": "+1 412 555 1212", # This should be redacted in logs + "action": "ANONYMIZED", + } + ] + } + } + ], + } + + request_data = { + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "Hello, my phone number is +1 412 555 1212"}, + ], + } + + # Mock AWS credentials to avoid credential loading issues in CI + mock_credentials = MagicMock() + mock_credentials.access_key = "test-access-key" + mock_credentials.secret_key = "test-secret-key" + mock_credentials.token = None + + # Mock AWS-related methods to ensure test runs without external dependencies + with patch.object( + guardrail.async_handler, "post", new_callable=AsyncMock + ) as mock_post, patch( + "litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails.verbose_proxy_logger.debug" + ) as mock_debug, patch.object( + guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ) as mock_load_creds, patch.object( + guardrail, "_prepare_request", return_value=MagicMock() + ) as mock_prepare_request: + + mock_post.return_value = mock_bedrock_response + + # Call the method that should log the redacted response + await guardrail.make_bedrock_api_request( + source="INPUT", + messages=request_data.get("messages"), + request_data=request_data, + ) + + # Verify that debug logging was called + mock_debug.assert_called() + + # Get the logged response (second argument to debug call) + logged_calls = mock_debug.call_args_list + bedrock_response_log_call = None + + for call in logged_calls: + args, kwargs = call + if len(args) >= 2 and "Bedrock AI response" in str(args[0]): + bedrock_response_log_call = call + break + + assert ( + bedrock_response_log_call is not None + ), "Should have logged Bedrock AI response" + + # Extract the logged response data + logged_response = bedrock_response_log_call[0][ + 1 + ] # Second argument to debug call + + # Verify that the logged response has redacted PII + assert ( + logged_response["assessments"][0]["sensitiveInformationPolicy"][ + "piiEntities" + ][0]["match"] + == "[REDACTED]" + ) + + # Verify other fields are preserved + assert logged_response["action"] == "GUARDRAIL_INTERVENED" + assert ( + logged_response["assessments"][0]["sensitiveInformationPolicy"][ + "piiEntities" + ][0]["type"] + == "PHONE" + ) + + print("Bedrock guardrail logging redaction test passed") + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_original_response_not_modified(): + """Test that the original response is not modified by redaction, only the logged version""" + + # Create proper mock objects + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = BedrockGuardrail( + guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT" + ) + + # Mock the Bedrock API response with PII + original_response_data = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{"text": "Hello, my phone number is {PHONE}"}], + "assessments": [ + { + "sensitiveInformationPolicy": { + "piiEntities": [ + { + "type": "PHONE", + "match": "+1 412 555 1212", # This should NOT be modified in original + "action": "ANONYMIZED", + } + ] + } + } + ], + } + + mock_bedrock_response = MagicMock() + mock_bedrock_response.status_code = 200 + mock_bedrock_response.json.return_value = original_response_data + + request_data = { + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "Hello, my phone number is +1 412 555 1212"}, + ], + } + + # Mock AWS credentials to avoid credential loading issues in CI + mock_credentials = MagicMock() + mock_credentials.access_key = "test-access-key" + mock_credentials.secret_key = "test-secret-key" + mock_credentials.token = None + + # Mock AWS-related methods to ensure test runs without external dependencies + with patch.object( + guardrail.async_handler, "post", new_callable=AsyncMock + ) as mock_post, patch.object( + guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ) as mock_load_creds, patch.object( + guardrail, "_prepare_request", return_value=MagicMock() + ) as mock_prepare_request: + + mock_post.return_value = mock_bedrock_response + + # Call the method + result = await guardrail.make_bedrock_api_request( + source="INPUT", + messages=request_data.get("messages"), + request_data=request_data, + ) + + # Verify that the original response data was not modified + # (The json() method should return the original data) + original_data = mock_bedrock_response.json() + assert ( + original_data["assessments"][0]["sensitiveInformationPolicy"][ + "piiEntities" + ][0]["match"] + == "+1 412 555 1212" + ) + + # Verify that the returned BedrockGuardrailResponse contains original data + assert ( + result["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][ + "match" + ] + == "+1 412 555 1212" + ) + + print("Original response not modified test passed") + + +@pytest.mark.asyncio +async def test__redact_pii_matches_preserves_non_pii_entities(): + """Test that _redact_pii_matches only affects PII-related entities and preserves other assessment data""" + + response_with_mixed_data = { + "action": "GUARDRAIL_INTERVENED", + "assessments": [ + { + "sensitiveInformationPolicy": { + "piiEntities": [ + { + "type": "EMAIL", + "match": "user@example.com", + "action": "ANONYMIZED", + "confidence": "HIGH", + } + ], + "regexes": [ + { + "name": "custom_pattern", + "match": "some_pattern_match", + "action": "BLOCKED", + } + ], + }, + "contentPolicy": { + "filters": [ + { + "type": "VIOLENCE", + "confidence": "MEDIUM", + "action": "BLOCKED", + } + ] + }, + "topicPolicy": { + "topics": [ + { + "name": "Restricted Topic", + "type": "DENY", + "action": "BLOCKED", + } + ] + }, + } + ], + "outputs": [{"text": "Content blocked"}], + } + + # Call the redaction function + redacted_response = _redact_pii_matches(response_with_mixed_data) + + # Verify that PII entity matches are redacted + pii_entities = redacted_response["assessments"][0]["sensitiveInformationPolicy"][ + "piiEntities" + ] + assert pii_entities[0]["match"] == "[REDACTED]", "PII match should be redacted" + assert pii_entities[0]["type"] == "EMAIL", "PII type should be preserved" + assert pii_entities[0]["action"] == "ANONYMIZED", "PII action should be preserved" + assert pii_entities[0]["confidence"] == "HIGH", "PII confidence should be preserved" + + # Verify that regex matches are also redacted (updated behavior) + regexes = redacted_response["assessments"][0]["sensitiveInformationPolicy"][ + "regexes" + ] + assert regexes[0]["match"] == "[REDACTED]", "Regex match should be redacted" + assert regexes[0]["name"] == "custom_pattern", "Regex name should be preserved" + assert regexes[0]["action"] == "BLOCKED", "Regex action should be preserved" + + # Verify that other policies are completely unchanged + content_policy = redacted_response["assessments"][0]["contentPolicy"] + assert content_policy["filters"][0]["type"] == "VIOLENCE" + assert content_policy["filters"][0]["confidence"] == "MEDIUM" + + topic_policy = redacted_response["assessments"][0]["topicPolicy"] + assert topic_policy["topics"][0]["name"] == "Restricted Topic" + + # Verify top-level fields are unchanged + assert redacted_response["action"] == "GUARDRAIL_INTERVENED" + assert redacted_response["outputs"][0]["text"] == "Content blocked" + + print("Preserves non-PII entities test passed") + + +@pytest.mark.asyncio +async def test_pii_redaction_matches_debug_output_format(): + """Test that demonstrates the exact behavior shown in your debug output""" + + # This matches the structure from your debug output + original_response = { + "action": "GUARDRAIL_INTERVENED", + "actionReason": "Guardrail blocked.", + "assessments": [ + { + "invocationMetrics": { + "guardrailCoverage": { + "textCharacters": {"guarded": 84, "total": 84} + }, + "guardrailProcessingLatency": 322, + "usage": { + "contentPolicyImageUnits": 0, + "contentPolicyUnits": 0, + "contextualGroundingPolicyUnits": 0, + "sensitiveInformationPolicyFreeUnits": 0, + "sensitiveInformationPolicyUnits": 1, + "topicPolicyUnits": 0, + "wordPolicyUnits": 0, + }, + }, + "sensitiveInformationPolicy": { + "piiEntities": [ + { + "action": "BLOCKED", + "detected": True, + "match": "John Smith", + "type": "NAME", + }, + { + "action": "BLOCKED", + "detected": True, + "match": "324-12-3212", + "type": "US_SOCIAL_SECURITY_NUMBER", + }, + { + "action": "BLOCKED", + "detected": True, + "match": "607-456-7890", + "type": "PHONE", + }, + ] + }, + } + ], + "blockedResponse": "Input blocked by PII policy", + "guardrailCoverage": {"textCharacters": {"guarded": 84, "total": 84}}, + "output": [{"text": "Input blocked by PII policy"}], + "outputs": [{"text": "Input blocked by PII policy"}], + "usage": { + "contentPolicyImageUnits": 0, + "contentPolicyUnits": 0, + "contextualGroundingPolicyUnits": 0, + "sensitiveInformationPolicyFreeUnits": 0, + "sensitiveInformationPolicyUnits": 1, + "topicPolicyUnits": 0, + "wordPolicyUnits": 0, + }, + } + + # Apply redaction + redacted_response = _redact_pii_matches(original_response) + + # Verify the redacted response matches your expected debug output + pii_entities = redacted_response["assessments"][0]["sensitiveInformationPolicy"][ + "piiEntities" + ] + + # All PII matches should be redacted + assert pii_entities[0]["match"] == "[REDACTED]", "NAME should be redacted" + assert pii_entities[1]["match"] == "[REDACTED]", "SSN should be redacted" + assert pii_entities[2]["match"] == "[REDACTED]", "PHONE should be redacted" + + # But all other fields should be preserved + assert pii_entities[0]["type"] == "NAME" + assert pii_entities[1]["type"] == "US_SOCIAL_SECURITY_NUMBER" + assert pii_entities[2]["type"] == "PHONE" + assert pii_entities[0]["action"] == "BLOCKED" + assert pii_entities[0]["detected"] == True + + # Verify that the original response is unchanged + original_pii_entities = original_response["assessments"][0][ + "sensitiveInformationPolicy" + ]["piiEntities"] + assert ( + original_pii_entities[0]["match"] == "John Smith" + ), "Original should be unchanged" + assert ( + original_pii_entities[1]["match"] == "324-12-3212" + ), "Original should be unchanged" + assert ( + original_pii_entities[2]["match"] == "607-456-7890" + ), "Original should be unchanged" + + # Verify all other metadata is preserved in redacted response + assert redacted_response["action"] == "GUARDRAIL_INTERVENED" + assert redacted_response["actionReason"] == "Guardrail blocked." + assert redacted_response["blockedResponse"] == "Input blocked by PII policy" + assert ( + redacted_response["assessments"][0]["invocationMetrics"][ + "guardrailProcessingLatency" + ] + == 322 + ) + + print("PII redaction matches debug output format test passed") + print( + f"Original PII values preserved: {[e['match'] for e in original_pii_entities]}" + ) + print(f"Redacted PII values: {[e['match'] for e in pii_entities]}") + + +@pytest.mark.asyncio +async def test__redact_pii_matches_with_regex_matches(): + """Test redaction of regex matches in sensitive information policy""" + + response_with_regex = { + "action": "GUARDRAIL_INTERVENED", + "assessments": [ + { + "sensitiveInformationPolicy": { + "regexes": [ + { + "name": "SSN_PATTERN", + "match": "123-45-6789", + "action": "BLOCKED", + }, + { + "name": "CREDIT_CARD_PATTERN", + "match": "4111-1111-1111-1111", + "action": "ANONYMIZED", + }, + ] + } + } + ], + "outputs": [{"text": "Regex patterns detected"}], + } + + # Call the redaction function + redacted_response = _redact_pii_matches(response_with_regex) + + # Verify that regex matches are redacted + regexes = redacted_response["assessments"][0]["sensitiveInformationPolicy"][ + "regexes" + ] + + assert regexes[0]["match"] == "[REDACTED]", "SSN regex match should be redacted" + assert ( + regexes[1]["match"] == "[REDACTED]" + ), "Credit card regex match should be redacted" + + # Verify other fields are preserved + assert regexes[0]["name"] == "SSN_PATTERN", "Regex name should be preserved" + assert regexes[0]["action"] == "BLOCKED", "Regex action should be preserved" + assert regexes[1]["name"] == "CREDIT_CARD_PATTERN", "Regex name should be preserved" + assert regexes[1]["action"] == "ANONYMIZED", "Regex action should be preserved" + + # Verify original response is unchanged + original_regexes = response_with_regex["assessments"][0][ + "sensitiveInformationPolicy" + ]["regexes"] + assert original_regexes[0]["match"] == "123-45-6789", "Original should be unchanged" + assert ( + original_regexes[1]["match"] == "4111-1111-1111-1111" + ), "Original should be unchanged" + + print("Regex matches redaction test passed") + + +@pytest.mark.asyncio +async def test__redact_pii_matches_with_custom_words(): + """Test redaction of custom word matches in word policy""" + + response_with_custom_words = { + "action": "GUARDRAIL_INTERVENED", + "assessments": [ + { + "wordPolicy": { + "customWords": [ + { + "match": "confidential_data", + "action": "BLOCKED", + }, + { + "match": "secret_information", + "action": "ANONYMIZED", + }, + ] + } + } + ], + "outputs": [{"text": "Custom words detected"}], + } + + # Call the redaction function + redacted_response = _redact_pii_matches(response_with_custom_words) + + # Verify that custom word matches are redacted + custom_words = redacted_response["assessments"][0]["wordPolicy"]["customWords"] + + assert ( + custom_words[0]["match"] == "[REDACTED]" + ), "First custom word match should be redacted" + assert ( + custom_words[1]["match"] == "[REDACTED]" + ), "Second custom word match should be redacted" + + # Verify other fields are preserved + assert ( + custom_words[0]["action"] == "BLOCKED" + ), "Custom word action should be preserved" + assert ( + custom_words[1]["action"] == "ANONYMIZED" + ), "Custom word action should be preserved" + + # Verify original response is unchanged + original_custom_words = response_with_custom_words["assessments"][0]["wordPolicy"][ + "customWords" + ] + assert ( + original_custom_words[0]["match"] == "confidential_data" + ), "Original should be unchanged" + assert ( + original_custom_words[1]["match"] == "secret_information" + ), "Original should be unchanged" + + print("Custom words redaction test passed") + + +@pytest.mark.asyncio +async def test__redact_pii_matches_with_managed_words(): + """Test redaction of managed word matches in word policy""" + + response_with_managed_words = { + "action": "GUARDRAIL_INTERVENED", + "assessments": [ + { + "wordPolicy": { + "managedWordLists": [ + { + "match": "inappropriate_word", + "action": "BLOCKED", + "type": "PROFANITY", + }, + { + "match": "offensive_term", + "action": "ANONYMIZED", + "type": "HATE_SPEECH", + }, + ] + } + } + ], + "outputs": [{"text": "Managed words detected"}], + } + + # Call the redaction function + redacted_response = _redact_pii_matches(response_with_managed_words) + + # Verify that managed word matches are redacted + managed_words = redacted_response["assessments"][0]["wordPolicy"][ + "managedWordLists" + ] + + assert ( + managed_words[0]["match"] == "[REDACTED]" + ), "First managed word match should be redacted" + assert ( + managed_words[1]["match"] == "[REDACTED]" + ), "Second managed word match should be redacted" + + # Verify other fields are preserved + assert ( + managed_words[0]["action"] == "BLOCKED" + ), "Managed word action should be preserved" + assert ( + managed_words[0]["type"] == "PROFANITY" + ), "Managed word type should be preserved" + assert ( + managed_words[1]["action"] == "ANONYMIZED" + ), "Managed word action should be preserved" + assert ( + managed_words[1]["type"] == "HATE_SPEECH" + ), "Managed word type should be preserved" + + # Verify original response is unchanged + original_managed_words = response_with_managed_words["assessments"][0][ + "wordPolicy" + ]["managedWordLists"] + assert ( + original_managed_words[0]["match"] == "inappropriate_word" + ), "Original should be unchanged" + assert ( + original_managed_words[1]["match"] == "offensive_term" + ), "Original should be unchanged" + + print("Managed words redaction test passed") + + +@pytest.mark.asyncio +async def test__redact_pii_matches_comprehensive_coverage(): + """Test redaction across all supported policy types in a single response""" + + comprehensive_response = { + "action": "GUARDRAIL_INTERVENED", + "assessments": [ + { + "sensitiveInformationPolicy": { + "piiEntities": [ + { + "type": "EMAIL", + "match": "user@example.com", + "action": "ANONYMIZED", + } + ], + "regexes": [ + { + "name": "PHONE_PATTERN", + "match": "555-123-4567", + "action": "BLOCKED", + } + ], + }, + "wordPolicy": { + "customWords": [ + { + "match": "confidential", + "action": "BLOCKED", + } + ], + "managedWordLists": [ + { + "match": "inappropriate", + "action": "ANONYMIZED", + "type": "PROFANITY", + } + ], + }, + } + ], + "outputs": [{"text": "Multiple policy violations detected"}], + } + + # Call the redaction function + redacted_response = _redact_pii_matches(comprehensive_response) + + # Verify all match fields are redacted + assessment = redacted_response["assessments"][0] + + # PII entities + pii_entities = assessment["sensitiveInformationPolicy"]["piiEntities"] + assert ( + pii_entities[0]["match"] == "[REDACTED]" + ), "PII entity match should be redacted" + + # Regex matches + regexes = assessment["sensitiveInformationPolicy"]["regexes"] + assert regexes[0]["match"] == "[REDACTED]", "Regex match should be redacted" + + # Custom words + custom_words = assessment["wordPolicy"]["customWords"] + assert ( + custom_words[0]["match"] == "[REDACTED]" + ), "Custom word match should be redacted" + + # Managed words + managed_words = assessment["wordPolicy"]["managedWordLists"] + assert ( + managed_words[0]["match"] == "[REDACTED]" + ), "Managed word match should be redacted" + + # Verify all other fields are preserved + assert pii_entities[0]["type"] == "EMAIL" + assert regexes[0]["name"] == "PHONE_PATTERN" + assert managed_words[0]["type"] == "PROFANITY" + + # Verify original response is unchanged + original_assessment = comprehensive_response["assessments"][0] + assert ( + original_assessment["sensitiveInformationPolicy"]["piiEntities"][0]["match"] + == "user@example.com" + ) + assert ( + original_assessment["sensitiveInformationPolicy"]["regexes"][0]["match"] + == "555-123-4567" + ) + assert ( + original_assessment["wordPolicy"]["customWords"][0]["match"] == "confidential" + ) + assert ( + original_assessment["wordPolicy"]["managedWordLists"][0]["match"] + == "inappropriate" + ) + + print("Comprehensive coverage redaction test passed") diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index c5fdbf4925a..bd37e9cbe41 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -451,3 +451,119 @@ class TestClearCache: mock_config.add_deployment.assert_called_once_with( prisma_client=mock_prisma, proxy_logging_obj=mock_logging ) + + +class TestModelInfoEndpoint: + """Test the model_info endpoint for retrieving individual model information""" + + @pytest.mark.asyncio + async def test_model_info_accessible_model_success(self): + """Test model_info returns model data for accessible models""" + from litellm.proxy.proxy_server import model_info + + # Mock user with access to specific models + user_api_key_dict = UserAPIKeyAuth( + user_id="test_user", + api_key="test_key", + models=["gpt-4", "claude-3"], + team_models=["gpt-3.5-turbo"] + ) + + with patch("litellm.proxy.proxy_server.llm_router") as mock_router, \ + patch("litellm.proxy.proxy_server.get_key_models") as mock_get_key_models, \ + patch("litellm.proxy.proxy_server.get_team_models") as mock_get_team_models, \ + patch("litellm.proxy.proxy_server.get_complete_model_list") as mock_get_complete_models, \ + patch("litellm.get_llm_provider") as mock_get_provider: + + # Setup mocks + mock_router.get_model_names.return_value = ["gpt-4", "claude-3", "gpt-3.5-turbo"] + mock_router.get_model_access_groups.return_value = {} + mock_get_key_models.return_value = ["gpt-4", "claude-3"] + mock_get_team_models.return_value = ["gpt-3.5-turbo"] + mock_get_complete_models.return_value = ["gpt-4", "claude-3", "gpt-3.5-turbo"] + mock_get_provider.return_value = (None, "openai", None, None) + + # Test accessible model + result = await model_info( + model_id="gpt-4", + user_api_key_dict=user_api_key_dict + ) + + assert result["id"] == "gpt-4" + assert result["object"] == "model" + assert result["owned_by"] == "openai" + assert "created" in result + + @pytest.mark.asyncio + async def test_model_info_inaccessible_model_returns_404(self): + """Test model_info returns 404 for inaccessible models""" + from litellm.proxy.proxy_server import model_info + from fastapi import HTTPException + + # Mock user with limited access + user_api_key_dict = UserAPIKeyAuth( + user_id="test_user", + api_key="test_key", + models=["gpt-4"], # Only has access to gpt-4 + team_models=[] + ) + + with patch("litellm.proxy.proxy_server.llm_router") as mock_router, \ + patch("litellm.proxy.proxy_server.get_key_models") as mock_get_key_models, \ + patch("litellm.proxy.proxy_server.get_team_models") as mock_get_team_models, \ + patch("litellm.proxy.proxy_server.get_complete_model_list") as mock_get_complete_models: + + # Setup mocks - user only has access to gpt-4 + mock_router.get_model_names.return_value = ["gpt-4", "claude-3"] + mock_router.get_model_access_groups.return_value = {} + mock_get_key_models.return_value = ["gpt-4"] + mock_get_team_models.return_value = [] + mock_get_complete_models.return_value = ["gpt-4"] # Only gpt-4 accessible + + # Test inaccessible model should raise 404 + with pytest.raises(HTTPException) as exc_info: + await model_info( + model_id="claude-3", # Not in user's accessible models + user_api_key_dict=user_api_key_dict + ) + + assert exc_info.value.status_code == 404 + assert "does not exist or is not accessible" in exc_info.value.detail + + @pytest.mark.asyncio + async def test_model_info_team_model_access(self): + """Test model_info works with team model access""" + from litellm.proxy.proxy_server import model_info + + # Mock user with team access + user_api_key_dict = UserAPIKeyAuth( + user_id="test_user", + api_key="test_key", + team_id="test_team", + models=[], # No direct key models + team_models=["team-model-1"] + ) + + with patch("litellm.proxy.proxy_server.llm_router") as mock_router, \ + patch("litellm.proxy.proxy_server.get_key_models") as mock_get_key_models, \ + patch("litellm.proxy.proxy_server.get_team_models") as mock_get_team_models, \ + patch("litellm.proxy.proxy_server.get_complete_model_list") as mock_get_complete_models, \ + patch("litellm.get_llm_provider") as mock_get_provider: + + # Setup mocks + mock_router.get_model_names.return_value = ["team-model-1"] + mock_router.get_model_access_groups.return_value = {} + mock_get_key_models.return_value = [] + mock_get_team_models.return_value = ["team-model-1"] + mock_get_complete_models.return_value = ["team-model-1"] + mock_get_provider.return_value = (None, "custom", None, None) + + # Test team model access + result = await model_info( + model_id="team-model-1", + user_api_key_dict=user_api_key_dict + ) + + assert result["id"] == "team-model-1" + assert result["object"] == "model" + assert result["owned_by"] == "custom" diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index c7c67cbba71..245f350be1b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -1245,3 +1245,47 @@ class TestCustomUISSO: # Verify the result is the redirect response assert result == mock_redirect_response assert result.status_code == 303 + + +@pytest.mark.asyncio +async def test_serve_login_page_server_root_path(): + """ + Test that serve_login_page includes SERVER_ROOT_PATH in the SSO login URL + when SERVER_ROOT_PATH is set. + """ + # Arrange + mock_request = MagicMock(spec=Request) + captured_html = "" + + # Mock environment variables + env_vars = { + "PROXY_BASE_URL": "https://example.com", + "SERVER_ROOT_PATH": "/api/v1", + "GOOGLE_CLIENT_ID": "mock_google_client_id", # Enable SSO + "DATABASE_URL": "mock_db_url", # Satisfy show_missing_vars_in_env + "LITELLM_MASTER_KEY": "mock_master_key", # Satisfy show_missing_vars_in_env + } + + # Patch HTMLResponse to capture the content + def mock_html_response(content, status_code=200): + nonlocal captured_html + captured_html = content + return MagicMock() + + with patch.dict(os.environ, env_vars): + with patch("litellm.proxy.proxy_server.premium_user", True): + with patch("litellm.proxy.proxy_server.prisma_client", MagicMock()): + with patch("litellm.proxy.proxy_server.master_key", "mock_master_key"): + with patch("fastapi.responses.HTMLResponse", side_effect=mock_html_response): + # Import the function to test + from litellm.proxy.management_endpoints.ui_sso import ( + serve_login_page, + ) + + # Act + result = await serve_login_page(request=mock_request) + + # Assert + assert result is not None + expected_url = "https://example.com/api/v1/sso/login" + assert expected_url in captured_html, f"Expected URL '{expected_url}' not found in HTML content" diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 93be9717e96..5b1e784caa3 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -29,6 +29,7 @@ ignored_keys = [ "endTime", "metadata.model_map_information", "metadata.usage_object", + "metadata.cold_storage_object_key", ] diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 26f41b76d3b..eac631bdb0d 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -16,7 +16,7 @@ sys.path.insert( from unittest.mock import MagicMock, patch import litellm -from litellm.constants import REDACTED_BY_LITELM_STRING +from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, REDACTED_BY_LITELM_STRING from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy.spend_tracking.spend_tracking_utils import ( _get_vector_store_request_for_spend_logs_payload, @@ -35,7 +35,7 @@ def test_sanitize_request_body_for_spend_logs_payload_long_string(): long_string = "a" * 2000 # Create a string longer than MAX_STRING_LENGTH request_body = {"text": long_string, "normal_text": "short text"} sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) - assert len(sanitized["text"]) == 1000 + len("... (truncated 1000 chars)") + assert len(sanitized["text"]) == 1000 + len(f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} 1000 chars)") assert sanitized["normal_text"] == "short text" @@ -43,7 +43,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_dict(): request_body = {"outer": {"inner": {"text": "a" * 2000, "normal": "short"}}} sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) assert len(sanitized["outer"]["inner"]["text"]) == 1000 + len( - "... (truncated 1000 chars)" + f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} 1000 chars)" ) assert sanitized["outer"]["inner"]["normal"] == "short" @@ -54,11 +54,11 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_list(): } sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) assert len(sanitized["items"][0]["text"]) == 1000 + len( - "... (truncated 1000 chars)" + f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} 1000 chars)" ) assert sanitized["items"][1]["text"] == "short" assert len(sanitized["items"][2][0]["text"]) == 1000 + len( - "... (truncated 1000 chars)" + f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} 1000 chars)" ) @@ -81,14 +81,14 @@ def test_sanitize_request_body_for_spend_logs_payload_mixed_types(): "nested": {"list": ["short", "a" * 2000], "dict": {"key": "a" * 2000}}, } sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) - assert len(sanitized["text"]) == 1000 + len("... (truncated 1000 chars)") + assert len(sanitized["text"]) == 1000 + len(f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} 1000 chars)") assert sanitized["number"] == 42 assert sanitized["nested"]["list"][0] == "short" assert len(sanitized["nested"]["list"][1]) == 1000 + len( - "... (truncated 1000 chars)" + f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} 1000 chars)" ) assert len(sanitized["nested"]["dict"]["key"]) == 1000 + len( - "... (truncated 1000 chars)" + f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} 1000 chars)" ) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 5840d52813f..3e2e8fba2e8 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1043,3 +1043,150 @@ async def test_async_data_generator_midstream_error(): # Verify that post_call_failure_hook was NOT called (since this is not an exception case) mock_proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def _has_nested_none_values(obj, path="root"): + """ + Recursively check if an object contains nested None values. + + Args: + obj: The object to check + path: Current path in the object tree (for debugging) + + Returns: + List of paths where None values were found + """ + none_paths = [] + + if obj is None: + none_paths.append(path) + elif isinstance(obj, dict): + for key, value in obj.items(): + none_paths.extend(_has_nested_none_values(value, f"{path}.{key}")) + elif isinstance(obj, (list, tuple)): + for i, item in enumerate(obj): + none_paths.extend(_has_nested_none_values(item, f"{path}[{i}]")) + elif hasattr(obj, "__dict__"): + # Handle object attributes + for key, value in obj.__dict__.items(): + if not key.startswith("_"): # Skip private attributes + none_paths.extend(_has_nested_none_values(value, f"{path}.{key}")) + + return none_paths + + +@pytest.mark.asyncio +async def test_chat_completion_result_no_nested_none_values(): + """ + Test that chat_completion result doesn't have nested None values when using exclude_none=True + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from fastapi import Request, Response + from pydantic import BaseModel + + import litellm + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.proxy_server import chat_completion + + # Create a mock ModelResponse with nested None values + mock_model_response = litellm.ModelResponse() + mock_model_response.id = "test-id" + mock_model_response.model = "gpt-3.5-turbo" + mock_model_response.object = "chat.completion" + mock_model_response.created = 1234567890 + + # Create message with None values that should be excluded + mock_message = litellm.Message( + content="Hello, world!", + role="assistant", + function_call=None, # This should be excluded + tool_calls=None, # This should be excluded + audio=None, # This should be excluded + reasoning_content=None, # This should be excluded + thinking_blocks=None, # This should be excluded + annotations=None, # This should be excluded + ) + + # Create choice with potential None values + mock_choice = litellm.Choices( + finish_reason="stop", + index=0, + message=mock_message, + logprobs=None, # This should be excluded when exclude_none=True + ) + + mock_model_response.choices = [mock_choice] + mock_model_response.usage = litellm.Usage( + prompt_tokens=10, completion_tokens=5, total_tokens=15 + ) + + # Verify the mock has None values before serialization + raw_dict = mock_model_response.model_dump() + none_paths_before = _has_nested_none_values(raw_dict) + assert ( + len(none_paths_before) > 0 + ), "Mock should have None values before exclude_none=True" + + # Mock the request processing to return our mock response + mock_base_processor = MagicMock() + mock_base_processor.base_process_llm_request = AsyncMock( + return_value=mock_model_response + ) + + # Mock other dependencies + mock_request = MagicMock(spec=Request) + mock_response = MagicMock(spec=Response) + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + + with patch( + "litellm.proxy.proxy_server._read_request_body", + return_value={"model": "gpt-3.5-turbo", "messages": []}, + ), patch( + "litellm.proxy.proxy_server.ProxyBaseLLMRequestProcessing", + return_value=mock_base_processor, + ): + + # Call the chat_completion function + result = await chat_completion( + request=mock_request, + fastapi_response=mock_response, + user_api_key_dict=mock_user_api_key_dict, + ) + + # Verify the result is a dict (since isinstance(result, BaseModel) was True) + assert isinstance(result, dict), f"Expected dict result, got {type(result)}" + + # Check that there are no nested None values in the result + none_paths_after = _has_nested_none_values(result) + assert ( + len(none_paths_after) == 0 + ), f"Result should not contain nested None values. Found None at: {none_paths_after}" + + # Verify essential fields are present + assert "id" in result + assert "model" in result + assert "object" in result + assert "created" in result + assert "choices" in result + assert "usage" in result + + # Verify that the choices contain the expected message content + assert len(result["choices"]) == 1 + assert result["choices"][0]["message"]["content"] == "Hello, world!" + assert result["choices"][0]["message"]["role"] == "assistant" + + # Verify that None fields were excluded (should not be present in the dict) + message = result["choices"][0]["message"] + excluded_fields = [ + "function_call", + "tool_calls", + "audio", + "reasoning_content", + "thinking_blocks", + "annotations", + ] + for field in excluded_fields: + assert ( + field not in message + ), f"Field '{field}' should be excluded when it's None" diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index d4a65d8847d..9d8aebd2d17 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -49,7 +49,7 @@ async def test_route_request_dynamic_credentials(route_type): @pytest.mark.asyncio async def test_route_request_no_model_required(): """Test route types that don't require model parameter""" - test_cases = ["amoderation", "aget_responses", "adelete_responses"] + test_cases = ["amoderation", "aget_responses", "adelete_responses", "avector_store_create", "avector_store_search"] for route_type in test_cases: # Test data without model parameter @@ -72,7 +72,7 @@ async def test_route_request_no_model_required(): @pytest.mark.asyncio async def test_route_request_no_model_required_with_router_settings(): """Test route types that don't require model parameter with router settings""" - test_cases = ["amoderation", "aget_responses", "adelete_responses"] + test_cases = ["amoderation", "aget_responses", "adelete_responses", "avector_store_create", "avector_store_search"] for route_type in test_cases: # Test data with model parameter (it will be ignored for these route types) diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 752b6ab8f4f..8e38011279d 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -8,7 +8,11 @@ sys.path.insert( from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) -from litellm.types.utils import ModelResponse, Choices, Message +from litellm.types.llms.openai import ( + ChatCompletionResponseMessage, + ChatCompletionToolMessage, +) +from litellm.types.utils import Choices, Message, ModelResponse class TestLiteLLMCompletionResponsesConfig: @@ -130,9 +134,9 @@ class TestLiteLLMCompletionResponsesConfig: ) # Assert - expected = {"type": "image", "image_url": {"url": image_url, "detail": "high"}} + expected = {"type": "image_url", "image_url": {"url": image_url, "detail": "high"}} assert result == expected - assert result["type"] == "image" + assert result["type"] == "image_url" assert result["image_url"]["url"] == image_url assert result["image_url"]["detail"] == "high" @@ -150,9 +154,9 @@ class TestLiteLLMCompletionResponsesConfig: ) # Assert - expected = {"type": "image", "image_url": {"url": image_url, "detail": "high"}} + expected = {"type": "image_url", "image_url": {"url": image_url, "detail": "high"}} assert result == expected - assert result["type"] == "image" + assert result["type"] == "image_url" assert result["image_url"]["url"] == image_url assert result["image_url"]["detail"] == "high" @@ -170,9 +174,9 @@ class TestLiteLLMCompletionResponsesConfig: ) # Assert - expected = {"type": "image", "image_url": {"url": image_url, "detail": "auto"}} + expected = {"type": "image_url", "image_url": {"url": image_url, "detail": "auto"}} assert result == expected - assert result["type"] == "image" + assert result["type"] == "image_url" assert result["image_url"]["url"] == image_url assert result["image_url"]["detail"] == "auto" @@ -189,9 +193,9 @@ class TestLiteLLMCompletionResponsesConfig: ) # Assert - expected = {"type": "image", "image_url": {"url": "", "detail": "auto"}} + expected = {"type": "image_url", "image_url": {"url": "", "detail": "auto"}} assert result == expected - assert result["type"] == "image" + assert result["type"] == "image_url" assert result["image_url"]["url"] == "" assert result["image_url"]["detail"] == "auto" @@ -213,9 +217,9 @@ class TestLiteLLMCompletionResponsesConfig: ) # Assert - expected = {"type": "image", "image_url": {"url": "https://example.com/image.png", "detail": "auto"}} + expected = {"type": "image_url", "image_url": {"url": "https://example.com/image.png", "detail": "auto"}} assert result == expected - assert result["type"] == "image" + assert result["type"] == "image_url" assert result["image_url"]["url"] == "https://example.com/image.png" assert result["image_url"]["detail"] == "auto" assert "extra_field" not in result @@ -364,3 +368,218 @@ class TestLiteLLMCompletionResponsesConfig: item for item in responses_api_response.output if item.type == "message" ] assert len(message_items) == 2, "Should have two message items" + + + + +class TestFunctionCallTransformation: + """Test cases for function_call input transformation""" + + def test_function_call_detection(self): + """Test that function_call items are correctly detected""" + function_call_item = { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "test"}', + "call_id": "test_id" + } + + function_call_output_item = { + "type": "function_call_output", + "call_id": "test_id", + "output": "result" + } + + regular_message = { + "type": "message", + "role": "user", + "content": "Hello" + } + + # Test function_call detection + assert LiteLLMCompletionResponsesConfig._is_input_item_function_call(function_call_item) + assert not LiteLLMCompletionResponsesConfig._is_input_item_function_call(function_call_output_item) + assert not LiteLLMCompletionResponsesConfig._is_input_item_function_call(regular_message) + + # Test function_call_output detection (should still work) + assert LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(function_call_output_item) + assert not LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(function_call_item) + assert not LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(regular_message) + + def test_function_call_transformation(self): + """Test that function_call items are correctly transformed to assistant messages with tool calls""" + function_call_item = { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "SĆ£o Paulo, Brazil"}', + "call_id": "call_123", + "id": "call_123", + "status": "completed" + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( + function_call=function_call_item + ) + + assert len(result) == 1 + message = result[0] + + # Should be an assistant message + assert message.get("role") == "assistant" + assert message.get("content") is None # Function calls don't have content + + # Should have tool calls + tool_calls = message.get("tool_calls", []) + assert len(tool_calls) == 1 + + tool_call = tool_calls[0] + assert tool_call.get("id") == "call_123" + assert tool_call.get("type") == "function" + + function = tool_call.get("function", {}) + assert function.get("name") == "get_weather" + assert function.get("arguments") == '{"location": "SĆ£o Paulo, Brazil"}' + + def test_complete_input_transformation_with_function_calls(self): + """Test the complete transformation with the exact input from the issue""" + test_input = [ + { + "type": "message", + "role": "user", + "content": "How is the weather in SĆ£o Paulo today ?" + }, + { + "type": "function_call", + "arguments": '{"location": "SĆ£o Paulo, Brazil"}', + "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "name": "get_weather", + "id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "status": "completed" + }, + { + "type": "function_call_output", + "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "output": "Rainy" + } + ] + + # This should not raise an error (previously would raise "Invalid content type: ") + messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( + input=test_input + ) + + assert len(messages) == 3 + + # First message: user message + user_msg = messages[0] + assert user_msg.get("role") == "user" + assert user_msg.get("content") == "How is the weather in SĆ£o Paulo today ?" + + # Second message: assistant message with tool call + assistant_msg = messages[1] + assert assistant_msg.get("role") == "assistant" + assert assistant_msg.get("tool_calls") is not None + assert len(assistant_msg.get("tool_calls", [])) == 1 + + tool_call = assistant_msg.get("tool_calls")[0] + assert tool_call.get("function", {}).get("name") == "get_weather" + + # Third message: tool output + tool_msg = messages[2] + assert tool_msg.get("role") == "tool" + assert tool_msg.get("content") == "Rainy" + assert tool_msg.get("tool_call_id") == "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5" + + def test_complete_request_transformation_with_function_calls(self): + """Test the complete request transformation that would be used by the responses API""" + test_input = [ + { + "type": "message", + "role": "user", + "content": "How is the weather in SĆ£o Paulo today ?" + }, + { + "type": "function_call", + "arguments": '{"location": "SĆ£o Paulo, Brazil"}', + "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "name": "get_weather", + "id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "status": "completed" + }, + { + "type": "function_call_output", + "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", + "output": "Rainy" + } + ] + + tools = [ + { + "type": "function", + "name": "get_weather", + "description": "Get current temperature for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and country e.g. BogotĆ”, Colombia" + } + }, + "required": ["location"], + "additionalProperties": False + } + } + ] + + responses_api_request = { + "store": False, + "tools": tools + } + + # This should work without errors for non-OpenAI models + result = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini/gemini-2.0-flash", + input=test_input, + responses_api_request=responses_api_request + ) + + assert "messages" in result + assert "model" in result + assert "tools" in result + + messages = result["messages"] + assert len(messages) == 3 + assert result["model"] == "gemini/gemini-2.0-flash" + + # Verify the structure is correct for chat completion + user_msg = messages[0] + assert user_msg["role"] == "user" + + assistant_msg = messages[1] + assert assistant_msg["role"] == "assistant" + assert "tool_calls" in assistant_msg + + tool_msg = messages[2] + assert tool_msg["role"] == "tool" + + def test_function_call_without_call_id_fallback_to_id(self): + """Test that function_call items can use 'id' field when 'call_id' is missing""" + function_call_item = { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "test"}', + "id": "fallback_id" # Only has 'id', not 'call_id' + } + + result = LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( + function_call=function_call_item + ) + + assert len(result) == 1 + message = result[0] + tool_calls = message.get("tool_calls", []) + assert len(tool_calls) == 1 + + tool_call = tool_calls[0] + assert tool_call.get("id") == "fallback_id" \ No newline at end of file diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_session_handler.py b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py similarity index 58% rename from tests/enterprise/litellm_enterprise/enterprise_callbacks/test_session_handler.py rename to tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py index dbe163560b0..27edfef3eb5 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_session_handler.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py @@ -11,9 +11,9 @@ sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path - -from enterprise.litellm_enterprise.enterprise_callbacks.session_handler import ( - _ENTERPRISE_ResponsesSessionHandler, +from litellm.responses.litellm_completion_transformation import session_handler +from litellm.responses.litellm_completion_transformation.session_handler import ( + ResponsesSessionHandler, ) @@ -111,7 +111,7 @@ async def test_get_chat_completion_message_history_for_previous_response_id(): # Mock the get_all_spend_logs_for_previous_response_id method with patch.object( - _ENTERPRISE_ResponsesSessionHandler, + ResponsesSessionHandler, "get_all_spend_logs_for_previous_response_id", new_callable=AsyncMock, ) as mock_get_spend_logs: @@ -119,7 +119,7 @@ async def test_get_chat_completion_message_history_for_previous_response_id(): # Test the function previous_response_id = "chatcmpl-935b8dad-fdc2-466e-a8ca-e26e5a8a21bb" - result = await _ENTERPRISE_ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( previous_response_id ) @@ -166,17 +166,130 @@ async def test_get_chat_completion_message_history_empty_spend_logs(): Test get_chat_completion_message_history_for_previous_response_id with empty spend logs """ with patch.object( - _ENTERPRISE_ResponsesSessionHandler, + ResponsesSessionHandler, "get_all_spend_logs_for_previous_response_id", new_callable=AsyncMock, ) as mock_get_spend_logs: mock_get_spend_logs.return_value = [] previous_response_id = "non-existent-id" - result = await _ENTERPRISE_ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( previous_response_id ) # Verify empty result structure assert result.get("messages") == [] assert result.get("litellm_session_id") is None + + +@pytest.mark.asyncio +async def test_e2e_cold_storage_successful_retrieval(): + """ + Test end-to-end cold storage functionality with successful retrieval of full proxy request from cold storage. + """ + # Mock spend logs with cold storage object key in metadata + mock_spend_logs = [ + { + "request_id": "chatcmpl-test-123", + "session_id": "session-456", + "metadata": '{"cold_storage_object_key": "s3://test-bucket/requests/session_456_req1.json"}', + "proxy_server_request": '{"litellm_truncated": true}', # Truncated payload + "response": { + "id": "chatcmpl-test-123", + "object": "chat.completion", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "I am an AI assistant." + } + } + ] + } + } + ] + + # Full proxy request data from cold storage + full_proxy_request = { + "input": "Hello, who are you?", + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello, who are you?"}] + } + + with patch.object( + ResponsesSessionHandler, + "get_all_spend_logs_for_previous_response_id", + new_callable=AsyncMock, + ) as mock_get_spend_logs, \ + patch.object(session_handler, "COLD_STORAGE_HANDLER") as mock_cold_storage, \ + patch("litellm.proxy.spend_tracking.cold_storage_handler.ColdStorageHandler._get_configured_cold_storage_custom_logger", return_value="s3"): + + # Setup mocks + mock_get_spend_logs.return_value = mock_spend_logs + mock_cold_storage.get_proxy_server_request_from_cold_storage_with_object_key = AsyncMock(return_value=full_proxy_request) + + # Call the main function + result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + "chatcmpl-test-123" + ) + + # Verify cold storage was called with correct object key + mock_cold_storage.get_proxy_server_request_from_cold_storage_with_object_key.assert_called_once_with( + object_key="s3://test-bucket/requests/session_456_req1.json" + ) + + # Verify result structure + assert result.get("litellm_session_id") == "session-456" + assert len(result.get("messages", [])) >= 1 # At least the assistant response + + +@pytest.mark.asyncio +async def test_e2e_cold_storage_fallback_to_truncated_payload(): + """ + Test end-to-end cold storage functionality when object key is missing, falling back to truncated payload. + """ + # Mock spend logs without cold storage object key + mock_spend_logs = [ + { + "request_id": "chatcmpl-test-789", + "session_id": "session-999", + "metadata": '{"user_api_key": "test-key"}', # No cold storage object key + "proxy_server_request": '{"input": "Truncated message", "model": "gpt-4"}', # Regular payload + "response": { + "id": "chatcmpl-test-789", + "object": "chat.completion", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "This is a response." + } + } + ] + } + } + ] + + with patch.object( + ResponsesSessionHandler, + "get_all_spend_logs_for_previous_response_id", + new_callable=AsyncMock, + ) as mock_get_spend_logs, \ + patch.object(session_handler, "COLD_STORAGE_HANDLER") as mock_cold_storage: + + # Setup mocks + mock_get_spend_logs.return_value = mock_spend_logs + + # Call the main function + result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + "chatcmpl-test-789" + ) + + # Verify cold storage was NOT called since no object key in metadata + mock_cold_storage.get_proxy_server_request_from_cold_storage_with_object_key.assert_not_called() + + # Verify result structure + assert result.get("litellm_session_id") == "session-999" + assert len(result.get("messages", [])) >= 1 # At least the assistant response diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler_with_cold_storage.py b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler_with_cold_storage.py new file mode 100644 index 00000000000..976152db353 --- /dev/null +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler_with_cold_storage.py @@ -0,0 +1,186 @@ +""" +Unit tests for cold storage object key integration. + +Tests for the changes to integrate cold storage handling across different components: +1. Add cold_storage_object_key field to StandardLoggingMetadata and SpendLogsMetadata +2. S3Logger generates object key when cold storage is enabled +3. Store object key in SpendLogsMetadata via spend_tracking_utils +4. Session handler uses object key from spend logs metadata +5. S3Logger supports retrieval using provided object key +""" + +import json +from datetime import datetime, timezone +from typing import Optional +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.integrations.s3_v2 import S3Logger +from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload +from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler +from litellm.proxy.spend_tracking.spend_tracking_utils import _get_spend_logs_metadata +from litellm.responses.litellm_completion_transformation.session_handler import ( + ResponsesSessionHandler, +) +from litellm.types.utils import StandardLoggingMetadata, StandardLoggingPayload + + +class TestColdStorageObjectKeyIntegration: + """Test suite for cold storage object key integration.""" + + def test_standard_logging_metadata_has_cold_storage_object_key_field(self): + """ + Test: Add cold_storage_object_key field to StandardLoggingMetadata. + + This test verifies that the StandardLoggingMetadata TypedDict has the + cold_storage_object_key field for storing S3/GCS object keys. + """ + from litellm.types.utils import StandardLoggingMetadata + + # Create a StandardLoggingMetadata instance with cold_storage_object_key + metadata = StandardLoggingMetadata( + user_api_key_hash="test_hash", + cold_storage_object_key="test/path/to/object.json" + ) + + # Verify the field can be set and accessed + assert metadata.get("cold_storage_object_key") == "test/path/to/object.json" + + assert "cold_storage_object_key" in StandardLoggingMetadata.__annotations__ + + def test_spend_logs_metadata_has_cold_storage_object_key_field(self): + """ + Test: Add cold_storage_object_key field to SpendLogsMetadata. + + This test verifies that the SpendLogsMetadata TypedDict has the + cold_storage_object_key field for storing S3/GCS object keys. + """ + # Create a SpendLogsMetadata instance with cold_storage_object_key + metadata = SpendLogsMetadata( + user_api_key="test_key", + cold_storage_object_key="test/path/to/object.json" + ) + + # Verify the field can be set and accessed + assert metadata.get("cold_storage_object_key") == "test/path/to/object.json" + + # Verify it's part of the SpendLogsMetadata annotations + assert "cold_storage_object_key" in SpendLogsMetadata.__annotations__ + + + def test_spend_tracking_utils_stores_object_key_in_metadata(self): + """ + Test: Store object key in SpendLogsMetadata via spend_tracking_utils. + + This test verifies that the _get_spend_logs_metadata function extracts + the cold_storage_object_key from StandardLoggingPayload and stores it + in SpendLogsMetadata. + """ + # Create test data + metadata = { + "user_api_key": "test_key", + "user_api_key_team_id": "test_team" + } + + + # Call the function + result = _get_spend_logs_metadata( + metadata=metadata, + cold_storage_object_key="test/path/to/object.json" + ) + + # Verify the object key is stored in the result + assert result.get("cold_storage_object_key") == "test/path/to/object.json" + + + def test_session_handler_extracts_object_key_from_spend_log(self): + """ + Test: Session handler extracts object key from spend logs metadata. + + This test verifies that the ResponsesSessionHandler can extract the + cold_storage_object_key from spend log metadata. + """ + # Create test spend log + spend_log = { + "request_id": "test_request_id", + "metadata": json.dumps({ + "cold_storage_object_key": "test/path/to/object.json", + "user_api_key": "test_key" + }) + } + + # Test the extraction method + object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log) + + assert object_key == "test/path/to/object.json" + + def test_session_handler_handles_dict_metadata_in_spend_log(self): + """ + Test: Session handler handles dict metadata in spend log. + + This test verifies that the method works when metadata is already a dict. + """ + # Create test spend log with dict metadata + spend_log = { + "request_id": "test_request_id", + "metadata": { + "cold_storage_object_key": "test/path/to/object.json", + "user_api_key": "test_key" + } + } + + # Test the extraction method + object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log) + + assert object_key == "test/path/to/object.json" + + + @pytest.mark.asyncio + async def test_cold_storage_handler_supports_object_key_retrieval(self): + """ + Test: ColdStorageHandler supports object key retrieval. + + This test verifies that the ColdStorageHandler has the new method + for retrieving objects using object keys directly. + """ + handler = ColdStorageHandler() + + # Mock the custom logger + mock_logger = AsyncMock() + mock_logger.get_proxy_server_request_from_cold_storage_with_object_key = AsyncMock( + return_value={"test": "data"} + ) + + with patch.object(handler, '_select_custom_logger_for_cold_storage', return_value="s3_v2"), \ + patch('litellm.logging_callback_manager.get_active_custom_logger_for_callback_name', return_value=mock_logger): + + result = await handler.get_proxy_server_request_from_cold_storage_with_object_key( + object_key="test/path/to/object.json" + ) + + assert result == {"test": "data"} + mock_logger.get_proxy_server_request_from_cold_storage_with_object_key.assert_called_once_with( + object_key="test/path/to/object.json" + ) + + @pytest.mark.asyncio + @patch('asyncio.create_task') # Mock asyncio.create_task to avoid event loop issues + async def test_s3_logger_supports_object_key_retrieval(self, mock_create_task): + """ + Test: S3Logger supports retrieval using provided object key. + + This test verifies that the S3Logger can retrieve objects using + the object key directly without generating it from request_id and start_time. + """ + # Create S3Logger instance + s3_logger = S3Logger(s3_bucket_name="test-bucket") + + # Mock the _download_object_from_s3 method + with patch.object(s3_logger, '_download_object_from_s3', return_value={"test": "data"}) as mock_download: + result = await s3_logger.get_proxy_server_request_from_cold_storage_with_object_key( + object_key="test/path/to/object.json" + ) + + assert result == {"test": "data"} + mock_download.assert_called_once_with("test/path/to/object.json") \ No newline at end of file diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index e8b6567a97d..0a6ea31a1ba 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1357,3 +1357,47 @@ async def test_async_function_with_fallbacks_common_utils(): args=(), kwargs={}, # No model key ) + + +def test_should_include_deployment(): + """Test that Router.should_include_deployment returns the correct response""" + router = litellm.Router( + model_list=[ + { + "model_name": "model_name_a28a12f9-3e44-4861-bd4f-325f2d309ce8_cd5dc6fb-b046-4e05-ae1d-32ba4d936266", + "litellm_params": {"model": "openai/*"}, + "model_info": { + "team_id": "a28a12f9-3e44-4861-bd4f-325f2d309ce8", + "team_public_model_name": "openai/*", + }, + } + ], + ) + + model = { + "model_name": "model_name_a28a12f9-3e44-4861-bd4f-325f2d309ce8_cd5dc6fb-b046-4e05-ae1d-32ba4d936266", + "litellm_params": { + "api_key": "sk-proj-1234567890", + "custom_llm_provider": "openai", + "use_in_pass_through": False, + "use_litellm_proxy": False, + "merge_reasoning_content_in_choices": False, + "model": "openai/*", + }, + "model_info": { + "id": "95f58039-d54a-4d1c-b700-5e32e99a1120", + "db_model": True, + "updated_by": "64a2f787-0863-4d76-9516-2dc49c1598e8", + "created_by": "64a2f787-0863-4d76-9516-2dc49c1598e8", + "team_id": "a28a12f9-3e44-4861-bd4f-325f2d309ce8", + "team_public_model_name": "openai/*", + "mode": "completion", + "access_groups": ["restricted-models-openai"], + }, + } + model_name = "openai/o4-mini-deep-research" + team_id = "a28a12f9-3e44-4861-bd4f-325f2d309ce8" + assert router.get_model_list( + model_name=model_name, + team_id=team_id, + ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 1f59bb94dbd..c17535bf0ec 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -373,6 +373,115 @@ def test_cohere_embedding_optional_params(): assert optional_params is not None +def validate_model_cost_values(model_data, exceptions=None): + """ + Validates that cost values in model data do not exceed 1. + + Args: + model_data (dict): The model data dictionary + exceptions (list, optional): List of model IDs that are allowed to have costs > 1 + + Returns: + tuple: (is_valid, violations) where is_valid is a boolean and violations is a list of error messages + """ + if exceptions is None: + exceptions = [] + + violations = [] + + # Define all cost-related fields to check + cost_fields = [ + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_character", + "output_cost_per_character", + "input_cost_per_image", + "output_cost_per_image", + "input_cost_per_pixel", + "output_cost_per_pixel", + "input_cost_per_second", + "output_cost_per_second", + "input_cost_per_query", + "input_cost_per_request", + "input_cost_per_audio_token", + "output_cost_per_audio_token", + "input_cost_per_audio_per_second", + "input_cost_per_video_per_second", + "input_cost_per_token_above_128k_tokens", + "output_cost_per_token_above_128k_tokens", + "input_cost_per_token_above_200k_tokens", + "output_cost_per_token_above_200k_tokens", + "input_cost_per_character_above_128k_tokens", + "output_cost_per_character_above_128k_tokens", + "input_cost_per_image_above_128k_tokens", + "input_cost_per_video_per_second_above_8s_interval", + "input_cost_per_video_per_second_above_15s_interval", + "input_cost_per_video_per_second_above_128k_tokens", + "input_cost_per_token_batch_requests", + "input_cost_per_token_batches", + "output_cost_per_token_batches", + "input_cost_per_token_cache_hit", + "cache_creation_input_token_cost", + "cache_creation_input_audio_token_cost", + "cache_read_input_token_cost", + "cache_read_input_audio_token_cost", + "input_dbu_cost_per_token", + "output_db_cost_per_token", + "output_dbu_cost_per_token", + "output_cost_per_reasoning_token", + "citation_cost_per_token", + ] + + # Also check nested cost fields + nested_cost_fields = [ + "search_context_cost_per_query", + ] + + for model_id, model_info in model_data.items(): + # Skip if this model is in exceptions + if model_id in exceptions: + continue + + # Check direct cost fields + for field in cost_fields: + if field in model_info and model_info[field] is not None: + cost_value = model_info[field] + + # Convert string values to float if needed + if isinstance(cost_value, str): + try: + cost_value = float(cost_value) + except (ValueError, TypeError): + # Skip if we can't convert to float + continue + + if isinstance(cost_value, (int, float)) and cost_value > 1: + violations.append( + f"Model '{model_id}' has {field} = {cost_value} which exceeds 1" + ) + + # Check nested cost fields + for field in nested_cost_fields: + if field in model_info and model_info[field] is not None: + nested_costs = model_info[field] + if isinstance(nested_costs, dict): + for nested_field, nested_value in nested_costs.items(): + # Convert string values to float if needed + if isinstance(nested_value, str): + try: + nested_value = float(nested_value) + except (ValueError, TypeError): + # Skip if we can't convert to float + continue + + if isinstance(nested_value, (int, float)) and nested_value > 1: + violations.append( + f"Model '{model_id}' has {field}.{nested_field} = {nested_value} which exceeds 1" + ) + + return len(violations) == 0, violations + + def test_aaamodel_prices_and_context_window_json_is_valid(): """ Validates the `model_prices_and_context_window.json` file. @@ -542,7 +651,23 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "sample_spec", None ) # remove the sample, whose schema is inconsistent with the real data + # Validate schema validate(actual_json, INTENDED_SCHEMA) + + # Validate cost values + # Define exceptions for models that are allowed to have costs > 1 + # Add model IDs here if they legitimately have costs > 1 + exceptions = [ + # Add any model IDs that should be exempt from the cost validation + # Example: "expensive-model-id", + ] + + is_valid, violations = validate_model_cost_values(actual_json, exceptions) + + if not is_valid: + error_message = "Cost validation failed:\n" + "\n".join(violations) + error_message += "\n\nTo add exceptions, add the model ID to the 'exceptions' list in the test function." + raise AssertionError(error_message) def test_get_model_info_gemini(): @@ -2225,6 +2350,67 @@ def test_block_key_hashing_logic(): print("āœ… All block_key hashing logic tests passed!") +def test_generate_gcp_iam_access_token(): + """ + Test the _generate_gcp_iam_access_token function with mocked GCP IAM client. + """ + from unittest.mock import Mock, patch + + service_account = "projects/-/serviceAccounts/test@project.iam.gserviceaccount.com" + expected_token = "test-access-token-12345" + + # Mock the GCP IAM client and its response + mock_response = Mock() + mock_response.access_token = expected_token + + mock_client = Mock() + mock_client.generate_access_token.return_value = mock_response + + # Mock the iam_credentials_v1 module + mock_iam_credentials_v1 = Mock() + mock_iam_credentials_v1.IAMCredentialsClient = Mock(return_value=mock_client) + mock_iam_credentials_v1.GenerateAccessTokenRequest = Mock() + + # Test successful token generation by mocking sys.modules + with patch.dict('sys.modules', {'google.cloud.iam_credentials_v1': mock_iam_credentials_v1}): + from litellm._redis import _generate_gcp_iam_access_token + + result = _generate_gcp_iam_access_token(service_account) + + assert result == expected_token + mock_iam_credentials_v1.IAMCredentialsClient.assert_called_once() + mock_client.generate_access_token.assert_called_once() + + # Verify the request was created with correct parameters + mock_iam_credentials_v1.GenerateAccessTokenRequest.assert_called_once_with( + name=service_account, + scope=['https://www.googleapis.com/auth/cloud-platform'] + ) + + +def test_generate_gcp_iam_access_token_import_error(): + """ + Test that _generate_gcp_iam_access_token raises ImportError when google-cloud-iam is not available. + """ + # Import the function first, before mocking + from litellm._redis import _generate_gcp_iam_access_token + + # Mock the import to fail when the function tries to import google.cloud.iam_credentials_v1 + original_import = __builtins__['__import__'] + + def mock_import(name, *args, **kwargs): + if name == 'google.cloud.iam_credentials_v1': + raise ImportError("No module named 'google.cloud.iam_credentials_v1'") + return original_import(name, *args, **kwargs) + + with patch('builtins.__import__', side_effect=mock_import): + with pytest.raises(ImportError) as exc_info: + _generate_gcp_iam_access_token("test-service-account") + + assert "google-cloud-iam is required" in str(exc_info.value) + assert "pip install google-cloud-iam" in str(exc_info.value) + + if __name__ == "__main__": # Allow running this test file directly for debugging pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index 1bf005db503..71b92475186 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -73,3 +73,56 @@ def test_usage_dump(): new_usage = Usage(**current_usage.model_dump()) assert new_usage.prompt_tokens_details.web_search_requests == 1 + + +def test_usage_completion_tokens_details_text_tokens(): + from litellm.types.utils import Usage + + # Test data from the reported issue + usage_data = { + 'completion_tokens': 77, + 'prompt_tokens': 11937, + 'total_tokens': 12014, + 'completion_tokens_details': { + 'accepted_prediction_tokens': None, + 'audio_tokens': None, + 'reasoning_tokens': 65, + 'rejected_prediction_tokens': None, + 'text_tokens': 12 + }, + 'prompt_tokens_details': { + 'audio_tokens': None, + 'cached_tokens': None, + 'text_tokens': 11937, + 'image_tokens': None + } + } + + # Create Usage object + u = Usage(**usage_data) + + # Verify the object has the text_tokens field + assert hasattr(u.completion_tokens_details, 'text_tokens') + assert u.completion_tokens_details.text_tokens == 12 + + # Get model_dump output + dump_result = u.model_dump() + + # Verify text_tokens is present in the model_dump output + assert 'completion_tokens_details' in dump_result + assert 'text_tokens' in dump_result['completion_tokens_details'] + assert dump_result['completion_tokens_details']['text_tokens'] == 12 + + # Verify the full completion_tokens_details structure + expected_completion_details = { + 'accepted_prediction_tokens': None, + 'audio_tokens': None, + 'reasoning_tokens': 65, + 'rejected_prediction_tokens': None, + 'text_tokens': 12 + } + assert dump_result['completion_tokens_details'] == expected_completion_details + + # Verify round-trip serialization works + new_usage = Usage(**dump_result) + assert new_usage.completion_tokens_details.text_tokens == 12 diff --git a/tests/unified_google_tests/test_google_ai_studio.py b/tests/unified_google_tests/test_google_ai_studio.py index e6859647b69..385a070e1cb 100644 --- a/tests/unified_google_tests/test_google_ai_studio.py +++ b/tests/unified_google_tests/test_google_ai_studio.py @@ -15,7 +15,7 @@ class TestGoogleGenAIStudio(BaseGoogleGenAITest): @property def model_config(self): return { - "model": "gemini/gemini-1.5-flash", + "model": "gemini/gemini-2.5-flash-lite", } @pytest.mark.asyncio @@ -86,7 +86,7 @@ async def test_mock_stream_generate_content_with_tools(): print("\n--- Testing async agenerate_content_stream with function call parsing ---") response = await litellm.google_genai.agenerate_content_stream( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash-lite", contents=contents, tools=[ { @@ -299,7 +299,7 @@ async def test_validate_post_request_parameters(): # Make the API call response = await litellm.google_genai.agenerate_content_stream( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash-lite", contents=contents, tools=tools ) @@ -341,9 +341,9 @@ async def test_validate_post_request_parameters(): # Validate model field assert "model" in request_data, "Expected 'model' field in request data" - # Model might be transformed, but should contain gemini-1.5-flash + # Model might be transformed, but should contain gemini-2.5-flash-lite model_value = request_data["model"] - assert "gemini-1.5-flash" in model_value, f"Expected model to contain 'gemini-1.5-flash', got: {model_value}" + assert "gemini-2.5-flash-lite" in model_value, f"Expected model to contain 'gemini-2.5-flash-lite', got: {model_value}" print(f"āœ… Model validation passed: {model_value}") # Validate contents field diff --git a/tests/unified_google_tests/test_vertex_ai_native.py b/tests/unified_google_tests/test_vertex_ai_native.py index 3dd40380375..5c8f8575c42 100644 --- a/tests/unified_google_tests/test_vertex_ai_native.py +++ b/tests/unified_google_tests/test_vertex_ai_native.py @@ -6,5 +6,5 @@ class TestVertexAIGenerateContent(BaseGoogleGenAITest): @property def model_config(self): return { - "model": "vertex_ai/gemini-1.5-flash", + "model": "vertex_ai/gemini-2.5-flash-lite", } \ No newline at end of file diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 2194bbaec48..7106a6f884f 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -6,6 +6,7 @@ import { jwtDecode } from "jwt-decode"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { Team } from "@/components/key_team_helpers/key_list"; import Navbar from "@/components/navbar"; +import { ThemeProvider } from "@/contexts/ThemeContext"; import UserDashboard from "@/components/user_dashboard"; import ModelDashboard from "@/components/model_dashboard"; import ViewUserDashboard from "@/components/view_users"; @@ -39,6 +40,7 @@ import { fetchTeams } from "@/components/common_components/fetch_teams"; import { MCPServers } from "@/components/mcp_tools"; import TagManagement from "@/components/tag_management"; import VectorStoreManagement from "@/components/vector_store_management"; +import UIThemeSettings from "@/components/ui_theme_settings"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { cx } from "@/lib/cva.config"; @@ -232,8 +234,9 @@ export default function CreateKeyPage() { return ( }> - {invitation_id ? ( - + {invitation_id ? ( + + ) : page == "ui-theme" ? ( + ) : page == "model-hub-table" ? (
)} + ); diff --git a/ui/litellm-dashboard/src/components/common_components/ModelAliasManager.tsx b/ui/litellm-dashboard/src/components/common_components/ModelAliasManager.tsx new file mode 100644 index 00000000000..db49e1d999c --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/ModelAliasManager.tsx @@ -0,0 +1,342 @@ +import React, { useState, useEffect } from "react"; +import { message } from "antd"; +import { PlusCircleIcon, PencilIcon, TrashIcon } from "@heroicons/react/outline"; +import { + Card, + Title, + Text, + Table, + TableHead, + TableHeaderCell, + TableBody, + TableRow, + TableCell +} from "@tremor/react"; +import ModelSelector from "./ModelSelector"; + +interface ModelAliasManagerProps { + accessToken: string; + initialModelAliases?: { [key: string]: string }; + onAliasUpdate?: (updatedAliases: { [key: string]: string }) => void; + showExampleConfig?: boolean; +} + +interface AliasItem { + id: string; + aliasName: string; + targetModel: string; +} + +const ModelAliasManager: React.FC = ({ + accessToken, + initialModelAliases = {}, + onAliasUpdate, + showExampleConfig = true, +}) => { + const [aliases, setAliases] = useState([]); + const [newAlias, setNewAlias] = useState({ aliasName: "", targetModel: "" }); + const [editingAlias, setEditingAlias] = useState(null); + + useEffect(() => { + // Convert object to array for display + const aliasArray = Object.entries(initialModelAliases).map(([aliasName, targetModel], index) => ({ + id: `${index}-${aliasName}`, + aliasName, + targetModel, + })); + setAliases(aliasArray); + }, [initialModelAliases]); + + const handleAddAlias = () => { + if (!newAlias.aliasName || !newAlias.targetModel) { + message.error("Please provide both alias name and target model"); + return; + } + + // Check for duplicate alias names + if (aliases.some(alias => alias.aliasName === newAlias.aliasName)) { + message.error("An alias with this name already exists"); + return; + } + + const newAliasObj: AliasItem = { + id: `${Date.now()}-${newAlias.aliasName}`, + aliasName: newAlias.aliasName, + targetModel: newAlias.targetModel, + }; + + const updatedAliases = [...aliases, newAliasObj]; + setAliases(updatedAliases); + setNewAlias({ aliasName: "", targetModel: "" }); + + // Convert array back to object format and notify parent + const aliasObject: { [key: string]: string } = {}; + updatedAliases.forEach(alias => { + aliasObject[alias.aliasName] = alias.targetModel; + }); + + if (onAliasUpdate) { + onAliasUpdate(aliasObject); + } + + message.success("Alias added successfully"); + }; + + const handleEditAlias = (alias: AliasItem) => { + setEditingAlias({ ...alias }); + }; + + const handleUpdateAlias = () => { + if (!editingAlias) return; + + if (!editingAlias.aliasName || !editingAlias.targetModel) { + message.error("Please provide both alias name and target model"); + return; + } + + // Check for duplicate alias names (excluding current alias) + if (aliases.some(alias => alias.id !== editingAlias.id && alias.aliasName === editingAlias.aliasName)) { + message.error("An alias with this name already exists"); + return; + } + + const updatedAliases = aliases.map(alias => + alias.id === editingAlias.id ? editingAlias : alias + ); + + setAliases(updatedAliases); + setEditingAlias(null); + + // Convert array back to object format and notify parent + const aliasObject: { [key: string]: string } = {}; + updatedAliases.forEach(alias => { + aliasObject[alias.aliasName] = alias.targetModel; + }); + + if (onAliasUpdate) { + onAliasUpdate(aliasObject); + } + + message.success("Alias updated successfully"); + }; + + const handleCancelEdit = () => { + setEditingAlias(null); + }; + + const deleteAlias = (aliasId: string) => { + const updatedAliases = aliases.filter(alias => alias.id !== aliasId); + setAliases(updatedAliases); + + // Convert array back to object format and notify parent + const aliasObject: { [key: string]: string } = {}; + updatedAliases.forEach(alias => { + aliasObject[alias.aliasName] = alias.targetModel; + }); + + if (onAliasUpdate) { + onAliasUpdate(aliasObject); + } + + message.success("Alias deleted successfully"); + }; + + // Convert current aliases to object for config example + const aliasObject = aliases.reduce((acc, alias) => { + acc[alias.aliasName] = alias.targetModel; + return acc; + }, {} as { [key: string]: string }); + + return ( +
+
+ Add New Alias +
+
+ + + setNewAlias({ + ...newAlias, + aliasName: e.target.value, + }) + } + placeholder="e.g., gpt-4o" + className="w-full px-3 py-2 border border-gray-300 rounded-md text-sm" + /> +
+
+ + + setNewAlias({ + ...newAlias, + targetModel: value, + }) + } + showLabel={false} + /> +
+
+ +
+
+
+ + + Manage Existing Aliases + +
+
+ + + + + Alias Name + + + Target Model + + + Actions + + + + + {aliases.map((alias) => ( + + {editingAlias && editingAlias.id === alias.id ? ( + <> + + + setEditingAlias({ + ...editingAlias, + aliasName: e.target.value, + }) + } + className="w-full px-2 py-1 border border-gray-300 rounded-md text-sm" + /> + + + + setEditingAlias({ + ...editingAlias, + targetModel: value, + }) + } + showLabel={false} + style={{ height: '32px' }} + /> + + +
+ + +
+ + + ) : ( + <> + + {alias.aliasName} + + + {alias.targetModel} + + +
+ + +
+ + + )} + + ))} + {aliases.length === 0 && ( + + + No aliases added yet. Add a new alias above. + + + )} + +
+
+
+ + {/* Configuration Example */} + {showExampleConfig && ( + + Configuration Example + + Here's how your current aliases would look in the config: + +
+
+ model_aliases: + {Object.keys(aliasObject).length === 0 ? ( + +
+   # No aliases configured yet + + ) : ( + Object.entries(aliasObject).map(([key, value]) => ( + +
+   "{key}": "{value}" + + )) + )} +
+
+ + )} +
+ ); +}; + +export default ModelAliasManager; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/common_components/ModelSelector.tsx b/ui/litellm-dashboard/src/components/common_components/ModelSelector.tsx new file mode 100644 index 00000000000..a8a8087cb62 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/ModelSelector.tsx @@ -0,0 +1,122 @@ +import React, { useState, useEffect, useRef } from "react"; +import { TextInput, Text } from "@tremor/react"; +import { Select } from "antd"; +import { RobotOutlined } from "@ant-design/icons"; +import { fetchAvailableModels, ModelGroup } from "../chat_ui/llm_calls/fetch_models"; + +interface ModelSelectorProps { + accessToken: string; + value?: string; + placeholder?: string; + onChange?: (value: string) => void; + disabled?: boolean; + style?: React.CSSProperties; + className?: string; + showLabel?: boolean; + labelText?: string; +} + +const ModelSelector: React.FC = ({ + accessToken, + value, + placeholder = "Select a Model", + onChange, + disabled = false, + style, + className, + showLabel = true, + labelText = "Select Model" +}) => { + const [selectedModel, setSelectedModel] = useState(value); + const [showCustomModelInput, setShowCustomModelInput] = useState(false); + const [modelInfo, setModelInfo] = useState([]); + const customModelTimeout = useRef(null); + + useEffect(() => { + setSelectedModel(value); + }, [value]); + + useEffect(() => { + if (!accessToken) return; + + const loadModels = async () => { + try { + const uniqueModels = await fetchAvailableModels(accessToken); + console.log("Fetched models for selector:", uniqueModels); + + if (uniqueModels.length > 0) { + setModelInfo(uniqueModels); + } + } catch (error) { + console.error("Error fetching model info:", error); + } + }; + + loadModels(); + }, [accessToken]); + + const onModelChange = (value: string) => { + if (value === 'custom') { + setShowCustomModelInput(true); + setSelectedModel(undefined); + } else { + setShowCustomModelInput(false); + setSelectedModel(value); + if (onChange) { + onChange(value); + } + } + }; + + const handleCustomModelChange = (value: string) => { + // Using setTimeout to create a simple debounce effect + if (customModelTimeout.current) { + clearTimeout(customModelTimeout.current); + } + + customModelTimeout.current = setTimeout(() => { + setSelectedModel(value); + if (onChange) { + onChange(value); + } + }, 500); // 500ms delay after typing stops + }; + + return ( +
+ {showLabel && ( + + {labelText} + + )} + setDeleteConfirmInput(e.target.value)} + placeholder="Enter team name exactly" + className="w-full px-4 py-3 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500 text-base" + autoFocus + /> +
+ + +
+ + +
+ + + ); +})()} @@ -1440,6 +1477,7 @@ const Teams: React.FC = ({ placeholder="Select MCP servers or access groups (optional)" /> + @@ -1457,6 +1495,26 @@ const Teams: React.FC = ({ + + + + + Model Aliases + + +
+ + Create custom aliases for models that can be used by team members in API calls. This allows you to create shortcuts for specific models. + + +
+ +
Create Team diff --git a/ui/litellm-dashboard/src/components/ui_theme_settings.tsx b/ui/litellm-dashboard/src/components/ui_theme_settings.tsx new file mode 100644 index 00000000000..8bdafe01483 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ui_theme_settings.tsx @@ -0,0 +1,207 @@ +import React, { useState, useEffect } from "react" +import { + Card, + Title, + Text, + TextInput, + Button, +} from "@tremor/react" +import { message } from "antd" +import { useTheme } from "@/contexts/ThemeContext" +import { getProxyBaseUrl } from "@/components/networking" + +interface UIThemeSettingsProps { + userID: string | null; + userRole: string | null; + accessToken: string | null; +} + +const UIThemeSettings: React.FC = ({ + userID, + userRole, + accessToken, +}) => { + const { logoUrl, setLogoUrl } = useTheme(); + const [logoUrlInput, setLogoUrlInput] = useState(""); + const [loading, setLoading] = useState(false); + + // Load current settings when component mounts + useEffect(() => { + if (accessToken) { + fetchLogoSettings(); + } + }, [accessToken]); + + const fetchLogoSettings = async () => { + try { + const proxyBaseUrl = getProxyBaseUrl(); + const url = proxyBaseUrl ? `${proxyBaseUrl}/get/ui_theme_settings` : "/get/ui_theme_settings"; + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (response.ok) { + const data = await response.json(); + const logoUrl = data.values?.logo_url || ''; + setLogoUrlInput(logoUrl); + setLogoUrl(logoUrl || null); + } + } catch (error) { + console.error("Error fetching theme settings:", error); + } + }; + + const handleSave = async () => { + setLoading(true); + try { + const proxyBaseUrl = getProxyBaseUrl(); + const url = proxyBaseUrl ? `${proxyBaseUrl}/update/ui_theme_settings` : "/update/ui_theme_settings"; + const response = await fetch(url, { + method: "PATCH", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + logo_url: logoUrlInput || null, + }), + }); + + if (response.ok) { + message.success("Logo settings updated successfully!"); + setLogoUrl(logoUrlInput || null); + } else { + throw new Error("Failed to update settings"); + } + } catch (error) { + console.error("Error updating logo settings:", error); + message.error("Failed to update logo settings"); + } finally { + setLoading(false); + } + }; + + const handleReset = async () => { + setLogoUrlInput(""); + setLogoUrl(null); + + // Save null to backend to clear the logo + setLoading(true); + try { + const proxyBaseUrl = getProxyBaseUrl(); + const url = proxyBaseUrl ? `${proxyBaseUrl}/update/ui_theme_settings` : "/update/ui_theme_settings"; + const response = await fetch(url, { + method: "PATCH", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + logo_url: null, + }), + }); + + if (response.ok) { + message.success("Logo reset to default!"); + } else { + throw new Error("Failed to reset logo"); + } + } catch (error) { + console.error("Error resetting logo:", error); + message.error("Failed to reset logo"); + } finally { + setLoading(false); + } + }; + + if (!accessToken) { + return null; + } + + return ( +
+
+ Logo Customization + + Customize your LiteLLM admin dashboard with a custom logo. + +
+ + +
+
+ + Custom Logo URL + + { + setLogoUrlInput(value); + // Update logo in real-time for preview + setLogoUrl(value || null); + }} + className="w-full" + /> + + Enter a URL for your custom logo or leave empty to use the default LiteLLM logo + +
+ + {/* Logo Preview */} +
+ + Current Logo + +
+ {logoUrlInput ? ( + Custom logo { + const target = e.target as HTMLImageElement; + target.style.display = 'none'; + const fallbackText = document.createElement('div'); + fallbackText.className = 'text-gray-500 text-sm'; + fallbackText.textContent = 'Failed to load image'; + target.parentElement?.appendChild(fallbackText); + }} + /> + ) : ( + Default LiteLLM logo will be used + )} +
+
+ + {/* Action Buttons */} +
+ + +
+
+ +
+ ); +}; + +export default UIThemeSettings; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/view_logs/SessionView.tsx b/ui/litellm-dashboard/src/components/view_logs/SessionView.tsx index 1bd8a51192f..2d05680040b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/SessionView.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/SessionView.tsx @@ -9,6 +9,7 @@ import { ArrowLeftIcon } from "@heroicons/react/outline" import { Button } from "antd" import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils" import { CheckIcon, CopyIcon } from "lucide-react" +import { Tooltip } from "antd" interface SessionViewProps { sessionId: string @@ -24,6 +25,21 @@ export const SessionView: React.FC = ({ sessionId, logs, onBac // Calculate session metrics const totalCost = logs.reduce((sum, log) => sum + (log.spend || 0), 0) const totalTokens = logs.reduce((sum, log) => sum + (log.total_tokens || 0), 0) + + // Calculate cache token totals from metadata + const totalCacheReadTokens = logs.reduce((sum, log) => { + const cacheReadTokens = log.metadata?.additional_usage_values?.cache_read_input_tokens || 0 + return sum + cacheReadTokens + }, 0) + + const totalCacheCreationTokens = logs.reduce((sum, log) => { + const cacheCreationTokens = log.metadata?.additional_usage_values?.cache_creation_input_tokens || 0 + return sum + cacheCreationTokens + }, 0) + + // Calculate total tokens including cache tokens + const totalTokensWithCache = totalTokens + totalCacheReadTokens + totalCacheCreationTokens + const startTime = logs.length > 0 ? new Date(logs[0].startTime) : new Date() const endTime = logs.length > 0 ? new Date(logs[logs.length - 1].endTime) : new Date() const durationMs = endTime.getTime() - startTime.getTime() @@ -100,10 +116,61 @@ export const SessionView: React.FC = ({ sessionId, logs, onBac Total Cost ${formatNumberWithCommas(totalCost, 6)} - - Total Tokens - {totalTokens} - + +
Usage breakdown
+
+
+
Input usage:
+
+
+ input: + {formatNumberWithCommas(logs.reduce((sum, log) => sum + (log.prompt_tokens || 0), 0))} +
+ {totalCacheReadTokens > 0 && ( +
+ input_cached_tokens: + {formatNumberWithCommas(totalCacheReadTokens)} +
+ )} + {totalCacheCreationTokens > 0 && ( +
+ input_cache_creation_tokens: + {formatNumberWithCommas(totalCacheCreationTokens)} +
+ )} +
+
+
+
Output usage:
+
+
+ output: + {formatNumberWithCommas(logs.reduce((sum, log) => sum + (log.completion_tokens || 0), 0))} +
+
+
+
+
+ Total usage: + {formatNumberWithCommas(totalTokensWithCache)} +
+
+
+
+ } + placement="top" + overlayStyle={{ minWidth: '300px' }} + > + +
+ Total Tokens + ā“˜ +
+ {formatNumberWithCommas(totalTokensWithCache)} + + {/* Request Timeline */} Session Logs diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index ea97d54a6f1..d2041b389fd 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -925,7 +925,15 @@ export function RequestViewer({ row }: { row: Row }) {
Tokens: - {row.original.total_tokens} ({row.original.prompt_tokens}+{row.original.completion_tokens}) + {row.original.total_tokens} ({row.original.prompt_tokens} prompt tokens + {row.original.completion_tokens} completion tokens) +
+
+ Cache Read Tokens: + {formatNumberWithCommas(row.original.metadata?.additional_usage_values?.cache_read_input_tokens || 0)} +
+
+ Cache Creation Tokens: + {formatNumberWithCommas(row.original.metadata?.additional_usage_values.cache_creation_input_tokens)}
Cost: diff --git a/ui/litellm-dashboard/src/contexts/ThemeContext.tsx b/ui/litellm-dashboard/src/contexts/ThemeContext.tsx new file mode 100644 index 00000000000..619d6a1004a --- /dev/null +++ b/ui/litellm-dashboard/src/contexts/ThemeContext.tsx @@ -0,0 +1,62 @@ +import React, { createContext, useContext, useState, useEffect, ReactNode } from 'react' +import { getProxyBaseUrl } from '@/components/networking' + +interface ThemeContextType { + logoUrl: string | null; + setLogoUrl: (url: string | null) => void; +} + +const ThemeContext = createContext(undefined) + +export const useTheme = () => { + const context = useContext(ThemeContext); + if (!context) { + throw new Error('useTheme must be used within a ThemeProvider'); + } + return context; +} + +interface ThemeProviderProps { + children: ReactNode; + accessToken?: string | null; +} + +export const ThemeProvider: React.FC = ({ children, accessToken }) => { + const [logoUrl, setLogoUrl] = useState(null); + + // Load logo URL from backend on mount + useEffect(() => { + const loadLogoSettings = async () => { + if (accessToken) { + try { + const proxyBaseUrl = getProxyBaseUrl(); + const url = proxyBaseUrl ? `${proxyBaseUrl}/get/ui_theme_settings` : '/get/ui_theme_settings'; + const response = await fetch(url, { + method: 'GET', + headers: { + 'Authorization': `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + }); + + if (response.ok) { + const data = await response.json(); + if (data.values?.logo_url) { + setLogoUrl(data.values.logo_url); + } + } + } catch (error) { + console.warn('Failed to load logo settings from backend:', error); + } + } + }; + + loadLogoSettings(); + }, [accessToken]); + + return ( + + {children} + + ); +}; \ No newline at end of file