diff --git a/.circleci/config.yml b/.circleci/config.yml index b1c630cdcb3..c9de5adcc9f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1913,6 +1913,7 @@ jobs: -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index 7e67aee8d73..0d3a9f2b5d4 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -31,6 +31,7 @@ jobs: poetry run pip install "pytest-retry==1.6.3" poetry run pip install pytest-xdist poetry run pip install "google-genai==1.22.0" + poetry run pip install "google-cloud-aiplatform>=1.38" poetry run pip install "fastapi-offline==1.7.3" - name: Setup litellm-enterprise as local package run: | diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index 26629a0b8f8..9699d97b352 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -106,6 +106,7 @@ def completion( parallel_tool_calls: Optional[bool] = None, logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, + safety_identifier: Optional[str] = None, deployment_id=None, # soon to be deprecated params by OpenAI functions: Optional[List] = None, @@ -196,6 +197,8 @@ def completion( - `top_logprobs`: *int (optional)* - An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to true if this parameter is used. +- `safety_identifier`: *string (optional)* - A unique identifier for tracking and managing safety-related requests. This parameter helps with safety monitoring and compliance tracking. + - `headers`: *dict (optional)* - A dictionary of headers to be sent with the request. - `extra_headers`: *dict (optional)* - Alternative to `headers`, used to send extra headers in LLM API request. diff --git a/docs/my-website/docs/providers/vertex_partner.md b/docs/my-website/docs/providers/vertex_partner.md index cf780e35dbd..856f054b8e6 100644 --- a/docs/my-website/docs/providers/vertex_partner.md +++ b/docs/my-website/docs/providers/vertex_partner.md @@ -15,6 +15,7 @@ import TabItem from '@theme/TabItem'; | Mistral | `vertex_ai/mistral-*` | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) | | AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) | | Qwen | `vertex_ai/qwen/*` | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) | +| OpenAI (GPT-OSS) | `vertex_ai/openai/gpt-oss-*` | [Vertex AI - GPT-OSS Models](https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/) | | Model Garden | `vertex_ai/openai/{MODEL_ID}` or `vertex_ai/{MODEL_ID}` | [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) | ## Vertex AI - Anthropic (Claude) @@ -658,6 +659,141 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ +## VertexAI GPT-OSS Models + +| Property | Details | +|----------|---------| +| Provider Route | `vertex_ai/openai/{MODEL}` | +| Vertex Documentation | [Vertex AI - GPT-OSS Models](https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/) | + +**LiteLLM Supports all Vertex AI GPT-OSS Models.** Ensure you use the `vertex_ai/openai/` prefix for all Vertex AI GPT-OSS models. + +| Model Name | Usage | +|------------------|------------------------------| +| vertex_ai/openai/gpt-oss-20b-maas | `completion('vertex_ai/openai/gpt-oss-20b-maas', messages)` | + +#### Usage + + + + +```python +from litellm import completion +import os + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" + +model = "openai/gpt-oss-20b-maas" + +vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] +vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] + +response = completion( + model="vertex_ai/" + model, + messages=[{"role": "user", "content": "hi"}], + vertex_ai_project=vertex_ai_project, + vertex_ai_location=vertex_ai_location, +) +print("\nModel Response", response) +``` + + + +**1. Add to config** + +```yaml +model_list: + - model_name: gpt-oss + litellm_params: + model: vertex_ai/openai/gpt-oss-20b-maas + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-central1" +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-oss", # πŸ‘ˆ the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + +#### Usage - `reasoning_effort` + +GPT-OSS models support the `reasoning_effort` parameter for enhanced reasoning capabilities. + + + + +```python +from litellm import completion + +response = completion( + model="vertex_ai/openai/gpt-oss-20b-maas", + messages=[{"role": "user", "content": "Solve this complex problem step by step"}], + reasoning_effort="low", # Options: "minimal", "low", "medium", "high" + vertex_ai_project="your-vertex-project", + vertex_ai_location="us-central1", +) +``` + + + + + +1. Setup config.yaml + +```yaml +model_list: +- model_name: gpt-oss + litellm_params: + model: vertex_ai/openai/gpt-oss-20b-maas + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-central1" +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{ + "model": "gpt-oss", + "messages": [{"role": "user", "content": "Solve this complex problem step by step"}], + "reasoning_effort": "low" + }' +``` + + + + ## Model Garden :::tip diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 541dc6fb3c8..7eb355d39f2 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -573,6 +573,10 @@ router_settings: | LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM | LITELLM_LOG | Enable detailed logging for LiteLLM | LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file +| LITELLM_LOGGER_NAME | Name for OTEL logger +| LITELLM_METER_NAME | Name for OTEL Meter +| LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL +| LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL | LITELLM_MASTER_KEY | Master key for proxy authentication | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60 diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index 468bcad2cf8..7d50aedb424 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -439,6 +439,33 @@ response = client.chat.completions.create( print(response) ``` + +**Using Headers:** + +```python +import openai +client = openai.OpenAI( + api_key="sk-1234", + base_url="http://0.0.0.0:4000" +) + +# Pass spend logs metadata via headers +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_headers={ + "x-litellm-spend-logs-metadata": '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}' + } +) + +print(response) +``` + @@ -478,6 +505,43 @@ async function runOpenAI() { // Call the asynchronous function runOpenAI(); ``` + +**Using Headers:** + +```js +const openai = require('openai'); + +async function runOpenAI() { + const client = new openai.OpenAI({ + apiKey: 'sk-1234', + baseURL: 'http://0.0.0.0:4000' + }); + + try { + const response = await client.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [ + { + role: 'user', + content: "this is a test request, write a short poem" + }, + ] + }, { + headers: { + 'x-litellm-spend-logs-metadata': '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}' + } + }); + console.log(response); + } catch (error) { + console.log("got this exception from server"); + console.error(error); + } +} + +// Call the asynchronous function +runOpenAI(); +``` + @@ -502,6 +566,29 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } }' ``` + + + + + +Pass `x-litellm-spend-logs-metadata` as a request header with JSON string + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'x-litellm-spend-logs-metadata: {"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + diff --git a/docs/my-website/docs/proxy/request_headers.md b/docs/my-website/docs/proxy/request_headers.md index c250d42f7bb..eea66e5fa93 100644 --- a/docs/my-website/docs/proxy/request_headers.md +++ b/docs/my-website/docs/proxy/request_headers.md @@ -14,6 +14,8 @@ Special headers that are supported by LiteLLM. `x-litellm-num-retries`: Optional[int]: The number of retries for the request. +`x-litellm-spend-logs-metadata`: Optional[str]: JSON string containing custom metadata to include in spend logs. Example: `{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}`. [Learn More](../proxy/enterprise#tracking-spend-with-custom-metadata) + ## Anthropic Headers `anthropic-version` Optional[str]: The version of the Anthropic API to use. diff --git a/litellm/__init__.py b/litellm/__init__.py index 79865c83513..6a184d70b5a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -67,6 +67,7 @@ from litellm.constants import ( bedrock_embedding_models, known_tokenizer_config, BEDROCK_INVOKE_PROVIDERS_LITERAL, + BEDROCK_CONVERSE_MODELS, DEFAULT_MAX_TOKENS, DEFAULT_SOFT_BUDGET, DEFAULT_ALLOWED_FAILS, @@ -432,40 +433,6 @@ organization = None project = None config_path = None vertex_ai_safety_settings: Optional[dict] = None -BEDROCK_CONVERSE_MODELS = [ - "openai.gpt-oss-20b-1:0", - "openai.gpt-oss-120b-1:0", - "anthropic.claude-opus-4-1-20250805-v1:0", - "anthropic.claude-opus-4-20250514-v1:0", - "anthropic.claude-sonnet-4-20250514-v1:0", - "anthropic.claude-3-7-sonnet-20250219-v1:0", - "anthropic.claude-3-5-haiku-20241022-v1:0", - "anthropic.claude-3-5-sonnet-20241022-v2:0", - "anthropic.claude-3-5-sonnet-20240620-v1:0", - "anthropic.claude-3-opus-20240229-v1:0", - "anthropic.claude-3-sonnet-20240229-v1:0", - "anthropic.claude-3-haiku-20240307-v1:0", - "anthropic.claude-v2", - "anthropic.claude-v2:1", - "anthropic.claude-v1", - "anthropic.claude-instant-v1", - "ai21.jamba-instruct-v1:0", - "ai21.jamba-1-5-mini-v1:0", - "ai21.jamba-1-5-large-v1:0", - "meta.llama3-70b-instruct-v1:0", - "meta.llama3-8b-instruct-v1:0", - "meta.llama3-1-8b-instruct-v1:0", - "meta.llama3-1-70b-instruct-v1:0", - "meta.llama3-1-405b-instruct-v1:0", - "meta.llama3-70b-instruct-v1:0", - "mistral.mistral-large-2407-v1:0", - "mistral.mistral-large-2402-v1:0", - "mistral.mistral-small-2402-v1:0", - "meta.llama3-2-1b-instruct-v1:0", - "meta.llama3-2-3b-instruct-v1:0", - "meta.llama3-2-11b-instruct-v1:0", - "meta.llama3-2-90b-instruct-v1:0", -] ####### COMPLETION MODELS ################### from typing import Set @@ -491,6 +458,7 @@ vertex_llama3_models: Set = set() vertex_deepseek_models: Set = set() vertex_ai_ai21_models: Set = set() vertex_mistral_models: Set = set() +vertex_openai_models: Set = set() ai21_models: Set = set() ai21_chat_models: Set = set() nlp_cloud_models: Set = set() @@ -637,6 +605,9 @@ def add_known_models(): elif value.get("litellm_provider") == "vertex_ai-image-models": key = key.replace("vertex_ai/", "") vertex_ai_image_models.add(key) + elif value.get("litellm_provider") == "vertex_ai-openai_models": + key = key.replace("vertex_ai/", "") + vertex_openai_models.add(key) elif value.get("litellm_provider") == "ai21": if value.get("mode") == "chat": ai21_chat_models.add(key) diff --git a/litellm/constants.py b/litellm/constants.py index 0655473301f..21e30bef32b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -14,7 +14,9 @@ DEFAULT_S3_BATCH_SIZE = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int( os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10) ) -DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 4)) +DEFAULT_NUM_WORKERS_LITELLM_PROXY = int( + os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 4) +) DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512)) SQS_SEND_MESSAGE_ACTION = "SendMessage" SQS_API_VERSION = "2012-11-05" @@ -395,6 +397,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES = { "reasoning_effort": None, "thinking": None, "web_search_options": None, + "safety_identifier": None, } openai_compatible_endpoints: List = [ @@ -745,6 +748,42 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ "deepseek_r1", ] +BEDROCK_CONVERSE_MODELS = [ + "openai.gpt-oss-20b-1:0", + "openai.gpt-oss-120b-1:0", + "anthropic.claude-opus-4-1-20250805-v1:0", + "anthropic.claude-opus-4-20250514-v1:0", + "anthropic.claude-sonnet-4-20250514-v1:0", + "anthropic.claude-3-7-sonnet-20250219-v1:0", + "anthropic.claude-3-5-haiku-20241022-v1:0", + "anthropic.claude-3-5-sonnet-20241022-v2:0", + "anthropic.claude-3-5-sonnet-20240620-v1:0", + "anthropic.claude-3-opus-20240229-v1:0", + "anthropic.claude-3-sonnet-20240229-v1:0", + "anthropic.claude-3-haiku-20240307-v1:0", + "anthropic.claude-v2", + "anthropic.claude-v2:1", + "anthropic.claude-v1", + "anthropic.claude-instant-v1", + "ai21.jamba-instruct-v1:0", + "ai21.jamba-1-5-mini-v1:0", + "ai21.jamba-1-5-large-v1:0", + "meta.llama3-70b-instruct-v1:0", + "meta.llama3-8b-instruct-v1:0", + "meta.llama3-1-8b-instruct-v1:0", + "meta.llama3-1-70b-instruct-v1:0", + "meta.llama3-1-405b-instruct-v1:0", + "meta.llama3-70b-instruct-v1:0", + "mistral.mistral-large-2407-v1:0", + "mistral.mistral-large-2402-v1:0", + "mistral.mistral-small-2402-v1:0", + "meta.llama3-2-1b-instruct-v1:0", + "meta.llama3-2-3b-instruct-v1:0", + "meta.llama3-2-11b-instruct-v1:0", + "meta.llama3-2-90b-instruct-v1:0", +] + + open_ai_embedding_models: set = set(["text-embedding-ada-002"]) cohere_embedding_models: set = set( [ diff --git a/litellm/google_genai/adapters/handler.py b/litellm/google_genai/adapters/handler.py index 1f575f27591..dcf707ebd51 100644 --- a/litellm/google_genai/adapters/handler.py +++ b/litellm/google_genai/adapters/handler.py @@ -37,6 +37,10 @@ class GenerateContentToCompletionHandler: completion_kwargs: Dict[str, Any] = dict(completion_request) + # feed metadata for custom callback + if extra_kwargs is not None and "metadata" in extra_kwargs: + completion_kwargs["metadata"] = extra_kwargs["metadata"] + if stream: completion_kwargs["stream"] = stream diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 41db4a551bd..7da38e193b6 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -805,9 +805,9 @@ class SlackAlerting(CustomBatchLogger): ### UNIQUE CACHE KEY ### cache_key = provider + region_name - outage_value: Optional[ProviderRegionOutageModel] = ( - await self.internal_usage_cache.async_get_cache(key=cache_key) - ) + outage_value: Optional[ + ProviderRegionOutageModel + ] = await self.internal_usage_cache.async_get_cache(key=cache_key) if ( getattr(exception, "status_code", None) is None @@ -1367,12 +1367,13 @@ Model Info: # Get the current timestamp current_time = datetime.now().strftime("%H:%M:%S") _proxy_base_url = os.getenv("PROXY_BASE_URL", None) + # Use .name if it's an enum, otherwise use as is + alert_type_name = getattr(alert_type, 'name', alert_type) + alert_type_formatted = f"Alert type: `{alert_type_name}`" if alert_type == "daily_reports" or alert_type == "new_model_added": - formatted_message = message + formatted_message = alert_type_formatted + message else: - formatted_message = ( - f"Level: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}" - ) + formatted_message = f"{alert_type_formatted}\nLevel: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}" if kwargs: for key, value in kwargs.items(): @@ -1388,9 +1389,9 @@ Model Info: self.alert_to_webhook_url is not None and alert_type in self.alert_to_webhook_url ): - slack_webhook_url: Optional[Union[str, List[str]]] = ( - self.alert_to_webhook_url[alert_type] - ) + slack_webhook_url: Optional[ + Union[str, List[str]] + ] = self.alert_to_webhook_url[alert_type] elif self.default_webhook_url is not None: slack_webhook_url = self.default_webhook_url else: diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 531da933fcc..5bc6afb6dbc 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -1,13 +1,11 @@ # What is this? ## Log success + failure events to Braintrust -import copy import os from datetime import datetime from typing import Dict, Optional import httpx -from pydantic import BaseModel import litellm from litellm import verbose_logger @@ -24,7 +22,6 @@ API_BASE = "https://api.braintrustdata.com/v1" def get_utc_datetime(): import datetime as dt - from datetime import datetime if hasattr(dt, "UTC"): return datetime.now(dt.UTC) # type: ignore @@ -45,9 +42,9 @@ class BraintrustLogger(CustomLogger): "Authorization": "Bearer " + self.api_key, "Content-Type": "application/json", } - self._project_id_cache: Dict[ - str, str - ] = {} # Cache mapping project names to IDs + self._project_id_cache: Dict[str, str] = ( + {} + ) # Cache mapping project names to IDs self.global_braintrust_http_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback ) @@ -108,43 +105,6 @@ class BraintrustLogger(CustomLogger): except httpx.HTTPStatusError as e: raise Exception(f"Failed to register project: {e.response.text}") - @staticmethod - def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict: - """ - Adds metadata from proxy request headers to Braintrust logging if keys start with "braintrust_" - and overwrites litellm_params.metadata if already included. - - For example if you want to append your trace to an existing `trace_id` via header, send - `headers: { ..., langfuse_existing_trace_id: your-existing-trace-id }` via proxy request. - """ - if litellm_params is None: - return metadata - - if litellm_params.get("proxy_server_request") is None: - return metadata - - if metadata is None: - metadata = {} - - proxy_headers = ( - litellm_params.get("proxy_server_request", {}).get("headers", {}) or {} - ) - - for metadata_param_key in proxy_headers: - if metadata_param_key.startswith("braintrust"): - trace_param_key = metadata_param_key.replace("braintrust", "", 1) - if trace_param_key in metadata: - verbose_logger.warning( - f"Overwriting Braintrust `{trace_param_key}` from request header" - ) - else: - verbose_logger.debug( - f"Found Braintrust `{trace_param_key}` in request header" - ) - metadata[trace_param_key] = proxy_headers.get(metadata_param_key) - - return metadata - async def create_default_project_and_experiment(self): project = await self.global_braintrust_http_handler.post( f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"} @@ -169,7 +129,9 @@ class BraintrustLogger(CustomLogger): verbose_logger.debug("REACHES BRAINTRUST SUCCESS") try: litellm_call_id = kwargs.get("litellm_call_id") + standard_logging_object = kwargs.get("standard_logging_object", {}) prompt = {"messages": kwargs.get("messages")} + output = None choices = [] if response_obj is not None and ( @@ -192,33 +154,13 @@ class BraintrustLogger(CustomLogger): ): output = response_obj["data"] - litellm_params = kwargs.get("litellm_params", {}) - metadata = ( - litellm_params.get("metadata", {}) or {} - ) # if litellm_params['metadata'] == None - metadata = self.add_metadata_from_header(litellm_params, metadata) - clean_metadata = {} - try: - metadata = copy.deepcopy( - metadata - ) # Avoid modifying the original metadata - except Exception: - new_metadata = {} - for key, value in metadata.items(): - if ( - isinstance(value, list) - or isinstance(value, dict) - or isinstance(value, str) - or isinstance(value, int) - or isinstance(value, float) - ): - new_metadata[key] = copy.deepcopy(value) - metadata = new_metadata + litellm_params = kwargs.get("litellm_params", {}) or {} + dynamic_metadata = litellm_params.get("metadata", {}) or {} # Get project_id from metadata or create default if needed - project_id = metadata.get("project_id") + project_id = dynamic_metadata.get("project_id") if project_id is None: - project_name = metadata.get("project_name") + project_name = dynamic_metadata.get("project_name") project_id = ( self.get_project_id_sync(project_name) if project_name else None ) @@ -229,8 +171,9 @@ class BraintrustLogger(CustomLogger): project_id = self.default_project_id tags = [] - if isinstance(metadata, dict): - for key, value in metadata.items(): + + if isinstance(dynamic_metadata, dict): + for key, value in dynamic_metadata.items(): # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy if ( litellm.langfuse_default_tags is not None @@ -239,25 +182,12 @@ class BraintrustLogger(CustomLogger): ): tags.append(f"{key}:{value}") - # clean litellm metadata before logging - if key in [ - "headers", - "endpoint", - "caching_groups", - "previous_models", - ]: - continue - else: - clean_metadata[key] = value + if ( + isinstance(value, str) and key not in standard_logging_object + ): # support logging dynamic metadata to braintrust + standard_logging_object[key] = value cost = kwargs.get("response_cost", None) - if cost is not None: - clean_metadata["litellm_response_cost"] = cost - - # metadata.model is required for braintrust to calculate the "Estimated cost" metric - litellm_model = kwargs.get("model", None) - if litellm_model is not None: - clean_metadata["model"] = litellm_model metrics: Optional[dict] = None usage_obj = getattr(response_obj, "usage", None) @@ -275,12 +205,12 @@ class BraintrustLogger(CustomLogger): } # Allow metadata override for span name - span_name = metadata.get("span_name", "Chat Completion") - + span_name = dynamic_metadata.get("span_name", "Chat Completion") + request_data = { "id": litellm_call_id, "input": prompt["messages"], - "metadata": clean_metadata, + "metadata": standard_logging_object, "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } @@ -312,6 +242,7 @@ class BraintrustLogger(CustomLogger): verbose_logger.debug("REACHES BRAINTRUST SUCCESS") try: litellm_call_id = kwargs.get("litellm_call_id") + standard_logging_object = kwargs.get("standard_logging_object", {}) prompt = {"messages": kwargs.get("messages")} output = None choices = [] @@ -336,32 +267,12 @@ class BraintrustLogger(CustomLogger): output = response_obj["data"] litellm_params = kwargs.get("litellm_params", {}) - metadata = ( - litellm_params.get("metadata", {}) or {} - ) # if litellm_params['metadata'] == None - metadata = self.add_metadata_from_header(litellm_params, metadata) - clean_metadata = {} - new_metadata = {} - for key, value in metadata.items(): - if ( - isinstance(value, list) - or isinstance(value, str) - or isinstance(value, int) - or isinstance(value, float) - ): - new_metadata[key] = value - elif isinstance(value, BaseModel): - new_metadata[key] = value.model_dump_json() - elif isinstance(value, dict): - for k, v in value.items(): - if isinstance(v, datetime): - value[k] = v.isoformat() - new_metadata[key] = value + dynamic_metadata = litellm_params.get("metadata", {}) or {} # Get project_id from metadata or create default if needed - project_id = metadata.get("project_id") + project_id = dynamic_metadata.get("project_id") if project_id is None: - project_name = metadata.get("project_name") + project_name = dynamic_metadata.get("project_name") project_id = ( await self.get_project_id_async(project_name) if project_name @@ -374,8 +285,9 @@ class BraintrustLogger(CustomLogger): project_id = self.default_project_id tags = [] - if isinstance(metadata, dict): - for key, value in metadata.items(): + + if isinstance(dynamic_metadata, dict): + for key, value in dynamic_metadata.items(): # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy if ( litellm.langfuse_default_tags is not None @@ -384,25 +296,12 @@ class BraintrustLogger(CustomLogger): ): tags.append(f"{key}:{value}") - # clean litellm metadata before logging - if key in [ - "headers", - "endpoint", - "caching_groups", - "previous_models", - ]: - continue - else: - clean_metadata[key] = value + if ( + isinstance(value, str) and key not in standard_logging_object + ): # support logging dynamic metadata to braintrust + standard_logging_object[key] = value cost = kwargs.get("response_cost", None) - if cost is not None: - clean_metadata["litellm_response_cost"] = cost - - # metadata.model is required for braintrust to calculate the "Estimated cost" metric - litellm_model = kwargs.get("model", None) - if litellm_model is not None: - clean_metadata["model"] = litellm_model metrics: Optional[dict] = None usage_obj = getattr(response_obj, "usage", None) @@ -430,13 +329,13 @@ class BraintrustLogger(CustomLogger): ) # Allow metadata override for span name - span_name = metadata.get("span_name", "Chat Completion") - + span_name = dynamic_metadata.get("span_name", "Chat Completion") + request_data = { "id": litellm_call_id, "input": prompt["messages"], "output": output, - "metadata": clean_metadata, + "metadata": standard_logging_object, "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 22ab3092901..e6f265ded58 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -15,6 +15,8 @@ from litellm.types.utils import ( StandardLoggingPayload, ) +# OpenTelemetry imports moved to individual functions to avoid import errors when not installed + if TYPE_CHECKING: from opentelemetry.sdk.trace.export import SpanExporter as _SpanExporter from opentelemetry.trace import Context as _Context @@ -41,6 +43,8 @@ else: Context = Any LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm") +LITELLM_METER_NAME = os.getenv("LITELLM_METER_NAME", "litellm") +LITELLM_LOGGER_NAME = os.getenv("LITELLM_LOGGER_NAME", "litellm") # Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" @@ -83,6 +87,8 @@ class OpenTelemetryConfig: exporter: Union[str, SpanExporter] = "console" endpoint: Optional[str] = None headers: Optional[str] = None + enable_metrics: bool = False + enable_events: bool = False @classmethod def from_env(cls): @@ -104,6 +110,14 @@ class OpenTelemetryConfig: headers = os.getenv( "OTEL_EXPORTER_OTLP_HEADERS", os.getenv("OTEL_HEADERS") ) # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***" + enable_metrics: bool = ( + os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", "false").lower() + == "true" + ) + enable_events: bool = ( + os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", "false").lower() + == "true" + ) if exporter == "in_memory": return cls(exporter=InMemorySpanExporter()) @@ -111,6 +125,8 @@ class OpenTelemetryConfig: exporter=exporter, endpoint=endpoint, headers=headers, # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***" + enable_metrics=enable_metrics, + enable_events=enable_events, ) @@ -119,27 +135,22 @@ class OpenTelemetry(CustomLogger): self, config: Optional[OpenTelemetryConfig] = None, callback_name: Optional[str] = None, + # injection points for testing + tracer_provider: Optional[Any] = None, + logger_provider: Optional[Any] = None, + meter_provider: Optional[Any] = None, **kwargs, ): - from opentelemetry import trace - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.trace import SpanKind if config is None: config = OpenTelemetryConfig.from_env() self.config = config + self.callback_name = callback_name self.OTEL_EXPORTER = self.config.exporter self.OTEL_ENDPOINT = self.config.endpoint self.OTEL_HEADERS = self.config.headers - provider = TracerProvider(resource=_get_litellm_resource()) - provider.add_span_processor(self._get_span_processor()) - self.callback_name = callback_name - - trace.set_tracer_provider(provider) - self.tracer = trace.get_tracer(LITELLM_TRACER_NAME) - - self.span_kind = SpanKind + self._init_tracing(tracer_provider) _debug_otel = str(os.getenv("DEBUG_OTEL", "False")).lower() @@ -156,6 +167,8 @@ class OpenTelemetry(CustomLogger): # init CustomLogger params super().__init__(**kwargs) + self._init_metrics(meter_provider) + self._init_logs(logger_provider) self._init_otel_logger_on_litellm_proxy() def _init_otel_logger_on_litellm_proxy(self): @@ -178,14 +191,109 @@ class OpenTelemetry(CustomLogger): litellm.service_callback.append("otel") setattr(proxy_server, "open_telemetry_logger", self) + def _init_tracing(self, tracer_provider): + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.trace import SpanKind + + # use provided tracer or create a new one + if tracer_provider is None: + tracer_provider = TracerProvider(resource=_get_litellm_resource()) + # Only add OTLP span processor if we created the tracer provider ourselves + tracer_provider.add_span_processor(self._get_span_processor()) + + # register global provider and grab our tracer + trace.set_tracer_provider(tracer_provider) + self.tracer = trace.get_tracer(LITELLM_TRACER_NAME) + self.span_kind = SpanKind + + def _init_metrics(self, meter_provider): + if not self.config.enable_metrics: + self._operation_duration_histogram = None + self._token_usage_histogram = None + self._cost_histogram = None + return + + from opentelemetry import metrics + from opentelemetry.sdk.metrics import Histogram, MeterProvider + + # Only create OTLP infrastructure if no custom meter provider is provided + if meter_provider is None: + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, + ) + from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + PeriodicExportingMetricReader, + ) + + _metric_exporter = OTLPMetricExporter( + endpoint=self.config.endpoint, + headers=OpenTelemetry._get_headers_dictionary(self.config.headers), + preferred_temporality={Histogram: AggregationTemporality.DELTA}, + ) + _metric_reader = PeriodicExportingMetricReader( + _metric_exporter, export_interval_millis=10000 + ) + + meter_provider = MeterProvider( + metric_readers=[_metric_reader], resource=_get_litellm_resource() + ) + meter = meter_provider.get_meter(__name__) + else: + # Use the provided meter provider as-is, without creating additional OTLP infrastructure + meter = meter_provider.get_meter(__name__) + + metrics.set_meter_provider(meter_provider) + + self._operation_duration_histogram = meter.create_histogram( + name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38 + description="GenAI operation duration", + unit="s", + ) + self._token_usage_histogram = meter.create_histogram( + name="gen_ai.client.token.usage", # Replace with semconv constant in otel 1.38 + description="GenAI token usage", + unit="{token}", + ) + self._cost_histogram = meter.create_histogram( + name="gen_ai.client.token.cost", + description="GenAI request cost", + unit="USD", + ) + + def _init_logs(self, logger_provider): + # nothing to do if events disabled + if not self.config.enable_events: + return + + from opentelemetry._logs import set_logger_provider + from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter + from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + + # set up log pipeline + if logger_provider is None: + logger_provider = OTLoggerProvider() + # Only add OTLP exporter if we created the logger provider ourselves + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter( + endpoint=self.config.endpoint, + headers=self._get_headers_dictionary(self.config.headers), + ) + ) + ) + set_logger_provider(logger_provider) + def log_success_event(self, kwargs, response_obj, start_time, end_time): - self._handle_sucess(kwargs, response_obj, start_time, end_time) + self._handle_success(kwargs, response_obj, start_time, end_time) def log_failure_event(self, kwargs, response_obj, start_time, end_time): self._handle_failure(kwargs, response_obj, start_time, end_time) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - self._handle_sucess(kwargs, response_obj, start_time, end_time) + self._handle_success(kwargs, response_obj, start_time, end_time) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): self._handle_failure(kwargs, response_obj, start_time, end_time) @@ -372,9 +480,9 @@ class OpenTelemetry(CustomLogger): def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]: """Extract dynamic headers from kwargs if available.""" - standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = ( - kwargs.get("standard_callback_dynamic_params") - ) + standard_callback_dynamic_params: Optional[ + StandardCallbackDynamicParams + ] = kwargs.get("standard_callback_dynamic_params") if not standard_callback_dynamic_params: return None @@ -414,50 +522,185 @@ class OpenTelemetry(CustomLogger): # End of Team/Key Based Logging Control Flow ######################################################### - def _handle_sucess(self, kwargs, response_obj, start_time, end_time): - from opentelemetry import trace - from opentelemetry.trace import Status, StatusCode + def _handle_success(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug( "OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s", kwargs, self.config, ) + ctx, parent_span = self._get_span_context(kwargs) + + # 1. Primary span + span = self._start_primary_span(kwargs, response_obj, start_time, end_time, ctx) + + # 2. Raw‐request sub-span (if enabled) + self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span) + + # 3. Guardrail span + self._create_guardrail_span(kwargs=kwargs, context=ctx) + + # 4. Metrics & cost recording + self._record_metrics(kwargs, response_obj, start_time, end_time) + + # 5. Semantic logs. + if self.config.enable_events: + self._emit_semantic_logs(kwargs, response_obj, span) + + # 6. End parent span + if parent_span is not None: + parent_span.end(end_time=self._to_ns(datetime.now())) + + def _start_primary_span(self, kwargs, response_obj, start_time, end_time, context): + from opentelemetry.trace import Status, StatusCode - _parent_context, parent_otel_span = self._get_span_context(kwargs) - # Span 1: Request sent to litellm SDK otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs) span = otel_tracer.start_span( name=self._get_span_name(kwargs), start_time=self._to_ns(start_time), - context=_parent_context, + context=context, ) span.set_status(Status(StatusCode.OK)) self.set_attributes(span, kwargs, response_obj) + span.end(end_time=self._to_ns(end_time)) + return span - if litellm.turn_off_message_logging is True: - pass - elif self.message_logging is not True: - pass - else: - # Span 2: Raw Request / Response to LLM - raw_request_span = otel_tracer.start_span( - name=RAW_REQUEST_SPAN_NAME, - start_time=self._to_ns(start_time), - context=trace.set_span_in_context(span), + def _maybe_log_raw_request( + self, kwargs, response_obj, start_time, end_time, parent_span + ): + from opentelemetry import trace + from opentelemetry.trace import Status, StatusCode + + # only log raw LLM request/response if message_logging is on and not globally turned off + if litellm.turn_off_message_logging or not self.message_logging: + return + + otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs) + raw_span = otel_tracer.start_span( + name=RAW_REQUEST_SPAN_NAME, + start_time=self._to_ns(start_time), + context=trace.set_span_in_context(parent_span), + ) + raw_span.set_status(Status(StatusCode.OK)) + self.set_raw_request_attributes(raw_span, kwargs, response_obj) + raw_span.end(end_time=self._to_ns(end_time)) + + def _record_metrics(self, kwargs, response_obj, start_time, end_time): + duration_s = (end_time - start_time).total_seconds() + params = kwargs.get("litellm_params") or {} + provider = params.get("custom_llm_provider", "Unknown") + + common_attrs = { + "gen_ai.operation.name": "chat", + "gen_ai.system": provider, + "gen_ai.request.model": kwargs.get("model"), + "gen_ai.framework": "litellm", + } + + std_log = kwargs.get("standard_logging_object") + md = getattr(std_log, "metadata", None) or (std_log or {}).get("metadata", {}) + for key in [ + "user_api_key_hash", + "user_api_key_alias", + "user_api_key_team_id", + "user_api_key_org_id", + "user_api_key_user_id", + "user_api_key_team_alias", + "user_api_key_user_email", + "spend_logs_metadata", + "requester_ip_address", + "requester_metadata", + "user_api_key_end_user_id", + "prompt_management_metadata", + "applied_guardrails", + "mcp_tool_call_metadata", + "vector_store_request_metadata", + ]: + if md.get(key) is not None: + common_attrs[f"metadata.{key}"] = str(md[key]) + + if self._operation_duration_histogram: + self._operation_duration_histogram.record( + duration_s, attributes=common_attrs + ) + if ( + response_obj + and (usage := response_obj.get("usage")) + and self._token_usage_histogram + ): + in_attrs = {**common_attrs, "gen_ai.token.type": "input"} + out_attrs = {**common_attrs, "gen_ai.token.type": "completion"} + self._token_usage_histogram.record( + usage.get("prompt_tokens", 0), attributes=in_attrs + ) + self._token_usage_histogram.record( + usage.get("completion_tokens", 0), attributes=out_attrs + ) + + cost = kwargs.get("response_cost") + if self._cost_histogram and cost: + self._cost_histogram.record(cost, attributes=common_attrs) + + def _emit_semantic_logs(self, kwargs, response_obj, span: Span): + if not self.config.enable_events: + return + + from opentelemetry._logs import get_logger, LogRecord + otel_logger = get_logger(LITELLM_LOGGER_NAME) + + parent_ctx = span.get_span_context() + provider = (kwargs.get("litellm_params") or {}).get( + "custom_llm_provider", "Unknown" + ) + + # per-message events + for msg in kwargs.get("messages", []): + role = msg.get("role", "user") + attrs = {"event_name": "gen_ai.content.prompt", "gen_ai.system": provider} + if role == "tool" and msg.get("id"): + attrs["id"] = msg["id"] + if self.message_logging and msg.get("content"): + attrs["gen_ai.prompt"] = msg["content"] + + otel_logger.emit( + LogRecord( + attributes=attrs, + body=msg.copy(), + trace_id=parent_ctx.trace_id, + span_id=parent_ctx.span_id, + trace_flags=parent_ctx.trace_flags, + ) ) - raw_request_span.set_status(Status(StatusCode.OK)) - self.set_raw_request_attributes(raw_request_span, kwargs, response_obj) - raw_request_span.end(end_time=self._to_ns(end_time)) + # per-choice events + for idx, choice in enumerate(response_obj.get("choices", [])): + attrs = { + "event_name": "gen_ai.content.completion", + "gen_ai.system": provider, + "index": idx, + "finish_reason": choice.get("finish_reason"), + } + body_msg = choice.get("message", {}) + if self.message_logging and body_msg.get("content"): + attrs["message.content"] = body_msg["content"] + body = { + "index": idx, + "finish_reason": choice.get("finish_reason"), + "message": {"role": body_msg.get("role", "assistant")}, + } + if self.message_logging and body_msg.get("content"): + body["message"]["content"] = body_msg["content"] - span.end(end_time=self._to_ns(end_time)) + otel_logger.emit( + LogRecord( + attributes=attrs, + body=body, + trace_id=parent_ctx.trace_id, + span_id=parent_ctx.span_id, + trace_flags=parent_ctx.trace_flags, + ) + ) - # Create span for guardrail information - self._create_guardrail_span(kwargs=kwargs, context=_parent_context) - - if parent_otel_span is not None: - parent_otel_span.end(end_time=self._to_ns(datetime.now())) def _create_guardrail_span( self, kwargs: Optional[dict], context: Optional[Context] diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index 7ad0038ecb2..c714e36b5f9 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -1,5 +1,6 @@ import json from typing import Any, Union + from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 915d2029afe..3be373ca5e5 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -772,7 +772,14 @@ def adapt_messages_to_generic_oci_standard( tool_calls = message.get("tool_calls") tool_call_id = message.get("tool_call_id") - if role in ["system", "user", "assistant"] and content is not None: + if role == "assistant" and tool_calls is not None: + if not isinstance(tool_calls, list): + raise Exception("Prop `tool_calls` must be a list of tool calls") + new_messages.append( + adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls) + ) + + elif role in ["system", "user", "assistant"] and content is not None: if not isinstance(content, (str, list)): raise Exception( "Prop `content` must be a string or a list of content items" @@ -781,13 +788,6 @@ def adapt_messages_to_generic_oci_standard( adapt_messages_to_generic_oci_standard_content_message(role, content) ) - elif role == "assistant" and tool_calls is not None: - if not isinstance(tool_calls, list): - raise Exception("Prop `tool_calls` must be a list of tool calls") - new_messages.append( - adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls) - ) - elif role == "tool": if not isinstance(tool_call_id, str): raise Exception("Prop `tool_call_id` is required and must be a string") diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index d4ce4052a7e..c70fb97af74 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -137,6 +137,7 @@ class OllamaChatConfig(BaseConfig): "tool_choice", "functions", "response_format", + "reasoning_effort", ] def map_openai_params( @@ -175,6 +176,8 @@ class OllamaChatConfig(BaseConfig): if value.get("json_schema") and value["json_schema"].get("schema"): optional_params["format"] = value["json_schema"]["schema"] ### FUNCTION CALLING LOGIC ### + if param == "reasoning_effort" and value is not None: + optional_params["think"] = True if param == "tools": ## CHECK IF MODEL SUPPORTS TOOL CALLING ## try: @@ -212,9 +215,9 @@ class OllamaChatConfig(BaseConfig): litellm.add_function_to_prompt = ( True # so that main.py adds the function call to the prompt ) - optional_params[ - "functions_unsupported_model" - ] = non_default_params.get("functions") + optional_params["functions_unsupported_model"] = ( + non_default_params.get("functions") + ) non_default_params.pop("tool_choice", None) # causes ollama requests to hang non_default_params.pop("functions", None) # causes ollama requests to hang return optional_params @@ -346,11 +349,31 @@ class OllamaChatConfig(BaseConfig): ## RESPONSE OBJECT model_response.choices[0].finish_reason = "stop" + response_json_message = response_json.get("message") + if response_json_message is not None: + if "thinking" in response_json_message: + # remap 'thinking' to 'reasoning_content' + response_json_message["reasoning_content"] = response_json_message[ + "thinking" + ] + del response_json_message["thinking"] + elif response_json_message.get("content") is not None: + # parse reasoning content from content + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + _parse_content_for_reasoning, + ) + + reasoning_content, content = _parse_content_for_reasoning( + response_json_message["content"] + ) + response_json_message["reasoning_content"] = reasoning_content + response_json_message["content"] = content + if ( request_data.get("format", "") == "json" and litellm_params.get("function_name") is not None ): - function_call = json.loads(response_json["message"]["content"]) + function_call = json.loads(response_json_message["content"]) message = litellm.Message( content=None, tool_calls=[ @@ -367,11 +390,13 @@ class OllamaChatConfig(BaseConfig): "type": "function", } ], + reasoning_content=response_json_message.get("reasoning_content"), ) model_response.choices[0].message = message # type: ignore model_response.choices[0].finish_reason = "tool_calls" else: - _message = litellm.Message(**response_json["message"]) + + _message = litellm.Message(**response_json_message) model_response.choices[0].message = _message # type: ignore model_response.created = int(time.time()) model_response.model = "ollama_chat/" + model @@ -412,6 +437,9 @@ class OllamaChatConfig(BaseConfig): class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): + started_reasoning_content: bool = False + finished_reasoning_content: bool = False + def _is_function_call_complete(self, function_args: Union[str, dict]) -> bool: if isinstance(function_args, dict): return True @@ -465,8 +493,38 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): if is_function_call_complete: tool_call["id"] = str(uuid.uuid4()) + # PROCESS REASONING CONTENT + reasoning_content: Optional[str] = None + content: Optional[str] = None + if chunk["message"].get("thinking") is not None: + if self.started_reasoning_content is False: + reasoning_content = chunk["message"].get("thinking") + self.started_reasoning_content = True + elif self.finished_reasoning_content is False: + reasoning_content = chunk["message"].get("thinking") + self.finished_reasoning_content = True + elif chunk["message"].get("content") is not None: + message_content = chunk["message"].get("content") + if "" in message_content: + message_content = message_content.replace("", "") + + self.started_reasoning_content = True + + if "" in message_content and self.started_reasoning_content: + message_content = message_content.replace("", "") + self.finished_reasoning_content = True + + if ( + self.started_reasoning_content + and not self.finished_reasoning_content + ): + reasoning_content = message_content + else: + content = message_content + delta = Delta( - content=chunk["message"].get("content", ""), + content=content, + reasoning_content=reasoning_content, tool_calls=tool_calls, ) diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 4f7be507cc2..4a491c88963 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -19,13 +19,13 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock from litellm.types.utils import ( + Delta, GenericStreamingChunk, ModelInfoBase, ModelResponse, ModelResponseStream, ProviderField, StreamingChoices, - Delta, ) from ..common_utils import OllamaError, _convert_image @@ -92,9 +92,9 @@ class OllamaConfig(BaseConfig): repeat_penalty: Optional[float] = None temperature: Optional[float] = None seed: Optional[int] = None - stop: Optional[ - list - ] = None # stop is a list based on this - https://github.com/ollama/ollama/pull/442 + stop: Optional[list] = ( + None # stop is a list based on this - https://github.com/ollama/ollama/pull/442 + ) tfs_z: Optional[float] = None num_predict: Optional[int] = None top_k: Optional[int] = None @@ -154,6 +154,7 @@ class OllamaConfig(BaseConfig): "stop", "response_format", "max_completion_tokens", + "reasoning_effort", ] def map_openai_params( @@ -166,19 +167,21 @@ class OllamaConfig(BaseConfig): for param, value in non_default_params.items(): if param == "max_tokens" or param == "max_completion_tokens": optional_params["num_predict"] = value - if param == "stream": + elif param == "stream": optional_params["stream"] = value - if param == "temperature": + elif param == "temperature": optional_params["temperature"] = value - if param == "seed": + elif param == "seed": optional_params["seed"] = value - if param == "top_p": + elif param == "top_p": optional_params["top_p"] = value - if param == "frequency_penalty": + elif param == "frequency_penalty": optional_params["frequency_penalty"] = value - if param == "stop": + elif param == "stop": optional_params["stop"] = value - if param == "response_format" and isinstance(value, dict): + elif param == "reasoning_effort" and value is not None: + optional_params["think"] = True + elif param == "response_format" and isinstance(value, dict): if value["type"] == "json_object": optional_params["format"] = "json" elif value["type"] == "json_schema": @@ -258,12 +261,17 @@ class OllamaConfig(BaseConfig): api_key: Optional[str] = None, json_mode: Optional[bool] = None, ) -> ModelResponse: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + _parse_content_for_reasoning, + ) + response_json = raw_response.json() ## RESPONSE OBJECT model_response.choices[0].finish_reason = "stop" if request_data.get("format", "") == "json": # Check if response field exists and is not empty before parsing JSON response_text = response_json.get("response", "") + if not response_text or not response_text.strip(): # Handle empty response gracefully - set empty content message = litellm.Message(content="") @@ -288,7 +296,9 @@ class OllamaConfig(BaseConfig): "id": f"call_{str(uuid.uuid4())}", "function": { "name": function_call["name"], - "arguments": json.dumps(function_call["arguments"]), + "arguments": json.dumps( + function_call["arguments"] + ), }, "type": "function", } @@ -305,11 +315,28 @@ class OllamaConfig(BaseConfig): model_response.choices[0].finish_reason = "stop" except json.JSONDecodeError: # If JSON parsing fails, treat as regular text response - message = litellm.Message(content=response_text) + ## output parse reasoning content from response_text + reasoning_content: Optional[str] = None + content: Optional[str] = None + if response_text is not None: + reasoning_content, content = _parse_content_for_reasoning( + response_text + ) + message = litellm.Message( + content=content, reasoning_content=reasoning_content + ) model_response.choices[0].message = message # type: ignore model_response.choices[0].finish_reason = "stop" else: - model_response.choices[0].message.content = response_json["response"] # type: ignore + response_text = response_json.get("response", "") + content = None + reasoning_content = None + if response_text is not None and isinstance(response_text, str): + reasoning_content, content = _parse_content_for_reasoning(response_text) + else: + content = response_text # type: ignore + model_response.choices[0].message.content = content # type: ignore + model_response.choices[0].message.reasoning_content = reasoning_content # type: ignore model_response.created = int(time.time()) model_response.model = "ollama/" + model _prompt = request_data.get("prompt", "") @@ -434,12 +461,21 @@ class OllamaConfig(BaseConfig): class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): + def __init__( + self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False + ): + super().__init__(streaming_response, sync_stream, json_mode) + self.started_reasoning_content: bool = False + self.finished_reasoning_content: bool = False + def _handle_string_chunk( self, str_line: str ) -> Union[GenericStreamingChunk, ModelResponseStream]: return self.chunk_parser(json.loads(str_line)) - def chunk_parser(self, chunk: dict) -> Union[GenericStreamingChunk, ModelResponseStream]: + def chunk_parser( + self, chunk: dict + ) -> Union[GenericStreamingChunk, ModelResponseStream]: try: if "error" in chunk: raise Exception(f"Ollama Error - {chunk}") @@ -469,12 +505,42 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): ) elif chunk["response"]: text = chunk["response"] - return GenericStreamingChunk( - text=text, - is_finished=is_finished, - finish_reason="stop", + reasoning_content: Optional[str] = None + content: Optional[str] = None + if text is not None: + if "" in text: + text = text.replace("", "") + self.started_reasoning_content = True + elif "" in text: + text = text.replace("", "") + self.finished_reasoning_content = True + + if ( + self.started_reasoning_content + and not self.finished_reasoning_content + ): + reasoning_content = text + else: + content = text + + return ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + reasoning_content=reasoning_content, content=content + ), + ) + ], + finish_reason=finish_reason, usage=None, ) + # return GenericStreamingChunk( + # text=text, + # is_finished=is_finished, + # finish_reason="stop", + # usage=None, + # ) elif "thinking" in chunk and not chunk["response"]: # Return reasoning content as ModelResponseStream so UIs can render it thinking_content = chunk.get("thinking") or "" diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index be0ca3a7086..204916e3a48 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -158,6 +158,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): "parallel_tool_calls", "audio", "web_search_options", + "safety_identifier", ] # works across all models model_specific_params = [] diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 8ab212e2558..267ca61ef5d 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -105,6 +105,64 @@ def _process_gemini_image(image_url: str, format: Optional[str] = None) -> PartT raise e +def _snake_to_camel(snake_str: str) -> str: + """Convert snake_case to camelCase""" + components = snake_str.split("_") + return components[0] + "".join(x.capitalize() for x in components[1:]) + + +def _camel_to_snake(camel_str: str) -> str: + """Convert camelCase to snake_case""" + import re + + return re.sub(r"(? Optional[str]: + """ + Get the equivalent key from available keys, checking both camelCase and snake_case variants + """ + if key in available_keys: + return key + + # Try camelCase version + camel_key = _snake_to_camel(key) + if camel_key in available_keys: + return camel_key + + # Try snake_case version + snake_key = _camel_to_snake(key) + if snake_key in available_keys: + return snake_key + + return None + + +def check_if_part_exists_in_parts( + parts: List[PartType], part: PartType, excluded_keys: List[str] = [] +) -> bool: + """ + Check if a part exists in a list of parts + Handles both camelCase and snake_case key variations (e.g., function_call vs functionCall) + """ + keys_to_compare = set(part.keys()) - set(excluded_keys) + for p in parts: + p_keys = set(p.keys()) + # Check if all keys in part have equivalent values in p + match_found = True + for key in keys_to_compare: + equivalent_key = _get_equivalent_key(key, p_keys) + if equivalent_key is None or p.get(equivalent_key, None) != part.get( + key, None + ): + match_found = False + break + + if match_found: + return True + return False + + def _gemini_convert_messages_with_history( # noqa: PLR0915 messages: List[AllMessageValues], ) -> List[ContentType]: @@ -236,10 +294,33 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 assistant_msg = ChatCompletionAssistantMessage(**msg_dict) # type: ignore _message_content = assistant_msg.get("content", None) reasoning_content = assistant_msg.get("reasoning_content", None) + thinking_blocks = assistant_msg.get("thinking_blocks") if reasoning_content is not None: assistant_content.append( PartType(thought=True, text=reasoning_content) ) + if thinking_blocks is not None: + for block in thinking_blocks: + block_thinking_str = block.get("thinking") + block_signature = block.get("signature") + if ( + block_thinking_str is not None + and block_signature is not None + ): + try: + assistant_content.append( + PartType( + thoughtSignature=block_signature, + **json.loads(block_thinking_str), + ) + ) + except Exception: + assistant_content.append( + PartType( + thoughtSignature=block_signature, + text=block_thinking_str, + ) + ) if _message_content is not None and isinstance(_message_content, list): _parts = [] for element in _message_content: @@ -262,9 +343,17 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 assistant_msg.get("tool_calls", []) is not None or assistant_msg.get("function_call") is not None ): # support assistant tool invoke conversion - assistant_content.extend( - convert_to_gemini_tool_call_invoke(assistant_msg) + gemini_tool_call_parts = convert_to_gemini_tool_call_invoke( + assistant_msg ) + ## check if gemini_tool_call already exists in assistant_content + for gemini_tool_call_part in gemini_tool_call_parts: + if not check_if_part_exists_in_parts( + assistant_content, + gemini_tool_call_part, + excluded_keys=["thoughtSignature"], + ): + assistant_content.append(gemini_tool_call_part) last_message_with_tool_calls = assistant_msg msg_i += 1 @@ -476,6 +565,7 @@ async def async_transform_request_body( optional_params=optional_params, ) + def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage: """ Returns a default user message when a "system" message is passed in gemini fails. @@ -484,6 +574,7 @@ def _default_user_message_when_system_message_passed() -> ChatCompletionUserMess """ return ChatCompletionUserMessage(content=".", role="user") + def _transform_system_message( supports_system_message: bool, messages: List[AllMessageValues] ) -> Tuple[Optional[SystemInstructions], List[AllMessageValues]]: diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 99a04c20fba..37470a6ee09 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -43,6 +43,7 @@ from litellm.types.llms.gemini import BidiGenerateContentServerMessage from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionResponseMessage, + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, @@ -792,7 +793,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): content_str += _content_str return content_str, reasoning_content_str - + + def _extract_thinking_blocks_from_parts( + self, parts: List[HttpxPartType] + ) -> List[ChatCompletionThinkingBlock]: + """Extract thinking blocks from parts if present""" + thinking_blocks: List[ChatCompletionThinkingBlock] = [] + for part in parts: + if "thoughtSignature" in part: + part_copy = part.copy() + part_copy.pop("thoughtSignature") + thinking_blocks.append( + ChatCompletionThinkingBlock( + type="thinking", + thinking=json.dumps(part_copy), + signature=part["thoughtSignature"], + ) + ) + return thinking_blocks + def _extract_image_response_from_parts( self, parts: List[HttpxPartType] ) -> Optional[ImageURLObject]: @@ -804,10 +823,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if mime_type.startswith("image/"): # Convert base64 data to data URI format data_uri = f"data:{mime_type};base64,{data}" - return ImageURLObject( - url=data_uri, - detail="auto" - ) + return ImageURLObject(url=data_uri, detail="auto") return None def _extract_audio_response_from_parts( @@ -1127,7 +1143,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif web_search_queries: web_search_requests = len(grounding_metadata) return web_search_requests - + @staticmethod def _create_streaming_choice( chat_completion_message: ChatCompletionResponseMessage, @@ -1151,9 +1167,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): index=candidate.get("index", idx), delta=Delta( content=chat_completion_message.get("content"), - reasoning_content=chat_completion_message.get( - "reasoning_content" - ), + reasoning_content=chat_completion_message.get("reasoning_content"), tool_calls=tools, image=image_response, function_call=functions, @@ -1164,13 +1178,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return choice @staticmethod - def _extract_candidate_metadata(candidate: Candidates) -> Tuple[List[dict], List[dict], List, List]: + def _extract_candidate_metadata( + candidate: Candidates, + ) -> Tuple[List[dict], List[dict], List, List]: """ Extract metadata from a single candidate response. - + Returns: grounding_metadata: List[dict] - url_context_metadata: List[dict] + url_context_metadata: List[dict] safety_ratings: List citation_metadata: List """ @@ -1178,7 +1194,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): url_context_metadata: List[dict] = [] safety_ratings: List = [] citation_metadata: List = [] - + if "groundingMetadata" in candidate: if isinstance(candidate["groundingMetadata"], list): grounding_metadata.extend(candidate["groundingMetadata"]) # type: ignore @@ -1194,8 +1210,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "urlContextMetadata" in candidate: # Add URL context metadata to grounding metadata url_context_metadata.append(cast(dict, candidate["urlContextMetadata"])) - - return grounding_metadata, url_context_metadata, safety_ratings, citation_metadata + + return ( + grounding_metadata, + url_context_metadata, + safety_ratings, + citation_metadata, + ) @staticmethod def _process_candidates( @@ -1227,6 +1248,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): tools: Optional[List[ChatCompletionToolCallChunk]] = [] functions: Optional[ChatCompletionToolCallFunctionChunk] = None cumulative_tool_call_index: int = 0 + thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None for idx, candidate in enumerate(_candidates): if "content" not in candidate: @@ -1239,7 +1261,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): candidate_safety_ratings, candidate_citation_metadata, ) = VertexGeminiConfig._extract_candidate_metadata(candidate) - + grounding_metadata.extend(candidate_grounding_metadata) url_context_metadata.extend(candidate_url_context_metadata) safety_ratings.extend(candidate_safety_ratings) @@ -1264,6 +1286,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) ) + thinking_blocks = ( + VertexGeminiConfig()._extract_thinking_blocks_from_parts( + parts=candidate["content"]["parts"] + ) + ) + if audio_response is not None: cast(Dict[str, Any], chat_completion_message)[ "audio" @@ -1271,7 +1299,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_message["content"] = None # OpenAI spec if image_response is not None: # Handle image response - combine with text content into structured format - cast(Dict[str, Any], chat_completion_message)["image"] = image_response + cast(Dict[str, Any], chat_completion_message)[ + "image" + ] = image_response if content is not None: chat_completion_message["content"] = content @@ -1298,15 +1328,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if functions is not None: chat_completion_message["function_call"] = functions + if thinking_blocks is not None: + chat_completion_message["thinking_blocks"] = thinking_blocks # type: ignore + if isinstance(model_response, ModelResponseStream): choice = VertexGeminiConfig._create_streaming_choice( chat_completion_message=chat_completion_message, - candidate=candidate, - idx=idx, - tools=tools, - functions=functions, + candidate=candidate, + idx=idx, + tools=tools, + functions=functions, chat_completion_logprobs=chat_completion_logprobs, - image_response=image_response + image_response=image_response, ) model_response.choices.append(choice) elif isinstance(model_response, ModelResponse): diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/transformation.py new file mode 100644 index 00000000000..86e36e802ed --- /dev/null +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/transformation.py @@ -0,0 +1,27 @@ +import litellm +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + +class VertexAIGPTOSSTransformation(OpenAIGPTConfig): + """ + Transformation for GPT-OSS model from VertexAI + + https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas?hl=id + """ + def __init__(self): + super().__init__() + + def get_supported_openai_params(self, model: str) -> list: + base_gpt_series_params = super().get_supported_openai_params(model=model) + gpt_oss_only_params = ["reasoning_effort"] + base_gpt_series_params.extend(gpt_oss_only_params) + + ######################################################### + # VertexAI - GPT-OSS does not support tool calls + ######################################################### + if litellm.supports_function_calling(model=model) is False: + TOOL_CALLING_PARAMS_TO_REMOVE = ["tool", "tool_choice", "function_call", "functions"] + base_gpt_series_params = [param for param in base_gpt_series_params if param not in TOOL_CALLING_PARAMS_TO_REMOVE] + + return base_gpt_series_params + diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index f281cab3b58..ee30c1749a8 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -49,6 +49,7 @@ class VertexAIPartnerModels(VertexBase): or model.startswith("jamba") or model.startswith("claude") or model.startswith("qwen") + or model.startswith("openai") ): return True return False @@ -59,6 +60,7 @@ class VertexAIPartnerModels(VertexBase): "llama", "deepseek-ai", "qwen", + "openai", ] if any(provider in model for provider in OPENAI_LIKE_VERTEX_PROVIDERS): return True diff --git a/litellm/main.py b/litellm/main.py index 786a0196e5e..9c2aa678369 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -357,6 +357,7 @@ async def acompletion( top_logprobs: Optional[int] = None, deployment_id=None, reasoning_effort: Optional[Literal["minimal", "low", "medium", "high"]] = None, + safety_identifier: Optional[str] = None, # set api_base, api_version, api_key base_url: Optional[str] = None, api_version: Optional[str] = None, @@ -493,6 +494,7 @@ async def acompletion( "api_key": api_key, "model_list": model_list, "reasoning_effort": reasoning_effort, + "safety_identifier": safety_identifier, "extra_headers": extra_headers, "acompletion": True, # assuming this is a required parameter "thinking": thinking, @@ -906,6 +908,7 @@ def completion( # type: ignore # noqa: PLR0915 web_search_options: Optional[OpenAIWebSearchOptions] = None, deployment_id=None, extra_headers: Optional[dict] = None, + safety_identifier: Optional[str] = None, # soon to be deprecated params by OpenAI functions: Optional[List] = None, function_call: Optional[str] = None, @@ -1243,6 +1246,7 @@ def completion( # type: ignore # noqa: PLR0915 "reasoning_effort": reasoning_effort, "thinking": thinking, "web_search_options": web_search_options, + "safety_identifier": safety_identifier, "allowed_openai_params": kwargs.get("allowed_openai_params"), } optional_params = get_optional_params( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 57309554621..7cedffd93f3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9884,6 +9884,28 @@ "supports_tool_choice": true, "supports_prompt_caching": true }, + "vertex_ai/openai/gpt-oss-20b-maas": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 0.075e-06, + "output_cost_per_token": 0.30e-06, + "litellm_provider": "vertex_ai-openai_models", + "mode": "chat", + "supports_reasoning": true, + "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas" + }, + "vertex_ai/openai/gpt-oss-120b-maas": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 0.15e-06, + "output_cost_per_token": 0.60e-06, + "litellm_provider": "vertex_ai-openai_models", + "mode": "chat", + "supports_reasoning": true, + "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas" + }, "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": { "max_tokens": 32768, "max_input_tokens": 262144, diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index 5c5f1cfe908..00000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index b3653c31435..b38272560d1 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,30 +1,27 @@ model_list: - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - - model_name: gpt-5-mini - litellm_params: - model: azure/gpt-5-mini - api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE") - api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY") - stream_timeout: 60 - merge_reasoning_content_in_choices: true - model_info: - mode: chat + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + - model_name: gpt-5-mini + litellm_params: + model: azure/gpt-5-mini + api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE") + api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY") + stream_timeout: 60 + merge_reasoning_content_in_choices: true + model_info: + mode: chat + - model_name: ollama-deepseek-r1 + litellm_params: + model: ollama/deepseek-r1:1.5b + model_info: + mode: chat router_settings: model_group_alias: {"my-fake-gpt-4": "fake-openai-endpoint"} - + litellm_settings: callbacks: ["otel"] - cache: true - cache_params: - type: redis - ttl: 600 - supported_call_types: ["acompletion", "completion"] - - model_group_settings: - forward_client_headers_to_llm_api: - - fake-openai-endpoint \ No newline at end of file + success_callback: ["braintrust"] diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 87c5bd7a7c5..7206d3dcb0e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2908,6 +2908,12 @@ class LitellmDataForBackendLLMCall(TypedDict, total=False): user: Optional[str] num_retries: Optional[int] +class LitellmMetadataFromRequestHeaders(TypedDict, total=False): + """ + Headers a user can pass that will get added to litellm metadata for the request + """ + spend_logs_metadata: Optional[dict] + class JWTKeyItem(TypedDict, total=False): kid: str diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index e718255750a..d52592952bc 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -317,17 +317,26 @@ def get_remaining_tokens_and_requests_from_request_data(data: Dict) -> Dict[str, _metadata = data.get("metadata", None) or {} model_group = get_model_group_from_request_data(data) + # The h11 package considers "/" or ":" invalid and raise a LocalProtocolError + h11_model_group_name = ( + model_group.replace("/", "-").replace(":", "-") if model_group else None + ) + # Remaining Requests remaining_requests_variable_name = f"litellm-key-remaining-requests-{model_group}" remaining_requests = _metadata.get(remaining_requests_variable_name, None) if remaining_requests: - headers[f"x-litellm-key-remaining-requests-{model_group}"] = remaining_requests + headers[f"x-litellm-key-remaining-requests-{h11_model_group_name}"] = ( + remaining_requests + ) # Remaining Tokens remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}" remaining_tokens = _metadata.get(remaining_tokens_variable_name, None) if remaining_tokens: - headers[f"x-litellm-key-remaining-tokens-{model_group}"] = remaining_tokens + headers[f"x-litellm-key-remaining-tokens-{h11_model_group_name}"] = ( + remaining_tokens + ) return headers diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 373232e22d2..eb481b0a4f0 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -173,15 +173,24 @@ async def google_count_tokens(request: Request, model_name: str): """ from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.proxy_server import token_counter as internal_token_counter + from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter data = await _read_request_body(request=request) contents = data.get("contents", []) #Create TokenCountRequest for the internal endpoint from litellm.proxy._types import TokenCountRequest + # Translate contents to openai format messages using the adapter + messages = ( + GoogleGenAIAdapter() + .translate_generate_content_to_completion(model_name, contents) + .get("messages", []) + ) + token_request = TokenCountRequest( model=model_name, - contents=contents + contents=contents, + messages=messages, # compatibility when use openai-like endpoint ) # Call the internal token counter function with direct request flag set to False @@ -192,11 +201,17 @@ async def google_count_tokens(request: Request, model_name: str): if token_response is not None: # cast the response to the well known format original_response: dict = token_response.original_response or {} - return TokenCountDetailsResponse( - totalTokens=original_response.get("totalTokens", 0), - promptTokensDetails=original_response.get("promptTokensDetails", []), - ) - + if original_response: + return TokenCountDetailsResponse( + totalTokens=original_response.get("totalTokens", 0), + promptTokensDetails=original_response.get("promptTokensDetails", []), + ) + else: + return TokenCountDetailsResponse( + totalTokens=token_response.total_tokens or 0, + promptTokensDetails=[], + ) + ######################################################### # Return the response in the well known format ######################################################### diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 32a283066ed..557aa3ceb9a 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -291,6 +291,17 @@ class LiteLLMProxyRequestSetup: if num_retries_header is not None: return int(num_retries_header) return None + + @staticmethod + def _get_spend_logs_metadata_from_request_headers(headers: dict) -> Optional[dict]: + """ + Get the `spend_logs_metadata` from the request headers. + """ + from litellm.litellm_core_utils.safe_json_loads import safe_json_loads + spend_logs_metadata_header = headers.get("x-litellm-spend-logs-metadata", None) + if spend_logs_metadata_header is not None: + return safe_json_loads(spend_logs_metadata_header) + return None @staticmethod def _get_forwardable_headers( @@ -459,6 +470,30 @@ class LiteLLMProxyRequestSetup: data["num_retries"] = num_retries return data + + @staticmethod + def add_litellm_metadata_from_request_headers( + headers: dict, + data: dict, + _metadata_variable_name: str, + ) -> dict: + """ + Add litellm metadata from request headers + + Relevant issue: https://github.com/BerriAI/litellm/issues/14008 + """ + from litellm.proxy._types import LitellmMetadataFromRequestHeaders + metadata_from_headers = LitellmMetadataFromRequestHeaders() + spend_logs_metadata = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers) + if spend_logs_metadata is not None: + metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata + + ######################################################################################### + # Finally update the requests metadata with the `metadata_from_headers` + ######################################################################################### + if isinstance(data[_metadata_variable_name], dict): + data[_metadata_variable_name].update(metadata_from_headers) + return data @staticmethod def get_sanitized_user_information_from_key( @@ -643,6 +678,10 @@ async def add_litellm_data_to_request( # noqa: PLR0915 from litellm.types.proxy.litellm_pre_call_utils import SecretFields safe_add_api_version_from_query_params(data, request) + _metadata_variable_name = _get_metadata_variable_name(request) + if data.get(_metadata_variable_name, None) is None: + data[_metadata_variable_name] = {} + _headers = clean_headers( request.headers, @@ -661,6 +700,14 @@ async def add_litellm_data_to_request( # noqa: PLR0915 ) ) + data.update( + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=_headers, + data=data, + _metadata_variable_name=_metadata_variable_name, + ) + ) + # check for forwardable headers data = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group( data=data, headers=_headers, user_api_key_dict=user_api_key_dict @@ -711,11 +758,6 @@ async def add_litellm_data_to_request( # noqa: PLR0915 verbose_proxy_logger.debug("receiving data: %s", data) - _metadata_variable_name = _get_metadata_variable_name(request) - - if data.get(_metadata_variable_name, None) is None: - data[_metadata_variable_name] = {} - # Parse metadata if it's a string (e.g., from multipart/form-data) if "metadata" in data and data["metadata"] is not None: if isinstance(data["metadata"], str): diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index acff522196e..72c69a28e95 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -3,4 +3,3 @@ model_list: litellm_params: model: openai/* api_base: https://exampleopenaiendpoint-production-0ee2.up.railway.app/ - mock_response: "hi" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a0c8e5b6295..5b58c232964 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -43,10 +43,14 @@ from openai.types.responses.response import ( # Handle OpenAI SDK version compatibility for Text type try: - from openai.types.responses.response_create_params import Text as ResponseText + from openai.types.responses.response_create_params import ( + Text as ResponseText, # type: ignore + ) except (ImportError, AttributeError): # Fall back to the concrete config type available in all SDK versions - from openai.types.responses.response_text_config_param import ResponseTextConfigParam as ResponseText + from openai.types.responses.response_text_config_param import ( + ResponseTextConfigParam as ResponseText, + ) from openai.types.responses.response_create_params import ( Reasoning, @@ -784,6 +788,7 @@ class ChatCompletionRequest(TypedDict, total=False): response_format: dict seed: int service_tier: str + safety_identifier: str stop: Union[str, List[str]] stream_options: dict temperature: float @@ -1025,29 +1030,29 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject): class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): id: str created_at: int - error: Optional[dict] - incomplete_details: Optional[IncompleteDetails] - instructions: Optional[str] - metadata: Optional[Dict] - model: Optional[str] - object: Optional[str] + error: Optional[dict] = None + incomplete_details: Optional[IncompleteDetails] = None + instructions: Optional[str] = None + metadata: Optional[Dict] = None + model: Optional[str] = None + object: Optional[str] = None output: Union[ List[Union[ResponseOutputItem, Dict]], List[Union[GenericResponseOutputItem, OutputFunctionToolCall]], ] parallel_tool_calls: bool - temperature: Optional[float] + temperature: Optional[float] = None tool_choice: ToolChoice tools: Union[List[Tool], List[ResponseFunctionToolCall], List[Dict[str, Any]]] top_p: Optional[float] - max_output_tokens: Optional[int] - previous_response_id: Optional[str] - reasoning: Optional[Reasoning] - status: Optional[str] - text: Optional[Union["ResponseText", Dict[str, Any]]] - truncation: Optional[Literal["auto", "disabled"]] - usage: Optional[ResponseAPIUsage] - user: Optional[str] + max_output_tokens: Optional[int] = None + previous_response_id: Optional[str] = None + reasoning: Optional[Reasoning] = None + status: Optional[str] = None + text: Optional[Union["ResponseText", Dict[str, Any]]] = None + truncation: Optional[Literal["auto", "disabled"]] = None + usage: Optional[ResponseAPIUsage] = None + user: Optional[str] = None store: Optional[bool] = None # Define private attributes using PrivateAttr _hidden_params: dict = PrivateAttr(default_factory=dict) diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 2931770cd6e..1b74ee25803 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -41,6 +41,7 @@ class PartType(TypedDict, total=False): function_call: FunctionCall function_response: FunctionResponse thought: bool + thoughtSignature: str class HttpxFunctionCall(TypedDict): @@ -72,6 +73,7 @@ class HttpxPartType(TypedDict, total=False): executableCode: HttpxExecutableCode codeExecutionResult: HttpxCodeExecutionResult thought: bool + thoughtSignature: str class HttpxContentType(TypedDict, total=False): @@ -245,10 +247,11 @@ class UsageMetadata(TypedDict, total=False): class TokenCountDetailsResponse(TypedDict): """ Response structure for token count details with modality breakdown. - + Example: {'totalTokens': 12, 'promptTokensDetails': [{'modality': 'TEXT', 'tokenCount': 12}]} """ + totalTokens: int promptTokensDetails: List[PromptTokensDetails] diff --git a/litellm/utils.py b/litellm/utils.py index 69f4603fea0..405d4cb98e4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -837,15 +837,13 @@ async def _client_async_logging_helper( # Async Logging Worker ################################################ from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( - async_coroutine = logging_obj.async_success_handler( - result=result, - start_time=start_time, - end_time=end_time + async_coroutine=logging_obj.async_success_handler( + result=result, start_time=start_time, end_time=end_time ) ) - ################################################ # Sync Logging Worker ################################################ @@ -3304,6 +3302,7 @@ def get_optional_params( # noqa: PLR0915 messages: Optional[List[AllMessageValues]] = None, thinking: Optional[AnthropicThinkingParam] = None, web_search_options: Optional[OpenAIWebSearchOptions] = None, + safety_identifier: Optional[str] = None, **kwargs, ): passed_params = locals().copy() @@ -3602,6 +3601,17 @@ def get_optional_params( # noqa: PLR0915 else False ), ) + elif provider_config is not None: + optional_params = provider_config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=( + drop_params + if drop_params is not None and isinstance(drop_params, bool) + else False + ), + ) else: # use generic openai-like param mapping optional_params = litellm.VertexAILlama3Config().map_openai_params( non_default_params=non_default_params, @@ -6865,6 +6875,11 @@ class ProviderConfigManager: return litellm.VertexGeminiConfig() elif "claude" in model: return litellm.VertexAIAnthropicConfig() + elif "gpt-oss" in model: + from litellm.llms.vertex_ai.vertex_ai_partner_models.gpt_oss.transformation import ( + VertexAIGPTOSSTransformation, + ) + return VertexAIGPTOSSTransformation() elif model in litellm.vertex_mistral_models: if "codestral" in model: return litellm.CodestralTextCompletionConfig() diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 57309554621..7cedffd93f3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9884,6 +9884,28 @@ "supports_tool_choice": true, "supports_prompt_caching": true }, + "vertex_ai/openai/gpt-oss-20b-maas": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 0.075e-06, + "output_cost_per_token": 0.30e-06, + "litellm_provider": "vertex_ai-openai_models", + "mode": "chat", + "supports_reasoning": true, + "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas" + }, + "vertex_ai/openai/gpt-oss-120b-maas": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 0.15e-06, + "output_cost_per_token": 0.60e-06, + "litellm_provider": "vertex_ai-openai_models", + "mode": "chat", + "supports_reasoning": true, + "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas" + }, "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": { "max_tokens": 32768, "max_input_tokens": 262144, diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 22a54b8a56b..b3f16ecd838 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -436,7 +436,10 @@ def test_gemini_with_empty_function_call_arguments(): async def test_claude_tool_use_with_gemini(): response = await litellm.anthropic.messages.acreate( messages=[ - {"role": "user", "content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?"} + { + "role": "user", + "content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?", + } ], model="gemini/gemini-2.5-flash", stream=True, @@ -578,11 +581,17 @@ def test_gemini_tool_use(): assert stop_reason is not None assert stop_reason == "tool_calls" + @pytest.mark.asyncio async def test_gemini_image_generation_async(): litellm._turn_on_debug() response = await litellm.acompletion( - messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}], + messages=[ + { + "role": "user", + "content": "Generate an image of a banana wearing a costume that says LiteLLM", + } + ], model="gemini/gemini-2.5-flash-image-preview", ) @@ -597,12 +606,16 @@ async def test_gemini_image_generation_async(): assert IMAGE_URL["url"].startswith("data:image/png;base64,") - @pytest.mark.asyncio async def test_gemini_image_generation_async_stream(): - #litellm._turn_on_debug() + # litellm._turn_on_debug() response = await litellm.acompletion( - messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}], + messages=[ + { + "role": "user", + "content": "Generate an image of a banana wearing a costume that says LiteLLM", + } + ], model="gemini/gemini-2.5-flash-image-preview", stream=True, ) @@ -611,35 +624,144 @@ async def test_gemini_image_generation_async_stream(): model_response_image = None async for chunk in response: print("CHUNK: ", chunk) - if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None: + if ( + hasattr(chunk.choices[0].delta, "image") + and chunk.choices[0].delta.image is not None + ): model_response_image = chunk.choices[0].delta.image print("MODEL_RESPONSE_IMAGE: ", model_response_image) assert model_response_image is not None assert model_response_image["url"].startswith("data:image/png;base64,") break - + ######################################################### # Important: Validate we did get an image in the response ######################################################### assert model_response_image is not None assert model_response_image["url"].startswith("data:image/png;base64,") - + def test_system_message_with_no_user_message(): - """ - Test that the system message is translated correctly for non-OpenAI providers. - """ - messages = [ - { - "role": "system", - "content": "Be a good bot!", - }, - ] + """ + Test that the system message is translated correctly for non-OpenAI providers. + """ + messages = [ + { + "role": "system", + "content": "Be a good bot!", + }, + ] - response = litellm.completion( - model="gemini/gemini-2.5-flash", - messages=messages, + response = litellm.completion( + model="gemini/gemini-2.5-flash", + messages=messages, + ) + assert response is not None + + assert response.choices[0].message.content is not None + + +def get_current_weather(location, unit="fahrenheit"): + """Get the current weather in a given location""" + if "tokyo" in location.lower(): + return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"}) + elif "san francisco" in location.lower(): + return json.dumps( + {"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"} ) - assert response is not None + elif "paris" in location.lower(): + return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"}) + else: + return json.dumps({"location": location, "temperature": "unknown"}) - assert response.choices[0].message.content is not None \ No newline at end of file + +def test_gemini_with_thinking(): + from litellm import completion + + litellm._turn_on_debug() + litellm.modify_params = True + model = "gemini/gemini-2.5-flash" + messages = [ + { + "role": "user", + "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", + } + ] + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + }, + } + ] + response = litellm.completion( + model=model, + messages=messages, + tools=tools, + tool_choice="auto", # auto is default, but we'll be explicit + reasoning_effort="low", + ) + print("Response\n", response) + response_message = response.choices[0].message + tool_calls = response_message.tool_calls + + print("Expecting there to be 3 tool calls") + assert len(tool_calls) > 0 # this has to call the function for SF, Tokyo and paris + + # Step 2: check if the model wanted to call a function + print(f"tool_calls: {tool_calls}") + if tool_calls: + # Step 3: call the function + # Note: the JSON response may not always be valid; be sure to handle errors + available_functions = { + "get_current_weather": get_current_weather, + } # only one function in this example, but you can have multiple + messages.append(response_message) # extend conversation with assistant's reply + print("Response message\n", response_message) + # Step 4: send the info for each function call and function response to the model + for tool_call in tool_calls: + function_name = tool_call.function.name + if function_name not in available_functions: + # the model called a function that does not exist in available_functions - don't try calling anything + return + function_to_call = available_functions[function_name] + function_args = json.loads(tool_call.function.arguments) + function_response = function_to_call( + location=function_args.get("location"), + unit=function_args.get("unit"), + ) + messages.append( + { + "tool_call_id": tool_call.id, + "role": "tool", + "name": function_name, + "content": function_response, + } + ) # extend conversation with function response + print(f"messages: {messages}") + second_response = litellm.completion( + model=model, + messages=messages, + seed=22, + reasoning_effort="low", + tools=tools, + drop_params=True, + ) # get a new response from the model where it can see the function response + print("second response\n", second_response) diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 0121eccaac3..619ae338e50 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -664,3 +664,62 @@ async def test_openai_gpt5_reasoning(): ) print("response: ", response) assert response.choices[0].message.content is not None + + +@pytest.mark.asyncio +async def test_openai_safety_identifier_parameter(): + """Test that safety_identifier parameter is correctly passed to the OpenAI API.""" + from openai import AsyncOpenAI + + litellm.set_verbose = True + client = AsyncOpenAI(api_key="fake-api-key") + + with patch.object( + client.chat.completions.with_raw_response, "create" + ) as mock_client: + try: + await litellm.acompletion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello, how are you?"}], + safety_identifier="user_code_123456", + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_client.assert_called_once() + request_body = mock_client.call_args.kwargs + + # Verify the request contains the safety_identifier parameter + assert "safety_identifier" in request_body + # Verify safety_identifier is correctly sent to the API + assert request_body["safety_identifier"] == "user_code_123456" + + +def test_openai_safety_identifier_parameter_sync(): + """Test that safety_identifier parameter is correctly passed to the OpenAI API.""" + from openai import OpenAI + + litellm.set_verbose = True + client = OpenAI(api_key="fake-api-key") + + with patch.object( + client.chat.completions.with_raw_response, "create" + ) as mock_client: + try: + litellm.completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello, how are you?"}], + safety_identifier="user_code_123456", + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_client.assert_called_once() + request_body = mock_client.call_args.kwargs + + # Verify the request contains the safety_identifier parameter + assert "safety_identifier" in request_body + # Verify safety_identifier is correctly sent to the API + assert request_body["safety_identifier"] == "user_code_123456" diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index b908eabd0cf..9b6fa868677 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -840,7 +840,8 @@ from test_completion import response_format_tests [ ("vertex_ai/mistral-large-2411", "us-central1"), ("vertex_ai/mistral-nemo@2407", "us-central1"), - ("vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", "us-south1") + ("vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", "us-south1"), + ("vertex_ai/openai/gpt-oss-20b-maas", "us-central1"), ], ) @pytest.mark.parametrize( @@ -911,6 +912,7 @@ async def test_partner_models_httpx(model, region, sync_mode): ("vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas", "us-east5"), ("vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", "us-south1"), ("vertex_ai/mistral-large-2411", "us-central1"), # critical - we had this issue: https://github.com/BerriAI/litellm/issues/13888 + ("vertex_ai/openai/gpt-oss-20b-maas", "us-central1"), ], ) @pytest.mark.parametrize( diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py index d389be79618..9cccdb51799 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py @@ -172,3 +172,27 @@ class TestSlackAlerting(unittest.TestCase): self.slack_alerting.update_values(alerting_args={"slack_alerting": "True"}) assert self.slack_alerting.periodic_started == True + + @patch("litellm.integrations.SlackAlerting.slack_alerting.datetime") + def test_alert_type_in_formatted_message(self, mock_datetime): + # Setup mocks + mock_datetime.now.return_value.strftime.return_value = "12:34:56" + + # Import required types + from litellm.types.integrations.slack_alerting import AlertType + + # Create a simple test message to check formatting + alert_type = AlertType.llm_exceptions + level = "Medium" + message = "Test alert message" + current_time = "12:34:56" + + # Test the specific formatting logic we're interested in + alert_type_formatted = f"Alert type: `{alert_type.name}`\n" + formatted_message = f"{alert_type_formatted}\n Level: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}" + + # Verify alert_type is in the formatted message as expected + self.assertIn("Alert type: `llm_exceptions`", formatted_message) + self.assertIn("Level: `Medium`", formatted_message) + self.assertIn("Timestamp: `12:34:56`", formatted_message) + self.assertIn("Message: Test alert message", formatted_message) diff --git a/tests/test_litellm/integrations/open_telemetry/data/captured_kwargs.json b/tests/test_litellm/integrations/open_telemetry/data/captured_kwargs.json new file mode 100644 index 00000000000..913e3bfedae --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/data/captured_kwargs.json @@ -0,0 +1 @@ +{"litellm_trace_id": null, "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "input": [{"role": "user", "content": "What is the capital of France?"}], "litellm_params": {"acompletion": true, "api_key": null, "force_timeout": 600, "logger_fn": null, "verbose": false, "custom_llm_provider": "bedrock", "api_base": "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-west-2%3A1234567890123%3Ainference-profile%2Fus.anthropic.claude-3-7-sonnet-20250219-v1%3A0/converse", "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "model_alias_map": {}, "completion_call_id": null, "aembedding": null, "metadata": {"requester_metadata": {}, "user_api_key_hash": "unused-for-aws-bedrock", "user_api_key_alias": null, "user_api_key_team_id": null, "user_api_key_user_id": null, "user_api_key_org_id": null, "user_api_key_team_alias": null, "user_api_key_end_user_id": null, "user_api_key_user_email": null, "user_api_key": "unused-for-aws-bedrock", "user_api_end_user_max_budget": null, "litellm_api_version": "1.72.3", "global_max_parallel_requests": null, "user_api_key_team_max_budget": null, "user_api_key_team_spend": null, "user_api_key_spend": 0.0, "user_api_key_max_budget": null, "user_api_key_model_max_budget": {}, "user_api_key_metadata": {}, "headers": {"host": "0.0.0.0:44444", "accept-encoding": "gzip, deflate, zstd", "connection": "keep-alive", "accept": "application/json", "content-type": "application/json", "user-agent": "AsyncOpenAI/Python 1.84.0", "x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600", "content-length": "116"}, "endpoint": "http://0.0.0.0:44444/chat/completions", "litellm_parent_otel_span": null, "requester_ip_address": "", "model_group": "claude-3-7-sonnet", "model_group_size": 1, "deployment": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "model_info": {"id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "db_model": false}, "api_base": null, "caching_groups": null, "hidden_params": {"custom_llm_provider": "bedrock", "region_name": null, "optional_params": {"stream": false, "max_retries": 0, "provider": "aws", "region": "us-west-2"}, "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "api_base": null, "model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "response_cost": 0.001047, "additional_headers": {"x-litellm-model-group": "claude-3-7-sonnet", "x-litellm-attempted-retries": 0, "x-litellm-attempted-fallbacks": 0}, "litellm_model_name": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "litellm_overhead_time_ms": 231.156, "_response_ms": 236.798}}, "model_info": {"id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "db_model": false}, "proxy_server_request": {"url": "http://0.0.0.0:44444/chat/completions", "method": "POST", "headers": {"host": "0.0.0.0:44444", "accept-encoding": "gzip, deflate, zstd", "connection": "keep-alive", "accept": "application/json", "content-type": "application/json", "user-agent": "AsyncOpenAI/Python 1.84.0", "x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600", "content-length": "116"}, "body": {"messages": [{"role": "user", "content": "What is the capital of France?"}], "model": "claude-3-7-sonnet", "stream": false}}, "preset_cache_key": null, "no-log": null, "stream_response": {}, "input_cost_per_token": null, "input_cost_per_second": null, "output_cost_per_token": null, "output_cost_per_second": null, "cooldown_time": null, "text_completion": null, "azure_ad_token_provider": null, "user_continue_message": null, "base_model": null, "litellm_trace_id": "4c97150b-b1a3-4dec-bd7a-734786b1b3bc", "litellm_session_id": null, "hf_model_name": null, "custom_prompt_dict": {}, "litellm_metadata": null, "disable_add_transform_inline_image_block": null, "drop_params": null, "prompt_id": null, "prompt_variables": null, "async_call": null, "ssl_verify": null, "merge_reasoning_content_in_choices": false, "api_version": null, "azure_ad_token": null, "tenant_id": null, "client_id": null, "client_secret": null, "azure_username": null, "azure_password": null, "max_retries": 0, "timeout": 6000.0, "bucket_name": null, "vertex_credentials": null, "vertex_project": null, "use_litellm_proxy": false}, "applied_guardrails": [], "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "messages": [{"role": "user", "content": "What is the capital of France?"}], "optional_params": {"stream": false, "max_retries": 0, "provider": "aws", "region": "us-west-2"}, "start_time": "2025-06-22 10:59:08.159939", "stream": false, "user": null, "call_type": "acompletion", "completion_start_time": "2025-06-22 10:59:08.399523", "standard_callback_dynamic_params": {}, "stream_options": null, "max_retries": 0, "provider": "aws", "region": "us-west-2", "custom_llm_provider": "bedrock", "api_key": "", "additional_args": {"complete_input_dict": "{\"messages\": [{\"role\": \"user\", \"content\": [{\"text\": \"What is the capital of France?\"}]}], \"additionalModelRequestFields\": {\"provider\": \"aws\", \"region\": \"us-west-2\"}, \"system\": [], \"inferenceConfig\": {}}"}, "log_event_type": "post_api_call", "api_call_start_time": "2025-06-22 10:59:08.387641", "llm_api_duration_ms": 5.642, "original_response": "{\"metrics\":{\"latencyMs\":1513},\"output\":{\"message\":{\"content\":[{\"text\":\"The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.\"}],\"role\":\"assistant\"}},\"stopReason\":\"end_turn\",\"usage\":{\"cacheReadInputTokenCount\":0,\"cacheReadInputTokens\":0,\"cacheWriteInputTokenCount\":0,\"cacheWriteInputTokens\":0,\"inputTokens\":14,\"outputTokens\":67,\"totalTokens\":81}}", "end_time": "2025-06-22 10:59:08.399523", "cache_hit": null, "response_cost": 0.001047, "standard_logging_object": {"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "trace_id": "4c97150b-b1a3-4dec-bd7a-734786b1b3bc", "call_type": "acompletion", "cache_hit": null, "stream": true, "status": "success", "custom_llm_provider": "bedrock", "saved_cache_cost": 0.0, "startTime": 1750615148.162725, "endTime": 1750615148.399523, "completionStartTime": 1750615148.399523, "response_time": 0.23679804801940918, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "metadata": {"user_api_key_hash": "unused-for-aws-bedrock", "user_api_key_alias": null, "user_api_key_team_id": null, "user_api_key_org_id": null, "user_api_key_user_id": null, "user_api_key_team_alias": null, "user_api_key_user_email": null, "spend_logs_metadata": null, "requester_ip_address": "", "requester_metadata": {}, "user_api_key_end_user_id": null, "prompt_management_metadata": null, "applied_guardrails": [], "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "usage_object": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "requester_custom_headers": {"x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600"}}, "cache_key": null, "response_cost": 0.001047, "total_tokens": 81, "prompt_tokens": 14, "completion_tokens": 67, "request_tags": [], "end_user": "", "api_base": "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-west-2%3A1234567890123%3Ainference-profile%2Fus.anthropic.claude-3-7-sonnet-20250219-v1%3A0/converse", "model_group": "claude-3-7-sonnet", "model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "requester_ip_address": "", "messages": [{"role": "user", "content": "What is the capital of France?"}], "response": {"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "created": 1750615148, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "object": "chat.completion", "system_fingerprint": null, "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.", "role": "assistant", "tool_calls": null, "function_call": null}}], "usage": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}, "model_parameters": {"stream": false}, "hidden_params": {"model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "cache_key": null, "api_base": null, "response_cost": 0.001047, "additional_headers": {}, "litellm_overhead_time_ms": 231.156, "batch_models": null, "litellm_model_name": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "usage_object": null}, "model_map_information": {"model_map_key": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "model_map_value": {"key": "anthropic.claude-3-7-sonnet-20250219-v1:0", "max_tokens": 8192, "max_input_tokens": 200000, "max_output_tokens": 8192, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_reasoning_token": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "bedrock_converse", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": null, "supports_audio_output": null, "supports_pdf_input": true, "supports_embedding_image_input": null, "supports_native_streaming": null, "supports_web_search": null, "supports_url_context": null, "supports_reasoning": true, "supports_computer_use": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["max_tokens", "max_completion_tokens", "stream", "stream_options", "stop", "temperature", "top_p", "extra_headers", "response_format", "tools", "tool_choice", "thinking", "reasoning_effort"]}}, "error_str": null, "error_information": {"error_code": "", "error_class": "", "llm_provider": "", "traceback": "", "error_message": ""}, "response_cost_failure_debug_info": null, "guardrail_information": null, "standard_built_in_tools_params": {"web_search_options": null, "file_search": null}}, "async_complete_streaming_response": "ModelResponse(id='chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1', created=1750615148, model='arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.', role='assistant', tool_calls=None, function_call=None, provider_specific_fields=None))], usage=Usage(completion_tokens=67, prompt_tokens=14, total_tokens=81, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=0))"} \ No newline at end of file diff --git a/tests/test_litellm/integrations/open_telemetry/data/captured_response.json b/tests/test_litellm/integrations/open_telemetry/data/captured_response.json new file mode 100644 index 00000000000..3cf77781cc2 --- /dev/null +++ b/tests/test_litellm/integrations/open_telemetry/data/captured_response.json @@ -0,0 +1 @@ +{"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "created": 1750615148, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "object": "chat.completion", "system_fingerprint": null, "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.", "role": "assistant", "tool_calls": null, "function_call": null}}], "usage": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}} \ No newline at end of file diff --git a/tests/test_litellm/integrations/test_braintrust_span_name.py b/tests/test_litellm/integrations/test_braintrust_span_name.py index 10e512fc0ca..30381e99783 100644 --- a/tests/test_litellm/integrations/test_braintrust_span_name.py +++ b/tests/test_litellm/integrations/test_braintrust_span_name.py @@ -11,7 +11,7 @@ from litellm.integrations.braintrust_logging import BraintrustLogger class TestBraintrustSpanName(unittest.TestCase): """Test custom span_name functionality in Braintrust logging.""" - @patch('litellm.integrations.braintrust_logging.HTTPHandler') + @patch("litellm.integrations.braintrust_logging.HTTPHandler") def test_default_span_name(self, MockHTTPHandler): """Test that default span name is 'Chat Completion' when not provided.""" # Mock HTTP response @@ -22,39 +22,43 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], "litellm_params": {"metadata": {}}, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, } - + # Execute logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Chat Completion') + json_data = call_args.kwargs["json"] + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Chat Completion" + ) - @patch('litellm.integrations.braintrust_logging.HTTPHandler') + @patch("litellm.integrations.braintrust_logging.HTTPHandler") def test_custom_span_name(self, MockHTTPHandler): """Test that custom span name is used when provided in metadata.""" # Mock HTTP response @@ -65,39 +69,43 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], "litellm_params": {"metadata": {"span_name": "Custom Operation"}}, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, } - + # Execute logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Custom Operation') + json_data = call_args.kwargs["json"] + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Custom Operation" + ) - @patch('litellm.integrations.braintrust_logging.HTTPHandler') + @patch("litellm.integrations.braintrust_logging.HTTPHandler") def test_span_name_with_other_metadata(self, MockHTTPHandler): """Test that span_name works alongside other metadata fields.""" # Mock HTTP response @@ -108,21 +116,23 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], @@ -132,34 +142,40 @@ class TestBraintrustSpanName(unittest.TestCase): "project_id": "custom-project", "user_id": "user123", "session_id": "session456", - "environment": "production" + "environment": "production", } }, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, + "standard_logging_object": { + "user_id": "user123", + }, } - + # Execute logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - - # Check span name - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Multi Metadata Test') - - # Check that other metadata is preserved (except for filtered keys) - event_metadata = json_data['events'][0]['metadata'] - self.assertEqual(event_metadata['user_id'], 'user123') - self.assertEqual(event_metadata['session_id'], 'session456') - self.assertEqual(event_metadata['environment'], 'production') - - # Span name should be in span_attributes, not in metadata - self.assertIn('span_name', event_metadata) # span_name is also kept in metadata + json_data = call_args.kwargs["json"] - @patch('litellm.integrations.braintrust_logging.get_async_httpx_client') + # Check span name + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Multi Metadata Test" + ) + + # Check that other metadata is preserved (except for filtered keys) + event_metadata = json_data["events"][0]["metadata"] + print(event_metadata) + self.assertEqual(event_metadata["user_id"], "user123") + self.assertEqual(event_metadata["session_id"], "session456") + self.assertEqual(event_metadata["environment"], "production") + + # Span name should be in span_attributes, not in metadata + self.assertIn("span_name", event_metadata) # span_name is also kept in metadata + + @patch("litellm.integrations.braintrust_logging.get_async_httpx_client") async def test_async_custom_span_name(self, mock_get_http_handler): """Test async logging with custom span name.""" # Mock async HTTP response @@ -170,38 +186,44 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], "litellm_params": {"metadata": {"span_name": "Async Custom Operation"}}, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, } - + # Execute - await logger.async_log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + await logger.async_log_success_event( + kwargs, response_obj, datetime.now(), datetime.now() + ) + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Async Custom Operation') + json_data = call_args.kwargs["json"] + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Async Custom Operation" + ) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index e11895e30ea..7fb91f274d0 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -1,15 +1,142 @@ +import json import os import sys import unittest from unittest.mock import MagicMock, patch +from datetime import datetime, timedelta +import time # Adds the grandparent directory to sys.path to allow importing project modules sys.path.insert(0, os.path.abspath("../..")) from litellm.integrations.opentelemetry import OpenTelemetry from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider +from opentelemetry.sdk._logs.export import SimpleLogRecordProcessor, InMemoryLogExporter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + +class TestOpenTelemetryGuardrails(unittest.TestCase): + @patch("litellm.integrations.opentelemetry.datetime") + def test_create_guardrail_span_with_valid_info(self, mock_datetime): + # Setup + otel = OpenTelemetry() + otel.tracer = MagicMock() + mock_span = MagicMock() + otel.tracer.start_span.return_value = mock_span + + # Create guardrail information + guardrail_info = { + "guardrail_name": "test_guardrail", + "guardrail_mode": "input", + "masked_entity_count": {"CREDIT_CARD": 2}, + "guardrail_response": "filtered_content", + "start_time": 1609459200.0, + "end_time": 1609459201.0, + } + + # Create a kwargs dict with standard_logging_object containing guardrail information + kwargs = {"standard_logging_object": {"guardrail_information": guardrail_info}} + + # Call the method + otel._create_guardrail_span(kwargs=kwargs, context=None) + + # Assertions + otel.tracer.start_span.assert_called_once() + + # print all calls to mock_span.set_attribute + print("Calls to mock_span.set_attribute:") + for call in mock_span.set_attribute.call_args_list: + print(call) + + # Check that the span has the correct attributes set + mock_span.set_attribute.assert_any_call("guardrail_name", "test_guardrail") + mock_span.set_attribute.assert_any_call("guardrail_mode", "input") + mock_span.set_attribute.assert_any_call( + "guardrail_response", "filtered_content" + ) + mock_span.set_attribute.assert_any_call( + "masked_entity_count", safe_dumps({"CREDIT_CARD": 2}) + ) + + # Verify that the span was ended + mock_span.end.assert_called_once() + + def test_create_guardrail_span_with_no_info(self): + # Setup + otel = OpenTelemetry() + otel.tracer = MagicMock() + + # Test with no guardrail information + kwargs = {"standard_logging_object": {}} + otel._create_guardrail_span(kwargs=kwargs, context=None) + + # Verify that start_span was never called + otel.tracer.start_span.assert_not_called() + class TestOpenTelemetry(unittest.TestCase): + POLL_INTERVAL = 0.05 + POLL_TIMEOUT = 2.0 + MODEL = "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0" + HERE = os.path.dirname(__file__) + + def wait_for_spans(self, exporter: InMemorySpanExporter, prefix: str): + """Poll until we see at least one span with an attribute key starting with `prefix`.""" + deadline = time.time() + self.POLL_TIMEOUT + while time.time() < deadline: + spans = exporter.get_finished_spans() + matches = [ + s + for s in spans + if s.attributes and any(str(k).startswith(prefix) for k in s.attributes) + ] + if matches: + return matches + time.sleep(self.POLL_INTERVAL) + return [] + + def wait_for_metric(self, reader: InMemoryMetricReader, name: str): + """Poll until we see a metric with the given name.""" + deadline = time.time() + self.POLL_TIMEOUT + while time.time() < deadline: + data = reader.get_metrics_data() + # guard against None or missing attribute + if not data or not hasattr(data, "resource_metrics"): + time.sleep(self.POLL_INTERVAL) + continue + + for rm in data.resource_metrics: + for sm in rm.scope_metrics: + for m in sm.metrics: + if m.name == name: + return m + + time.sleep(self.POLL_INTERVAL) + return None + + def wait_for_log(self, reader: InMemoryLogExporter, name: str): + """Poll until we see a log with the given name.""" + deadline = time.time() + self.POLL_TIMEOUT + while time.time() < deadline: + logs = reader.get_finished_logs() + if not logs: + time.sleep(self.POLL_INTERVAL) + continue + matches = [ + log + for log in logs + # if log.attributes and any(str(k).startswith(prefix) for k in log.attributes) + ] + if matches: + return matches + time.sleep(self.POLL_INTERVAL) + return [] + @patch("litellm.integrations.opentelemetry.datetime") def test_create_guardrail_span_with_valid_info(self, mock_datetime): # Setup @@ -79,7 +206,6 @@ class TestOpenTelemetry(unittest.TestCase): ) as mock_get_headers, patch.object( otel, "_get_tracer_with_dynamic_headers" ) as mock_get_tracer: - # Test case 1: With dynamic headers mock_get_headers.return_value = { "arize-space-id": "test-space", @@ -399,3 +525,229 @@ class TestOpenTelemetry(unittest.TestCase): self.assertEqual(attributes.get("service.name"), "litellm-service") # But other attributes from OTEL_RESOURCE_ATTRIBUTES should still be present self.assertEqual(attributes.get("extra.attr"), "extra-value") + + def test_handle_success_generates_spans_metrics_and_events(self): + # force both metrics & events on + os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS"] = "true" + os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true" + + # ─── build in‐memory OTEL providers/exporters ───────────────────────────── + span_exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + + log_exporter = InMemoryLogExporter() + logger_provider = OTLoggerProvider() + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + + metric_reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[metric_reader]) + + # ─── instantiate our OpenTelemetry logger with test providers ─────────── + otel = OpenTelemetry( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + ) + + # OpenTelemetry attempts to set a global tracer provider, which can be set only once. + # so we hack here to set a local tracer deriver from the provider we created. + otel.tracer = tracer_provider.get_tracer(__name__) + + # ─── minimal input / output for a chat call ────────────────────────────── + start = datetime.utcnow() + end = start + timedelta(seconds=1) + + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json") + ) as f: + kwargs = json.load(f) + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json") + ) as f: + response_obj = json.load(f) + + # ─── exercise the hook ─────────────────────────────────────────────────── + otel._handle_success(kwargs, response_obj, start, end) + + # ─── assert spans ──────────────────────────────────────────────────────── + spans = self.wait_for_spans(span_exporter, "gen_ai.") + self.assertTrue(spans, "Expected at least one gen_ai span") + + # verify our top‐level litellm_request span is present + names = [s.name for s in spans] + self.assertIn("litellm_request", names) + + # ─── assert metrics ────────────────────────────────────────────────────── + duration_metric = self.wait_for_metric( + metric_reader, "gen_ai.client.operation.duration" + ) + self.assertIsNotNone(duration_metric, "duration histogram was not recorded") + + # check that our model attribute made it onto at least one data point + found_dp = False + if ( + duration_metric + and hasattr(duration_metric, "data") + and hasattr(duration_metric.data, "data_points") + ): + found_dp = any( + dp.attributes.get("gen_ai.request.model") == self.MODEL + for dp in duration_metric.data.data_points + ) + self.assertTrue( + found_dp, "expected gen_ai.request.model attribute on a data point" + ) + + # ─── assert logs ─────────────────────────────────────────────────────── + logs = [] + logs = self.wait_for_log(log_exporter, "gen_ai.") + self.assertTrue(logs, "Expected at least one gen_ai log") + + user_logs = [log for log in logs if log.log_record.attributes.get("event_name") == "gen_ai.content.prompt"] + self.assertTrue(user_logs, "did not see a gen_ai.content.prompt log") + # check log bodies + user_prompt = user_logs[0].log_record.attributes.get("gen_ai.prompt") + self.assertEqual("What is the capital of France?", user_prompt, "did not see a prompt message") + + choice_logs = [log for log in logs if log.log_record.attributes.get("event_name") == "gen_ai.content.completion"] + self.assertTrue(choice_logs, "did not see a gen_ai.content.completion event") + + choice_response = choice_logs[0].log_record.body + self.assertIsNotNone(choice_response, "did not see a response message") + self.assertEqual("stop", choice_response.get("finish_reason"), "did not see expected finish reason") + + + def test_handle_success_spans_only(self): + # make sure neither events nor metrics is on + os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) + os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", None) + + # ─── build in‐memory OTEL providers/exporters ───────────────────────────── + span_exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + + # no logs / no metrics + log_exporter = InMemoryLogExporter() + logger_provider = OTLoggerProvider() + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + metric_reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[metric_reader]) + + # ─── instantiate our OpenTelemetry logger with test providers ─────────── + otel = OpenTelemetry( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, # pass even if events disabled (safe) + ) + # bind our tracer to the test tracer provider (global registration is a no-op after the first time) + otel.tracer = tracer_provider.get_tracer(__name__) + + # ─── minimal input / output for a chat call ────────────────────────────── + start = datetime.utcnow() + end = start + timedelta(seconds=1) + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json") + ) as f: + kwargs = json.load(f) + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json") + ) as f: + response_obj = json.load(f) + + # ─── exercise the hook ─────────────────────────────────────────────────── + otel._handle_success(kwargs, response_obj, start, end) + + # ─── assert spans only ─────────────────────────────────────────────────── + spans = span_exporter.get_finished_spans() + self.assertTrue(spans, "Expected at least one span") + # must have the top‐level litellm_request span + # self.assertIn( + # LITELLM_REQUEST_SPAN_NAME, + # [s.name for s in spans], + # "litellm_request span missing", + # ) + # model attribute should be on that span + found = any( + s.attributes + and s.attributes.get("gen_ai.request.model") == self.MODEL + for s in spans + ) + self.assertTrue(found, "expected gen_ai.request.model on span attributes") + + # no metrics recorded + self.assertIsNone( + self.wait_for_metric(metric_reader, "gen_ai.client.operation.duration"), + "Did not expect any metrics", + ) + # no logs emitted + logs = log_exporter.get_finished_logs() + self.assertFalse(logs, "Did not expect any logs") + + def test_handle_success_spans_and_metrics(self): + # only metrics on + os.environ.pop("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", None) + os.environ["LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"] = "true" + + # ─── build in‐memory OTEL providers/exporters ───────────────────────────── + span_exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + + log_exporter = InMemoryLogExporter() + logger_provider = OTLoggerProvider() + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + metric_reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[metric_reader]) + + # ─── instantiate our OpenTelemetry logger with test providers ─────────── + otel = OpenTelemetry( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, # needed if events were enabled + ) + otel.tracer = tracer_provider.get_tracer(__name__) + + # ─── minimal input / output for a chat call ────────────────────────────── + start = datetime.utcnow() + end = start + timedelta(seconds=1) + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json") + ) as f: + kwargs = json.load(f) + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json") + ) as f: + response_obj = json.load(f) + + # ─── exercise the hook ─────────────────────────────────────────────────── + otel._handle_success(kwargs, response_obj, start, end) + + # ─── assert spans ──────────────────────────────────────────────────────── + spans = span_exporter.get_finished_spans() + self.assertTrue(spans, "Expected at least one span") + + # ─── assert metrics ────────────────────────────────────────────────────── + duration_metric = self.wait_for_metric( + metric_reader, "gen_ai.client.operation.duration" + ) + self.assertIsNotNone(duration_metric, "duration histogram was not recorded") + # model attribute should be present on a data point + found_dp = False + if ( + duration_metric + and hasattr(duration_metric, "data") + and hasattr(duration_metric.data, "data_points") + ): + found_dp = any( + dp.attributes.get("gen_ai.request.model") == self.MODEL + for dp in duration_metric.data.data_points + ) + self.assertTrue( + found_dp, "expected gen_ai.request.model attribute on a data point" + ) + + # ─── no events when only metrics enabled ───────────────────────────────── + logs = log_exporter.get_finished_logs() + self.assertFalse(logs, "Did not expect any logs") diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation_for_14158.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation_for_14158.py new file mode 100644 index 00000000000..950c1fcb4c9 --- /dev/null +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation_for_14158.py @@ -0,0 +1,229 @@ +import pytest +from litellm.llms.oci.chat.transformation import adapt_messages_to_generic_oci_standard + +def test_adapt_messages_with_empty_content_and_tool_calls(): + """Test that assistant messages with empty content and tool_calls are processed correctly.""" + # Arrange + messages_with_empty_content = [ + {"role": "user", "content": "Tell me the weather in Tokyo."}, + { + "role": "assistant", + "content": "", # Empty string + "tool_calls": [ + { + "id": "call_test_empty", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Tokyo"}' + } + } + ] + }, + { + "role": "tool", + "content": '{"weather": "Sunny", "temperature": "25Β°C"}', + "tool_call_id": "call_test_empty" + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages_with_empty_content) + + # Assert + assert len(result) == 3 + + # Check user message + assert result[0].role == "USER" + assert result[0].content[0].type == "TEXT" + assert result[0].content[0].text == "Tell me the weather in Tokyo." + + # Check assistant message with tool_calls (should prioritize tool_calls over empty content) + assert result[1].role == "ASSISTANT" + assert result[1].toolCalls is not None + assert len(result[1].toolCalls) == 1 + assert result[1].toolCalls[0].id == "call_test_empty" + assert result[1].toolCalls[0].name == "get_weather" + + # Check tool response message + assert result[2].role == "TOOL" # Tool responses have TOOL role, not USER + assert result[2].content[0].type == "TEXT" + assert "weather" in result[2].content[0].text + assert result[2].toolCallId == "call_test_empty" # Tool call ID is in separate field + +def test_adapt_messages_with_none_content_and_tool_calls(): + """Test that assistant messages with None content and tool_calls are processed correctly.""" + # Arrange + messages_with_none_content = [ + {"role": "user", "content": "Tell me the weather in Tokyo."}, + { + "role": "assistant", + "content": None, # None value + "tool_calls": [ + { + "id": "call_test_none", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Tokyo"}' + } + } + ] + }, + { + "role": "tool", + "content": '{"weather": "Sunny", "temperature": "25Β°C"}', + "tool_call_id": "call_test_none" + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages_with_none_content) + + # Assert + assert len(result) == 3 + + # Check assistant message prioritizes tool_calls over None content + assert result[1].role == "ASSISTANT" + assert result[1].toolCalls is not None + assert len(result[1].toolCalls) == 1 + assert result[1].toolCalls[0].id == "call_test_none" + +def test_adapt_messages_with_tool_calls_only(): + """Test that assistant messages with only tool_calls (no content field) are processed correctly.""" + # Arrange + messages_no_content = [ + {"role": "user", "content": "Tell me the weather in Tokyo."}, + { + "role": "assistant", + # No content field at all + "tool_calls": [ + { + "id": "call_test_no_content", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Tokyo"}' + } + } + ] + }, + { + "role": "tool", + "content": '{"weather": "Sunny", "temperature": "25Β°C"}', + "tool_call_id": "call_test_no_content" + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages_no_content) + + # Assert + assert len(result) == 3 + + # Check assistant message processes tool_calls correctly + assert result[1].role == "ASSISTANT" + assert result[1].toolCalls is not None + assert len(result[1].toolCalls) == 1 + assert result[1].toolCalls[0].id == "call_test_no_content" + +def test_adapt_messages_with_content_only(): + """Test that assistant messages with only content (no tool_calls) are processed correctly.""" + # Arrange + messages_content_only = [ + {"role": "user", "content": "Hello"}, + { + "role": "assistant", + "content": "Hello! How can I help you today?" + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages_content_only) + + # Assert + assert len(result) == 2 + + # Check assistant message with content only + assert result[1].role == "ASSISTANT" + assert result[1].content[0].type == "TEXT" + assert result[1].content[0].text == "Hello! How can I help you today?" + assert result[1].toolCalls is None + +def test_adapt_messages_tool_id_tracking(): + """Test that tool call IDs are properly tracked for validation.""" + # Arrange + messages = [ + {"role": "user", "content": "Test"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": { + "name": "test_func", + "arguments": '{"param": "value"}' + } + } + ] + }, + { + "role": "tool", + "content": "Result", + "tool_call_id": "call_123" + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages) + + # Assert + # Tool call should be processed and ID should be available for validation + assert result[1].toolCalls[0].id == "call_123" + + # Tool response should reference the same ID + tool_response_text = result[2].content[0].text + # Tool response text is just the content, tool_call_id is separate + assert tool_response_text == "Result" # The actual content + assert result[2].toolCallId == "call_123" # Tool call ID is in separate field + +def test_adapt_messages_multiple_tool_calls(): + """Test that multiple tool calls in a single message are processed correctly.""" + # Arrange + messages = [ + {"role": "user", "content": "Test multiple tools"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "func1", + "arguments": '{"param": "value1"}' + } + }, + { + "id": "call_2", + "type": "function", + "function": { + "name": "func2", + "arguments": '{"param": "value2"}' + } + } + ] + } + ] + + # Act + result = adapt_messages_to_generic_oci_standard(messages) + + # Assert + assert len(result) == 2 + assert result[1].role == "ASSISTANT" + assert len(result[1].toolCalls) == 2 + assert result[1].toolCalls[0].id == "call_1" + assert result[1].toolCalls[1].id == "call_2" + diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py index 985d51f99da..452f5a94024 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py @@ -159,6 +159,261 @@ class TestOllamaConfig: assert result.choices[0]["finish_reason"] == "stop" # No usage assertions here as we don't need to test them in every case + def test_transform_response_with_thinking_tags(self): + """Test that responses with ... tags parse reasoning content correctly.""" + # Initialize config + config = OllamaConfig() + + # Create mock response with thinking tags + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "I need to think about this problem step by stepHere is my answer", + "prompt_eval_count": 15, + "eval_count": 8, + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify reasoning content is extracted + assert ( + result.choices[0]["message"].reasoning_content + == "I need to think about this problem step by step" + ) + assert result.choices[0]["message"].content == "Here is my answer" + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_with_thinking_tags_alternative(self): + """Test that responses with ... tags parse reasoning content correctly.""" + # Initialize config + config = OllamaConfig() + + # Create mock response with thinking tags (alternative format) + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "Let me analyze this carefullyThe solution is X", + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify reasoning content is extracted + assert ( + result.choices[0]["message"].reasoning_content + == "Let me analyze this carefully" + ) + assert result.choices[0]["message"].content == "The solution is X" + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_with_multiline_thinking_tags(self): + """Test that responses with multiline thinking content work correctly.""" + # Initialize config + config = OllamaConfig() + + # Create mock response with multiline thinking content + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "\nThis is a complex problem.\nI need to break it down:\n1. First step\n2. Second step\nBased on my analysis, the answer is Y", + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify multiline reasoning content is extracted + expected_reasoning = "\nThis is a complex problem.\nI need to break it down:\n1. First step\n2. Second step\n" + assert result.choices[0]["message"].reasoning_content == expected_reasoning + assert ( + result.choices[0]["message"].content + == "Based on my analysis, the answer is Y" + ) + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_thinking_only(self): + """Test response with only thinking content and no additional content.""" + # Initialize config + config = OllamaConfig() + + # Create mock response with only thinking content + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "Just internal thoughts, no response", + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify reasoning content is extracted and content is empty + assert ( + result.choices[0]["message"].reasoning_content + == "Just internal thoughts, no response" + ) + assert result.choices[0]["message"].content == "" + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_json_mode_with_thinking_tags(self): + """Test JSON mode with thinking tags - should handle as text when JSON parsing fails.""" + # Initialize config + config = OllamaConfig() + + # Create mock response with thinking tags in JSON mode + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "Planning my JSON responseThis is not valid JSON", + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={"format": "json"}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify reasoning content is extracted even in JSON mode when JSON parsing fails + assert ( + result.choices[0]["message"].reasoning_content + == "Planning my JSON response" + ) + assert result.choices[0]["message"].content == "This is not valid JSON" + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_no_thinking_tags(self): + """Test that responses without thinking tags work normally.""" + # Initialize config + config = OllamaConfig() + + # Create mock response without thinking tags + raw_response = MagicMock() + raw_response.json.return_value = { + "response": "Regular response without any thinking tags", + } + + # Create properly structured model response object + model_response = ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ) + + # Create mock encoding + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + # Transform response + result = config.transform_response( + model="llama2", + raw_response=raw_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + # Verify no reasoning content is extracted + assert result.choices[0]["message"].reasoning_content is None + assert ( + result.choices[0]["message"].content + == "Regular response without any thinking tags" + ) + assert result.choices[0]["finish_reason"] == "stop" + class TestOllamaTextCompletionResponseIterator: def test_chunk_parser_with_thinking_field(self): @@ -199,10 +454,11 @@ class TestOllamaTextCompletionResponseIterator: result = iterator.chunk_parser(normal_chunk) - assert result["text"] == "Hello world" - assert result["is_finished"] is False - assert result["finish_reason"] == "stop" - assert result["usage"] is None + # Updated to handle ModelResponseStream return type + assert isinstance(result, ModelResponseStream) + assert result.choices and result.choices[0].delta is not None + assert result.choices[0].delta.content == "Hello world" + assert getattr(result.choices[0].delta, "reasoning_content", None) is None def test_chunk_parser_done_chunk(self): """Test that done chunks work correctly.""" diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py new file mode 100644 index 00000000000..d6d33258576 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -0,0 +1,75 @@ +from litellm.llms.vertex_ai.gemini.transformation import check_if_part_exists_in_parts + + +def test_check_if_part_exists_in_parts(): + parts = [ + {"text": "Hello", "thought": True}, + {"text": "World", "thought": False}, + ] + part = {"text": "Hello", "thought": True} + new_part = {"text": "Hello World", "thought": True} + assert check_if_part_exists_in_parts(parts, part) + assert not check_if_part_exists_in_parts(parts, new_part, ["thought"]) + assert check_if_part_exists_in_parts(parts, new_part, ["text"]) + + +def test_check_if_part_exists_in_parts_camel_case_snake_case(): + """Test that function handles both camelCase and snake_case key variations""" + # Test snake_case to camelCase matching + parts_with_snake_case = [ + { + "function_call": { + "name": "get_current_weather", + "args": {"location": "San Francisco, CA"}, + } + }, + {"text": "Some other content"}, + ] + + part_with_camel_case = { + "functionCall": { + "name": "get_current_weather", + "args": {"location": "San Francisco, CA"}, + } + } + + # Should find match between function_call and functionCall + assert check_if_part_exists_in_parts(parts_with_snake_case, part_with_camel_case) + + # Test camelCase to snake_case matching + parts_with_camel_case = [ + {"functionCall": {"name": "calculate_sum", "args": {"a": 1, "b": 2}}} + ] + + part_with_snake_case = { + "function_call": {"name": "calculate_sum", "args": {"a": 1, "b": 2}} + } + + # Should find match between functionCall and function_call + assert check_if_part_exists_in_parts(parts_with_camel_case, part_with_snake_case) + + # Test no match when values differ + part_with_different_values = { + "function_call": {"name": "different_function", "args": {"x": 5}} + } + + assert not check_if_part_exists_in_parts( + parts_with_snake_case, part_with_different_values + ) + + # Test multiple keys with mixed casing + parts_mixed = [ + { + "function_call": {"name": "test"}, + "thoughtSignature": "reasoning", + "text": "content", + } + ] + + part_mixed_casing = { + "functionCall": {"name": "test"}, + "thought_signature": "reasoning", + "text": "content", + } + + assert check_if_part_exists_in_parts(parts_mixed, part_mixed_casing) diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py new file mode 100644 index 00000000000..34046a00ee8 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py @@ -0,0 +1,238 @@ +import json +import os +import sys +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../../..") +) # Adds the parent directory to the system path + +import litellm +from litellm.llms.vertex_ai.vertex_ai_partner_models.gpt_oss.transformation import ( + VertexAIGPTOSSTransformation, +) + + +class TestVertexAIGPTOSSTransformation: + """Test class for VertexAI GPT-OSS transformation functionality.""" + + def test_supports_reasoning_effort(self): + """Test that reasoning_effort parameter is supported for GPT-OSS models.""" + config = VertexAIGPTOSSTransformation() + supported_params = config.get_supported_openai_params(model="openai/gpt-oss-20b-maas") + + assert "reasoning_effort" in supported_params + + def test_removes_tool_calling_params_when_not_supported(self): + """Test that tool calling parameters are removed when function calling is not supported.""" + config = VertexAIGPTOSSTransformation() + + # Mock litellm.supports_function_calling to return False + with patch('litellm.supports_function_calling', return_value=False): + supported_params = config.get_supported_openai_params(model="openai/gpt-oss-20b-maas") + + # Tool calling params should be removed + assert "tool" not in supported_params + assert "tool_choice" not in supported_params + assert "function_call" not in supported_params + assert "functions" not in supported_params + + # But reasoning_effort should still be there + assert "reasoning_effort" in supported_params + + +@pytest.mark.asyncio +async def test_vertex_ai_gpt_oss_simple_request(): + """ + Test that a simple request to vertex_ai/openai/gpt-oss-20b-maas lands at the correct URL + with the correct request body. + """ + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexLLM, + ) + + # Mock response + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = {} + mock_response.json.return_value = { + "id": "chatcmpl-test123", + "object": "chat.completion", + "created": 1234567890, + "model": "openai/gpt-oss-20b-maas", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm Litellm Bot, a helpful assistant. I don't have access to real-time weather information, but I'd be happy to help you with other questions or tasks!" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 42, + "completion_tokens": 28, + "total_tokens": 70 + } + } + + client = AsyncHTTPHandler() + + async def mock_post_func(*args, **kwargs): + return mock_response + + with patch.object(client, "post", side_effect=mock_post_func) as mock_post, \ + patch.object(VertexLLM, "_ensure_access_token", return_value=("fake-token", "pathrise-convert-1606954137718")): + response = await litellm.acompletion( + model="vertex_ai/openai/gpt-oss-20b-maas", + messages=[ + { + "role": "system", + "content": "Your name is Litellm Bot, you are a helpful assistant" + }, + { + "role": "user", + "content": "Hello, what is your name and can you tell me the weather?" + } + ], + vertex_ai_location="us-central1", + vertex_ai_project="pathrise-convert-1606954137718", + client=client + ) + + # Verify the mock was called + mock_post.assert_called_once() + + # Get the call arguments + call_args = mock_post.call_args + # For side_effect, the URL is passed as kwargs['url'] + called_url = call_args.kwargs["url"] + request_body = json.loads(call_args.kwargs["data"]) + + # Verify the URL + expected_url = "https://us-central1-aiplatform.googleapis.com/v1/projects/pathrise-convert-1606954137718/locations/us-central1/endpoints/openapi/chat/completions" + assert called_url == expected_url + + # Verify the request body + expected_request_body = { + 'model': 'openai/gpt-oss-20b-maas', + 'messages': [ + { + 'role': 'system', + 'content': 'Your name is Litellm Bot, you are a helpful assistant' + }, + { + 'role': 'user', + 'content': 'Hello, what is your name and can you tell me the weather?' + } + ], + 'stream': False + } + assert request_body == expected_request_body + + # Verify response structure + assert response.model == "openai/gpt-oss-20b-maas" + assert len(response.choices) == 1 + assert response.choices[0].message.role == "assistant" + + +@pytest.mark.asyncio +async def test_vertex_ai_gpt_oss_reasoning_effort(): + """ + Test that reasoning_effort parameter is correctly passed in the request body + for GPT-OSS models. + """ + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexLLM, + ) + + # Mock response + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = {} + mock_response.json.return_value = { + "id": "chatcmpl-test456", + "object": "chat.completion", + "created": 1234567890, + "model": "openai/gpt-oss-20b-maas", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "I need to think about this carefully. The weather varies by location and time, so I would need to know your specific location to provide accurate weather information." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 35, + "completion_tokens": 32, + "total_tokens": 67 + } + } + + client = AsyncHTTPHandler() + + async def mock_post_func(*args, **kwargs): + return mock_response + + with patch.object(client, "post", side_effect=mock_post_func) as mock_post, \ + patch.object(VertexLLM, "_ensure_access_token", return_value=("fake-token", "pathrise-convert-1606954137718")): + response = await litellm.acompletion( + model="vertex_ai/openai/gpt-oss-20b-maas", + messages=[ + { + "role": "system", + "content": "Your name is Litellm Bot, you are a helpful assistant" + }, + { + "role": "user", + "content": "Hello, what is your name and can you tell me the weather?" + } + ], + reasoning_effort="low", + vertex_ai_location="us-central1", + vertex_ai_project="pathrise-convert-1606954137718", + client=client + ) + + # Verify the mock was called + mock_post.assert_called_once() + + # Get the call arguments + call_args = mock_post.call_args + request_body = json.loads(call_args.kwargs["data"]) + + # Verify reasoning_effort is in the request body + assert "reasoning_effort" in request_body + assert request_body["reasoning_effort"] == "low" + + # Verify other expected fields + expected_request_body = { + 'model': 'openai/gpt-oss-20b-maas', + 'messages': [ + { + 'role': 'system', + 'content': 'Your name is Litellm Bot, you are a helpful assistant' + }, + { + 'role': 'user', + 'content': 'Hello, what is your name and can you tell me the weather?' + } + ], + 'reasoning_effort': 'low', + 'stream': False + } + assert request_body == expected_request_body + + # Verify response structure + assert response.model == "openai/gpt-oss-20b-maas" + assert len(response.choices) == 1 + assert response.choices[0].message.role == "assistant" diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py new file mode 100644 index 00000000000..b9ed4b9b508 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -0,0 +1,29 @@ +import sys +import os + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +from litellm.proxy.common_utils.callback_utils import ( + get_remaining_tokens_and_requests_from_request_data, +) + + +def test_get_remaining_tokens_and_requests_from_request_data(): + model_group = "openrouter/google/gemini-2.0-flash-001" + casedata = { + "metadata": { + "model_group": model_group, + f"litellm-key-remaining-requests-{model_group}": 100, + f"litellm-key-remaining-tokens-{model_group}": 200, + } + } + + headers = get_remaining_tokens_and_requests_from_request_data(casedata) + + expected_name = "openrouter-google-gemini-2.0-flash-001" + assert headers == { + f"x-litellm-key-remaining-requests-{expected_name}": 100, + f"x-litellm-key-remaining-tokens-{expected_name}": 200, + } diff --git a/tests/test_litellm/proxy/google_endpoints/__init__.py b/tests/test_litellm/proxy/google_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/proxy/google_endpoints/test_endpoints.py b/tests/test_litellm/proxy/google_endpoints/test_endpoints.py new file mode 100644 index 00000000000..2f2538bf9aa --- /dev/null +++ b/tests/test_litellm/proxy/google_endpoints/test_endpoints.py @@ -0,0 +1,49 @@ +""" +Test for google_endpoints/endpoints.py +""" +import pytest +import sys, os +from dotenv import load_dotenv + + +from litellm.proxy.google_endpoints.endpoints import google_count_tokens +from litellm.types.llms.vertex_ai import TokenCountDetailsResponse +from starlette.requests import Request + +load_dotenv() + +sys.path.insert( + 0, os.path.abspath("../../../..") +) + +@pytest.mark.asyncio +async def test_proxy_gemini_to_openai_like_model_token_counting(): + """ + Test the token counting endpoint for proxing gemini to openai-like models. + """ + response: TokenCountDetailsResponse = await google_count_tokens( + request=Request( + scope={ + "type": "http", + "parsed_body": ( + [ + "contents" + ], + { + "contents": [ + { + "parts": [ + { + "text": "Hello, how are you?" + } + ] + } + ] + } + ) + } + ), + model_name="volcengine/foo", + ) + + assert response.get("totalTokens") > 0 \ No newline at end of file diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 5104ffd80de..817f19d8d7d 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -8,6 +8,7 @@ from unittest.mock import MagicMock, patch import pytest from fastapi import Request +import litellm from litellm.proxy._types import TeamCallbackMetadata, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import ( KeyAndTeamLoggingSettings, @@ -935,3 +936,126 @@ def test_add_headers_to_llm_call_by_model_group_existing_headers_in_data(): finally: # Restore original model_group_settings litellm.model_group_settings = original_model_group_settings + +import json +import time +from typing import Optional +from unittest.mock import AsyncMock + +from fastapi.responses import Response + +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.utils import ProxyLogging +from litellm.types.utils import StandardLoggingPayload + + +class TestCustomLogger(CustomLogger): + def __init__(self): + self.standard_logging_object: Optional[StandardLoggingPayload] = None + super().__init__() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + print(f"SUCCESS CALLBACK CALLED! kwargs keys: {list(kwargs.keys())}") + self.standard_logging_object = kwargs.get("standard_logging_object") + print(f"Captured standard_logging_object: {self.standard_logging_object}") + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + print(f"FAILURE CALLBACK CALLED! kwargs keys: {list(kwargs.keys())}") + +@pytest.mark.asyncio +async def test_add_litellm_metadata_from_request_headers(): + """ + Test that add_litellm_metadata_from_request_headers properly adds litellm metadata from request headers, + makes an LLM request using base_process_llm_request, sleeps for 3 seconds, and checks standard_logging_payload has spend_logs_metadata from headers + + Relevant issue: https://github.com/BerriAI/litellm/issues/14008 + """ + # Set up test logger + litellm._turn_on_debug() + test_logger = TestCustomLogger() + litellm.callbacks = [test_logger] + + # Prepare test data (ensure no streaming, add mock_response and api_key to route to litellm.acompletion) + headers = {"x-litellm-spend-logs-metadata": '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion", "timestamp": "2025-09-02T10:30:00Z"}'} + data = {"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}], "stream": False, "mock_response": "Hi", "api_key": "fake-key"} + + # Create mock request with headers + mock_request = MagicMock(spec=Request) + mock_request.headers = headers + mock_request.url.path = "/chat/completions" + + # Create mock response + mock_fastapi_response = MagicMock(spec=Response) + + # Create mock user API key dict + mock_user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + user_id="test-user", + org_id="test-org" + ) + + # Create mock proxy logging object + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + + # Create async functions for the hooks + async def mock_during_call_hook(*args, **kwargs): + return None + + async def mock_pre_call_hook(*args, **kwargs): + return data + + async def mock_post_call_success_hook(*args, **kwargs): + # Return the response unchanged + return kwargs.get('response', args[2] if len(args) > 2 else None) + + mock_proxy_logging_obj.during_call_hook = mock_during_call_hook + mock_proxy_logging_obj.pre_call_hook = mock_pre_call_hook + mock_proxy_logging_obj.post_call_success_hook = mock_post_call_success_hook + + # Create mock proxy config + mock_proxy_config = MagicMock() + + # Create mock general settings + general_settings = {} + + # Create mock select_data_generator with correct signature + def mock_select_data_generator(response=None, user_api_key_dict=None, request_data=None): + async def mock_generator(): + yield "data: " + json.dumps({"choices": [{"delta": {"content": "Hello"}}]}) + "\n\n" + yield "data: [DONE]\n\n" + return mock_generator() + + # Create the processor + processor = ProxyBaseLLMRequestProcessing(data=data) + + # Call base_process_llm_request (it will use the mock_response="Hi" parameter) + result = await processor.base_process_llm_request( + request=mock_request, + fastapi_response=mock_fastapi_response, + user_api_key_dict=mock_user_api_key_dict, + route_type="acompletion", + proxy_logging_obj=mock_proxy_logging_obj, + general_settings=general_settings, + proxy_config=mock_proxy_config, + select_data_generator=mock_select_data_generator, + llm_router=None, + model="gpt-4", + is_streaming_request=False + ) + + # Sleep for 3 seconds to allow logging to complete + await asyncio.sleep(3) + + # Check if standard_logging_object was set + assert test_logger.standard_logging_object is not None, "standard_logging_object should be populated after LLM request" + + # Verify the logging object contains expected metadata + standard_logging_obj = test_logger.standard_logging_object + + print(f"Standard logging object captured: {json.dumps(standard_logging_obj, indent=4, default=str)}") + + SPEND_LOGS_METADATA = standard_logging_obj["metadata"]["spend_logs_metadata"] + assert SPEND_LOGS_METADATA == dict(json.loads(headers["x-litellm-spend-logs-metadata"])), "spend_logs_metadata should be the same as the headers" + + diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index ed9a2729282..9e487939284 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -981,8 +981,8 @@ class TestProxyFunctionCalling: ( "groq/llama-3.3-70b-versatile", "litellm_proxy/groq/llama-3.3-70b-versatile", - False, - ), # This model doesn't support function calling + True, + ), # Cohere models (generally don't support function calling) ("command-nightly", "litellm_proxy/command-nightly", False), ], diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index c45116c0ddd..6c9ef66e95b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -80,6 +80,7 @@ export default function SpendLogsTable({ const [selectedKeyInfo, setSelectedKeyInfo] = useState(null) const [selectedKeyIdInfoView, setSelectedKeyIdInfoView] = useState(null) const [selectedStatus, setSelectedStatus] = useState("") + const [selectedEndUser, setSelectedEndUser] = useState("") const [filterByCurrentUser, setFilterByCurrentUser] = useState(userRole && internalUserRoles.includes(userRole)) const [activeTab, setActiveTab] = useState("request logs") @@ -193,6 +194,7 @@ export default function SpendLogsTable({ currentPage, pageSize, filterByCurrentUser ? userID : undefined, + selectedEndUser, selectedStatus, selectedModel, ) @@ -280,6 +282,7 @@ export default function SpendLogsTable({ } setSelectedStatus(filters["Status"] || "") setSelectedModel(filters["Model"] || "") + setSelectedEndUser(filters["End User"] || "") if (filters["Key Hash"]) { setSelectedKeyHash(filters["Key Hash"]) diff --git a/ui/litellm-dashboard/src/components/view_users.tsx b/ui/litellm-dashboard/src/components/view_users.tsx index 1d80e513e35..7c828f756b5 100644 --- a/ui/litellm-dashboard/src/components/view_users.tsx +++ b/ui/litellm-dashboard/src/components/view_users.tsx @@ -79,7 +79,6 @@ const ViewUserDashboard: React.FC = ({ accessToken, toke const [activeTab, setActiveTab] = useState("users") const [filters, setFilters] = useState(initialFilters) const [debouncedFilters, setDebouncedFilters, debouncer] = useDebouncedState(filters, { wait: 300 }) - const [showFilters, setShowFilters] = useState(false) const [isInvitationLinkModalVisible, setIsInvitationLinkModalVisible] = useState(false) const [invitationLinkData, setInvitationLinkData] = useState(null) const [baseUrl, setBaseUrl] = useState(null) @@ -330,209 +329,35 @@ const ViewUserDashboard: React.FC = ({ accessToken, toke -
-
-
- {/* Search and Filter Controls */} -
- {/* Email Search */} -
- updateFilters({ email: e.target.value })} - /> - - - -
- - {/* Filter Button */} - - - {/* Reset Filters Button */} - -
- - {/* Additional Filters */} - {showFilters && ( -
- {/* User ID Search */} -
- updateFilters({ user_id: e.target.value })} - /> - - - -
- - {/* Role Dropdown */} -
- -
- - {/* Team Dropdown */} -
- -
- - {/* SSO ID Search */} -
- updateFilters({ sso_user_id: e.target.value })} - /> -
-
- )} - - {/* Results Count and Pagination */} -
- - Showing{" "} - {userListResponse && userListResponse.users && userListResponse.users.length > 0 - ? (userListResponse.page - 1) * userListResponse.page_size + 1 - : 0}{" "} - -{" "} - {userListResponse && userListResponse.users - ? Math.min(userListResponse.page * userListResponse.page_size, userListResponse.total) - : 0}{" "} - of {userListResponse ? userListResponse.total : 0} results - - - {/* Pagination Buttons */} -
- - -
-
-
-
-
- { - setSelectedUser(user) - setEditModalVisible(true) - }} - handleDelete={handleDelete} - handleResetPassword={handleResetPassword} - enableSelection={selectionMode} - selectedUsers={selectedUsers} - onSelectionChange={handleSelectionChange} - /> -
- -
+ { + setSelectedUser(user) + setEditModalVisible(true) + }} + handleDelete={handleDelete} + handleResetPassword={handleResetPassword} + enableSelection={selectionMode} + selectedUsers={selectedUsers} + onSelectionChange={handleSelectionChange} + filters={filters} + updateFilters={updateFilters} + initialFilters={initialFilters} + teams={teams} + userListResponse={userListResponse} + currentPage={currentPage} + handlePageChange={handlePageChange} + /> diff --git a/ui/litellm-dashboard/src/components/view_users/table.tsx b/ui/litellm-dashboard/src/components/view_users/table.tsx index 0f1193c6e87..0b422ead7b3 100644 --- a/ui/litellm-dashboard/src/components/view_users/table.tsx +++ b/ui/litellm-dashboard/src/components/view_users/table.tsx @@ -15,12 +15,27 @@ import { TableBody, TableRow, TableCell, + Select, + SelectItem, } from "@tremor/react"; import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; import { UserInfo } from "./types"; import UserInfoView from "./user_info_view"; import { columns as createColumns } from "./columns"; +interface FilterState { + email: string; + user_id: string; + user_role: string; + sso_user_id: string; + team: string; + model: string; + min_spend: number | null; + max_spend: number | null; + sort_by: string; + sort_order: "asc" | "desc"; +} + interface UserDataTableProps { data: UserInfo[]; columns: ColumnDef[]; @@ -39,6 +54,15 @@ interface UserDataTableProps { selectedUsers?: UserInfo[]; onSelectionChange?: (selectedUsers: UserInfo[]) => void; enableSelection?: boolean; + // Filter-related props + filters: FilterState; + updateFilters: (update: Partial) => void; + initialFilters: FilterState; + teams: any[] | null; + // Pagination props + userListResponse: any; + currentPage: number; + handlePageChange: (newPage: number) => void; } export function UserDataTable({ @@ -56,6 +80,13 @@ export function UserDataTable({ selectedUsers = [], onSelectionChange, enableSelection = false, + filters, + updateFilters, + initialFilters, + teams, + userListResponse, + currentPage, + handlePageChange, }: UserDataTableProps) { const [sorting, setSorting] = React.useState([ { @@ -65,6 +96,7 @@ export function UserDataTable({ ]); const [selectedUserId, setSelectedUserId] = React.useState(null); const [openInEditMode, setOpenInEditMode] = React.useState(false); + const [showFilters, setShowFilters] = React.useState(false); const handleUserClick = (userId: string, openInEditMode: boolean = false) => { setSelectedUserId(userId); @@ -171,9 +203,190 @@ export function UserDataTable({ } return ( -
-
- +
+ {/* Filter Section */} +
+
+ {/* Search and Filter Controls */} +
+ {/* Email Search */} +
+ updateFilters({ email: e.target.value })} + /> + + + +
+ + {/* Filter Button */} + + + {/* Reset Filters Button */} + +
+ + {/* Additional Filters */} + {showFilters && ( +
+ {/* User ID Search */} +
+ updateFilters({ user_id: e.target.value })} + /> + + + +
+ + {/* Role Dropdown */} +
+ +
+ + {/* Team Dropdown */} +
+ +
+ + {/* SSO ID Search */} +
+ updateFilters({ sso_user_id: e.target.value })} + /> +
+
+ )} + + {/* Results Count and Pagination */} +
+ + Showing{" "} + {userListResponse && userListResponse.users && userListResponse.users.length > 0 + ? (userListResponse.page - 1) * userListResponse.page_size + 1 + : 0}{" "} + -{" "} + {userListResponse && userListResponse.users + ? Math.min(userListResponse.page * userListResponse.page_size, userListResponse.total) + : 0}{" "} + of {userListResponse ? userListResponse.total : 0} results + + + {/* Pagination Buttons */} +
+ + +
+
+
+
+ + {/* Table Section */} +
+
+
+
{table.getHeaderGroups().map((headerGroup) => ( @@ -260,6 +473,8 @@ export function UserDataTable({ )}
+
+
);