diff --git a/.circleci/config.yml b/.circleci/config.yml index faf43ff0b8b..b1c630cdcb3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1477,6 +1477,7 @@ jobs: docker run -d \ -p 4000:4000 \ -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DISABLE_SCHEMA_UPDATE="True" \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ @@ -2962,6 +2963,7 @@ jobs: command: | docker run --name my-app \ -p 4000:4000 \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ myapp:latest \ --port 4000 > docker_output.log 2>&1 || true diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index b6ac264a228..e361ee226b7 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.5 +version: 0.4.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/deploy/charts/litellm-helm/README.md b/deploy/charts/litellm-helm/README.md index 86b5918f01c..352c3e9ddff 100644 --- a/deploy/charts/litellm-helm/README.md +++ b/deploy/charts/litellm-helm/README.md @@ -41,6 +41,11 @@ If `db.useStackgresOperator` is used (not yet implemented): | `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` | | `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` | | `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. +| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` | +| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | +| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | +| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` | +| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` | #### Example `proxy_config` ConfigMap from values (default): diff --git a/deploy/charts/litellm-helm/templates/NOTES.txt b/deploy/charts/litellm-helm/templates/NOTES.txt index e72c9916080..017bbfa78bd 100644 --- a/deploy/charts/litellm-helm/templates/NOTES.txt +++ b/deploy/charts/litellm-helm/templates/NOTES.txt @@ -20,3 +20,4 @@ echo "Visit http://127.0.0.1:8080 to use your application" kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT {{- end }} +PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.* \ No newline at end of file diff --git a/deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml b/deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml new file mode 100644 index 00000000000..1715b94c1f6 --- /dev/null +++ b/deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml @@ -0,0 +1,33 @@ +{{- /* +PodDisruptionBudget for LiteLLM proxy +Controlled via .Values.pdb.enabled and .Values.pdb.{minAvailable|maxUnavailable} +Only one of minAvailable / maxUnavailable should be set. If both are set, minAvailable wins. +*/ -}} +{{- if .Values.pdb.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "litellm.fullname" . }} + labels: + {{- include "litellm.labels" . | nindent 4 }} + {{- with .Values.pdb.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.pdb.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- /* Match the Deployment selector to target the same pod set */ -}} + {{- include "litellm.selectorLabels" . | nindent 6 }} + {{- if .Values.pdb.minAvailable }} + minAvailable: {{ .Values.pdb.minAvailable }} + {{- else if .Values.pdb.maxUnavailable }} + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + {{- else }} + # Safe default if enabled but not configured + maxUnavailable: 1 + {{- end }} +{{- end }} diff --git a/deploy/charts/litellm-helm/tests/pdb_tests.yaml b/deploy/charts/litellm-helm/tests/pdb_tests.yaml new file mode 100644 index 00000000000..5e042e80bd3 --- /dev/null +++ b/deploy/charts/litellm-helm/tests/pdb_tests.yaml @@ -0,0 +1,45 @@ +suite: "pdb enabled" +templates: + - poddisruptionbudget.yaml +tests: + - it: "renders a PDB with maxUnavailable=1" + set: + pdb.enabled: true + pdb.maxUnavailable: 1 + asserts: + - hasDocuments: { count: 1 } + - isKind: { of: PodDisruptionBudget } + - equal: { path: apiVersion, value: policy/v1 } + - equal: { path: spec.maxUnavailable, value: 1 } + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME + +--- +suite: "pdb disabled" +templates: + - poddisruptionbudget.yaml +tests: + - it: "does not render when disabled" + set: + pdb.enabled: false + asserts: + - hasDocuments: { count: 0 } + +--- +suite: "pdb minAvailable precedence" +templates: + - poddisruptionbudget.yaml +tests: + - it: "uses minAvailable when both are set" + set: + pdb.enabled: true + pdb.minAvailable: "50%" + pdb.maxUnavailable: 1 + asserts: + - isKind: { of: PodDisruptionBudget } + - equal: { path: apiVersion, value: policy/v1 } + - equal: { path: spec.minAvailable, value: "50%" } + - isNull: { path: spec.maxUnavailable } diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index 58d1880cd4c..c1792497d29 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -240,4 +240,11 @@ extraEnvVars: { # value: EXTRA_ENV_VAR_VALUE } - +# Pod Disruption Budget +pdb: + enabled: false + # Set exactly one of the following. If both are set, minAvailable takes precedence. + minAvailable: null # e.g. "50%" or 1 + maxUnavailable: null # e.g. 1 or "20%" + annotations: {} + labels: {} diff --git a/docs/my-website/docs/exception_mapping.md b/docs/my-website/docs/exception_mapping.md index 13eda5b405a..2342f444e17 100644 --- a/docs/my-website/docs/exception_mapping.md +++ b/docs/my-website/docs/exception_mapping.md @@ -12,6 +12,7 @@ All exceptions can be imported from `litellm` - e.g. `from litellm import BadReq | 400 | UnsupportedParamsError | litellm.BadRequestError | Raised when unsupported params are passed | | 400 | ContextWindowExceededError| litellm.BadRequestError | Special error type for context window exceeded error messages - enables context window fallbacks | | 400 | ContentPolicyViolationError| litellm.BadRequestError | Special error type for content policy violation error messages - enables content policy fallbacks | +| 400 | ImageFetchError | litellm.BadRequestError | Raised when there are errors fetching or processing images | | 400 | InvalidRequestError | openai.BadRequestError | Deprecated error, use BadRequestError instead | | 401 | AuthenticationError | openai.AuthenticationError | | 403 | PermissionDeniedError | openai.PermissionDeniedError | diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index b03d7ab0328..541dc6fb3c8 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -431,6 +431,7 @@ router_settings: | DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20 | DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10 | DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602 +| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available** | DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7 | DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03 | DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0 diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 7f893068645..cdb6f7018fc 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -12,10 +12,7 @@ To start using Litellm, run the following commands in a shell: ```bash # Get the code -git clone https://github.com/BerriAI/litellm - -# Go to folder -cd litellm +curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml # Add the master key - you can change this after setup echo 'LITELLM_MASTER_KEY="sk-1234"' > .env diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index 09f7dfbaf78..b21105bb829 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -35,6 +35,30 @@ $ pip install 'litellm[proxy]' + + +Use this docker compose to spin up the proxy with a postgres database running locally. + +```bash +# Get the docker compose file +curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml + +# Add the master key - you can change this after setup +echo 'LITELLM_MASTER_KEY="sk-1234"' > .env + +# Add the litellm salt key - you cannot change this after adding a model +# It is used to encrypt / decrypt your LLM API Key credentials +# We recommend - https://1password.com/password-generator/ +# password generator to get a random hash for litellm salt key +echo 'LITELLM_SALT_KEY="sk-1234"' >> .env + +source .env + +# Start +docker-compose up +``` + + ## 1. Add a model @@ -43,6 +67,8 @@ Control LiteLLM Proxy with a config.yaml file. Setup your config.yaml with your azure model. +Note: When using the proxy with a database, you can also **just add models via UI** (UI is available on `/ui` route). + ```yaml model_list: - model_name: gpt-4o diff --git a/docs/my-website/docs/proxy/request_headers.md b/docs/my-website/docs/proxy/request_headers.md index 246d917d00c..c250d42f7bb 100644 --- a/docs/my-website/docs/proxy/request_headers.md +++ b/docs/my-website/docs/proxy/request_headers.md @@ -6,6 +6,8 @@ Special headers that are supported by LiteLLM. `x-litellm-timeout` Optional[float]: The timeout for the request in seconds. +`x-litellm-stream-timeout` Optional[float]: The timeout for getting the first chunk of the response in seconds (only applies for streaming requests). [Demo Video](https://www.loom.com/share/8da67e4845ce431a98c901d4e45db0e5) + `x-litellm-enable-message-redaction`: Optional[bool]: Don't log the message content to logging integrations. Just track spend. [Learn More](./logging#redact-messages-response-content) `x-litellm-tags`: Optional[str]: A comma separated list (e.g. `tag1,tag2,tag3`) of tags to use for [tag-based routing](./tag_routing) **OR** [spend-tracking](./enterprise.md#tracking-spend-for-custom-tags). diff --git a/enterprise/litellm_enterprise/integrations/prometheus.py b/enterprise/litellm_enterprise/integrations/prometheus.py index a2d781fa1c4..efee1a7783e 100644 --- a/enterprise/litellm_enterprise/integrations/prometheus.py +++ b/enterprise/litellm_enterprise/integrations/prometheus.py @@ -95,13 +95,14 @@ class PrometheusLogger(CustomLogger): self.litellm_llm_api_time_to_first_token_metric = self._histogram_factory( "litellm_llm_api_time_to_first_token_metric", "Time to first token for a models LLM API call", - labelnames=[ - "model", - "hashed_api_key", - "api_key_alias", - "team", - "team_alias", - ], + # labelnames=[ + # "model", + # "hashed_api_key", + # "api_key_alias", + # "team", + # "team_alias", + # ], + labelnames=self.get_labels_for_metric("litellm_llm_api_time_to_first_token_metric"), buckets=LATENCY_BUCKETS, ) @@ -109,15 +110,7 @@ class PrometheusLogger(CustomLogger): self.litellm_spend_metric = self._counter_factory( "litellm_spend_metric", "Total spend on LLM requests", - labelnames=[ - "end_user", - "hashed_api_key", - "api_key_alias", - "model", - "team", - "team_alias", - "user", - ], + labelnames=self.get_labels_for_metric("litellm_spend_metric"), ) # Counter for total_output_tokens @@ -243,25 +236,18 @@ class PrometheusLogger(CustomLogger): labelnames=["api_provider"], ) - # Get all keys - _logged_llm_labels = [ - UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, - UserAPIKeyLabelNames.MODEL_ID.value, - UserAPIKeyLabelNames.API_BASE.value, - UserAPIKeyLabelNames.API_PROVIDER.value, - ] - # Metric for deployment state self.litellm_deployment_state = self._gauge_factory( "litellm_deployment_state", "LLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage", - labelnames=_logged_llm_labels, + labelnames=self.get_labels_for_metric("litellm_deployment_state") ) self.litellm_deployment_cooled_down = self._counter_factory( "litellm_deployment_cooled_down", "LLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down", - labelnames=_logged_llm_labels + [EXCEPTION_STATUS], + # labelnames=_logged_llm_labels + [EXCEPTION_STATUS], + labelnames=self.get_labels_for_metric("litellm_deployment_cooled_down") ) self.litellm_deployment_success_responses = self._counter_factory( @@ -327,6 +313,7 @@ class PrometheusLogger(CustomLogger): documentation="deprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, user", labelnames=self.get_labels_for_metric("litellm_requests_metric"), ) + except Exception as e: print_verbose(f"Got exception on init prometheus client {str(e)}") raise e diff --git a/litellm/__init__.py b/litellm/__init__.py index fb280c34101..79865c83513 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1261,6 +1261,7 @@ from .exceptions import ( AuthenticationError, InvalidRequestError, BadRequestError, + ImageFetchError, NotFoundError, RateLimitError, ServiceUnavailableError, diff --git a/litellm/constants.py b/litellm/constants.py index 7ddd16c880c..0655473301f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -14,6 +14,7 @@ DEFAULT_S3_BATCH_SIZE = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int( os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10) ) +DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 4)) DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512)) SQS_SEND_MESSAGE_ACTION = "SendMessage" SQS_API_VERSION = "2012-11-05" diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6c6a09cd73e..01f3e2472f8 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -57,6 +57,7 @@ from litellm.llms.vertex_ai.cost_calculator import ( cost_per_token as google_cost_per_token, ) from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_router +from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.llms.openai import ( HttpxBinaryResponseContent, @@ -341,6 +342,8 @@ def cost_per_token( # noqa: PLR0915 return deepseek_cost_per_token(model=model, usage=usage_block) elif custom_llm_provider == "perplexity": return perplexity_cost_per_token(model=model, usage=usage_block) + elif custom_llm_provider == "xai": + return xai_cost_per_token(model=model, usage=usage_block) else: model_info = _cached_get_model_info_helper( model=model, custom_llm_provider=custom_llm_provider @@ -675,9 +678,9 @@ def completion_cost( # noqa: PLR0915 or isinstance(completion_response, dict) ): # tts returns a custom class if isinstance(completion_response, dict): - usage_obj: Optional[Union[dict, Usage]] = ( - completion_response.get("usage", {}) - ) + usage_obj: Optional[ + Union[dict, Usage] + ] = completion_response.get("usage", {}) else: usage_obj = getattr(completion_response, "usage", {}) if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects( @@ -1279,7 +1282,9 @@ class BaseTokenUsageProcessor: not hasattr(combined, "completion_tokens_details") or not combined.completion_tokens_details ): - combined.completion_tokens_details = CompletionTokensDetailsWrapper() + combined.completion_tokens_details = ( + CompletionTokensDetailsWrapper() + ) # Check what keys exist in the model's completion_tokens_details for attr in usage.completion_tokens_details.model_fields: diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 153230518cc..77fb9c1faef 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -153,6 +153,29 @@ class BadRequestError(openai.BadRequestError): # type: ignore _message += f", LiteLLM Max Retries: {self.max_retries}" return _message +class ImageFetchError(BadRequestError): + def __init__( + self, + message, + model=None, + llm_provider=None, + response: Optional[httpx.Response] = None, + litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, + body: Optional[dict] = None, + ): + super().__init__( + message=message, + model=model, + llm_provider=llm_provider, + response=response, + litellm_debug_info=litellm_debug_info, + max_retries=max_retries, + num_retries=num_retries, + body=body, + ) + class UnprocessableEntityError(openai.UnprocessableEntityError): # type: ignore def __init__( diff --git a/litellm/google_genai/adapters/handler.py b/litellm/google_genai/adapters/handler.py index 1f575f27591..dcf707ebd51 100644 --- a/litellm/google_genai/adapters/handler.py +++ b/litellm/google_genai/adapters/handler.py @@ -37,6 +37,10 @@ class GenerateContentToCompletionHandler: completion_kwargs: Dict[str, Any] = dict(completion_request) + # feed metadata for custom callback + if extra_kwargs is not None and "metadata" in extra_kwargs: + completion_kwargs["metadata"] = extra_kwargs["metadata"] + if stream: completion_kwargs["stream"] = stream diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 531da933fcc..5bc6afb6dbc 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -1,13 +1,11 @@ # What is this? ## Log success + failure events to Braintrust -import copy import os from datetime import datetime from typing import Dict, Optional import httpx -from pydantic import BaseModel import litellm from litellm import verbose_logger @@ -24,7 +22,6 @@ API_BASE = "https://api.braintrustdata.com/v1" def get_utc_datetime(): import datetime as dt - from datetime import datetime if hasattr(dt, "UTC"): return datetime.now(dt.UTC) # type: ignore @@ -45,9 +42,9 @@ class BraintrustLogger(CustomLogger): "Authorization": "Bearer " + self.api_key, "Content-Type": "application/json", } - self._project_id_cache: Dict[ - str, str - ] = {} # Cache mapping project names to IDs + self._project_id_cache: Dict[str, str] = ( + {} + ) # Cache mapping project names to IDs self.global_braintrust_http_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback ) @@ -108,43 +105,6 @@ class BraintrustLogger(CustomLogger): except httpx.HTTPStatusError as e: raise Exception(f"Failed to register project: {e.response.text}") - @staticmethod - def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict: - """ - Adds metadata from proxy request headers to Braintrust logging if keys start with "braintrust_" - and overwrites litellm_params.metadata if already included. - - For example if you want to append your trace to an existing `trace_id` via header, send - `headers: { ..., langfuse_existing_trace_id: your-existing-trace-id }` via proxy request. - """ - if litellm_params is None: - return metadata - - if litellm_params.get("proxy_server_request") is None: - return metadata - - if metadata is None: - metadata = {} - - proxy_headers = ( - litellm_params.get("proxy_server_request", {}).get("headers", {}) or {} - ) - - for metadata_param_key in proxy_headers: - if metadata_param_key.startswith("braintrust"): - trace_param_key = metadata_param_key.replace("braintrust", "", 1) - if trace_param_key in metadata: - verbose_logger.warning( - f"Overwriting Braintrust `{trace_param_key}` from request header" - ) - else: - verbose_logger.debug( - f"Found Braintrust `{trace_param_key}` in request header" - ) - metadata[trace_param_key] = proxy_headers.get(metadata_param_key) - - return metadata - async def create_default_project_and_experiment(self): project = await self.global_braintrust_http_handler.post( f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"} @@ -169,7 +129,9 @@ class BraintrustLogger(CustomLogger): verbose_logger.debug("REACHES BRAINTRUST SUCCESS") try: litellm_call_id = kwargs.get("litellm_call_id") + standard_logging_object = kwargs.get("standard_logging_object", {}) prompt = {"messages": kwargs.get("messages")} + output = None choices = [] if response_obj is not None and ( @@ -192,33 +154,13 @@ class BraintrustLogger(CustomLogger): ): output = response_obj["data"] - litellm_params = kwargs.get("litellm_params", {}) - metadata = ( - litellm_params.get("metadata", {}) or {} - ) # if litellm_params['metadata'] == None - metadata = self.add_metadata_from_header(litellm_params, metadata) - clean_metadata = {} - try: - metadata = copy.deepcopy( - metadata - ) # Avoid modifying the original metadata - except Exception: - new_metadata = {} - for key, value in metadata.items(): - if ( - isinstance(value, list) - or isinstance(value, dict) - or isinstance(value, str) - or isinstance(value, int) - or isinstance(value, float) - ): - new_metadata[key] = copy.deepcopy(value) - metadata = new_metadata + litellm_params = kwargs.get("litellm_params", {}) or {} + dynamic_metadata = litellm_params.get("metadata", {}) or {} # Get project_id from metadata or create default if needed - project_id = metadata.get("project_id") + project_id = dynamic_metadata.get("project_id") if project_id is None: - project_name = metadata.get("project_name") + project_name = dynamic_metadata.get("project_name") project_id = ( self.get_project_id_sync(project_name) if project_name else None ) @@ -229,8 +171,9 @@ class BraintrustLogger(CustomLogger): project_id = self.default_project_id tags = [] - if isinstance(metadata, dict): - for key, value in metadata.items(): + + if isinstance(dynamic_metadata, dict): + for key, value in dynamic_metadata.items(): # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy if ( litellm.langfuse_default_tags is not None @@ -239,25 +182,12 @@ class BraintrustLogger(CustomLogger): ): tags.append(f"{key}:{value}") - # clean litellm metadata before logging - if key in [ - "headers", - "endpoint", - "caching_groups", - "previous_models", - ]: - continue - else: - clean_metadata[key] = value + if ( + isinstance(value, str) and key not in standard_logging_object + ): # support logging dynamic metadata to braintrust + standard_logging_object[key] = value cost = kwargs.get("response_cost", None) - if cost is not None: - clean_metadata["litellm_response_cost"] = cost - - # metadata.model is required for braintrust to calculate the "Estimated cost" metric - litellm_model = kwargs.get("model", None) - if litellm_model is not None: - clean_metadata["model"] = litellm_model metrics: Optional[dict] = None usage_obj = getattr(response_obj, "usage", None) @@ -275,12 +205,12 @@ class BraintrustLogger(CustomLogger): } # Allow metadata override for span name - span_name = metadata.get("span_name", "Chat Completion") - + span_name = dynamic_metadata.get("span_name", "Chat Completion") + request_data = { "id": litellm_call_id, "input": prompt["messages"], - "metadata": clean_metadata, + "metadata": standard_logging_object, "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } @@ -312,6 +242,7 @@ class BraintrustLogger(CustomLogger): verbose_logger.debug("REACHES BRAINTRUST SUCCESS") try: litellm_call_id = kwargs.get("litellm_call_id") + standard_logging_object = kwargs.get("standard_logging_object", {}) prompt = {"messages": kwargs.get("messages")} output = None choices = [] @@ -336,32 +267,12 @@ class BraintrustLogger(CustomLogger): output = response_obj["data"] litellm_params = kwargs.get("litellm_params", {}) - metadata = ( - litellm_params.get("metadata", {}) or {} - ) # if litellm_params['metadata'] == None - metadata = self.add_metadata_from_header(litellm_params, metadata) - clean_metadata = {} - new_metadata = {} - for key, value in metadata.items(): - if ( - isinstance(value, list) - or isinstance(value, str) - or isinstance(value, int) - or isinstance(value, float) - ): - new_metadata[key] = value - elif isinstance(value, BaseModel): - new_metadata[key] = value.model_dump_json() - elif isinstance(value, dict): - for k, v in value.items(): - if isinstance(v, datetime): - value[k] = v.isoformat() - new_metadata[key] = value + dynamic_metadata = litellm_params.get("metadata", {}) or {} # Get project_id from metadata or create default if needed - project_id = metadata.get("project_id") + project_id = dynamic_metadata.get("project_id") if project_id is None: - project_name = metadata.get("project_name") + project_name = dynamic_metadata.get("project_name") project_id = ( await self.get_project_id_async(project_name) if project_name @@ -374,8 +285,9 @@ class BraintrustLogger(CustomLogger): project_id = self.default_project_id tags = [] - if isinstance(metadata, dict): - for key, value in metadata.items(): + + if isinstance(dynamic_metadata, dict): + for key, value in dynamic_metadata.items(): # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy if ( litellm.langfuse_default_tags is not None @@ -384,25 +296,12 @@ class BraintrustLogger(CustomLogger): ): tags.append(f"{key}:{value}") - # clean litellm metadata before logging - if key in [ - "headers", - "endpoint", - "caching_groups", - "previous_models", - ]: - continue - else: - clean_metadata[key] = value + if ( + isinstance(value, str) and key not in standard_logging_object + ): # support logging dynamic metadata to braintrust + standard_logging_object[key] = value cost = kwargs.get("response_cost", None) - if cost is not None: - clean_metadata["litellm_response_cost"] = cost - - # metadata.model is required for braintrust to calculate the "Estimated cost" metric - litellm_model = kwargs.get("model", None) - if litellm_model is not None: - clean_metadata["model"] = litellm_model metrics: Optional[dict] = None usage_obj = getattr(response_obj, "usage", None) @@ -430,13 +329,13 @@ class BraintrustLogger(CustomLogger): ) # Allow metadata override for span name - span_name = metadata.get("span_name", "Chat Completion") - + span_name = dynamic_metadata.get("span_name", "Chat Completion") + request_data = { "id": litellm_call_id, "input": prompt["messages"], "output": output, - "metadata": clean_metadata, + "metadata": standard_logging_object, "tags": tags, "span_attributes": {"name": span_name, "type": "llm"}, } diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index a9ff14d6c82..4fa10e42111 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -17,7 +17,7 @@ in_memory_cache = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY) def _process_image_response(response: Response, url: str) -> str: if response.status_code != 200: - raise Exception( + raise litellm.ImageFetchError( f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}" ) @@ -57,9 +57,11 @@ async def async_convert_url_to_base64(url: str) -> str: try: response = await client.get(url, follow_redirects=True) return _process_image_response(response, url) + except litellm.ImageFetchError: + raise except Exception: pass - raise Exception( + raise litellm.ImageFetchError( f"Error: Unable to fetch image from URL after 3 attempts. url={url}" ) @@ -74,10 +76,11 @@ def convert_url_to_base64(url: str) -> str: try: response = client.get(url, follow_redirects=True) return _process_image_response(response, url) + except litellm.ImageFetchError: + raise except Exception as e: verbose_logger.exception(e) - # print(e) pass - raise Exception( - f"Error: Unable to fetch image from URL after 3 attempts. url={url}" + raise litellm.ImageFetchError( + f"Error: Unable to fetch image from URL after 3 attempts. url={url}", ) diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index 7ad0038ecb2..c714e36b5f9 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -1,5 +1,6 @@ import json from typing import Any, Union + from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 9a8bb74d447..3902304a3b4 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -28,7 +28,18 @@ class OpenAIGPT5Config(OpenAIGPTConfig): base_gpt_series_params.extend(gpt_5_only_params) if not supports_tool_choice(model=model): base_gpt_series_params.remove("tool_choice") - return base_gpt_series_params + + non_supported_params = [ + "logprobs", + "top_p", + "presence_penalty", + "frequency_penalty", + "top_logprobs", + ] + + return [ + param for param in base_gpt_series_params if param not in non_supported_params + ] def map_openai_params( self, diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 6def8faffe0..8588c3efa27 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -187,6 +187,25 @@ def _check_text_in_content(parts: List[PartType]) -> bool: return has_text_param +def _fix_enum_empty_strings(schema, depth=0): + """Fix empty strings in enum values by replacing them with None. Gemini doesn't accept empty strings in enums.""" + if depth > DEFAULT_MAX_RECURSE_DEPTH: + raise ValueError(f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema.") + + if "enum" in schema and isinstance(schema["enum"], list): + schema["enum"] = [None if value == "" else value for value in schema["enum"]] + + # Reuse existing recursion pattern from convert_anyof_null_to_nullable + properties = schema.get("properties", None) + if properties is not None: + for _, value in properties.items(): + _fix_enum_empty_strings(value, depth=depth + 1) + + items = schema.get("items", None) + if items is not None: + _fix_enum_empty_strings(items, depth=depth + 1) + + def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False): """ This is a modified version of https://github.com/google-gemini/generative-ai-python/blob/8f77cc6ac99937cd3a81299ecf79608b91b06bbb/google/generativeai/types/content_types.py#L419 @@ -215,6 +234,11 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False): # * https://github.com/pydantic/pydantic/discussions/4872 convert_anyof_null_to_nullable(parameters) + _convert_schema_types(parameters) + + # Handle empty strings in enum values - Gemini doesn't accept empty strings in enums + _fix_enum_empty_strings(parameters) + # Handle empty items objects process_items(parameters) add_object_type(parameters) @@ -439,6 +463,47 @@ def _convert_vertex_datetime_to_openai_datetime(vertex_datetime: str) -> int: return int(dt.timestamp()) +def _convert_schema_types(schema, depth=0): + """ + Convert type arrays and lowercase types for Vertex AI compatibility. + + Transforms OpenAI-style schemas to Vertex AI format by converting type arrays + like ["string", "number"] to anyOf format and converting all types to uppercase. + """ + if depth > DEFAULT_MAX_RECURSE_DEPTH: + raise ValueError( + f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema. Please check the schema for excessive nesting." + ) + + if not isinstance(schema, dict): + return + + + # Handle type field + if "type" in schema: + type_val = schema["type"] + if isinstance(type_val, list) and len(type_val) > 1: + # Convert ["string", "number"] -> {"anyOf": [{"type": "STRING"}, {"type": "NUMBER"}]} + schema["anyOf"] = [{"type": t} for t in type_val if isinstance(t, str)] + schema.pop("type") + elif isinstance(type_val, list) and len(type_val) == 1: + schema["type"] = type_val[0] + elif isinstance(type_val, str): + schema["type"] = type_val + + # Recursively process nested properties, items, and anyOf + for key in ["properties", "items", "anyOf"]: + if key in schema: + value = schema[key] + if key == "properties" and isinstance(value, dict): + for prop_schema in value.values(): + _convert_schema_types(prop_schema, depth + 1) + elif key == "items": + _convert_schema_types(value, depth + 1) + elif key == "anyOf" and isinstance(value, list): + for anyof_schema in value: + _convert_schema_types(anyof_schema, depth + 1) + def get_vertex_project_id_from_url(url: str) -> Optional[str]: """ Get the vertex project id from the url diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 8ab212e2558..267ca61ef5d 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -105,6 +105,64 @@ def _process_gemini_image(image_url: str, format: Optional[str] = None) -> PartT raise e +def _snake_to_camel(snake_str: str) -> str: + """Convert snake_case to camelCase""" + components = snake_str.split("_") + return components[0] + "".join(x.capitalize() for x in components[1:]) + + +def _camel_to_snake(camel_str: str) -> str: + """Convert camelCase to snake_case""" + import re + + return re.sub(r"(? Optional[str]: + """ + Get the equivalent key from available keys, checking both camelCase and snake_case variants + """ + if key in available_keys: + return key + + # Try camelCase version + camel_key = _snake_to_camel(key) + if camel_key in available_keys: + return camel_key + + # Try snake_case version + snake_key = _camel_to_snake(key) + if snake_key in available_keys: + return snake_key + + return None + + +def check_if_part_exists_in_parts( + parts: List[PartType], part: PartType, excluded_keys: List[str] = [] +) -> bool: + """ + Check if a part exists in a list of parts + Handles both camelCase and snake_case key variations (e.g., function_call vs functionCall) + """ + keys_to_compare = set(part.keys()) - set(excluded_keys) + for p in parts: + p_keys = set(p.keys()) + # Check if all keys in part have equivalent values in p + match_found = True + for key in keys_to_compare: + equivalent_key = _get_equivalent_key(key, p_keys) + if equivalent_key is None or p.get(equivalent_key, None) != part.get( + key, None + ): + match_found = False + break + + if match_found: + return True + return False + + def _gemini_convert_messages_with_history( # noqa: PLR0915 messages: List[AllMessageValues], ) -> List[ContentType]: @@ -236,10 +294,33 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 assistant_msg = ChatCompletionAssistantMessage(**msg_dict) # type: ignore _message_content = assistant_msg.get("content", None) reasoning_content = assistant_msg.get("reasoning_content", None) + thinking_blocks = assistant_msg.get("thinking_blocks") if reasoning_content is not None: assistant_content.append( PartType(thought=True, text=reasoning_content) ) + if thinking_blocks is not None: + for block in thinking_blocks: + block_thinking_str = block.get("thinking") + block_signature = block.get("signature") + if ( + block_thinking_str is not None + and block_signature is not None + ): + try: + assistant_content.append( + PartType( + thoughtSignature=block_signature, + **json.loads(block_thinking_str), + ) + ) + except Exception: + assistant_content.append( + PartType( + thoughtSignature=block_signature, + text=block_thinking_str, + ) + ) if _message_content is not None and isinstance(_message_content, list): _parts = [] for element in _message_content: @@ -262,9 +343,17 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 assistant_msg.get("tool_calls", []) is not None or assistant_msg.get("function_call") is not None ): # support assistant tool invoke conversion - assistant_content.extend( - convert_to_gemini_tool_call_invoke(assistant_msg) + gemini_tool_call_parts = convert_to_gemini_tool_call_invoke( + assistant_msg ) + ## check if gemini_tool_call already exists in assistant_content + for gemini_tool_call_part in gemini_tool_call_parts: + if not check_if_part_exists_in_parts( + assistant_content, + gemini_tool_call_part, + excluded_keys=["thoughtSignature"], + ): + assistant_content.append(gemini_tool_call_part) last_message_with_tool_calls = assistant_msg msg_i += 1 @@ -476,6 +565,7 @@ async def async_transform_request_body( optional_params=optional_params, ) + def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage: """ Returns a default user message when a "system" message is passed in gemini fails. @@ -484,6 +574,7 @@ def _default_user_message_when_system_message_passed() -> ChatCompletionUserMess """ return ChatCompletionUserMessage(content=".", role="user") + def _transform_system_message( supports_system_message: bool, messages: List[AllMessageValues] ) -> Tuple[Optional[SystemInstructions], List[AllMessageValues]]: diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 99a04c20fba..37470a6ee09 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -43,6 +43,7 @@ from litellm.types.llms.gemini import BidiGenerateContentServerMessage from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionResponseMessage, + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, @@ -792,7 +793,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): content_str += _content_str return content_str, reasoning_content_str - + + def _extract_thinking_blocks_from_parts( + self, parts: List[HttpxPartType] + ) -> List[ChatCompletionThinkingBlock]: + """Extract thinking blocks from parts if present""" + thinking_blocks: List[ChatCompletionThinkingBlock] = [] + for part in parts: + if "thoughtSignature" in part: + part_copy = part.copy() + part_copy.pop("thoughtSignature") + thinking_blocks.append( + ChatCompletionThinkingBlock( + type="thinking", + thinking=json.dumps(part_copy), + signature=part["thoughtSignature"], + ) + ) + return thinking_blocks + def _extract_image_response_from_parts( self, parts: List[HttpxPartType] ) -> Optional[ImageURLObject]: @@ -804,10 +823,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if mime_type.startswith("image/"): # Convert base64 data to data URI format data_uri = f"data:{mime_type};base64,{data}" - return ImageURLObject( - url=data_uri, - detail="auto" - ) + return ImageURLObject(url=data_uri, detail="auto") return None def _extract_audio_response_from_parts( @@ -1127,7 +1143,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif web_search_queries: web_search_requests = len(grounding_metadata) return web_search_requests - + @staticmethod def _create_streaming_choice( chat_completion_message: ChatCompletionResponseMessage, @@ -1151,9 +1167,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): index=candidate.get("index", idx), delta=Delta( content=chat_completion_message.get("content"), - reasoning_content=chat_completion_message.get( - "reasoning_content" - ), + reasoning_content=chat_completion_message.get("reasoning_content"), tool_calls=tools, image=image_response, function_call=functions, @@ -1164,13 +1178,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return choice @staticmethod - def _extract_candidate_metadata(candidate: Candidates) -> Tuple[List[dict], List[dict], List, List]: + def _extract_candidate_metadata( + candidate: Candidates, + ) -> Tuple[List[dict], List[dict], List, List]: """ Extract metadata from a single candidate response. - + Returns: grounding_metadata: List[dict] - url_context_metadata: List[dict] + url_context_metadata: List[dict] safety_ratings: List citation_metadata: List """ @@ -1178,7 +1194,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): url_context_metadata: List[dict] = [] safety_ratings: List = [] citation_metadata: List = [] - + if "groundingMetadata" in candidate: if isinstance(candidate["groundingMetadata"], list): grounding_metadata.extend(candidate["groundingMetadata"]) # type: ignore @@ -1194,8 +1210,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "urlContextMetadata" in candidate: # Add URL context metadata to grounding metadata url_context_metadata.append(cast(dict, candidate["urlContextMetadata"])) - - return grounding_metadata, url_context_metadata, safety_ratings, citation_metadata + + return ( + grounding_metadata, + url_context_metadata, + safety_ratings, + citation_metadata, + ) @staticmethod def _process_candidates( @@ -1227,6 +1248,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): tools: Optional[List[ChatCompletionToolCallChunk]] = [] functions: Optional[ChatCompletionToolCallFunctionChunk] = None cumulative_tool_call_index: int = 0 + thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None for idx, candidate in enumerate(_candidates): if "content" not in candidate: @@ -1239,7 +1261,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): candidate_safety_ratings, candidate_citation_metadata, ) = VertexGeminiConfig._extract_candidate_metadata(candidate) - + grounding_metadata.extend(candidate_grounding_metadata) url_context_metadata.extend(candidate_url_context_metadata) safety_ratings.extend(candidate_safety_ratings) @@ -1264,6 +1286,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) ) + thinking_blocks = ( + VertexGeminiConfig()._extract_thinking_blocks_from_parts( + parts=candidate["content"]["parts"] + ) + ) + if audio_response is not None: cast(Dict[str, Any], chat_completion_message)[ "audio" @@ -1271,7 +1299,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_message["content"] = None # OpenAI spec if image_response is not None: # Handle image response - combine with text content into structured format - cast(Dict[str, Any], chat_completion_message)["image"] = image_response + cast(Dict[str, Any], chat_completion_message)[ + "image" + ] = image_response if content is not None: chat_completion_message["content"] = content @@ -1298,15 +1328,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if functions is not None: chat_completion_message["function_call"] = functions + if thinking_blocks is not None: + chat_completion_message["thinking_blocks"] = thinking_blocks # type: ignore + if isinstance(model_response, ModelResponseStream): choice = VertexGeminiConfig._create_streaming_choice( chat_completion_message=chat_completion_message, - candidate=candidate, - idx=idx, - tools=tools, - functions=functions, + candidate=candidate, + idx=idx, + tools=tools, + functions=functions, chat_completion_logprobs=chat_completion_logprobs, - image_response=image_response + image_response=image_response, ) model_response.choices.append(choice) elif isinstance(model_response, ModelResponse): diff --git a/litellm/llms/xai/cost_calculator.py b/litellm/llms/xai/cost_calculator.py new file mode 100644 index 00000000000..62a48080d1c --- /dev/null +++ b/litellm/llms/xai/cost_calculator.py @@ -0,0 +1,54 @@ +""" +Helper util for handling XAI-specific cost calculation +- e.g.: reasoning tokens for grok models +""" + +from typing import Tuple, Union + +from litellm.types.utils import Usage +from litellm.utils import get_model_info + + +def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]: + """ + Calculates the cost per token for a given XAI model, prompt tokens, and completion tokens. + + Input: + - model: str, the model name without provider prefix + - usage: LiteLLM Usage block, containing XAI-specific usage information + + Returns: + Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd + """ + ## GET MODEL INFO + model_info = get_model_info(model=model, custom_llm_provider="xai") + + def _safe_float_cast( + value: Union[str, int, float, None, object], default: float = 0.0 + ) -> float: + """Safely cast a value to float with proper type handling for mypy.""" + if value is None: + return default + try: + return float(value) # type: ignore + except (ValueError, TypeError): + return default + + ## CALCULATE INPUT COST + input_cost_per_token = _safe_float_cast(model_info.get("input_cost_per_token")) + prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token + + ## CALCULATE OUTPUT COST + output_cost_per_token = _safe_float_cast(model_info.get("output_cost_per_token")) + + # For XAI models, completion is billed as (visible completion tokens + reasoning tokens) + completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0) + reasoning_tokens = 0 + if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details: + reasoning_tokens = int( + getattr(usage.completion_tokens_details, "reasoning_tokens", 0) or 0 + ) + + completion_cost = (completion_tokens + reasoning_tokens) * output_cost_per_token + + return prompt_cost, completion_cost diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4fb87dc1863..57309554621 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5817,16 +5817,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "groq/llama3-8b-8192": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 5e-08, - "output_cost_per_token": 8e-08, - "litellm_provider": "groq", - "mode": "chat", - "supports_tool_choice": true - }, "groq/llama-3.2-1b-preview": { "max_tokens": 8192, "max_input_tokens": 8192, @@ -5907,17 +5897,6 @@ "supports_tool_choice": true, "deprecation_date": "2025-04-14" }, - "groq/llama3-70b-8192": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 5.9e-07, - "output_cost_per_token": 7.9e-07, - "litellm_provider": "groq", - "mode": "chat", - "supports_response_schema": true, - "supports_tool_choice": true - }, "groq/llama-3.1-8b-instant": { "max_tokens": 8192, "max_input_tokens": 128000, @@ -11991,6 +11970,108 @@ "mode": "chat", "supports_tool_choice": true }, + "openrouter/openai/gpt-4.1": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-2025-04-14": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-mini": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-mini-2025-04-14": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-nano": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-nano-2025-04-14": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "openrouter/openai/gpt-5-mini": { "max_tokens": 128000, "max_input_tokens": 400000, @@ -14970,10 +15051,10 @@ "output_cost_per_token": 6e-06, "max_input_tokens": 262000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" }, "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { @@ -14981,10 +15062,10 @@ "output_cost_per_token": 2e-06, "max_input_tokens": 256000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" }, "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { @@ -14992,10 +15073,10 @@ "output_cost_per_token": 3e-06, "max_input_tokens": 256000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" }, "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { @@ -15038,10 +15119,10 @@ "output_cost_per_token": 2.19e-06, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/deepseek-r1-0528-throughput" }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { @@ -15066,9 +15147,9 @@ "output_cost_per_token": 6e-07, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/gpt-oss-120b" }, @@ -15077,9 +15158,9 @@ "output_cost_per_token": 2e-07, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/gpt-oss-20b" }, @@ -15088,12 +15169,24 @@ "output_cost_per_token": 1.1e-06, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/glm-4-5-air" }, + "together_ai/deepseek-ai/DeepSeek-V3.1": { + "input_cost_per_token": 0.6e-06, + "output_cost_per_token": 1.7e-06, + "max_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "mode": "chat", + "supports_tool_choice": true, + "source": "https://www.together.ai/models/deepseek-v3-1" + }, "ollama/codegemma": { "max_tokens": 8192, "max_input_tokens": 8192, diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 324b4866305..b38272560d1 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -21,3 +21,7 @@ model_list: router_settings: model_group_alias: {"my-fake-gpt-4": "fake-openai-endpoint"} + +litellm_settings: + callbacks: ["otel"] + success_callback: ["braintrust"] diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5a1952686bc..87c5bd7a7c5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2904,6 +2904,7 @@ class LitellmDataForBackendLLMCall(TypedDict, total=False): headers: dict organization: str timeout: Optional[float] + stream_timeout: Optional[float] user: Optional[str] num_retries: Optional[int] diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 373232e22d2..eb481b0a4f0 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -173,15 +173,24 @@ async def google_count_tokens(request: Request, model_name: str): """ from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.proxy_server import token_counter as internal_token_counter + from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter data = await _read_request_body(request=request) contents = data.get("contents", []) #Create TokenCountRequest for the internal endpoint from litellm.proxy._types import TokenCountRequest + # Translate contents to openai format messages using the adapter + messages = ( + GoogleGenAIAdapter() + .translate_generate_content_to_completion(model_name, contents) + .get("messages", []) + ) + token_request = TokenCountRequest( model=model_name, - contents=contents + contents=contents, + messages=messages, # compatibility when use openai-like endpoint ) # Call the internal token counter function with direct request flag set to False @@ -192,11 +201,17 @@ async def google_count_tokens(request: Request, model_name: str): if token_response is not None: # cast the response to the well known format original_response: dict = token_response.original_response or {} - return TokenCountDetailsResponse( - totalTokens=original_response.get("totalTokens", 0), - promptTokensDetails=original_response.get("promptTokensDetails", []), - ) - + if original_response: + return TokenCountDetailsResponse( + totalTokens=original_response.get("totalTokens", 0), + promptTokensDetails=original_response.get("promptTokensDetails", []), + ) + else: + return TokenCountDetailsResponse( + totalTokens=token_response.total_tokens or 0, + promptTokensDetails=[], + ) + ######################################################### # Return the response in the well known format ######################################################### diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d2bd7c8db29..32a283066ed 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -271,6 +271,16 @@ class LiteLLMProxyRequestSetup: if timeout_header is not None: return float(timeout_header) return None + + @staticmethod + def _get_stream_timeout_from_request(headers: dict) -> Optional[float]: + """ + Get the `stream_timeout` from the request headers. + """ + stream_timeout_header = headers.get("x-litellm-stream-timeout", None) + if stream_timeout_header is not None: + return float(stream_timeout_header) + return None @staticmethod def _get_num_retries_from_request(headers: dict) -> Optional[int]: @@ -439,6 +449,10 @@ class LiteLLMProxyRequestSetup: timeout = LiteLLMProxyRequestSetup._get_timeout_from_request(headers) if timeout is not None: data["timeout"] = timeout + + stream_timeout = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers) + if stream_timeout is not None: + data["stream_timeout"] = stream_timeout num_retries = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers) if num_retries is not None: diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index cc4b1652d08..7ddc5b96bfc 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -12,6 +12,8 @@ import click import httpx from dotenv import load_dotenv +from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY + if TYPE_CHECKING: from fastapi import FastAPI else: @@ -308,8 +310,8 @@ class ProxyInitializationHelpers: @click.option("--port", default=4000, help="Port to bind the server to.", envvar="PORT") @click.option( "--num_workers", - default=1, - help="Number of uvicorn / gunicorn workers to spin up. By default, 1 uvicorn is used.", + default=DEFAULT_NUM_WORKERS_LITELLM_PROXY, + help="Number of uvicorn / gunicorn workers to spin up. By default, 4 uvicorn workers are used.", envvar="NUM_WORKERS", ) @click.option("--api_base", default=None, help="API base URL.") diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index be6da159a37..acff522196e 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,4 +1,6 @@ model_list: - - model_name: xai/* + - model_name: db-openai-endpoint litellm_params: - model: xai/* + model: openai/* + api_base: https://exampleopenaiendpoint-production-0ee2.up.railway.app/ + mock_response: "hi" diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index e0ee950d260..955c1d888f2 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -154,6 +154,7 @@ class UserAPIKeyLabelNames(Enum): DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_llm_api_latency_metric", + "litellm_llm_api_time_to_first_token_metric", "litellm_request_total_latency_metric", "litellm_overhead_latency_metric", "litellm_remaining_requests_metric", @@ -162,6 +163,7 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_proxy_failed_requests_metric", "litellm_deployment_latency_per_output_token", "litellm_requests_metric", + "litellm_spend_metric", "litellm_total_tokens_metric", "litellm_input_tokens_metric", "litellm_output_tokens_metric", @@ -173,9 +175,11 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_remaining_api_key_budget_metric", "litellm_api_key_max_budget_metric", "litellm_api_key_budget_remaining_hours_metric", + "litellm_deployment_state", "litellm_deployment_failure_responses", "litellm_deployment_total_requests", "litellm_deployment_success_responses", + "litellm_deployment_cooled_down", "litellm_pod_lock_manager_size", "litellm_in_memory_daily_spend_update_queue_size", "litellm_redis_daily_spend_update_queue_size", @@ -196,6 +200,14 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.USER.value, ] + litellm_llm_api_time_to_first_token_metric = [ + UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.API_KEY_HASH.value, + UserAPIKeyLabelNames.API_KEY_ALIAS.value, + UserAPIKeyLabelNames.TEAM.value, + UserAPIKeyLabelNames.TEAM_ALIAS.value, + ] + litellm_request_total_latency_metric = [ UserAPIKeyLabelNames.END_USER.value, UserAPIKeyLabelNames.API_KEY_HASH.value, @@ -282,6 +294,16 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.USER_EMAIL.value, ] + litellm_spend_metric = [ + UserAPIKeyLabelNames.END_USER.value, + UserAPIKeyLabelNames.API_KEY_HASH.value, + UserAPIKeyLabelNames.API_KEY_ALIAS.value, + UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.TEAM.value, + UserAPIKeyLabelNames.TEAM_ALIAS.value, + UserAPIKeyLabelNames.USER.value, + ] + litellm_input_tokens_metric = [ UserAPIKeyLabelNames.END_USER.value, UserAPIKeyLabelNames.API_KEY_HASH.value, @@ -315,6 +337,20 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.REQUESTED_MODEL.value, ] + litellm_deployment_state = [ + UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.MODEL_ID.value, + UserAPIKeyLabelNames.API_BASE.value, + UserAPIKeyLabelNames.API_PROVIDER.value, + ] + + litellm_deployment_cooled_down = [ + UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.MODEL_ID.value, + UserAPIKeyLabelNames.API_BASE.value, + UserAPIKeyLabelNames.API_PROVIDER.value, + ] + litellm_deployment_successful_fallbacks = [ UserAPIKeyLabelNames.REQUESTED_MODEL.value, UserAPIKeyLabelNames.FALLBACK_MODEL.value, diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 2931770cd6e..1b74ee25803 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -41,6 +41,7 @@ class PartType(TypedDict, total=False): function_call: FunctionCall function_response: FunctionResponse thought: bool + thoughtSignature: str class HttpxFunctionCall(TypedDict): @@ -72,6 +73,7 @@ class HttpxPartType(TypedDict, total=False): executableCode: HttpxExecutableCode codeExecutionResult: HttpxCodeExecutionResult thought: bool + thoughtSignature: str class HttpxContentType(TypedDict, total=False): @@ -245,10 +247,11 @@ class UsageMetadata(TypedDict, total=False): class TokenCountDetailsResponse(TypedDict): """ Response structure for token count details with modality breakdown. - + Example: {'totalTokens': 12, 'promptTokensDetails': [{'modality': 'TEXT', 'tokenCount': 12}]} """ + totalTokens: int promptTokensDetails: List[PromptTokensDetails] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4fb87dc1863..57309554621 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5817,16 +5817,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "groq/llama3-8b-8192": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 5e-08, - "output_cost_per_token": 8e-08, - "litellm_provider": "groq", - "mode": "chat", - "supports_tool_choice": true - }, "groq/llama-3.2-1b-preview": { "max_tokens": 8192, "max_input_tokens": 8192, @@ -5907,17 +5897,6 @@ "supports_tool_choice": true, "deprecation_date": "2025-04-14" }, - "groq/llama3-70b-8192": { - "max_tokens": 8192, - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "input_cost_per_token": 5.9e-07, - "output_cost_per_token": 7.9e-07, - "litellm_provider": "groq", - "mode": "chat", - "supports_response_schema": true, - "supports_tool_choice": true - }, "groq/llama-3.1-8b-instant": { "max_tokens": 8192, "max_input_tokens": 128000, @@ -11991,6 +11970,108 @@ "mode": "chat", "supports_tool_choice": true }, + "openrouter/openai/gpt-4.1": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-2025-04-14": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 5e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-mini": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-mini-2025-04-14": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 1e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-nano": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "openrouter/openai/gpt-4.1-nano-2025-04-14": { + "max_tokens": 32768, + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 2.5e-08, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "openrouter/openai/gpt-5-mini": { "max_tokens": 128000, "max_input_tokens": 400000, @@ -14970,10 +15051,10 @@ "output_cost_per_token": 6e-06, "max_input_tokens": 262000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8" }, "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { @@ -14981,10 +15062,10 @@ "output_cost_per_token": 2e-06, "max_input_tokens": 256000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct" }, "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { @@ -14992,10 +15073,10 @@ "output_cost_per_token": 3e-06, "max_input_tokens": 256000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507" }, "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { @@ -15038,10 +15119,10 @@ "output_cost_per_token": 2.19e-06, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, "mode": "chat", - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.together.ai/models/deepseek-r1-0528-throughput" }, "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { @@ -15066,9 +15147,9 @@ "output_cost_per_token": 6e-07, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/gpt-oss-120b" }, @@ -15077,9 +15158,9 @@ "output_cost_per_token": 2e-07, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/gpt-oss-20b" }, @@ -15088,12 +15169,24 @@ "output_cost_per_token": 1.1e-06, "max_input_tokens": 128000, "litellm_provider": "together_ai", - "supports_function_calling": false, - "supports_tool_choice": false, - "supports_parallel_function_calling": false, + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_parallel_function_calling": true, "mode": "chat", "source": "https://www.together.ai/models/glm-4-5-air" }, + "together_ai/deepseek-ai/DeepSeek-V3.1": { + "input_cost_per_token": 0.6e-06, + "output_cost_per_token": 1.7e-06, + "max_tokens": 128000, + "litellm_provider": "together_ai", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "mode": "chat", + "supports_tool_choice": true, + "source": "https://www.together.ai/models/deepseek-v3-1" + }, "ollama/codegemma": { "max_tokens": 8192, "max_input_tokens": 8192, diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 158399305b5..ee948f13170 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -25,7 +25,9 @@ IGNORE_FUNCTIONS = [ "filter_value_from_dict", # max depth set. "normalize_json_schema_types", # max depth set. "_extract_fields_recursive", # max depth set. - "_remove_json_schema_refs", # max depth set. + "_remove_json_schema_refs", # max depth set., + "_convert_schema_types", # max depth set., + "_fix_enum_empty_strings", # max depth set., ] diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index 3228bd92189..0818d0b4b08 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -1054,7 +1054,7 @@ def test_parse_content_for_reasoning(content, expected_reasoning, expected_conte ("gemini/gemini-1.5-pro", True), ("predibase/llama3-8b-instruct", True), ("gpt-3.5-turbo", False), - ("groq/llama3-70b-8192", True), + ("groq/llama-3.3-70b-versatile", True), ], ) def test_supports_response_schema(model, expected_bool): diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index d36ed944292..3c7d20ac950 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -141,6 +141,108 @@ class BaseLLMChatTest(ABC): # for OpenAI the content contains the JSON schema, so we need to assert that the content is not None assert response.choices[0].message.content is not None + + def test_tool_call_with_property_type_array(self): + litellm._turn_on_debug() + from litellm.utils import supports_function_calling + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + base_completion_call_args = self.get_base_completion_call_args() + if not supports_function_calling(base_completion_call_args["model"], None): + print("Model does not support function calling") + pytest.skip("Model does not support function calling") + base_completion_call_args = self.get_base_completion_call_args() + response = self.completion_function( + **base_completion_call_args, + messages = [ + { + "role": "user", + "content": "Tell me if the shoe brand Air Jordan has more models than the shoe brand Nike." + } + ], + tools = [ + { + "type": "function", + "function": { + "name": "shoe_get_id", + "description": "Get information about a show by its ID or name", + "parameters": { + "type": "object", + "properties": { + "shoe_id": { + "type": ["string", "number"], + "description": "The shoe ID or name" + } + }, + "required": ["shoe_id"], + "additionalProperties": False, + "$schema": "http://json-schema.org/draft-07/schema#" + } + } + }, + ] + ) + print(response) + print(json.dumps(response, indent=4, default=str)) + + def test_tool_call_with_empty_enum_property(self): + litellm._turn_on_debug() + from litellm.utils import supports_function_calling + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + base_completion_call_args = self.get_base_completion_call_args() + if not supports_function_calling(base_completion_call_args["model"], None): + print("Model does not support function calling") + pytest.skip("Model does not support function calling") + base_completion_call_args = self.get_base_completion_call_args() + response = self.completion_function( + **base_completion_call_args, + messages = [ + { + "role": "user", + "content": "Search for the latest iPhone models and tell me which storage options are available." + } + ], + tools = [ + { + "type": "function", + "function": { + "name": "litellm_product_search", + "description": "Search for product information and specifications.\n\nSupports filtering by category, brand, price range, and availability.\nCan retrieve detailed product specifications, pricing, and stock information.\nSupports different search modes and result formatting options.\n", + "parameters": { + "properties": { + "search_mode": { + "default": "", + "description": "The search strategy to use for finding products.", + "enum": [ + "", + "product_search", + "product_search_with_filters", + "product_search_with_sorting", + "product_search_with_pagination", + "product_search_with_aggregation", + ], + "title": "Search Mode", + "type": "string" + }, + }, + "required": [ + "search_mode" + ], + "title": "product_search_arguments", + "type": "object" + } + } + } + ] + ) + print(response) + print(json.dumps(response, indent=4, default=str)) + + + def test_streaming(self): """Check if litellm handles streaming correctly""" from litellm.types.utils import ModelResponseStream diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 22a54b8a56b..b3f16ecd838 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -436,7 +436,10 @@ def test_gemini_with_empty_function_call_arguments(): async def test_claude_tool_use_with_gemini(): response = await litellm.anthropic.messages.acreate( messages=[ - {"role": "user", "content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?"} + { + "role": "user", + "content": "Hello, can you tell me the weather in Boston. Please respond with a tool call?", + } ], model="gemini/gemini-2.5-flash", stream=True, @@ -578,11 +581,17 @@ def test_gemini_tool_use(): assert stop_reason is not None assert stop_reason == "tool_calls" + @pytest.mark.asyncio async def test_gemini_image_generation_async(): litellm._turn_on_debug() response = await litellm.acompletion( - messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}], + messages=[ + { + "role": "user", + "content": "Generate an image of a banana wearing a costume that says LiteLLM", + } + ], model="gemini/gemini-2.5-flash-image-preview", ) @@ -597,12 +606,16 @@ async def test_gemini_image_generation_async(): assert IMAGE_URL["url"].startswith("data:image/png;base64,") - @pytest.mark.asyncio async def test_gemini_image_generation_async_stream(): - #litellm._turn_on_debug() + # litellm._turn_on_debug() response = await litellm.acompletion( - messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}], + messages=[ + { + "role": "user", + "content": "Generate an image of a banana wearing a costume that says LiteLLM", + } + ], model="gemini/gemini-2.5-flash-image-preview", stream=True, ) @@ -611,35 +624,144 @@ async def test_gemini_image_generation_async_stream(): model_response_image = None async for chunk in response: print("CHUNK: ", chunk) - if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None: + if ( + hasattr(chunk.choices[0].delta, "image") + and chunk.choices[0].delta.image is not None + ): model_response_image = chunk.choices[0].delta.image print("MODEL_RESPONSE_IMAGE: ", model_response_image) assert model_response_image is not None assert model_response_image["url"].startswith("data:image/png;base64,") break - + ######################################################### # Important: Validate we did get an image in the response ######################################################### assert model_response_image is not None assert model_response_image["url"].startswith("data:image/png;base64,") - + def test_system_message_with_no_user_message(): - """ - Test that the system message is translated correctly for non-OpenAI providers. - """ - messages = [ - { - "role": "system", - "content": "Be a good bot!", - }, - ] + """ + Test that the system message is translated correctly for non-OpenAI providers. + """ + messages = [ + { + "role": "system", + "content": "Be a good bot!", + }, + ] - response = litellm.completion( - model="gemini/gemini-2.5-flash", - messages=messages, + response = litellm.completion( + model="gemini/gemini-2.5-flash", + messages=messages, + ) + assert response is not None + + assert response.choices[0].message.content is not None + + +def get_current_weather(location, unit="fahrenheit"): + """Get the current weather in a given location""" + if "tokyo" in location.lower(): + return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"}) + elif "san francisco" in location.lower(): + return json.dumps( + {"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"} ) - assert response is not None + elif "paris" in location.lower(): + return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"}) + else: + return json.dumps({"location": location, "temperature": "unknown"}) - assert response.choices[0].message.content is not None \ No newline at end of file + +def test_gemini_with_thinking(): + from litellm import completion + + litellm._turn_on_debug() + litellm.modify_params = True + model = "gemini/gemini-2.5-flash" + messages = [ + { + "role": "user", + "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", + } + ] + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + }, + } + ] + response = litellm.completion( + model=model, + messages=messages, + tools=tools, + tool_choice="auto", # auto is default, but we'll be explicit + reasoning_effort="low", + ) + print("Response\n", response) + response_message = response.choices[0].message + tool_calls = response_message.tool_calls + + print("Expecting there to be 3 tool calls") + assert len(tool_calls) > 0 # this has to call the function for SF, Tokyo and paris + + # Step 2: check if the model wanted to call a function + print(f"tool_calls: {tool_calls}") + if tool_calls: + # Step 3: call the function + # Note: the JSON response may not always be valid; be sure to handle errors + available_functions = { + "get_current_weather": get_current_weather, + } # only one function in this example, but you can have multiple + messages.append(response_message) # extend conversation with assistant's reply + print("Response message\n", response_message) + # Step 4: send the info for each function call and function response to the model + for tool_call in tool_calls: + function_name = tool_call.function.name + if function_name not in available_functions: + # the model called a function that does not exist in available_functions - don't try calling anything + return + function_to_call = available_functions[function_name] + function_args = json.loads(tool_call.function.arguments) + function_response = function_to_call( + location=function_args.get("location"), + unit=function_args.get("unit"), + ) + messages.append( + { + "tool_call_id": tool_call.id, + "role": "tool", + "name": function_name, + "content": function_response, + } + ) # extend conversation with function response + print(f"messages: {messages}") + second_response = litellm.completion( + model=model, + messages=messages, + seed=22, + reasoning_effort="low", + tools=tools, + drop_params=True, + ) # get a new response from the model where it can see the function response + print("second response\n", second_response) diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index bf482ca7527..39d4536d7aa 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -565,7 +565,7 @@ def test_groq_response_cost_tracking(is_streaming): response_cost = litellm.response_cost_calculator( response_object=response, - model="groq/llama3-70b-8192", + model="groq/llama-3.3-70b-versatile", custom_llm_provider="groq", call_type=CallTypes.acompletion.value, optional_params={}, diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index 0c75e37134c..de1ba39e5bc 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -50,7 +50,7 @@ def get_current_weather(location, unit="fahrenheit"): "claude-3-haiku-20240307", "gemini/gemini-1.5-pro", "anthropic.claude-3-sonnet-20240229-v1:0", - "groq/llama3-8b-8192", + "groq/llama-3.1-8b-instant", "cohere_chat/command-r", ], ) diff --git a/tests/local_testing/test_lowest_cost_routing.py b/tests/local_testing/test_lowest_cost_routing.py index caca0070525..c0305f33fce 100644 --- a/tests/local_testing/test_lowest_cost_routing.py +++ b/tests/local_testing/test_lowest_cost_routing.py @@ -31,7 +31,7 @@ async def test_get_available_deployments(): }, { "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "groq/llama3-8b-8192"}, + "litellm_params": {"model": "groq/llama-3.1-8b-instant"}, "model_info": {"id": "groq-llama"}, }, ] @@ -182,7 +182,7 @@ async def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm): }, { "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "groq/llama3-8b-8192"}, + "litellm_params": {"model": "groq/llama-3.1-8b-instant"}, "model_info": {"id": "5678", "rpm": non_ans_rpm}, }, ] diff --git a/tests/local_testing/test_model_alias_map.py b/tests/local_testing/test_model_alias_map.py index 2d60644fde8..cf731d66283 100644 --- a/tests/local_testing/test_model_alias_map.py +++ b/tests/local_testing/test_model_alias_map.py @@ -15,7 +15,7 @@ from litellm import completion, embedding litellm.set_verbose = True -model_alias_map = {"good-model": "groq/llama3-8b-8192"} +model_alias_map = {"good-model": "groq/llama-3.1-8b-instant"} def test_model_alias_map(caplog): @@ -35,7 +35,7 @@ def test_model_alias_map(caplog): for log in captured_logs: assert "ERROR" not in log - assert "llama3-8b-8192" in response.model + assert "llama-3.1-8b-instant" in response.model except litellm.ServiceUnavailableError: pass except Exception as e: diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index 10634ba3a73..007822a93b2 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -119,7 +119,7 @@ async def test_router_provider_wildcard_routing(): print("response 2 = ", response2) response3 = await router.acompletion( - model="groq/llama3-8b-8192", + model="groq/llama-3.1-8b-instant", messages=[{"role": "user", "content": "hello"}], ) diff --git a/tests/local_testing/test_router_batch_completion.py b/tests/local_testing/test_router_batch_completion.py index 534db7ed010..f7a1b41ca29 100644 --- a/tests/local_testing/test_router_batch_completion.py +++ b/tests/local_testing/test_router_batch_completion.py @@ -44,7 +44,7 @@ async def test_batch_completion_multiple_models(mode): { "model_name": "groq-llama", "litellm_params": { - "model": "groq/llama3-8b-8192", + "model": "groq/llama-3.1-8b-instant", }, }, ] @@ -143,7 +143,7 @@ async def test_batch_completion_fastest_response_streaming(): { "model_name": "groq-llama", "litellm_params": { - "model": "groq/llama3-8b-8192", + "model": "groq/llama-3.1-8b-instant", }, }, ] @@ -179,7 +179,7 @@ async def test_batch_completion_multiple_models_multiple_messages(): { "model_name": "groq-llama", "litellm_params": { - "model": "groq/llama3-8b-8192", + "model": "groq/llama-3.1-8b-instant", }, }, ] diff --git a/tests/local_testing/test_spend_calculate_endpoint.py b/tests/local_testing/test_spend_calculate_endpoint.py index 8bdd4a54d8a..8f7434e40b9 100644 --- a/tests/local_testing/test_spend_calculate_endpoint.py +++ b/tests/local_testing/test_spend_calculate_endpoint.py @@ -43,7 +43,7 @@ async def test_spend_calc_model_on_router_messages(): { "model_name": "special-llama-model", "litellm_params": { - "model": "groq/llama3-8b-8192", + "model": "groq/llama-3.1-8b-instant", }, } ] @@ -86,7 +86,7 @@ async def test_spend_calc_using_response(): } ], "created": "1677652288", - "model": "groq/llama3-8b-8192", + "model": "groq/llama-3.1-8b-instant", "object": "chat.completion", "system_fingerprint": "fp_873a560973", "usage": { diff --git a/tests/test_litellm/integrations/test_braintrust_span_name.py b/tests/test_litellm/integrations/test_braintrust_span_name.py index 10e512fc0ca..30381e99783 100644 --- a/tests/test_litellm/integrations/test_braintrust_span_name.py +++ b/tests/test_litellm/integrations/test_braintrust_span_name.py @@ -11,7 +11,7 @@ from litellm.integrations.braintrust_logging import BraintrustLogger class TestBraintrustSpanName(unittest.TestCase): """Test custom span_name functionality in Braintrust logging.""" - @patch('litellm.integrations.braintrust_logging.HTTPHandler') + @patch("litellm.integrations.braintrust_logging.HTTPHandler") def test_default_span_name(self, MockHTTPHandler): """Test that default span name is 'Chat Completion' when not provided.""" # Mock HTTP response @@ -22,39 +22,43 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], "litellm_params": {"metadata": {}}, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, } - + # Execute logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Chat Completion') + json_data = call_args.kwargs["json"] + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Chat Completion" + ) - @patch('litellm.integrations.braintrust_logging.HTTPHandler') + @patch("litellm.integrations.braintrust_logging.HTTPHandler") def test_custom_span_name(self, MockHTTPHandler): """Test that custom span name is used when provided in metadata.""" # Mock HTTP response @@ -65,39 +69,43 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], "litellm_params": {"metadata": {"span_name": "Custom Operation"}}, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, } - + # Execute logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Custom Operation') + json_data = call_args.kwargs["json"] + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Custom Operation" + ) - @patch('litellm.integrations.braintrust_logging.HTTPHandler') + @patch("litellm.integrations.braintrust_logging.HTTPHandler") def test_span_name_with_other_metadata(self, MockHTTPHandler): """Test that span_name works alongside other metadata fields.""" # Mock HTTP response @@ -108,21 +116,23 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], @@ -132,34 +142,40 @@ class TestBraintrustSpanName(unittest.TestCase): "project_id": "custom-project", "user_id": "user123", "session_id": "session456", - "environment": "production" + "environment": "production", } }, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, + "standard_logging_object": { + "user_id": "user123", + }, } - + # Execute logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - - # Check span name - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Multi Metadata Test') - - # Check that other metadata is preserved (except for filtered keys) - event_metadata = json_data['events'][0]['metadata'] - self.assertEqual(event_metadata['user_id'], 'user123') - self.assertEqual(event_metadata['session_id'], 'session456') - self.assertEqual(event_metadata['environment'], 'production') - - # Span name should be in span_attributes, not in metadata - self.assertIn('span_name', event_metadata) # span_name is also kept in metadata + json_data = call_args.kwargs["json"] - @patch('litellm.integrations.braintrust_logging.get_async_httpx_client') + # Check span name + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Multi Metadata Test" + ) + + # Check that other metadata is preserved (except for filtered keys) + event_metadata = json_data["events"][0]["metadata"] + print(event_metadata) + self.assertEqual(event_metadata["user_id"], "user123") + self.assertEqual(event_metadata["session_id"], "session456") + self.assertEqual(event_metadata["environment"], "production") + + # Span name should be in span_attributes, not in metadata + self.assertIn("span_name", event_metadata) # span_name is also kept in metadata + + @patch("litellm.integrations.braintrust_logging.get_async_httpx_client") async def test_async_custom_span_name(self, mock_get_http_handler): """Test async logging with custom span name.""" # Mock async HTTP response @@ -170,38 +186,44 @@ class TestBraintrustSpanName(unittest.TestCase): # Setup logger = BraintrustLogger(api_key="test-key") logger.default_project_id = "test-project-id" - + # Create a properly structured mock response response_obj = litellm.ModelResponse( id="test-id", object="chat.completion", created=1234567890, model="gpt-3.5-turbo", - choices=[{ - "index": 0, - "message": {"role": "assistant", "content": "test response"}, - "finish_reason": "stop" - }], - usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "test response"}, + "finish_reason": "stop", + } + ], + usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) - + kwargs = { "litellm_call_id": "test-call-id", "messages": [{"role": "user", "content": "test"}], "litellm_params": {"metadata": {"span_name": "Async Custom Operation"}}, "model": "gpt-3.5-turbo", - "response_cost": 0.001 + "response_cost": 0.001, } - + # Execute - await logger.async_log_success_event(kwargs, response_obj, datetime.now(), datetime.now()) - + await logger.async_log_success_event( + kwargs, response_obj, datetime.now(), datetime.now() + ) + # Verify call_args = mock_http_handler.post.call_args self.assertIsNotNone(call_args) - json_data = call_args.kwargs['json'] - self.assertEqual(json_data['events'][0]['span_attributes']['name'], 'Async Custom Operation') + json_data = call_args.kwargs["json"] + self.assertEqual( + json_data["events"][0]["span_attributes"]["name"], "Async Custom Operation" + ) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_litellm/litellm_core_utils/test_image_handling.py b/tests/test_litellm/litellm_core_utils/test_image_handling.py new file mode 100644 index 00000000000..64ae81b5763 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_image_handling.py @@ -0,0 +1,41 @@ +import pytest +from httpx import Request, Response + +import litellm +from litellm.litellm_core_utils.prompt_templates.image_handling import ( + convert_url_to_base64, +) + + +class DummyClient: + def get(self, url, follow_redirects=True): + return Response(status_code=404, request=Request("GET", url)) + + +def test_invalid_image_url_raises_bad_request(monkeypatch): + monkeypatch.setattr(litellm, "module_level_client", DummyClient()) + with pytest.raises(litellm.ImageFetchError) as excinfo: + convert_url_to_base64("https://invalid.example/image.png") + assert "Unable to fetch image" in str(excinfo.value) + + +def test_completion_with_invalid_image_url(monkeypatch): + monkeypatch.setattr(litellm, "module_level_client", DummyClient()) + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hi"}, + { + "type": "image_url", + "image_url": {"url": "https://invalid.example/image.png"}, + }, + ], + } + ] + with pytest.raises(litellm.ImageFetchError) as excinfo: + litellm.completion( + model="gemini/gemini-pro", messages=messages, api_key="test" + ) + assert excinfo.value.status_code == 400 + assert "Unable to fetch image" in str(excinfo.value) diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 3bdab355977..3e6a6a23468 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -41,3 +41,14 @@ def test_gpt5_temperature_error(config: OpenAIConfig): model="gpt-5", drop_params=False, ) + + +def test_gpt5_unsupported_params_drop(config: OpenAIConfig): + assert "top_p" not in config.get_supported_openai_params(model="gpt-5") + params = config.map_openai_params( + non_default_params={"top_p": 0.5}, + optional_params={}, + model="gpt-5", + drop_params=True, + ) + assert "top_p" not in params diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py new file mode 100644 index 00000000000..d6d33258576 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -0,0 +1,75 @@ +from litellm.llms.vertex_ai.gemini.transformation import check_if_part_exists_in_parts + + +def test_check_if_part_exists_in_parts(): + parts = [ + {"text": "Hello", "thought": True}, + {"text": "World", "thought": False}, + ] + part = {"text": "Hello", "thought": True} + new_part = {"text": "Hello World", "thought": True} + assert check_if_part_exists_in_parts(parts, part) + assert not check_if_part_exists_in_parts(parts, new_part, ["thought"]) + assert check_if_part_exists_in_parts(parts, new_part, ["text"]) + + +def test_check_if_part_exists_in_parts_camel_case_snake_case(): + """Test that function handles both camelCase and snake_case key variations""" + # Test snake_case to camelCase matching + parts_with_snake_case = [ + { + "function_call": { + "name": "get_current_weather", + "args": {"location": "San Francisco, CA"}, + } + }, + {"text": "Some other content"}, + ] + + part_with_camel_case = { + "functionCall": { + "name": "get_current_weather", + "args": {"location": "San Francisco, CA"}, + } + } + + # Should find match between function_call and functionCall + assert check_if_part_exists_in_parts(parts_with_snake_case, part_with_camel_case) + + # Test camelCase to snake_case matching + parts_with_camel_case = [ + {"functionCall": {"name": "calculate_sum", "args": {"a": 1, "b": 2}}} + ] + + part_with_snake_case = { + "function_call": {"name": "calculate_sum", "args": {"a": 1, "b": 2}} + } + + # Should find match between functionCall and function_call + assert check_if_part_exists_in_parts(parts_with_camel_case, part_with_snake_case) + + # Test no match when values differ + part_with_different_values = { + "function_call": {"name": "different_function", "args": {"x": 5}} + } + + assert not check_if_part_exists_in_parts( + parts_with_snake_case, part_with_different_values + ) + + # Test multiple keys with mixed casing + parts_mixed = [ + { + "function_call": {"name": "test"}, + "thoughtSignature": "reasoning", + "text": "content", + } + ] + + part_mixed_casing = { + "functionCall": {"name": "test"}, + "thought_signature": "reasoning", + "text": "content", + } + + assert check_if_part_exists_in_parts(parts_mixed, part_mixed_casing) diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index 9ebb9004b7f..02cd51920da 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -677,3 +677,127 @@ def test_vertex_filter_format_uri(): ) assert "uri" not in json.dumps(new_parameters) + +def test_convert_schema_types_type_array_conversion(): + """ + Test _convert_schema_types function handles type arrays and case conversion. + + This test verifies the fix for the issue where type arrays like ["string", "number"] + would raise an exception in Vertex AI schema validation. + + Relevant issue: https://github.com/BerriAI/litellm/issues/14091 + """ + from litellm.llms.vertex_ai.common_utils import _convert_schema_types + + # Input: OpenAI-style schema with type array (the problematic case) + input_schema = { + "type": "object", + "properties": { + "studio": { + "type": ["string", "number"], + "description": "The studio ID or name" + } + }, + "required": ["studio"], + "additionalProperties": False, + "$schema": "http://json-schema.org/draft-07/schema#" + } + + # Expected output: Vertex AI compatible schema with anyOf and uppercase types + expected_output = { + "type": "object", + "properties": { + "studio": { + "anyOf": [ + {"type": "string"}, + {"type": "number"} + ], + "description": "The studio ID or name" + } + }, + "required": ["studio"], + "additionalProperties": False, + "$schema": "http://json-schema.org/draft-07/schema#" + } + + # Apply the transformation + _convert_schema_types(input_schema) + + # Verify the transformation + assert input_schema == expected_output + + # Verify specific transformations: + # 1. Root level type converted to uppercase + assert input_schema["type"] == "object" + + # 2. Type array converted to anyOf format + assert "anyOf" in input_schema["properties"]["studio"] + assert "type" not in input_schema["properties"]["studio"] + + # 3. Individual types in anyOf are uppercase + anyof_types = input_schema["properties"]["studio"]["anyOf"] + assert anyof_types[0]["type"] == "string" + assert anyof_types[1]["type"] == "number" + + # 4. Other properties preserved + assert input_schema["properties"]["studio"]["description"] == "The studio ID or name" + assert input_schema["required"] == ["studio"] + + +def test_fix_enum_empty_strings(): + """ + Test _fix_enum_empty_strings function replaces empty strings with None in enum arrays. + + This test verifies the fix for the issue where Gemini rejects tool definitions + with empty strings in enum values, causing API failures. + + Relevant issue: Gemini does not accept empty strings in enum values + """ + from litellm.llms.vertex_ai.common_utils import _fix_enum_empty_strings + + # Input: Schema with empty string in enum (the problematic case) + input_schema = { + "type": "object", + "properties": { + "user_agent_type": { + "enum": ["", "desktop", "mobile", "tablet"], + "type": "string", + "description": "Device type for user agent" + } + }, + "required": ["user_agent_type"] + } + + # Expected output: Empty strings replaced with None + expected_output = { + "type": "object", + "properties": { + "user_agent_type": { + "enum": [None, "desktop", "mobile", "tablet"], + "type": "string", + "description": "Device type for user agent" + } + }, + "required": ["user_agent_type"] + } + + # Apply the transformation + _fix_enum_empty_strings(input_schema) + + # Verify the transformation + assert input_schema == expected_output + + # Verify specific transformations: + # 1. Empty string replaced with None + enum_values = input_schema["properties"]["user_agent_type"]["enum"] + assert "" not in enum_values + assert None in enum_values + + # 2. Other enum values preserved + assert "desktop" in enum_values + assert "mobile" in enum_values + assert "tablet" in enum_values + + # 3. Other properties preserved + assert input_schema["properties"]["user_agent_type"]["type"] == "string" + assert input_schema["properties"]["user_agent_type"]["description"] == "Device type for user agent" diff --git a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py new file mode 100644 index 00000000000..5a26d808c5d --- /dev/null +++ b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py @@ -0,0 +1,189 @@ +""" +Test suite for XAI cost calculation functionality. +""" + +import math +import os +import sys + +import litellm +from litellm.types.utils import ( + CompletionTokensDetailsWrapper, + Usage, +) + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +from litellm.llms.xai.cost_calculator import cost_per_token + + +class TestXAICostCalculator: + """Test suite for XAI cost calculation functionality.""" + + def setup_method(self): + """Set up test environment.""" + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + def test_basic_cost_calculation(self): + """Test basic cost calculation without reasoning tokens.""" + usage = Usage(prompt_tokens=12, completion_tokens=125, total_tokens=137) + + prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage) + + # Expected costs for grok-3-mini: + # Input: 12 tokens * $3e-7 = $0.0000036 + # Output: 125 tokens * $5e-7 = $0.0000625 + expected_prompt_cost = 12 * 3e-7 + expected_completion_cost = 125 * 5e-7 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) + + def test_reasoning_tokens_cost_calculation(self): + """Test cost calculation with reasoning tokens from completion_tokens_details.""" + usage = Usage( + prompt_tokens=12, + completion_tokens=125, + total_tokens=1086, + completion_tokens_details=CompletionTokensDetailsWrapper( + accepted_prediction_tokens=0, + audio_tokens=0, + reasoning_tokens=949, + rejected_prediction_tokens=0, + text_tokens=None, # Not set, but doesn't matter for XAI billing + ), + ) + + prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage) + + # Expected costs for grok-3-mini: + # Input: 12 tokens * $3e-7 = $0.0000036 + # Completion: (125 + 949) tokens * $5e-7 = $0.000537 + expected_prompt_cost = 12 * 3e-7 + expected_completion_cost = (125 + 949) * 5e-7 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) + + def test_reasoning_and_text_tokens_cost_calculation(self): + """Test cost calculation with both reasoning and text tokens.""" + usage = Usage( + prompt_tokens=12, + completion_tokens=125, + total_tokens=1086, + completion_tokens_details=CompletionTokensDetailsWrapper( + accepted_prediction_tokens=0, + audio_tokens=0, + reasoning_tokens=949, + rejected_prediction_tokens=0, + text_tokens=76, # Explicitly set (but ignored in XAI billing) + ), + ) + + prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage) + + # Expected costs for grok-3-mini: + # Input: 12 tokens * $3e-7 = $0.0000036 + # Completion: (125 + 949) tokens * $5e-7 = $0.000537 + # Note: text_tokens field is ignored, only completion_tokens + reasoning_tokens matters + expected_prompt_cost = 12 * 3e-7 + expected_completion_cost = (125 + 949) * 5e-7 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) + + def test_grok_4_cost_calculation(self): + """Test cost calculation for grok-4 model.""" + usage = Usage( + prompt_tokens=10, + completion_tokens=200, + total_tokens=210, + completion_tokens_details=CompletionTokensDetailsWrapper( + accepted_prediction_tokens=0, + audio_tokens=0, + reasoning_tokens=150, + rejected_prediction_tokens=0, + text_tokens=50, # Ignored in XAI billing + ), + ) + + prompt_cost, completion_cost = cost_per_token(model="grok-4", usage=usage) + + # Expected costs for grok-4: + # Input: 10 tokens * $3e-6 = $0.00003 + # Completion: (200 + 150) tokens * $1.5e-5 = $0.00525 + expected_prompt_cost = 10 * 3e-6 + expected_completion_cost = (200 + 150) * 1.5e-5 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) + + def test_grok_3_fast_beta_cost_calculation(self): + """Test cost calculation for grok-3-fast-beta model.""" + usage = Usage( + prompt_tokens=20, + completion_tokens=300, + total_tokens=320, + completion_tokens_details=CompletionTokensDetailsWrapper( + accepted_prediction_tokens=0, + audio_tokens=0, + reasoning_tokens=200, + rejected_prediction_tokens=0, + text_tokens=100, # Ignored in XAI billing + ), + ) + + prompt_cost, completion_cost = cost_per_token( + model="grok-3-fast-beta", usage=usage + ) + + # Expected costs for grok-3-fast-beta: + # Input: 20 tokens * $5e-6 = $0.0001 + # Completion: (300 + 200) tokens * $2.5e-5 = $0.0125 + expected_prompt_cost = 20 * 5e-6 + expected_completion_cost = (300 + 200) * 2.5e-5 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) + + def test_edge_case_no_completion_tokens_details(self): + """Test cost calculation when completion_tokens_details is not present.""" + usage = Usage(prompt_tokens=12, completion_tokens=125, total_tokens=137) + + prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage) + + # Should fall back to basic calculation + expected_prompt_cost = 12 * 3e-7 + expected_completion_cost = 125 * 5e-7 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) + + def test_edge_case_large_reasoning_tokens(self): + """Test cost calculation when reasoning_tokens is larger than completion_tokens.""" + usage = Usage( + prompt_tokens=12, + completion_tokens=50, # Less than reasoning_tokens + total_tokens=62, + completion_tokens_details=CompletionTokensDetailsWrapper( + accepted_prediction_tokens=0, + audio_tokens=0, + reasoning_tokens=100, # More than completion_tokens + rejected_prediction_tokens=0, + text_tokens=None, + ), + ) + + prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage) + + # Expected costs: + # Input: 12 tokens * $3e-7 = $0.0000036 + # Completion: (50 + 100) tokens * $5e-7 = $0.000075 + expected_prompt_cost = 12 * 3e-7 + expected_completion_cost = (50 + 100) * 5e-7 + + assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) + assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) diff --git a/tests/test_litellm/proxy/google_endpoints/__init__.py b/tests/test_litellm/proxy/google_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/proxy/google_endpoints/test_endpoints.py b/tests/test_litellm/proxy/google_endpoints/test_endpoints.py new file mode 100644 index 00000000000..2f2538bf9aa --- /dev/null +++ b/tests/test_litellm/proxy/google_endpoints/test_endpoints.py @@ -0,0 +1,49 @@ +""" +Test for google_endpoints/endpoints.py +""" +import pytest +import sys, os +from dotenv import load_dotenv + + +from litellm.proxy.google_endpoints.endpoints import google_count_tokens +from litellm.types.llms.vertex_ai import TokenCountDetailsResponse +from starlette.requests import Request + +load_dotenv() + +sys.path.insert( + 0, os.path.abspath("../../../..") +) + +@pytest.mark.asyncio +async def test_proxy_gemini_to_openai_like_model_token_counting(): + """ + Test the token counting endpoint for proxing gemini to openai-like models. + """ + response: TokenCountDetailsResponse = await google_count_tokens( + request=Request( + scope={ + "type": "http", + "parsed_body": ( + [ + "contents" + ], + { + "contents": [ + { + "parts": [ + { + "text": "Hello, how are you?" + } + ] + } + ] + } + ) + } + ), + model_name="volcengine/foo", + ) + + assert response.get("totalTokens") > 0 \ No newline at end of file diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index e1bdccdde9d..6a8d1ae1339 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -74,6 +74,96 @@ class TestProxyBaseLLMRequestProcessing: pytest.fail("litellm_call_id is not a valid UUID") assert data_passed["litellm_call_id"] == returned_data["litellm_call_id"] + @pytest.mark.asyncio + async def test_stream_timeout_header_processing(self): + """ + Test that x-litellm-stream-timeout header gets processed and added to request data as stream_timeout. + """ + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + # Test with stream timeout header + headers_with_timeout = {"x-litellm-stream-timeout": "30.5"} + result = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_with_timeout) + assert result == 30.5 + + # Test without stream timeout header + headers_without_timeout = {} + result = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_without_timeout) + assert result is None + + # Test with invalid header value (should raise ValueError when converting to float) + headers_with_invalid = {"x-litellm-stream-timeout": "invalid"} + with pytest.raises(ValueError): + LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_with_invalid) + + @pytest.mark.asyncio + async def test_add_litellm_data_to_request_with_stream_timeout_header(self): + """ + Test that x-litellm-stream-timeout header gets processed and added to request data + when calling add_litellm_data_to_request. + """ + from litellm.integrations.opentelemetry import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + # Create test data with a basic completion request + test_data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + # Mock request with stream timeout header + mock_request = MagicMock(spec=Request) + mock_request.headers = {"x-litellm-stream-timeout": "45.0"} + mock_request.url.path = "/v1/chat/completions" + mock_request.method = "POST" + mock_request.query_params = {} + mock_request.client = None + + # Create a minimal mock with just the required attributes + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.api_key = "test_api_key_hash" + mock_user_api_key_dict.tpm_limit = None + mock_user_api_key_dict.rpm_limit = None + mock_user_api_key_dict.max_budget = None + mock_user_api_key_dict.spend = 0 + mock_user_api_key_dict.allowed_model_region = None + mock_user_api_key_dict.key_alias = None + mock_user_api_key_dict.user_id = None + mock_user_api_key_dict.team_id = None + mock_user_api_key_dict.metadata = {} # Prevent enterprise feature check + mock_user_api_key_dict.team_metadata = None + mock_user_api_key_dict.org_id = None + mock_user_api_key_dict.team_alias = None + mock_user_api_key_dict.end_user_id = None + mock_user_api_key_dict.user_email = None + mock_user_api_key_dict.request_route = None + mock_user_api_key_dict.team_max_budget = None + mock_user_api_key_dict.team_spend = None + mock_user_api_key_dict.model_max_budget = None + mock_user_api_key_dict.parent_otel_span = None + mock_user_api_key_dict.team_model_aliases = None + + general_settings = {} + mock_proxy_config = MagicMock() + + # Call the actual function that processes headers and adds data + result_data = await add_litellm_data_to_request( + data=test_data, + request=mock_request, + general_settings=general_settings, + user_api_key_dict=mock_user_api_key_dict, + version=None, + proxy_config=mock_proxy_config, + ) + + # Verify that stream_timeout was extracted from header and added to request data + assert "stream_timeout" in result_data + assert result_data["stream_timeout"] == 45.0 + + # Verify that the original test data is preserved + assert result_data["model"] == "gpt-3.5-turbo" + assert result_data["messages"] == [{"role": "user", "content": "Hello"}] + @pytest.mark.asyncio class TestCommonRequestProcessingHelpers: diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 60b42c18cbf..ed9a2729282 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -979,8 +979,8 @@ class TestProxyFunctionCalling: # Groq models (mixed support) ("groq/gemma-7b-it", "litellm_proxy/groq/gemma-7b-it", True), ( - "groq/llama3-70b-8192", - "litellm_proxy/groq/llama3-70b-8192", + "groq/llama-3.3-70b-versatile", + "litellm_proxy/groq/llama-3.3-70b-versatile", False, ), # This model doesn't support function calling # Cohere models (generally don't support function calling) @@ -1051,7 +1051,7 @@ class TestProxyFunctionCalling: ("litellm_proxy/claude-prod", "anthropic/claude-3-sonnet-20240229", False), ("litellm_proxy/claude-dev", "anthropic/claude-3-haiku-20240307", False), # Groq with custom names (cannot be resolved) - ("litellm_proxy/fast-llama", "groq/llama3-8b-8192", False), + ("litellm_proxy/fast-llama", "groq/llama-3.1-8b-instant", False), ("litellm_proxy/groq-gemma", "groq/gemma-7b-it", False), # Cohere with custom names (cannot be resolved) ("litellm_proxy/cohere-command", "cohere/command-r", False), diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index fe1b2bef4a2..c90fd91a5d7 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -550,7 +550,7 @@ async def test_proxy_all_models(): async with aiohttp.ClientSession() as session: # call chat/completions with a model that the key was not created for + the model is not on the config.yaml await chat_completion( - session=session, key=LITELLM_MASTER_KEY, model="groq/llama3-8b-8192" + session=session, key=LITELLM_MASTER_KEY, model="groq/llama-3.1-8b-instant" ) await chat_completion( diff --git a/ui/litellm-dashboard/src/components/organizations.tsx b/ui/litellm-dashboard/src/components/organizations.tsx index f2a8f407740..bc9399463f4 100644 --- a/ui/litellm-dashboard/src/components/organizations.tsx +++ b/ui/litellm-dashboard/src/components/organizations.tsx @@ -139,6 +139,7 @@ const OrganizationsTable: React.FC = ({ } await organizationCreateCall(accessToken, values) + NotificationsManager.success("Organization created successfully") setIsOrgModalVisible(false) form.resetFields() // Refresh organizations list diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index b4a74bf846b..f5e18b68046 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -43,6 +43,7 @@ interface KeyInfoViewProps { teams: any[] | null premiumUser: boolean setAccessToken?: (token: string) => void + backButtonText?: string } export default function KeyInfoView({ @@ -57,6 +58,7 @@ export default function KeyInfoView({ onDelete, premiumUser, setAccessToken, + backButtonText = "Back to Keys", }: KeyInfoViewProps) { const [isEditing, setIsEditing] = useState(false) const [form] = Form.useForm() @@ -92,7 +94,7 @@ export default function KeyInfoView({ return (
Key not found
@@ -261,7 +263,7 @@ export default function KeyInfoView({
{currentKeyData.key_alias || "API Key"} diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx index 4aaaabba58d..13f2547bd75 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestResponsePanel.tsx @@ -72,9 +72,9 @@ export function RequestResponsePanel({ }; return ( -
+
{/* Request Side */} -
+

Request

-
- 
{JSON.stringify(getRawRequest(), null, 2)}
+
+ 
{JSON.stringify(getRawRequest(), null, 2)}
{/* Response Side */} -
+

Response @@ -116,9 +116,9 @@ export function RequestResponsePanel({

-
+
{hasResponse ? ( - 
{JSON.stringify(formattedResponse(), null, 2)}
+ 
{JSON.stringify(formattedResponse(), null, 2)}
) : (
Response data not available
)} diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index b227d493aa0..c45116c0ddd 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -465,7 +465,7 @@ export default function SpendLogsTable({ const displayLabel = isCustomDate ? getTimeRangeDisplay(isCustomDate, startTime, endTime) : selectedOption?.label return ( -
+
setActiveTab(index === 0 ? "request logs" : "audit logs")}> Request Logs @@ -500,6 +500,7 @@ export default function SpendLogsTable({ teams={allTeams} onClose={() => setSelectedKeyIdInfoView(null)} premiumUser={premiumUser} + backButtonText="Back to Logs" /> ) : selectedSessionId ? (
@@ -518,11 +519,11 @@ export default function SpendLogsTable({ onApplyFilters={handleFilterChange} onResetFilters={handleFilterReset} /> -
-
-
-
-
+
+
+
+
+
-
-
+
+
{quickSelectOpen && ( -
+
{quickSelectOptions.map((option) => (