diff --git a/.circleci/config.yml b/.circleci/config.yml index 796ac8ce597..e5b1a2d0bf3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,7 +95,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.99.5 + pip install openai==1.100.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -220,7 +220,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.99.5 + pip install openai==1.100.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -327,7 +327,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.99.5 + pip install openai==1.100.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -602,7 +602,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.99.5 + pip install openai==1.100.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -1522,7 +1522,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.99.5" + pip install "openai==1.100.1" - run: name: Install dockerize command: | @@ -1679,7 +1679,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.99.5" + pip install "openai==1.100.1" # Run pytest and generate JUnit XML report - run: name: Install dockerize @@ -1819,7 +1819,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.99.5" + pip install "openai==1.100.1" - run: name: Install dockerize command: | @@ -2399,7 +2399,7 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "google-cloud-aiplatform==1.43.0" pip install aiohttp - pip install "openai==1.99.5" + pip install "openai==1.100.1" pip install "assemblyai==0.37.0" python -m pip install --upgrade pip pip install "pydantic==2.10.2" @@ -2790,7 +2790,7 @@ jobs: pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" pip install aiohttp - pip install "openai==1.99.5" + pip install "openai==1.100.1" python -m pip install --upgrade pip pip install "pydantic==2.10.2" pip install "pytest==7.3.1" diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index f610e1aa56f..f41a5291e50 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -1,5 +1,5 @@ # used by CI/CD testing -openai==1.99.5 +openai==1.100.1 python-dotenv tiktoken importlib_metadata diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 5a4323b3e9b..ffca305a0d0 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -22,11 +22,8 @@ jobs: - name: Install dependencies run: | - pip install openai==1.99.5 poetry install --with dev - pip install openai==1.99.5 - - + poetry run pip install openai==1.100.1 - name: Run Black formatting run: | @@ -40,6 +37,10 @@ jobs: poetry run ruff check . cd .. + - name: Print OpenAI version + run: | + poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" + - name: Run MyPy type checking run: | cd litellm diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index bd63ca6bfca..b6ac264a228 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.4 +version: 0.4.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index cf10be0a76b..ec80e86b21d 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -73,6 +73,10 @@ spec: volumeMounts: {{- toYaml . | nindent 12 }} {{- end }} + {{- with .Values.migrationJob.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} {{- with .Values.migrationJob.extraContainers }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index f99204cbb4b..e9a96b23de7 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -206,6 +206,10 @@ migrationJob: disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0. annotations: {} ttlSecondsAfterFinished: 120 + resources: {} + # requests: + # cpu: 100m + # memory: 100Mi extraContainers: [] # Hook configuration diff --git a/docker/build_from_pip/requirements.txt b/docker/build_from_pip/requirements.txt index 71e038b6267..cc14b99727f 100644 --- a/docker/build_from_pip/requirements.txt +++ b/docker/build_from_pip/requirements.txt @@ -2,4 +2,5 @@ litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use prometheus_client langfuse prisma +openai==1.99.9 ddtrace==2.19.0 # for advanced DD tracing / profiling diff --git a/docs/my-website/docs/providers/litellm_proxy.md b/docs/my-website/docs/providers/litellm_proxy.md index efe61e15445..bfefc8a787c 100644 --- a/docs/my-website/docs/providers/litellm_proxy.md +++ b/docs/my-website/docs/providers/litellm_proxy.md @@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem'; | Description | LiteLLM Proxy is an OpenAI-compatible gateway that allows you to interact with multiple LLM providers through a unified API. Simply use the `litellm_proxy/` prefix before the model name to route your requests through the proxy. | | Provider Route on LiteLLM | `litellm_proxy/` (add this prefix to the model name, to route any requests to litellm_proxy - e.g. `litellm_proxy/your-model-name`) | | Setup LiteLLM Gateway | [LiteLLM Gateway ↗](../simple_proxy) | -| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/rerank` | +| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/images/edits`, `/rerank` | @@ -111,6 +111,21 @@ response = litellm.image_generation( ) ``` +## Image Edit + +```python +import litellm + +with open("your-image.png", "rb") as f: + response = litellm.image_edit( + model="litellm_proxy/gpt-image-1", + prompt="Make this image a watercolor painting", + image=[f], + api_base="your-litellm-proxy-url", + api_key="your-litellm-proxy-api-key", + ) +``` + ## Audio Transcription ```python diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 3b903935a04..a1288a47a79 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -349,6 +349,7 @@ router_settings: | AZURE_CODE_INTERPRETER_COST_PER_SESSION | Cost per session for Azure Code Interpreter service | AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS | Input cost per 1K tokens for Azure Computer Use service | AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS | Output cost per 1K tokens for Azure Computer Use service +| AZURE_DEFAULT_RESPONSES_API_VERSION | Version of the Azure Default Responses API being used. Default is "preview" | AZURE_TENANT_ID | Tenant ID for Azure Active Directory | AZURE_USERNAME | Username for Azure services, use in conjunction with AZURE_PASSWORD for azure ad token with basic username/password workflow | AZURE_PASSWORD | Password for Azure services, use in conjunction with AZURE_USERNAME for azure ad token with basic username/password workflow diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index e64f922ac80..c0bb003c096 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -803,11 +803,18 @@ LiteLLM Proxy supports session management for non-OpenAI models. This allows you 1. Enable storing request / response content in the database -Set `store_prompts_in_spend_logs: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the database. +Set `store_prompts_in_cold_storage: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the s3 bucket you specify. ```yaml +litellm_settings: + callbacks: ["s3_v2"] + s3_callback_params: # learn more https://docs.litellm.ai/docs/proxy/logging#s3-buckets + s3_bucket_name: litellm-logs # AWS Bucket Name for S3 + s3_region_name: us-west-2 + general_settings: - store_prompts_in_spend_logs: true + cold_storage_custom_logger: s3_v2 + store_prompts_in_cold_storage: true ``` 2. Make request 1 with no `previous_response_id` (new session) diff --git a/docs/my-website/img/release_notes/faster_caching_calls.png b/docs/my-website/img/release_notes/faster_caching_calls.png new file mode 100644 index 00000000000..fb7409aec28 Binary files /dev/null and b/docs/my-website/img/release_notes/faster_caching_calls.png differ diff --git a/docs/my-website/img/release_notes/responses_api_session_mgt_images.jpg b/docs/my-website/img/release_notes/responses_api_session_mgt_images.jpg new file mode 100644 index 00000000000..852d2fdd6d0 Binary files /dev/null and b/docs/my-website/img/release_notes/responses_api_session_mgt_images.jpg differ diff --git a/docs/my-website/release_notes/v1.75.5-stable/index.md b/docs/my-website/release_notes/v1.75.5-stable/index.md index de3945cef6e..270be64190e 100644 --- a/docs/my-website/release_notes/v1.75.5-stable/index.md +++ b/docs/my-website/release_notes/v1.75.5-stable/index.md @@ -1,5 +1,5 @@ --- -title: "[PRE-RELEASE]v1.75.5-stable" +title: "v1.75.5-stable - Redis latency improvements" slug: "v1-75-5" date: 2025-08-10T10:00:00 authors: @@ -28,14 +28,14 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.75.5.rc.1 +ghcr.io/berriai/litellm:v1.75.5-stable ``` ``` showLineNumbers title="pip install litellm" -pip install litellm==1.75.5.post1 +pip install litellm==1.75.5.post2 ``` @@ -43,8 +43,49 @@ pip install litellm==1.75.5.post1 --- +## Key Highlights + +- **Redis - Latency Improvements** - Reduces P99 latency by 50% with Redis enabled. +- **Responses API Session Management** - Support for managing responses API sessions with images. +- **Oracle Cloud Infrastructure** - New LLM provider for calling models on Oracle Cloud Infrastructure. +- **Digital Ocean's Gradient AI** - New LLM provider for calling models on Digital Ocean's Gradient AI platform. + + +### Risk of Upgrade + +If you build the proxy from the pip package, you should hold off on upgrading. This version makes `prisma migrate deploy` our default for managing the DB. This is safer, as it doesn't reset the DB, but it requires a manual `prisma generate` step. + +Users of our Docker image, are **not** affected by this change. + --- +## Redis Latency Improvements + + + +
+ +This release adds in-memory caching for Redis requests, enabling faster response times in high-traffic. Now, LiteLLM instances will check their in-memory cache for a cache hit, before checking Redis. This reduces caching-related latency from 100ms for LLM API calls to sub-1ms, on cache hits. + +--- + +## Responses API Session Management w/ Images + + + +
+ +LiteLLM now supports session management for Responses API requests with images. This is great for use-cases like chatbots, that are using the Responses API to track the state of a conversation. LiteLLM session management works across **ALL** LLM API's (including Anthropic, Bedrock, OpenAI, etc). LiteLLM session management works by storing the request and response content in an s3 bucket, you can specify. + +--- + + ## New Models / Updated Models #### New Model Support diff --git a/litellm/caching/s3_cache.py b/litellm/caching/s3_cache.py index c02e1091369..6095d547aab 100644 --- a/litellm/caching/s3_cache.py +++ b/litellm/caching/s3_cache.py @@ -55,13 +55,17 @@ class S3Cache(BaseCache): **kwargs, ) + def _to_s3_key(self, key: str) -> str: + """Convert cache key to S3 key""" + return self.key_prefix + key.replace(":", "/") + def set_cache(self, key, value, **kwargs): try: print_verbose(f"LiteLLM SET Cache - S3. Key={key}. Value={value}") ttl = kwargs.get("ttl", None) # Convert value to JSON before storing in S3 serialized_value = json.dumps(value) - key = self.key_prefix + key + key = self._to_s3_key(key) if ttl is not None: cache_control = f"immutable, max-age={ttl}, s-maxage={ttl}" @@ -104,7 +108,7 @@ class S3Cache(BaseCache): import botocore try: - key = self.key_prefix + key + key = self._to_s3_key(key) print_verbose(f"Get S3 Cache: key: {key}") # Download the data from S3 diff --git a/litellm/constants.py b/litellm/constants.py index 18f384b4ffb..afdb2073627 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1,6 +1,9 @@ import os from typing import List, Literal +AZURE_DEFAULT_RESPONSES_API_VERSION = str( + os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview") +) ROUTER_MAX_FALLBACKS = int(os.getenv("ROUTER_MAX_FALLBACKS", 5)) DEFAULT_BATCH_SIZE = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) diff --git a/litellm/images/main.py b/litellm/images/main.py index ca14fabd1f9..70d9eb41ddd 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -369,6 +369,7 @@ def image_generation( # noqa: PLR0915 ) elif ( custom_llm_provider == "openai" + or custom_llm_provider == LlmProviders.LITELLM_PROXY.value or custom_llm_provider in litellm.openai_compatible_providers ): model_response = openai_chat_completions.image_generation( @@ -444,7 +445,6 @@ def image_generation( # noqa: PLR0915 elif custom_llm_provider in ( litellm.LlmProviders.RECRAFT, litellm.LlmProviders.GEMINI, - ): if image_generation_config is None: raise ValueError(f"image generation config is not supported for {custom_llm_provider}") diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 2577ed3ddf0..8b1838c1edd 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -27,7 +27,12 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) from litellm.types.integrations.datadog_llm_obs import * -from litellm.types.utils import CallTypes, StandardLoggingPayload +from litellm.types.utils import ( + CallTypes, + StandardLoggingGuardrailInformation, + StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, +) class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): @@ -102,6 +107,24 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): verbose_logger.exception( f"DataDogLLMObs: Error logging success event - {str(e)}" ) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + try: + verbose_logger.debug( + f"DataDogLLMObs: Logging failure event for model {kwargs.get('model', 'unknown')}" + ) + payload = self.create_llm_obs_payload( + kwargs, start_time, end_time + ) + verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}") + self.log_queue.append(payload) + + if len(self.log_queue) >= self.batch_size: + await self.async_send_batch() + except Exception as e: + verbose_logger.exception( + f"DataDogLLMObs: Error logging failure event - {str(e)}" + ) async def async_send_batch(self): try: @@ -174,11 +197,14 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): call_type=standard_logging_payload.get("call_type") )) + error_info = self._assemble_error_info(standard_logging_payload) + meta = Meta( kind=self._get_datadog_span_kind(standard_logging_payload.get("call_type")), input=input_meta, output=output_meta, metadata=self._get_dd_llm_obs_payload_metadata(standard_logging_payload), + error=error_info, ) # Calculate metrics (you may need to adjust these based on available data) @@ -199,11 +225,31 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): start_ns=int(start_time.timestamp() * 1e9), duration=int((end_time - start_time).total_seconds() * 1e9), metrics=metrics, + status="error" if error_info else "ok", tags=[ self._get_datadog_tags(standard_logging_object=standard_logging_payload) ], ) + def _assemble_error_info(self, standard_logging_payload: StandardLoggingPayload) -> Optional[DDLLMObsError]: + """ + Assemble error information for failure cases according to DD LLM Obs API spec + """ + # Handle error information for failure cases according to DD LLM Obs API spec + error_info: Optional[DDLLMObsError] = None + + if standard_logging_payload.get("status") == "failure": + # Try to get structured error information first + error_information: Optional[StandardLoggingPayloadErrorInformation] = standard_logging_payload.get("error_information") + + if error_information: + error_info = DDLLMObsError( + message=error_information.get("error_message") or standard_logging_payload.get("error_str") or "Unknown error", + type=error_information.get("error_class"), + stack=error_information.get("traceback") + ) + return error_info + def _get_time_to_first_token_seconds(self, standard_logging_payload: StandardLoggingPayload) -> float: """ Get the time to first token in seconds @@ -232,8 +278,20 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): for now this handles logging /chat/completions responses """ + if response_obj is None: + return [] + if call_type in [CallTypes.completion.value, CallTypes.acompletion.value]: - return [response_obj["choices"][0]["message"]] + try: + # Safely extract message from response_obj, handle failure cases + if isinstance(response_obj, dict) and "choices" in response_obj: + choices = response_obj["choices"] + if choices and len(choices) > 0 and "message" in choices[0]: + return [choices[0]["message"]] + return [] + except (KeyError, IndexError, TypeError): + # In case of any error accessing the response structure, return empty list + return [] return [] def _get_datadog_span_kind(self, call_type: Optional[str]) -> Literal["llm", "tool", "task", "embedding", "retrieval"]: @@ -350,11 +408,11 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): def _get_dd_llm_obs_payload_metadata( self, standard_logging_payload: StandardLoggingPayload - ) -> Dict: + ) -> Dict[str, Any]: """ Fields to track in DD LLM Observability metadata from litellm standard logging payload """ - _metadata = { + _metadata: Dict[str, Any] = { "model_name": standard_logging_payload.get("model", "unknown"), "model_provider": standard_logging_payload.get( "custom_llm_provider", "unknown" @@ -365,8 +423,42 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): "cache_key": standard_logging_payload.get("cache_key", "unknown"), "saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0), } + + ######################################################### + # Add latency metrics to metadata + ######################################################### + latency_metrics = self._get_latency_metrics(standard_logging_payload) + _metadata.update({"latency_metrics": dict(latency_metrics)}) + _standard_logging_metadata: dict = ( dict(standard_logging_payload.get("metadata", {})) or {} ) _metadata.update(_standard_logging_metadata) return _metadata + + def _get_latency_metrics(self, standard_logging_payload: StandardLoggingPayload) -> DDLLMObsLatencyMetrics: + """ + Get the latency metrics from the standard logging payload + """ + latency_metrics: DDLLMObsLatencyMetrics = DDLLMObsLatencyMetrics() + # Add latency metrics to metadata + # Time to first token (convert from seconds to milliseconds for consistency) + time_to_first_token_seconds = self._get_time_to_first_token_seconds(standard_logging_payload) + if time_to_first_token_seconds > 0: + latency_metrics["time_to_first_token_ms"] = time_to_first_token_seconds * 1000 + + # LiteLLM overhead time + hidden_params = standard_logging_payload.get("hidden_params", {}) + litellm_overhead_ms = hidden_params.get("litellm_overhead_time_ms") + if litellm_overhead_ms is not None: + latency_metrics["litellm_overhead_time_ms"] = litellm_overhead_ms + + # Guardrail overhead latency + guardrail_info: Optional[StandardLoggingGuardrailInformation] = standard_logging_payload.get("guardrail_information") + if guardrail_info is not None: + _guardrail_duration_seconds: Optional[float] = guardrail_info.get("duration") + if _guardrail_duration_seconds is not None: + # Convert from seconds to milliseconds for consistency + latency_metrics["guardrail_overhead_time_ms"] = _guardrail_duration_seconds * 1000 + + return latency_metrics \ No newline at end of file diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 9ba547b3600..a99883ef7b6 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -18,6 +18,7 @@ from typing import ( cast, ) +from litellm.router_utils.batch_utils import InMemoryFile from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, @@ -453,6 +454,10 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData: filename, file_content, content_type = file_data elif len(file_data) == 4: filename, file_content, content_type, file_headers = file_data + elif isinstance(file_data, InMemoryFile): + filename = file_data.name + file_content = file_data + content_type = file_data.content_type else: file_content = file_data # Convert content to bytes diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 900239602df..07f652ecb9b 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -33,7 +33,12 @@ class SensitiveDataMasker: value_str = str(value) masked_length = len(value_str) - (self.visible_prefix + self.visible_suffix) - return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}" + + # Handle the case where visible_suffix is 0 to avoid showing the entire string + if self.visible_suffix == 0: + return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}" + else: + return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}" def is_sensitive_key(self, key: str) -> bool: key_lower = str(key).lower() diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 94abd2f814e..09b1888e04d 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -365,14 +365,16 @@ def get_azure_ad_token( azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope) except ValueError: verbose_logger.debug("Azure AD Token Provider could not be used.") - + ######################################################### # If litellm.enable_azure_ad_token_refresh is True and no other token provider is available, # try to get DefaultAzureCredential provider ######################################################### if azure_ad_token_provider is None and azure_ad_token is None: - azure_ad_token_provider = BaseAzureLLM._try_get_default_azure_credential_provider( - scope=scope, + azure_ad_token_provider = ( + BaseAzureLLM._try_get_default_azure_credential_provider( + scope=scope, + ) ) # Execute the token provider to get the token if available @@ -403,27 +405,27 @@ class BaseAzureLLM(BaseOpenAILLM): ) -> Optional[Callable[[], str]]: """ Try to get DefaultAzureCredential provider - + Args: scope: Azure scope for the token - + Returns: Token provider callable if DefaultAzureCredential is enabled and available, None otherwise """ from litellm.types.secret_managers.get_azure_ad_token_provider import ( AzureCredentialType, ) - - verbose_logger.debug( - "Attempting to use DefaultAzureCredential for Azure Auth" - ) - + + verbose_logger.debug("Attempting to use DefaultAzureCredential for Azure Auth") + try: azure_ad_token_provider = get_azure_ad_token_provider( azure_scope=scope, azure_credential=AzureCredentialType.DefaultAzureCredential, ) - verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential") + verbose_logger.debug( + "Successfully obtained Azure AD token provider using DefaultAzureCredential" + ) return azure_ad_token_provider except Exception as e: verbose_logger.debug(f"DefaultAzureCredential failed: {str(e)}") @@ -656,17 +658,17 @@ class BaseAzureLLM(BaseOpenAILLM): else: client = AzureOpenAI(**azure_client_params) # type: ignore return client - + @staticmethod def _base_validate_azure_environment( - headers: dict, litellm_params: Optional[GenericLiteLLMParams] + headers: dict, litellm_params: Optional[GenericLiteLLMParams] ) -> dict: litellm_params = litellm_params or GenericLiteLLMParams() - + # If api-key is already in headers, preserve it if "api-key" in headers: return headers - + api_key = ( litellm_params.api_key or litellm.api_key @@ -686,13 +688,24 @@ class BaseAzureLLM(BaseOpenAILLM): headers["Authorization"] = f"Bearer {azure_ad_token}" return headers - + @staticmethod def _get_base_azure_url( api_base: Optional[str], litellm_params: Optional[Union[GenericLiteLLMParams, Dict[str, Any]]], - route: Literal["/openai/responses", "/openai/vector_stores"] + route: Literal["/openai/responses", "/openai/vector_stores"], + default_api_version: Optional[Union[str, Literal["latest", "preview"]]] = None, ) -> str: + """ + Get the base Azure URL for the given route and API version. + + Args: + api_base: The base URL of the Azure API. + litellm_params: The litellm parameters. + route: The route to the API. + default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route. + """ + api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") if api_base is None: raise ValueError( @@ -702,7 +715,10 @@ class BaseAzureLLM(BaseOpenAILLM): # Extract api_version or use default litellm_params = litellm_params or {} - api_version = cast(Optional[str], litellm_params.get("api_version")) + api_version = ( + cast(Optional[str], litellm_params.get("api_version")) + or default_api_version + ) # Create a new dictionary with existing params query_params = dict(original_url.params) @@ -710,27 +726,28 @@ class BaseAzureLLM(BaseOpenAILLM): # Add api_version if needed if "api-version" not in query_params and api_version: query_params["api-version"] = api_version - + # Add the path to the base URL if route not in api_base: - new_url = _add_path_to_api_base( - api_base=api_base, ending_path=route - ) + new_url = _add_path_to_api_base(api_base=api_base, ending_path=route) else: new_url = api_base - + if BaseAzureLLM._is_azure_v1_api_version(api_version): # ensure the request go to /openai/v1 and not just /openai if "/openai/v1" not in new_url: parsed_url = httpx.URL(new_url) - new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1"))) - + new_url = str( + parsed_url.copy_with( + path=parsed_url.path.replace("/openai", "/openai/v1") + ) + ) # Use the new query_params dictionary final_url = httpx.URL(new_url).copy_with(params=query_params) return str(final_url) - + @staticmethod def _is_azure_v1_api_version(api_version: Optional[str]) -> bool: if api_version is None: diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index e3d37c8a15a..063d1af9c33 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -70,8 +70,13 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): - A complete URL string, e.g., "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview" """ + from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION + return BaseAzureLLM._get_base_azure_url( - api_base=api_base, litellm_params=litellm_params, route="/openai/responses" + api_base=api_base, + litellm_params=litellm_params, + route="/openai/responses", + default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION, ) ######################################################### diff --git a/litellm/llms/litellm_proxy/image_edit/transformation.py b/litellm/llms/litellm_proxy/image_edit/transformation.py new file mode 100644 index 00000000000..5f5e2bdb24d --- /dev/null +++ b/litellm/llms/litellm_proxy/image_edit/transformation.py @@ -0,0 +1,26 @@ +from typing import Optional + +from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig +from litellm.secret_managers.main import get_secret_str + + +class LiteLLMProxyImageEditConfig(OpenAIImageEditConfig): + """Configuration for image edit requests routed through LiteLLM Proxy.""" + + def validate_environment( + self, headers: dict, model: str, api_key: Optional[str] = None + ) -> dict: + api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY") + headers.update({"Authorization": f"Bearer {api_key}"}) + return headers + + def get_complete_url( + self, model: str, api_base: Optional[str], litellm_params: dict + ) -> str: + api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE") + if api_base is None: + raise ValueError( + "api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`" + ) + api_base = api_base.rstrip("/") + return f"{api_base}/images/edits" diff --git a/litellm/llms/litellm_proxy/image_generation/transformation.py b/litellm/llms/litellm_proxy/image_generation/transformation.py new file mode 100644 index 00000000000..6174424154d --- /dev/null +++ b/litellm/llms/litellm_proxy/image_generation/transformation.py @@ -0,0 +1,40 @@ +from typing import Optional + +from litellm.llms.openai.image_generation.gpt_transformation import ( + GPTImageGenerationConfig, +) +from litellm.secret_managers.main import get_secret_str + + +class LiteLLMProxyImageGenerationConfig(GPTImageGenerationConfig): + """Configuration for image generation requests routed through LiteLLM Proxy.""" + def validate_environment( + self, + headers: dict, + model: str, + messages, + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY") + headers.update({"Authorization": f"Bearer {api_key}"}) + return headers + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE") + if api_base is None: + raise ValueError( + "api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`" + ) + api_base = api_base.rstrip("/") + return f"{api_base}/images/generations" diff --git a/litellm/llms/openai_like/chat/handler.py b/litellm/llms/openai_like/chat/handler.py index 821fc9b7f15..ae0a3cc3551 100644 --- a/litellm/llms/openai_like/chat/handler.py +++ b/litellm/llms/openai_like/chat/handler.py @@ -49,8 +49,15 @@ async def make_call( model_response = ModelResponse(**response.json()) completion_stream = MockResponseIterator(model_response=model_response) else: + # Use aiter_text with explicit UTF-8 encoding to avoid ASCII encoding errors + async def utf8_aiter_lines(): + async for line in response.aiter_text(encoding='utf-8'): + for line_part in line.splitlines(keepends=True): + if line_part.strip(): + yield line_part.rstrip('\r\n') + completion_stream = ModelResponseIterator( - streaming_response=response.aiter_lines(), sync_stream=False + streaming_response=utf8_aiter_lines(), sync_stream=False ) # LOGGING logging_obj.post_call( @@ -93,8 +100,15 @@ def make_sync_call( model_response = ModelResponse(**response.json()) completion_stream = MockResponseIterator(model_response=model_response) else: + # Use iter_text with explicit UTF-8 encoding to avoid ASCII encoding errors + def utf8_iter_lines(): + for line in response.iter_text(encoding='utf-8'): + for line_part in line.splitlines(keepends=True): + if line_part.strip(): + yield line_part.rstrip('\r\n') + completion_stream = ModelResponseIterator( - streaming_response=response.iter_lines(), sync_stream=True + streaming_response=utf8_iter_lines(), sync_stream=True ) # LOGGING diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 37a4ab84dda..c5c4f46c92f 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -305,9 +305,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return None for tool in value: - openai_function_object: Optional[ - ChatCompletionToolParamFunctionChunk - ] = None + openai_function_object: Optional[ChatCompletionToolParamFunctionChunk] = ( + None + ) if "function" in tool: # tools list _openai_function_object = ChatCompletionToolParamFunctionChunk( # type: ignore **tool["function"] @@ -597,14 +597,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif param == "seed": optional_params["seed"] = value elif param == "reasoning_effort" and isinstance(value, str): - optional_params[ - "thinkingConfig" - ] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(value) + optional_params["thinkingConfig"] = ( + VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(value) + ) elif param == "thinking": - optional_params[ - "thinkingConfig" - ] = VertexGeminiConfig._map_thinking_param( - cast(AnthropicThinkingParam, value) + optional_params["thinkingConfig"] = ( + VertexGeminiConfig._map_thinking_param( + cast(AnthropicThinkingParam, value) + ) ) elif param == "modalities" and isinstance(value, list): response_modalities = self.map_response_modalities(value) @@ -1000,6 +1000,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): GenerateContentResponseBody, BidiGenerateContentServerMessage ], ) -> Usage: + if ( completion_response is not None and "usageMetadata" not in completion_response @@ -1038,6 +1039,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): text_tokens = detail.get("tokenCount", 0) if "thoughtsTokenCount" in usage_metadata: reasoning_tokens = usage_metadata["thoughtsTokenCount"] + + ## adjust 'text_tokens' to subtract cached tokens + if ( + (audio_tokens is None or audio_tokens == 0) + and text_tokens is not None + and text_tokens > 0 + and cached_tokens is not None + ): + text_tokens = text_tokens - cached_tokens + prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=cached_tokens, audio_tokens=audio_tokens, @@ -1344,28 +1355,28 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ## ADD METADATA TO RESPONSE ## setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) - model_response._hidden_params[ - "vertex_ai_grounding_metadata" - ] = grounding_metadata + model_response._hidden_params["vertex_ai_grounding_metadata"] = ( + grounding_metadata + ) setattr( model_response, "vertex_ai_url_context_metadata", url_context_metadata ) - model_response._hidden_params[ - "vertex_ai_url_context_metadata" - ] = url_context_metadata + model_response._hidden_params["vertex_ai_url_context_metadata"] = ( + url_context_metadata + ) setattr(model_response, "vertex_ai_safety_results", safety_ratings) - model_response._hidden_params[ - "vertex_ai_safety_results" - ] = safety_ratings # older approach - maintaining to prevent regressions + model_response._hidden_params["vertex_ai_safety_results"] = ( + safety_ratings # older approach - maintaining to prevent regressions + ) ## ADD CITATION METADATA ## setattr(model_response, "vertex_ai_citation_metadata", citation_metadata) - model_response._hidden_params[ - "vertex_ai_citation_metadata" - ] = citation_metadata # older approach - maintaining to prevent regressions + model_response._hidden_params["vertex_ai_citation_metadata"] = ( + citation_metadata # older approach - maintaining to prevent regressions + ) except Exception as e: raise VertexAIError( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 96680538888..a48c26bc87c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -722,7 +722,7 @@ "/v1/batch", "/v1/responses" ], - "supported_modalities": [ + "supported_modalities": [ "text", "image" ], @@ -730,13 +730,13 @@ "text" ], "supports_pdf_input": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, + "supports_function_calling": false, + "supports_parallel_function_calling": false, "supports_response_schema": true, "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, + "supports_tool_choice": false, "supports_native_streaming": true, "supports_reasoning": true }, @@ -762,13 +762,13 @@ "text" ], "supports_pdf_input": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, + "supports_function_calling": false, + "supports_parallel_function_calling": false, "supports_response_schema": true, "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, + "supports_tool_choice": false, "supports_native_streaming": true, "supports_reasoning": true }, @@ -11476,9 +11476,9 @@ }, "openrouter/anthropic/claude-sonnet-4": { "supports_computer_use": true, - "max_tokens": 8192, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 8192, + "max_output_tokens": 64000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, "input_cost_per_image": 0.0048, diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index 4015c1b815a..00000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index b48fe5be1c3..d02002aeff8 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -3,7 +3,16 @@ model_list: litellm_params: model: openai/fake api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_base: https://webhook.site/4feb0d46-4b23-468c-bf55-7008b5deb36d + - model_name: gpt-5-mini + litellm_params: + model: azure/gpt-5-mini + api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE") + api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY") + stream_timeout: 60 + merge_reasoning_content_in_choices: true + model_info: + mode: chat litellm_settings: cache: true @@ -11,3 +20,7 @@ litellm_settings: type: redis ttl: 600 supported_call_types: ["acompletion", "completion"] + + model_group_settings: + forward_client_headers_to_llm_api: + - fake-openai-endpoint diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 15953f4229b..d02671d69d8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -16,6 +16,7 @@ import json import sys from typing import Any, AsyncGenerator, List, Literal, Optional, Tuple, Union +import httpx from fastapi import HTTPException import litellm @@ -284,6 +285,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): response: Optional[Union[Any, litellm.ModelResponse]] = None, request_data: Optional[dict] = None ) -> BedrockGuardrailResponse: + from datetime import datetime + start_time = datetime.now() credentials, aws_region_name = self._load_credentials() bedrock_request_data: dict = dict( self.convert_to_bedrock_format( @@ -317,6 +320,18 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): data=prepared_request.body, # type: ignore headers=prepared_request.headers, # type: ignore ) + ######################################################### + # Add guardrail information to request trace + ######################################################### + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=response.json(), + request_data=request_data or {}, + guardrail_status=self._get_bedrock_guardrail_response_status(response=response), + start_time=start_time.timestamp(), + end_time=datetime.now().timestamp(), + duration=(datetime.now() - start_time).total_seconds(), + ) + ######################################################### if response.status_code == 200: # check if the response was flagged _json_response = response.json() @@ -338,6 +353,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return bedrock_guardrail_response + def _get_bedrock_guardrail_response_status(self, response: httpx.Response) -> Literal["success", "failure"]: + """ + Get the status of the bedrock guardrail response. + """ + if response.status_code == 200: + return "success" + return "failure" def _get_http_exception_for_blocked_guardrail(self, response: BedrockGuardrailResponse) -> HTTPException: """ @@ -501,10 +523,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): add_guardrail_to_applied_guardrails_header( request_data=data, guardrail_name=self.guardrail_name ) - return data - @log_guardrail_information async def async_moderation_hook( self, data: dict, @@ -561,7 +581,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return data - @log_guardrail_information async def async_post_call_success_hook( self, data: dict, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index aaeb86e34cc..adec337351c 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -384,6 +384,29 @@ class LiteLLMProxyRequestSetup: return returned_headers + @staticmethod + def add_headers_to_llm_call_by_model_group( + data: dict, headers: dict, user_api_key_dict: UserAPIKeyAuth + ) -> dict: + """ + Add headers to the LLM call by model group + """ + data_model = data.get("model") + if ( + data_model is not None + and litellm.model_group_settings is not None + and litellm.model_group_settings.forward_client_headers_to_llm_api + is not None + and data_model + in litellm.model_group_settings.forward_client_headers_to_llm_api + ): + _headers = LiteLLMProxyRequestSetup.add_headers_to_llm_call( + headers, user_api_key_dict + ) + if _headers != {}: + data["headers"] = _headers + return data + @staticmethod def add_litellm_data_for_backend_llm_call( *, @@ -439,7 +462,7 @@ class LiteLLMProxyRequestSetup: user_api_key_request_route=user_api_key_dict.request_route, ) return user_api_key_logged_metadata - + @staticmethod def add_user_api_key_auth_to_request_metadata( data: dict, @@ -457,9 +480,7 @@ class LiteLLMProxyRequestSetup: data[_metadata_variable_name].update(user_api_key_logged_metadata) data[_metadata_variable_name][ "user_api_key" - ] = ( - user_api_key_dict.api_key - ) # this is just the hashed token + ] = user_api_key_dict.api_key # this is just the hashed token data[_metadata_variable_name]["user_api_end_user_max_budget"] = getattr( user_api_key_dict, "end_user_max_budget", None @@ -624,6 +645,11 @@ async def add_litellm_data_to_request( # noqa: PLR0915 ) ) + # check for forwardable headers + data = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group( + data=data, headers=_headers, user_api_key_dict=user_api_key_dict + ) + # Parse user info from headers user = LiteLLMProxyRequestSetup.get_user_from_headers(_headers, general_settings) if user is not None: diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 5e1e3ea28e7..28ba8cd093b 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -5,3 +5,13 @@ model_list: - model_name: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0 litellm_params: model: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0 +litellm_settings: + callbacks: ["datadog_llm_observability"] + +guardrails: + - guardrail_name: "bedrock-pre-guard" + litellm_params: + guardrail: bedrock # supported values: "aporia", "bedrock", "lakera" + mode: "during_call" + guardrailIdentifier: ff6ujrregl1q + guardrailVersion: "DRAFT" \ No newline at end of file diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 3706b2f7fc9..ba453edbf66 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -33,7 +33,6 @@ from litellm.types.llms.openai import ( ResponseInputParam, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, - ResponseTextConfig, ) from litellm.types.responses.main import ( GenericResponseOutputItem, @@ -659,7 +658,7 @@ class LiteLLMCompletionResponsesConfig: ), reasoning=Reasoning(), status=getattr(chat_completion_response, "status", "completed"), - text=ResponseTextConfig(), + text={}, truncation=getattr(chat_completion_response, "truncation", None), usage=LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( chat_completion_response=chat_completion_response diff --git a/litellm/responses/main.py b/litellm/responses/main.py index b5f18cf9e1a..2a26c54a56e 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -21,7 +21,7 @@ from litellm.types.llms.openai import ( ResponseInputParam, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, - ResponseTextConfigParam, + ResponseText, ToolChoice, ToolParam, ) @@ -104,7 +104,7 @@ async def aresponses_api_with_mcp( background: Optional[bool] = None, stream: Optional[bool] = None, temperature: Optional[float] = None, - text: Optional[ResponseTextConfigParam] = None, + text: Optional["ResponseText"] = None, tool_choice: Optional[ToolChoice] = None, tools: Optional[Iterable[ToolParam]] = None, top_p: Optional[float] = None, @@ -237,7 +237,7 @@ async def aresponses( background: Optional[bool] = None, stream: Optional[bool] = None, temperature: Optional[float] = None, - text: Optional[ResponseTextConfigParam] = None, + text: Optional["ResponseText"] = None, tool_choice: Optional[ToolChoice] = None, tools: Optional[Iterable[ToolParam]] = None, top_p: Optional[float] = None, @@ -348,7 +348,7 @@ def responses( background: Optional[bool] = None, stream: Optional[bool] = None, temperature: Optional[float] = None, - text: Optional[ResponseTextConfigParam] = None, + text: Optional["ResponseText"] = None, tool_choice: Optional[ToolChoice] = None, tools: Optional[Iterable[ToolParam]] = None, top_p: Optional[float] = None, diff --git a/litellm/router.py b/litellm/router.py index 38de80141e9..190d19598c3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -93,9 +93,6 @@ from litellm.router_utils.fallback_event_handlers import ( get_fallback_model_group, run_async_fallback, ) -from litellm.router_utils.forward_clientside_headers_by_model_group import ( - ForwardClientSideHeadersByModelGroup, -) from litellm.router_utils.get_retry_from_policy import ( get_num_retries_from_retry_policy as _get_num_retries_from_retry_policy, ) @@ -624,9 +621,7 @@ class Router: Apply the default settings to the router. """ - default_pre_call_checks: OptionalPreCallChecks = [ - "forward_client_headers_by_model_group", - ] + default_pre_call_checks: OptionalPreCallChecks = [] self.add_optional_pre_call_checks(default_pre_call_checks) return None @@ -892,8 +887,6 @@ class Router: ) elif pre_call_check == "responses_api_deployment_check": _callback = ResponsesApiDeploymentCheck() - elif pre_call_check == "forward_client_headers_by_model_group": - _callback = ForwardClientSideHeadersByModelGroup() if _callback is not None: if self.optional_callbacks is None: self.optional_callbacks = [] @@ -4323,7 +4316,9 @@ class Router: "deployment", None ) # stable name - works for wildcard routes as well # Get model_group and id from kwargs like the sync version does - model_group = kwargs["litellm_params"]["metadata"].get("model_group", None) + model_group = kwargs["litellm_params"]["metadata"].get( + "model_group", None + ) model_info = kwargs["litellm_params"].get("model_info", {}) or {} id = model_info.get("id", None) if model_group is None or id is None: diff --git a/litellm/router_utils/batch_utils.py b/litellm/router_utils/batch_utils.py index 6617ad1f68e..6c5d80afc1b 100644 --- a/litellm/router_utils/batch_utils.py +++ b/litellm/router_utils/batch_utils.py @@ -7,9 +7,10 @@ from litellm.types.llms.openai import FileTypes, OpenAIFilesPurpose class InMemoryFile(io.BytesIO): - def __init__(self, content: bytes, name: str): + def __init__(self, content: bytes, name: str, content_type: str = "application/jsonl"): super().__init__(content) self.name = name + self.content_type = content_type def should_replace_model_in_jsonl( @@ -63,7 +64,7 @@ def replace_model_in_jsonl(file_content: FileTypes, new_model_name: str) -> File # Reassemble the modified lines and return as bytes modified_file_content = "\n".join(modified_lines).encode("utf-8") - return InMemoryFile(modified_file_content, name="modified_file.jsonl") # type: ignore + return InMemoryFile(modified_file_content, name="modified_file.jsonl", content_type="application/jsonl") # type: ignore except (json.JSONDecodeError, UnicodeDecodeError, TypeError): # return the original file content if there is an error replacing the model name diff --git a/litellm/router_utils/cooldown_cache.py b/litellm/router_utils/cooldown_cache.py index d987ab9444f..0a199d4b757 100644 --- a/litellm/router_utils/cooldown_cache.py +++ b/litellm/router_utils/cooldown_cache.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, Union from litellm import verbose_logger from litellm.caching.caching import DualCache from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -29,6 +30,12 @@ class CooldownCache: self.cache = cache self.default_cooldown_time = default_cooldown_time self.in_memory_cache = InMemoryCache() + # Initialize the masker with custom settings for exception strings + self.exception_masker = SensitiveDataMasker( + visible_prefix=50, # Show first 50 characters + visible_suffix=0, # Show last 0 characters + mask_char="*", # Use * for masking + ) def _common_add_cooldown_logic( self, model_id: str, original_exception, exception_status, cooldown_time: float @@ -39,7 +46,9 @@ class CooldownCache: # Store the cooldown information for the deployment separately cooldown_data = CooldownCacheValue( - exception_received=str(original_exception), + exception_received=self.exception_masker._mask_value( + str(original_exception) + ), status_code=str(exception_status), timestamp=current_time, cooldown_time=cooldown_time, diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 101159ad120..88bf1c0b277 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -118,16 +118,16 @@ def _should_run_cooldown_logic( "Should Not Run Cooldown Logic: deployment id is none or model group can't be found." ) return False - + ######################################################### # If time_to_cooldown is 0 or 0.0000000, don't run cooldown logic ######################################################### if time_to_cooldown is not None and math.isclose( - a=time_to_cooldown, - b=0.0, - abs_tol=1e-9 + a=time_to_cooldown, b=0.0, abs_tol=1e-9 ): - verbose_router_logger.debug("Should Not Run Cooldown Logic: time_to_cooldown is effectively 0") + verbose_router_logger.debug( + "Should Not Run Cooldown Logic: time_to_cooldown is effectively 0" + ) return False if litellm_router_instance.disable_cooldowns: @@ -275,8 +275,8 @@ def _set_cooldown_deployments( if ( _should_run_cooldown_logic( litellm_router_instance=litellm_router_instance, - deployment=deployment, - exception_status=exception_status, + deployment=deployment, + exception_status=exception_status, original_exception=original_exception, time_to_cooldown=time_to_cooldown, ) @@ -290,9 +290,9 @@ def _set_cooldown_deployments( verbose_router_logger.debug(f"Attempting to add {deployment} to cooldown list") if _should_cooldown_deployment( - litellm_router_instance=litellm_router_instance, - deployment=deployment, - exception_status=exception_status, + litellm_router_instance=litellm_router_instance, + deployment=deployment, + exception_status=exception_status, original_exception=original_exception, ): litellm_router_instance.cooldown_cache.add_deployment_to_cooldown( diff --git a/litellm/router_utils/forward_clientside_headers_by_model_group.py b/litellm/router_utils/forward_clientside_headers_by_model_group.py deleted file mode 100644 index 2e1a066a6cc..00000000000 --- a/litellm/router_utils/forward_clientside_headers_by_model_group.py +++ /dev/null @@ -1,84 +0,0 @@ -from typing import Any, Dict, Optional, TypedDict - -from litellm.types.utils import CallTypes - -from ..integrations.custom_logger import CustomLogger - - -class PotentialModelGroups(TypedDict): - deployment_model_name: Optional[str] - model_group_alias: Optional[str] - - -class ForwardClientSideHeadersByModelGroup(CustomLogger): - def get_potential_model_groups_from_kwargs( - self, kwargs: Dict[str, Any] - ) -> Optional[PotentialModelGroups]: - """ - Get the model group from the kwargs. - - Returns the potential model groups from the kwargs. - - deployment_model_name (useful for wildcard model names) - - model_group_alias (if the model is an alias) - """ - metadata = kwargs.get("litellm_metadata") or kwargs.get("metadata") - if metadata is None: - return None - deployment_model_name = metadata.get("deployment_model_name", None) - model_group_alias = metadata.get("model_group_alias", None) - return { - "deployment_model_name": deployment_model_name, - "model_group_alias": model_group_alias, - } - - def filter_headers(self, headers: Dict[str, Any]) -> Dict[str, Any]: - """ - Filter the headers to only include the headers that are forwarded to the LLM API. - - E.g. passing 'connection': 'keep-alive' will cause the request to hang, and not be acknowledged on the other side. - """ - return { - k: v - for k, v in headers.items() - if k.lower() not in ["connection", "content-length"] - } - - async def async_pre_call_deployment_hook( - self, kwargs: Dict[str, Any], call_type: Optional[CallTypes] - ) -> Optional[dict]: - """ - if kwargs["proxy_server_request"]["headers"] is not None: - and kwargs["forward_client_headers_to_llm_api"] is not None: - - add the headers to the request - kwargs["headers"].update(kwargs["proxy_server_request"]["headers"]) - """ - import litellm - - if litellm.model_group_settings is None: - return None - - potential_model_groups = self.get_potential_model_groups_from_kwargs(kwargs) - - if potential_model_groups is None: - return None - - if ( - "secret_fields" in kwargs - and kwargs["secret_fields"]["raw_headers"] is not None - and isinstance(kwargs["secret_fields"]["raw_headers"], dict) - ): - for model_group in potential_model_groups.values(): - if model_group is None: - continue - if ( - litellm.model_group_settings.forward_client_headers_to_llm_api - is not None - and model_group - in litellm.model_group_settings.forward_client_headers_to_llm_api - ): - kwargs.setdefault("headers", {}).update( - self.filter_headers(kwargs["secret_fields"]["raw_headers"]) - ) - - return kwargs diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index ba12e1cbede..63231923f1a 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -82,9 +82,14 @@ async def async_raise_no_deployment_exception( litellm_router_instance=litellm_router_instance, parent_otel_span=parent_otel_span, ) + verbose_router_logger.info( + f"No deployment found for model: {model}, cooldown_list with debug info: {_cooldown_list}" + ) + + cooldown_list_ids = [cooldown_model[0] for cooldown_model in (_cooldown_list or [])] return RouterRateLimitError( model=model, cooldown_time=_cooldown_time, enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks, - cooldown_list=_cooldown_list, + cooldown_list=cooldown_list_ids, ) diff --git a/litellm/types/integrations/datadog_llm_obs.py b/litellm/types/integrations/datadog_llm_obs.py index 25685db4833..82fb4fe3887 100644 --- a/litellm/types/integrations/datadog_llm_obs.py +++ b/litellm/types/integrations/datadog_llm_obs.py @@ -18,12 +18,20 @@ class OutputMeta(TypedDict): messages: List[Any] -class Meta(TypedDict): +class DDLLMObsError(TypedDict, total=False): + """Error information on the span according to DD LLM Obs API spec""" + message: str # The error message + stack: Optional[str] # The stack trace + type: Optional[str] # The error type + + +class Meta(TypedDict, total=False): # The span kind: "agent", "workflow", "llm", "tool", "task", "embedding", or "retrieval". kind: Literal["llm", "tool", "task", "embedding", "retrieval"] - input: InputMeta # The span’s input information. - output: OutputMeta # The span’s output information. + input: InputMeta # The span's input information. + output: OutputMeta # The span's output information. metadata: Dict[str, Any] + error: Optional[DDLLMObsError] # Error information on the span class LLMMetrics(TypedDict, total=False): @@ -35,7 +43,7 @@ class LLMMetrics(TypedDict, total=False): total_cost: float -class LLMObsPayload(TypedDict): +class LLMObsPayload(TypedDict, total=False): parent_id: str trace_id: str span_id: str @@ -45,6 +53,7 @@ class LLMObsPayload(TypedDict): duration: int metrics: LLMMetrics tags: List + status: Literal["ok", "error"] # Error status ("ok" or "error"). Defaults to "ok". class DDSpanAttributes(TypedDict): @@ -62,4 +71,10 @@ class DatadogLLMObsInitParams(StandardCustomLoggerInitParams): """ Params for initializing a DatadogLLMObs logger on litellm """ - pass \ No newline at end of file + pass + + +class DDLLMObsLatencyMetrics(TypedDict, total=False): + time_to_first_token_ms: float + litellm_overhead_time_ms: float + guardrail_overhead_time_ms: float \ No newline at end of file diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index f2f64fa3c5d..30bf7f7c956 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -37,15 +37,21 @@ from openai.types.responses.response import ( IncompleteDetails, Response, ResponseOutputItem, - ResponseTextConfig, Tool, ToolChoice, ) + +# Handle OpenAI SDK version compatibility for Text type +try: + from openai.types.responses.response_create_params import Text as ResponseText +except (ImportError, AttributeError): + # Fall back to the concrete config type available in all SDK versions + from openai.types.responses.response_text_config_param import ResponseTextConfigParam as ResponseText + from openai.types.responses.response_create_params import ( Reasoning, ResponseIncludable, ResponseInputParam, - ResponseTextConfigParam, ToolChoice, ToolParam, ) @@ -959,7 +965,7 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): background: Optional[bool] stream: Optional[bool] temperature: Optional[float] - text: Optional[ResponseTextConfigParam] + text: Optional["ResponseText"] tool_choice: Optional[ToolChoice] tools: Optional[List[ALL_RESPONSES_API_TOOL_PARAMS]] top_p: Optional[float] @@ -1034,7 +1040,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): previous_response_id: Optional[str] reasoning: Optional[Reasoning] status: Optional[str] - text: Optional[ResponseTextConfig] + text: Optional[Union["ResponseText", Dict[str, Any]]] truncation: Optional[Literal["auto", "disabled"]] usage: Optional[ResponseAPIUsage] user: Optional[str] diff --git a/litellm/utils.py b/litellm/utils.py index 2ae038e6e7b..05055b7df8b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7305,6 +7305,12 @@ class ProviderConfigManager: ) return get_gemini_image_generation_config(model) + elif LlmProviders.LITELLM_PROXY == provider: + from litellm.llms.litellm_proxy.image_generation.transformation import ( + LiteLLMProxyImageGenerationConfig, + ) + + return LiteLLMProxyImageGenerationConfig() return None @staticmethod @@ -7341,6 +7347,12 @@ class ProviderConfigManager: ) return RecraftImageEditConfig() + elif LlmProviders.LITELLM_PROXY == provider: + from litellm.llms.litellm_proxy.image_edit.transformation import ( + LiteLLMProxyImageEditConfig, + ) + + return LiteLLMProxyImageEditConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 96680538888..a48c26bc87c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -722,7 +722,7 @@ "/v1/batch", "/v1/responses" ], - "supported_modalities": [ + "supported_modalities": [ "text", "image" ], @@ -730,13 +730,13 @@ "text" ], "supports_pdf_input": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, + "supports_function_calling": false, + "supports_parallel_function_calling": false, "supports_response_schema": true, "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, + "supports_tool_choice": false, "supports_native_streaming": true, "supports_reasoning": true }, @@ -762,13 +762,13 @@ "text" ], "supports_pdf_input": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, + "supports_function_calling": false, + "supports_parallel_function_calling": false, "supports_response_schema": true, "supports_vision": true, "supports_prompt_caching": true, "supports_system_messages": true, - "supports_tool_choice": true, + "supports_tool_choice": false, "supports_native_streaming": true, "supports_reasoning": true }, @@ -11476,9 +11476,9 @@ }, "openrouter/anthropic/claude-sonnet-4": { "supports_computer_use": true, - "max_tokens": 8192, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 8192, + "max_output_tokens": 64000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, "input_cost_per_image": 0.0048, diff --git a/pyproject.toml b/pyproject.toml index ac6563f9a14..574348979ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.75.8" +version = "1.75.9" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -155,7 +155,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.75.8" +version = "1.75.9" version_files = [ "pyproject.toml:^version" ] diff --git a/scripts/test_groq_streaming_issue.py b/scripts/test_groq_streaming_issue.py new file mode 100644 index 00000000000..0a996c0c209 --- /dev/null +++ b/scripts/test_groq_streaming_issue.py @@ -0,0 +1,54 @@ +""" +Test script to reproduce the Groq streaming ASCII encoding issue. + +This reproduces the issue described in #12660 where streaming responses +containing non-ASCII characters like µ cause encoding errors. +""" +import asyncio +import os +import traceback +from litellm import acompletion + +async def test_groq_streaming_with_special_chars(): + """Test that reproduces the ASCII encoding issue with Groq streaming.""" + try: + print("Testing acompletion + streaming with Groq...") + + # Test message that should trigger the µ character or similar non-ASCII content + test_messages = [ + {"content": "What is the symbol for micro? Please include the µ symbol in your response.", "role": "user"} + ] + + # This should trigger the ASCII encoding error described in the issue + response = await acompletion( + model="groq/llama-3.3-70b-versatile", + messages=test_messages, + stream=True + ) + + print(f"Response type: {type(response)}") + + # Try to iterate through the stream + async for chunk in response: + print(f"Chunk: {chunk}") + + print("✅ Test completed successfully - no encoding errors!") + + except Exception as e: + print(f"❌ Error occurred: {e}") + print(f"Error type: {type(e)}") + print(f"Traceback:\n{traceback.format_exc()}") + return False + + return True + +if __name__ == "__main__": + # Note: This requires GROQ_API_KEY to be set + if not os.getenv("GROQ_API_KEY"): + print("⚠️ GROQ_API_KEY not set. Skipping test.") + else: + success = asyncio.run(test_groq_streaming_with_special_chars()) + if success: + print("🎉 All tests passed!") + else: + print("💥 Test failed!") \ No newline at end of file diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index bf15e44879b..72f6fb8c9cf 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -21,7 +21,6 @@ from litellm.types.utils import StandardLoggingPayload from litellm.types.llms.openai import ( ResponseCompletedEvent, ResponsesAPIResponse, - ResponseTextConfig, ResponseAPIUsage, IncompleteDetails, ) @@ -78,7 +77,7 @@ def validate_responses_api_response(response, final_chunk: bool = False): "previous_response_id": (str, type(None)), "reasoning": dict, "status": str, - "text": ResponseTextConfig, + "text": dict, "truncation": (str, type(None)), "usage": ResponseAPIUsage, "user": (str, type(None)), diff --git a/tests/llm_responses_api_testing/test_anthropic_responses_api.py b/tests/llm_responses_api_testing/test_anthropic_responses_api.py index 1e24457145f..8f7a96a016d 100644 --- a/tests/llm_responses_api_testing/test_anthropic_responses_api.py +++ b/tests/llm_responses_api_testing/test_anthropic_responses_api.py @@ -17,7 +17,6 @@ from litellm.types.utils import StandardLoggingPayload from litellm.types.llms.openai import ( ResponseCompletedEvent, ResponsesAPIResponse, - ResponseTextConfig, ResponseAPIUsage, IncompleteDetails, ) diff --git a/tests/llm_responses_api_testing/test_azure_responses_api.py b/tests/llm_responses_api_testing/test_azure_responses_api.py index 82f68b4a065..692f6774ac4 100644 --- a/tests/llm_responses_api_testing/test_azure_responses_api.py +++ b/tests/llm_responses_api_testing/test_azure_responses_api.py @@ -13,7 +13,6 @@ from litellm.types.utils import StandardLoggingPayload from litellm.types.llms.openai import ( ResponseCompletedEvent, ResponsesAPIResponse, - ResponseTextConfig, ResponseAPIUsage, IncompleteDetails, ) diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 10cb65d59ff..d7042c4304f 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -18,7 +18,6 @@ from litellm.types.utils import StandardLoggingPayload from litellm.types.llms.openai import ( ResponseCompletedEvent, ResponsesAPIResponse, - ResponseTextConfig, ResponseAPIUsage, IncompleteDetails, ) diff --git a/tests/llm_translation/test_azure_openai.py b/tests/llm_translation/test_azure_openai.py index a27d0dd165d..a1b05cbb4ae 100644 --- a/tests/llm_translation/test_azure_openai.py +++ b/tests/llm_translation/test_azure_openai.py @@ -630,3 +630,17 @@ def test_azure_openai_responses_bridge(): == "test-azure-computer-use-preview" ) assert mock_responses.call_args.kwargs["custom_llm_provider"] == "azure" + + +def test_azure_openai_gpt_5_responses_api(): + from litellm import responses + + litellm._turn_on_debug() + + response = responses( + model="azure/gpt-5", + input="Hello world", + api_key=os.getenv("AZURE_SWEDEN_API_KEY"), + api_base=os.getenv("AZURE_SWEDEN_API_BASE"), + ) + print(f"response: {response}") diff --git a/tests/llm_translation/test_litellm_proxy_provider.py b/tests/llm_translation/test_litellm_proxy_provider.py index 1ae507a10d5..1a09441979c 100644 --- a/tests/llm_translation/test_litellm_proxy_provider.py +++ b/tests/llm_translation/test_litellm_proxy_provider.py @@ -2,6 +2,7 @@ import json import os import sys from datetime import datetime +from io import BytesIO from unittest.mock import AsyncMock sys.path.insert( @@ -184,6 +185,127 @@ async def test_litellm_gateway_from_sdk_image_generation(is_async): assert "dall-e-3" == mock_method.call_args.kwargs["model"] +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.asyncio +async def test_litellm_gateway_image_generation_direct(is_async): + """Test image generation using the litellm_proxy provider directly.""" + litellm._turn_on_debug() + + # Create mock response that matches OpenAI's response structure + mock_openai_response = MagicMock() + mock_openai_response.model_dump.return_value = { + "created": 1, + "data": [{"url": "https://example.com/image.png"}], + } + + if is_async: + # Mock the AsyncOpenAI client that gets created inside _get_openai_client + mock_async_client = AsyncMock() + mock_async_client.images.generate = AsyncMock(return_value=mock_openai_response) + + with patch("litellm.llms.openai.openai.AsyncOpenAI", return_value=mock_async_client) as mock_async_constructor: + response = await litellm.aimage_generation( + model="litellm_proxy/dall-e-3", + prompt="A beautiful sunset over mountains", + api_base="http://my-proxy", + api_key="sk-1234", + ) + + # Verify the AsyncOpenAI client constructor was called with correct parameters + mock_async_constructor.assert_called_once() + constructor_kwargs = mock_async_constructor.call_args.kwargs + print("KWARGS to Async OpenAI constructor=", constructor_kwargs) + assert constructor_kwargs["api_key"] == "sk-1234" + assert constructor_kwargs["base_url"] == "http://my-proxy" + + # Verify the AsyncOpenAI client was called correctly + mock_async_client.images.generate.assert_awaited_once() + call_kwargs = mock_async_client.images.generate.call_args.kwargs + assert call_kwargs["model"] == "dall-e-3" + assert call_kwargs["prompt"] == "A beautiful sunset over mountains" + else: + # Mock the sync OpenAI client that gets created inside _get_openai_client + mock_sync_client = MagicMock() + mock_sync_client.images.generate.return_value = mock_openai_response + + with patch("litellm.llms.openai.openai.OpenAI", return_value=mock_sync_client) as mock_sync_constructor: + response = litellm.image_generation( + model="litellm_proxy/dall-e-3", + prompt="A beautiful sunset over mountains", + api_base="http://my-proxy", + api_key="sk-1234", + ) + + # Verify the OpenAI client constructor was called with correct parameters + mock_sync_constructor.assert_called_once() + constructor_kwargs = mock_sync_constructor.call_args.kwargs + assert constructor_kwargs["api_key"] == "sk-1234" + assert constructor_kwargs["base_url"] == "http://my-proxy" + + # Verify the OpenAI client was called correctly + mock_sync_client.images.generate.assert_called_once() + call_kwargs = mock_sync_client.images.generate.call_args.kwargs + assert call_kwargs["model"] == "dall-e-3" + assert call_kwargs["prompt"] == "A beautiful sunset over mountains" + + # Verify the response structure + assert response is not None + assert hasattr(response, 'data') or isinstance(response, dict) + + +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.asyncio +async def test_litellm_gateway_from_sdk_image_edit(is_async): + litellm._turn_on_debug() + + mock_response = { + "created": 1, + "data": [{"b64_json": ""}], + } + + class MockResponse: + def __init__(self, json_data, status_code): + self._json_data = json_data + self.status_code = status_code + self.text = json.dumps(json_data) + + def json(self): + return self._json_data + + image_file = BytesIO(b"fake-image") + + if is_async: + mock_post = AsyncMock(return_value=MockResponse(mock_response, 200)) + patch_target = "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post" + else: + mock_post = MagicMock(return_value=MockResponse(mock_response, 200)) + patch_target = "litellm.llms.custom_httpx.http_handler.HTTPHandler.post" + + with patch(patch_target, new=mock_post): + if is_async: + await litellm.aimage_edit( + model="litellm_proxy/gpt-image-1", + prompt="A test prompt", + image=[image_file], + api_base="http://my-proxy", + api_key="sk-1234", + ) + mock_post.assert_awaited_once() + else: + litellm.image_edit( + model="litellm_proxy/gpt-image-1", + prompt="A test prompt", + image=[image_file], + api_base="http://my-proxy", + api_key="sk-1234", + ) + mock_post.assert_called_once() + + called_kwargs = mock_post.call_args.kwargs + assert called_kwargs["url"] == "http://my-proxy/images/edits" + assert called_kwargs["headers"]["Authorization"] == "Bearer sk-1234" + + @pytest.mark.parametrize("is_async", [False, True]) @pytest.mark.asyncio async def test_litellm_gateway_from_sdk_transcription(is_async): diff --git a/tests/router_unit_tests/test_router_batch_utils.py b/tests/router_unit_tests/test_router_batch_utils.py index 94cd6e001e4..adbab7ce881 100644 --- a/tests/router_unit_tests/test_router_batch_utils.py +++ b/tests/router_unit_tests/test_router_batch_utils.py @@ -18,7 +18,7 @@ from io import BytesIO from typing import Dict, List from litellm.router_utils.batch_utils import ( replace_model_in_jsonl, - _get_router_metadata_variable_name, + _get_router_metadata_variable_name, InMemoryFile, ) @@ -57,6 +57,9 @@ def test_bytes_input(sample_jsonl_bytes): result = replace_model_in_jsonl(sample_jsonl_bytes, new_model) assert result is not None + assert isinstance(result, InMemoryFile) + assert result.name == "modified_file.jsonl" + assert result.content_type == "application/jsonl" def test_tuple_input(sample_jsonl_bytes): @@ -66,6 +69,9 @@ def test_tuple_input(sample_jsonl_bytes): result = replace_model_in_jsonl(test_tuple, new_model) assert result is not None + assert isinstance(result, InMemoryFile) + assert result.name == "modified_file.jsonl" + assert result.content_type == "application/jsonl" def test_file_like_object(sample_file_like): @@ -74,6 +80,9 @@ def test_file_like_object(sample_file_like): result = replace_model_in_jsonl(sample_file_like, new_model) assert result is not None + assert isinstance(result, InMemoryFile) + assert result.name == "modified_file.jsonl" + assert result.content_type == "application/jsonl" def test_router_metadata_variable_name(): diff --git a/tests/router_unit_tests/test_router_handle_error.py b/tests/router_unit_tests/test_router_handle_error.py index 39b9814ccc8..660b3885126 100644 --- a/tests/router_unit_tests/test_router_handle_error.py +++ b/tests/router_unit_tests/test_router_handle_error.py @@ -1,6 +1,7 @@ import sys, os, time import traceback, asyncio import pytest +from typing import List sys.path.insert( 0, os.path.abspath("../..") @@ -111,3 +112,147 @@ async def test_send_llm_exception_alert_when_proxy_server_request_in_kwargs(): # Assert that no exception was raised and the function completed successfully mock_router.slack_alerting_logger.send_alert.assert_not_called() + + +@pytest.mark.asyncio +async def test_async_raise_no_deployment_exception(): + """ + Test that async_raise_no_deployment_exception returns a RouterRateLimitError + with cooldown_list containing just IDs (not tuples with debug info). + """ + from litellm.router_utils.handle_error import async_raise_no_deployment_exception + from litellm.types.router import RouterRateLimitError + from unittest.mock import patch + + # Create a mock LitellmRouter instance + mock_router = MagicMock() + mock_router.get_model_ids.return_value = ["deployment-1", "deployment-2"] + mock_router.cooldown_cache.get_min_cooldown.return_value = 30.0 + mock_router.enable_pre_call_checks = True + + # Mock the _async_get_cooldown_deployments_with_debug_info function + # It should return a list of tuples where each tuple contains (model_id, debug_info) + mock_cooldown_list = [ + ("deployment-1", {"error": "rate_limit", "time": "2024-01-01"}), + ("deployment-2", {"error": "server_error", "time": "2024-01-01"}), + ("deployment-3", {"error": "timeout", "time": "2024-01-01"}), + ] + + with patch( + "litellm.router_utils.handle_error._async_get_cooldown_deployments_with_debug_info", + return_value=mock_cooldown_list, + ): + # Call the function + result = await async_raise_no_deployment_exception( + litellm_router_instance=mock_router, + model="gpt-3.5-turbo", + parent_otel_span=None, + ) + + # Assert that the function returns a RouterRateLimitError + assert isinstance(result, RouterRateLimitError) + + # Assert that the error has the correct properties + assert result.model == "gpt-3.5-turbo" + assert result.cooldown_time == 30.0 + assert result.enable_pre_call_checks is True + + # Assert that cooldown_list contains only IDs (extracted from tuples) + expected_cooldown_list = ["deployment-1", "deployment-2", "deployment-3"] + assert result.cooldown_list == expected_cooldown_list + + # Verify that cooldown_list contains only strings (IDs), not tuples + for item in result.cooldown_list: + assert isinstance(item, str), f"Expected string ID, got {type(item)}: {item}" + + # Verify mock calls + mock_router.get_model_ids.assert_called_once_with(model_name="gpt-3.5-turbo") + mock_router.cooldown_cache.get_min_cooldown.assert_called_once_with( + model_ids=["deployment-1", "deployment-2"], parent_otel_span=None + ) + + +@pytest.mark.asyncio +async def test_async_raise_no_deployment_exception_empty_cooldown_list(): + """ + Test that async_raise_no_deployment_exception handles empty cooldown list correctly. + """ + from litellm.router_utils.handle_error import async_raise_no_deployment_exception + from litellm.types.router import RouterRateLimitError + from unittest.mock import patch + + # Create a mock LitellmRouter instance + mock_router = MagicMock() + mock_router.get_model_ids.return_value = ["deployment-1", "deployment-2"] + mock_router.cooldown_cache.get_min_cooldown.return_value = 15.0 + mock_router.enable_pre_call_checks = False + + # Mock empty cooldown list + mock_cooldown_list: List = [] + + with patch( + "litellm.router_utils.handle_error._async_get_cooldown_deployments_with_debug_info", + return_value=mock_cooldown_list, + ): + # Call the function + result = await async_raise_no_deployment_exception( + litellm_router_instance=mock_router, + model="claude-3-sonnet", + parent_otel_span=None, + ) + + # Assert that the function returns a RouterRateLimitError + assert isinstance(result, RouterRateLimitError) + + # Assert that the error has the correct properties + assert result.model == "claude-3-sonnet" + assert result.cooldown_time == 15.0 + assert result.enable_pre_call_checks is False + + # Assert that cooldown_list is an empty list when no cooldowns exist + assert result.cooldown_list == [] + assert isinstance(result.cooldown_list, list) + + +@pytest.mark.asyncio +async def test_async_raise_no_deployment_exception_none_cooldown_list(): + """ + Test that async_raise_no_deployment_exception handles None cooldown list correctly. + Note: In practice, _async_get_cooldown_deployments_with_debug_info should never return None + based on the implementation, but this tests defensive programming. + """ + from litellm.router_utils.handle_error import async_raise_no_deployment_exception + from litellm.types.router import RouterRateLimitError + from unittest.mock import patch + + # Create a mock LitellmRouter instance + mock_router = MagicMock() + mock_router.get_model_ids.return_value = [] + mock_router.cooldown_cache.get_min_cooldown.return_value = 45.0 + mock_router.enable_pre_call_checks = True + + # Mock None cooldown list (though this shouldn't happen in practice) + mock_cooldown_list = None + + with patch( + "litellm.router_utils.handle_error._async_get_cooldown_deployments_with_debug_info", + return_value=mock_cooldown_list, + ): + # After the defensive fix, this should handle None gracefully and return empty list + result = await async_raise_no_deployment_exception( + litellm_router_instance=mock_router, + model="gpt-4", + parent_otel_span=None, + ) + + # Assert that the function returns a RouterRateLimitError + assert isinstance(result, RouterRateLimitError) + + # Assert that the error has the correct properties + assert result.model == "gpt-4" + assert result.cooldown_time == 45.0 + assert result.enable_pre_call_checks is True + + # Assert that cooldown_list is an empty list when cooldown_list is None + assert result.cooldown_list == [] + assert isinstance(result.cooldown_list, list) diff --git a/tests/test_litellm/caching/test_s3_cache.py b/tests/test_litellm/caching/test_s3_cache.py new file mode 100644 index 00000000000..a37f7f57064 --- /dev/null +++ b/tests/test_litellm/caching/test_s3_cache.py @@ -0,0 +1,126 @@ +import os +import sys +from unittest.mock import MagicMock, patch +import json +import datetime + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +from litellm.caching.s3_cache import S3Cache + + +@pytest.fixture +def mock_s3_dependencies(): + mock_s3_client = MagicMock() + + with patch("boto3.client", return_value=mock_s3_client): + yield {"s3_client": mock_s3_client} + + +def test_s3_cache_set_cache(mock_s3_dependencies): + """Test basic set_cache functionality""" + cache = S3Cache("test-bucket") + test_value = {"key": "value", "number": 42} + + cache.set_cache("test_key", test_value) + + cache.s3_client.put_object.assert_called_once() + call_args = cache.s3_client.put_object.call_args + + assert call_args[1]["Bucket"] == "test-bucket" + assert call_args[1]["Key"] == "test_key" + assert call_args[1]["Body"] == json.dumps(test_value) + assert call_args[1]["ContentType"] == "application/json" + assert call_args[1]["ContentLanguage"] == "en" + assert call_args[1]["ContentDisposition"] == 'inline; filename="test_key.json"' + + +def test_s3_cache_set_cache_with_ttl(mock_s3_dependencies): + """Test set_cache with TTL functionality""" + cache = S3Cache("test-bucket") + test_value = {"key": "value"} + ttl = datetime.timedelta(seconds=3600) # 1 hour + + cache.set_cache("test_key", test_value, ttl=ttl) + + cache.s3_client.put_object.assert_called_once() + call_args = cache.s3_client.put_object.call_args + + assert "Expires" in call_args[1] + assert "CacheControl" in call_args[1] + assert "max-age=1:00:00" in call_args[1]["CacheControl"] + + +def test_s3_cache_get_cache(mock_s3_dependencies): + """Test basic get_cache functionality""" + cache = S3Cache("test-bucket") + + mock_response = { + "Body": MagicMock() + } + mock_response["Body"].read.return_value = b'{"key": "value", "number": 42}' + cache.s3_client.get_object.return_value = mock_response + + result = cache.get_cache("test_key") + + cache.s3_client.get_object.assert_called_once_with( + Bucket="test-bucket", + Key="test_key" + ) + + assert result == {"key": "value", "number": 42} + + +def test_s3_cache_get_cache_not_found(mock_s3_dependencies): + """Test get_cache when key is not found""" + import botocore.exceptions + + cache = S3Cache("test-bucket") + + error_response = {"Error": {"Code": "NoSuchKey"}} + cache.s3_client.get_object.side_effect = botocore.exceptions.ClientError( + error_response, "GetObject" + ) + + result = cache.get_cache("nonexistent_key") + + cache.s3_client.get_object.assert_called_once_with( + Bucket="test-bucket", + Key="nonexistent_key" + ) + assert result is None + + +def test_s3_key_transformation(): + """Test the _to_s3_key method for key transformation""" + cache = S3Cache("test-bucket") + + # Test basic key transformation (colon to slash) + result = cache._to_s3_key("user:123:session:456") + assert result == "user/123/session/456" + + # Test with s3_path prefix + cache_with_prefix = S3Cache("test-bucket", s3_path="cache/data") + result = cache_with_prefix._to_s3_key("namespace:key") + assert result == "cache/data/namespace/key" + + # Test with s3_path that has trailing slash + cache_with_slash = S3Cache("test-bucket", s3_path="cache/data/") + result = cache_with_slash._to_s3_key("namespace:key") + assert result == "cache/data/namespace/key" + + +def test_s3_cache_initialization(): + """Test S3Cache initialization with various parameters""" + # Test basic initialization + cache = S3Cache("test-bucket") + assert cache.bucket_name == "test-bucket" + assert cache.key_prefix == "" + + # Test with s3_path + cache_with_path = S3Cache("test-bucket", s3_path="my/cache/path") + assert cache_with_path.key_prefix == "my/cache/path/" \ No newline at end of file diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py index 18d3efdddd9..25263e31f9f 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py @@ -20,10 +20,12 @@ from litellm.types.integrations.datadog_llm_obs import ( LLMObsPayload, ) from litellm.types.utils import ( + StandardLoggingGuardrailInformation, StandardLoggingHiddenParams, StandardLoggingMetadata, StandardLoggingModelInformation, StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, ) @@ -81,6 +83,67 @@ def create_standard_logging_payload_with_cache() -> StandardLoggingPayload: ) +def create_standard_logging_payload_with_failure() -> StandardLoggingPayload: + """Create a StandardLoggingPayload object for failure testing""" + return StandardLoggingPayload( + id="test-request-id-failure-789", + call_type="completion", + response_cost=0.0, + response_cost_failure_debug_info=None, + status="failure", + total_tokens=0, + prompt_tokens=10, + completion_tokens=0, + startTime=1234567890.0, + endTime=1234567891.0, + completionStartTime=1234567890.5, + model_map_information=StandardLoggingModelInformation( + model_map_key="gpt-4", model_map_value=None + ), + model="gpt-4", + model_id="model-123", + model_group="openai-gpt", + api_base="https://api.openai.com", + metadata=StandardLoggingMetadata( + user_api_key_hash="test_hash", + user_api_key_org_id=None, + user_api_key_alias="test_alias", + user_api_key_team_id="test_team", + user_api_key_user_id="test_user", + user_api_key_team_alias="test_team_alias", + spend_logs_metadata=None, + requester_ip_address="127.0.0.1", + requester_metadata=None, + ), + cache_hit=False, + cache_key=None, + saved_cache_cost=0.0, + request_tags=[], + end_user=None, + requester_ip_address="127.0.0.1", + messages=[{"role": "user", "content": "Hello, world!"}], + response=None, + error_str="RateLimitError: You exceeded your current quota", + error_information=StandardLoggingPayloadErrorInformation( + error_code="rate_limit_exceeded", + error_class="RateLimitError", + llm_provider="openai", + traceback="Traceback (most recent call last):\n File test.py, line 1\n RateLimitError: You exceeded your current quota", + error_message="RateLimitError: You exceeded your current quota" + ), + model_parameters={"stream": False}, + hidden_params=StandardLoggingHiddenParams( + model_id="model-123", + cache_key=None, + api_base="https://api.openai.com", + response_cost="0.0", + additional_headers=None, + ), + trace_id="test-trace-id-failure-456", + custom_llm_provider="openai", + ) + + class TestDataDogLLMObsLogger: """Test suite for DataDog LLM Observability Logger""" @@ -118,7 +181,7 @@ class TestDataDogLLMObsLogger: start_time = datetime.now() end_time = datetime.now() - payload = logger.create_llm_obs_payload(kwargs, mock_response_obj, start_time, end_time) + payload = logger.create_llm_obs_payload(kwargs, start_time, end_time) # Test 1: Verify total_cost is correctly extracted from response_cost assert payload["metrics"].get("total_cost") == 0.05 @@ -148,7 +211,7 @@ class TestDataDogLLMObsLogger: start_time = datetime.now() end_time = datetime.now() - payload = logger.create_llm_obs_payload(kwargs, mock_response_obj, start_time, end_time) + payload = logger.create_llm_obs_payload(kwargs, start_time, end_time) # Test the _get_dd_llm_obs_payload_metadata method directly metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) @@ -217,9 +280,56 @@ class TestDataDogLLMObsLogger: assert logger._get_datadog_span_kind("unknown_call_type") == "llm" assert logger._get_datadog_span_kind(None) == "llm" + @pytest.mark.asyncio + async def test_async_log_failure_event(self, mock_env_vars): + """Test that async_log_failure_event correctly processes failure payloads according to DD LLM Obs API spec""" + with patch('litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client'), \ + patch('asyncio.create_task'): + logger = DataDogLLMObsLogger() + + # Ensure log_queue starts empty + logger.log_queue = [] + + standard_failure_payload = create_standard_logging_payload_with_failure() + + kwargs = { + "standard_logging_object": standard_failure_payload, + "model": "gpt-4", + "litellm_params": {"metadata": {}} + } + + start_time = datetime.now() + end_time = datetime.now() + timedelta(seconds=2) + + # Mock async_send_batch to prevent actual network calls + with patch.object(logger, 'async_send_batch') as mock_send_batch: + # Call the method under test + await logger.async_log_failure_event(kwargs, None, start_time, end_time) + + # Verify payload was added to queue + assert len(logger.log_queue) == 1 + + # Verify the payload has correct failure characteristics according to DD LLM Obs API spec + payload = logger.log_queue[0] + assert payload["trace_id"] == "test-trace-id-failure-456" + assert payload["meta"]["metadata"]["id"] == "test-request-id-failure-789" + assert payload["status"] == "error" + + # Verify error information follows DD LLM Obs API spec + assert payload["meta"]["error"]["message"] == "RateLimitError: You exceeded your current quota" + assert payload["meta"]["error"]["type"] == "RateLimitError" + assert payload["meta"]["error"]["stack"] == "Traceback (most recent call last):\n File test.py, line 1\n RateLimitError: You exceeded your current quota" + + assert payload["metrics"]["total_cost"] == 0.0 + assert payload["metrics"]["total_tokens"] == 0 + assert payload["metrics"]["output_tokens"] == 0 + + # Verify batch sending not triggered (queue size < batch_size) + mock_send_batch.assert_not_called() -class TestDataDogLLMObsLogger(DataDogLLMObsLogger): + +class TestDataDogLLMObsLoggerForRedaction(DataDogLLMObsLogger): """Test suite for DataDog LLM Observability Logger""" def __init__(self, **kwargs): super().__init__(**kwargs) @@ -245,7 +355,7 @@ async def test_dd_llms_obs_redaction(mock_env_vars): litellm._turn_on_debug() from litellm.types.utils import LiteLLMCommonStrings litellm.datadog_llm_observability_params = DatadogLLMObsInitParams(turn_off_message_logging=True) - dd_llms_obs_logger = TestDataDogLLMObsLogger() + dd_llms_obs_logger = TestDataDogLLMObsLoggerForRedaction() test_s3_logger = TestS3Logger() litellm.callbacks = [ dd_llms_obs_logger, @@ -315,3 +425,145 @@ async def test_create_llm_obs_payload(mock_env_vars): assert payload["metrics"]["input_tokens"] == 10 assert payload["metrics"]["output_tokens"] == 20 assert payload["metrics"]["total_tokens"] == 30 + + +def create_standard_logging_payload_with_latency_metrics() -> StandardLoggingPayload: + """Create a StandardLoggingPayload object with latency metrics for testing""" + guardrail_info = StandardLoggingGuardrailInformation( + guardrail_name="test_guardrail", + guardrail_status="success", + start_time=1234567890.0, + end_time=1234567890.5, + duration=0.5, # 500ms + ) + + hidden_params = StandardLoggingHiddenParams( + model_id="model-123", + cache_key="test-cache-key", + api_base="https://api.openai.com", + response_cost="0.05", + litellm_overhead_time_ms=150.0, # 150ms + additional_headers=None, + ) + + return StandardLoggingPayload( + id="test-request-id-latency", + call_type="completion", + response_cost=0.05, + response_cost_failure_debug_info=None, + status="success", + total_tokens=30, + prompt_tokens=10, + completion_tokens=20, + startTime=1234567890.0, + endTime=1234567892.0, + completionStartTime=1234567890.8, # 800ms after start + response_time=2.0, + model_map_information=StandardLoggingModelInformation( + model_map_key="gpt-4", model_map_value=None + ), + model="gpt-4", + model_id="model-123", + model_group="openai-gpt", + api_base="https://api.openai.com", + metadata=StandardLoggingMetadata( + user_api_key_hash="test_hash", + user_api_key_org_id=None, + user_api_key_alias="test_alias", + user_api_key_team_id="test_team", + user_api_key_user_id="test_user", + user_api_key_team_alias="test_team_alias", + spend_logs_metadata=None, + requester_ip_address="127.0.0.1", + requester_metadata=None, + ), + cache_hit=False, + cache_key=None, + saved_cache_cost=0.0, + request_tags=[], + end_user=None, + requester_ip_address="127.0.0.1", + messages=[{"role": "user", "content": "Hello, world!"}], + response={"choices": [{"message": {"content": "Hi there!"}}]}, + error_str=None, + error_information=None, + model_parameters={"stream": True}, + hidden_params=hidden_params, + guardrail_information=guardrail_info, + trace_id="test-trace-id-latency", + custom_llm_provider="openai", + ) + + +def test_latency_metrics_in_metadata(mock_env_vars): + """Test that time to first token, litellm overhead, and guardrail overhead are included in metadata""" + with patch('litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client'), \ + patch('asyncio.create_task'): + logger = DataDogLLMObsLogger() + + standard_payload = create_standard_logging_payload_with_latency_metrics() + + kwargs = { + "standard_logging_object": standard_payload, + "litellm_params": {"metadata": {}} + } + + start_time = datetime.now() + end_time = datetime.now() + + # Test the metadata generation directly + metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) + latency_metadata = metadata.get("latency_metrics", {}) + + # Verify time to first token is included (800ms) + assert "time_to_first_token_ms" in latency_metadata + assert abs(latency_metadata["time_to_first_token_ms"] - 800.0) < 0.001 # 0.8 seconds * 1000 with tolerance for floating-point precision + + # Verify litellm overhead is included (150ms) + assert "litellm_overhead_time_ms" in latency_metadata + assert latency_metadata["litellm_overhead_time_ms"] == 150.0 + + # Verify guardrail overhead is included (500ms) + assert "guardrail_overhead_time_ms" in latency_metadata + assert latency_metadata["guardrail_overhead_time_ms"] == 500.0 # 0.5 seconds * 1000 + + # Verify these metrics are also included in the full payload + payload = logger.create_llm_obs_payload(kwargs, start_time, end_time) + payload_metadata_latency = payload["meta"]["metadata"]["latency_metrics"] + + assert abs(payload_metadata_latency["time_to_first_token_ms"] - 800.0) < 0.001 + assert payload_metadata_latency["litellm_overhead_time_ms"] == 150.0 + assert payload_metadata_latency["guardrail_overhead_time_ms"] == 500.0 + + +def test_latency_metrics_edge_cases(mock_env_vars): + """Test latency metrics with edge cases (missing fields, zero values, etc.)""" + with patch('litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client'), \ + patch('asyncio.create_task'): + logger = DataDogLLMObsLogger() + + # Test case 1: No latency metrics present + standard_payload = create_standard_logging_payload_with_cache() + metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) + + # Should not have latency fields if data is missing/zero + assert "time_to_first_token_ms" not in metadata # Will be 0, so not included + assert "litellm_overhead_time_ms" not in metadata # Not present in hidden_params + assert "guardrail_overhead_time_ms" not in metadata # No guardrail_information + + # Test case 2: Zero time to first token should not be included + standard_payload = create_standard_logging_payload_with_cache() + standard_payload["startTime"] = 1000.0 + standard_payload["completionStartTime"] = 1000.0 # Same time = 0 difference + metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) + assert "time_to_first_token_ms" not in metadata + + # Test case 3: Missing guardrail duration should not crash + standard_payload = create_standard_logging_payload_with_cache() + standard_payload["guardrail_information"] = StandardLoggingGuardrailInformation( + guardrail_name="test", + guardrail_status="success", + # duration is missing + ) + metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload) + assert "guardrail_overhead_time_ms" not in metadata diff --git a/tests/test_litellm/llms/azure/response/test_azure_transformation.py b/tests/test_litellm/llms/azure/response/test_azure_transformation.py index 51edf91b70c..5a0db987eff 100644 --- a/tests/test_litellm/llms/azure/response/test_azure_transformation.py +++ b/tests/test_litellm/llms/azure/response/test_azure_transformation.py @@ -8,10 +8,14 @@ sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path +from unittest.mock import MagicMock + +from litellm.llms.azure.responses.o_series_transformation import ( + AzureOpenAIOSeriesResponsesAPIConfig, +) from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig -from litellm.llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig -from litellm.types.router import GenericLiteLLMParams from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams @pytest.mark.serial @@ -27,6 +31,7 @@ def test_validate_environment_api_key_within_litellm_params(): assert result == expected + @pytest.mark.serial def test_validate_environment_api_key_within_litellm(): azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig() @@ -41,6 +46,7 @@ def test_validate_environment_api_key_within_litellm(): assert result == expected + @pytest.mark.serial def test_validate_environment_azure_key_within_litellm(): azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig() @@ -55,6 +61,7 @@ def test_validate_environment_azure_key_within_litellm(): assert result == expected + @pytest.mark.serial def test_validate_environment_azure_key_within_headers(): azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig() @@ -93,10 +100,10 @@ def test_azure_o_series_responses_api_supported_params(): """Test that Azure OpenAI O-series responses API excludes temperature from supported parameters.""" config = AzureOpenAIOSeriesResponsesAPIConfig() supported_params = config.get_supported_openai_params("o_series/gpt-o1") - + # Temperature should not be in supported params for O-series models assert "temperature" not in supported_params - + # Other parameters should still be supported assert "input" in supported_params assert "max_output_tokens" in supported_params @@ -108,35 +115,32 @@ def test_azure_o_series_responses_api_supported_params(): def test_azure_o_series_responses_api_drop_temperature_param(): """Test that temperature parameter is dropped when drop_params is True for O-series models.""" config = AzureOpenAIOSeriesResponsesAPIConfig() - + # Create request params with temperature request_params = ResponsesAPIOptionalRequestParams( - temperature=0.7, - max_output_tokens=1000, - stream=False, - top_p=0.9 + temperature=0.7, max_output_tokens=1000, stream=False, top_p=0.9 ) - + # Test with drop_params=True mapped_params_with_drop = config.map_openai_params( response_api_optional_params=request_params, model="o_series/gpt-o1", - drop_params=True + drop_params=True, ) - + # Temperature should be dropped assert "temperature" not in mapped_params_with_drop # Other params should remain assert mapped_params_with_drop["max_output_tokens"] == 1000 assert mapped_params_with_drop["top_p"] == 0.9 - + # Test with drop_params=False mapped_params_without_drop = config.map_openai_params( response_api_optional_params=request_params, model="o_series/gpt-o1", - drop_params=False + drop_params=False, ) - + # Temperature should still be present when drop_params=False assert mapped_params_without_drop["temperature"] == 0.7 assert mapped_params_without_drop["max_output_tokens"] == 1000 @@ -147,21 +151,19 @@ def test_azure_o_series_responses_api_drop_temperature_param(): def test_azure_o_series_responses_api_drop_params_no_temperature(): """Test that map_openai_params works correctly when temperature is not present for O-series models.""" config = AzureOpenAIOSeriesResponsesAPIConfig() - + # Create request params without temperature request_params = ResponsesAPIOptionalRequestParams( - max_output_tokens=1000, - stream=False, - top_p=0.9 + max_output_tokens=1000, stream=False, top_p=0.9 ) - + # Should work fine even with drop_params=True mapped_params = config.map_openai_params( response_api_optional_params=request_params, model="o_series/gpt-o1", - drop_params=True + drop_params=True, ) - + assert "temperature" not in mapped_params assert mapped_params["max_output_tokens"] == 1000 assert mapped_params["top_p"] == 0.9 @@ -172,10 +174,10 @@ def test_azure_regular_responses_api_supports_temperature(): """Test that regular Azure OpenAI responses API (non-O-series) supports temperature parameter.""" config = AzureOpenAIResponsesAPIConfig() supported_params = config.get_supported_openai_params("gpt-4o") - + # Regular Azure models should support temperature assert "temperature" in supported_params - + # Other parameters should still be supported assert "input" in supported_params assert "max_output_tokens" in supported_params @@ -187,11 +189,11 @@ def test_azure_regular_responses_api_supports_temperature(): def test_o_series_model_detection(): """Test that the O-series configuration correctly identifies O-series models.""" config = AzureOpenAIOSeriesResponsesAPIConfig() - + # Test explicit o_series naming assert config.is_o_series_model("o_series/gpt-o1") == True assert config.is_o_series_model("azure/o_series/gpt-o3") == True - + # Test regular models assert config.is_o_series_model("gpt-4o") == False assert config.is_o_series_model("gpt-3.5-turbo") == False @@ -200,28 +202,94 @@ def test_o_series_model_detection(): @pytest.mark.serial def test_provider_config_manager_o_series_selection(): """Test that ProviderConfigManager returns the correct config for O-series vs regular models.""" - from litellm.utils import ProviderConfigManager import litellm - + from litellm.utils import ProviderConfigManager + # Test O-series model selection o_series_config = ProviderConfigManager.get_provider_responses_api_config( - provider=litellm.LlmProviders.AZURE, - model="o_series/gpt-o1" + provider=litellm.LlmProviders.AZURE, model="o_series/gpt-o1" ) assert isinstance(o_series_config, AzureOpenAIOSeriesResponsesAPIConfig) - + # Test regular model selection regular_config = ProviderConfigManager.get_provider_responses_api_config( - provider=litellm.LlmProviders.AZURE, - model="gpt-4o" + provider=litellm.LlmProviders.AZURE, model="gpt-4o" ) assert isinstance(regular_config, AzureOpenAIResponsesAPIConfig) assert not isinstance(regular_config, AzureOpenAIOSeriesResponsesAPIConfig) - + # Test with no model specified (should default to regular) default_config = ProviderConfigManager.get_provider_responses_api_config( - provider=litellm.LlmProviders.AZURE, - model=None + provider=litellm.LlmProviders.AZURE, model=None ) assert isinstance(default_config, AzureOpenAIResponsesAPIConfig) assert not isinstance(default_config, AzureOpenAIOSeriesResponsesAPIConfig) + + +class TestAzureResponsesAPIConfig: + def setup_method(self): + self.config = AzureOpenAIResponsesAPIConfig() + self.model = "gpt-4o" + self.logging_obj = MagicMock() + + def test_azure_get_complete_url_with_version_types(self): + """Test Azure get_complete_url with different API version types""" + base_url = "https://litellm8397336933.openai.azure.com" + + # Test with preview version - should use openai/v1/responses + result_preview = self.config.get_complete_url( + api_base=base_url, + litellm_params={"api_version": "preview"}, + ) + assert ( + result_preview + == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview" + ) + + # Test with latest version - should use openai/v1/responses + result_latest = self.config.get_complete_url( + api_base=base_url, + litellm_params={"api_version": "latest"}, + ) + assert ( + result_latest + == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest" + ) + + # Test with date-based version - should use openai/responses + result_date = self.config.get_complete_url( + api_base=base_url, + litellm_params={"api_version": "2025-01-01"}, + ) + assert ( + result_date + == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" + ) + + def test_azure_get_complete_url_with_default_api_version(self): + """Test Azure get_complete_url uses default API version when none is provided""" + from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION + + base_url = "https://litellm8397336933.openai.azure.com" + + # Test with no api_version provided - should use default + result_no_version = self.config.get_complete_url( + api_base=base_url, + litellm_params={}, + ) + expected_url = f"https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version={AZURE_DEFAULT_RESPONSES_API_VERSION}" + assert result_no_version == expected_url + + # Test with empty litellm_params - should use default + result_empty_params = self.config.get_complete_url( + api_base=base_url, + litellm_params={}, + ) + assert result_empty_params == expected_url + + # Test with None api_version - should use default + result_none_version = self.config.get_complete_url( + api_base=base_url, + litellm_params={"api_version": None}, + ) + assert result_none_version == expected_url diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 1ed2266d1f1..9b8e56ab499 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -147,7 +147,7 @@ class TestOpenAIResponsesAPIConfig: assert result.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED assert result.response.id == "resp_123" - + @pytest.mark.serial def test_validate_environment(self): """Test that validate_environment correctly sets the Authorization header""" @@ -283,38 +283,6 @@ class TestOpenAIResponsesAPIConfig: assert result.type == "test" -class TestAzureResponsesAPIConfig: - def setup_method(self): - self.config = AzureOpenAIResponsesAPIConfig() - self.model = "gpt-4o" - self.logging_obj = MagicMock() - - def test_azure_get_complete_url_with_version_types(self): - """Test Azure get_complete_url with different API version types""" - base_url = "https://litellm8397336933.openai.azure.com" - - # Test with preview version - should use openai/v1/responses - result_preview = self.config.get_complete_url( - api_base=base_url, - litellm_params={"api_version": "preview"}, - ) - assert result_preview == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview" - - # Test with latest version - should use openai/v1/responses - result_latest = self.config.get_complete_url( - api_base=base_url, - litellm_params={"api_version": "latest"}, - ) - assert result_latest == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest" - - # Test with date-based version - should use openai/responses - result_date = self.config.get_complete_url( - api_base=base_url, - litellm_params={"api_version": "2025-01-01"}, - ) - assert result_date == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" - - class TestTransformListInputItemsRequest: """Test suite for transform_list_input_items_request function""" diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 0ec3fd93936..5104ffd80de 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -606,7 +606,6 @@ def test_get_dynamic_logging_metadata_with_arize_team_logging(): assert result.callback_vars["arize_space_id"] == "test_arize_space_id" - def test_get_num_retries_from_request(): """ Test LiteLLMProxyRequestSetup._get_num_retries_from_request method @@ -668,6 +667,7 @@ def test_get_num_retries_from_request(): ) assert result == -1 + def test_add_user_api_key_auth_to_request_metadata(): """ Test that add_user_api_key_auth_to_request_metadata properly adds user API key authentication data to request metadata @@ -676,9 +676,9 @@ def test_add_user_api_key_auth_to_request_metadata(): data = { "model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}], - "litellm_metadata": {} # This will be the metadata variable name + "litellm_metadata": {}, # This will be the metadata variable name } - + user_api_key_dict = UserAPIKeyAuth( api_key="hashed-test-key-123", user_id="test-user-123", @@ -689,21 +689,21 @@ def test_add_user_api_key_auth_to_request_metadata(): team_alias="test-team-alias", end_user_id="test-end-user-123", request_route="/chat/completions", - end_user_max_budget=500.0 + end_user_max_budget=500.0, ) - + metadata_variable_name = "litellm_metadata" - + # Call the function result = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( data=data, user_api_key_dict=user_api_key_dict, - _metadata_variable_name=metadata_variable_name + _metadata_variable_name=metadata_variable_name, ) - + # Verify the metadata was properly added metadata = result[metadata_variable_name] - + # Check that user API key information was added assert metadata["user_api_key_hash"] == "hashed-test-key-123" assert metadata["user_api_key_alias"] == "test-key-alias" @@ -714,13 +714,224 @@ def test_add_user_api_key_auth_to_request_metadata(): assert metadata["user_api_key_end_user_id"] == "test-end-user-123" assert metadata["user_api_key_user_email"] == "test@example.com" assert metadata["user_api_key_request_route"] == "/chat/completions" - + # Check that the hashed API key was added assert metadata["user_api_key"] == "hashed-test-key-123" - + # Check that end user max budget was added assert metadata["user_api_end_user_max_budget"] == 500.0 - + # Verify original data is preserved assert result["model"] == "gpt-3.5-turbo" - assert result["messages"] == [{"role": "user", "content": "Hello"}] \ No newline at end of file + assert result["messages"] == [{"role": "user", "content": "Hello"}] + + +@pytest.mark.parametrize( + "data, model_group_settings, expected_headers_added", + [ + # Test case 1: Model is in forward_client_headers_to_llm_api list + ( + {"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}, + MagicMock(forward_client_headers_to_llm_api=["gpt-4"]), + True, + ), + # Test case 2: Model is not in forward_client_headers_to_llm_api list + ( + {"model": "claude-3", "messages": [{"role": "user", "content": "Hello"}]}, + MagicMock(forward_client_headers_to_llm_api=["gpt-4"]), + False, + ), + # Test case 3: Model group settings is None + ( + {"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}, + None, + False, + ), + # Test case 4: forward_client_headers_to_llm_api is None + ( + {"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}, + MagicMock(forward_client_headers_to_llm_api=None), + False, + ), + # Test case 5: Data has no model + ( + {"messages": [{"role": "user", "content": "Hello"}]}, + MagicMock(forward_client_headers_to_llm_api=["gpt-4"]), + False, + ), + # Test case 6: Model is None + ( + {"model": None, "messages": [{"role": "user", "content": "Hello"}]}, + MagicMock(forward_client_headers_to_llm_api=["gpt-4"]), + False, + ), + ], +) +def test_add_headers_to_llm_call_by_model_group( + data, model_group_settings, expected_headers_added +): + """ + Test LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group method + + This tests various scenarios: + 1. When model is in the forward_client_headers_to_llm_api list + 2. When model is not in the list + 3. When model_group_settings is None + 4. When forward_client_headers_to_llm_api is None + 5. When data has no model + 6. When model is None + """ + import litellm + + # Setup test headers and user API key + headers = { + "Authorization": "Bearer token123", + "User-Agent": "test-client/1.0", + "X-Custom-Header": "custom-value", + } + + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", user_id="test-user", org_id="test-org" + ) + + # Mock the model_group_settings + original_model_group_settings = getattr(litellm, "model_group_settings", None) + litellm.model_group_settings = model_group_settings + + try: + # Mock the add_headers_to_llm_call method to return expected headers + expected_returned_headers = { + "X-LiteLLM-User": "test-user", + "X-LiteLLM-Org": "test-org", + } + + with patch.object( + LiteLLMProxyRequestSetup, + "add_headers_to_llm_call", + return_value=expected_returned_headers if expected_headers_added else {}, + ) as mock_add_headers: + + # Make a copy of original data to verify it's not mutated unexpectedly + original_data = copy.deepcopy(data) + + # Call the method under test + result = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group( + data=data, headers=headers, user_api_key_dict=user_api_key_dict + ) + + # Verify the result + assert result is not None + assert isinstance(result, dict) + + if expected_headers_added: + # Verify that add_headers_to_llm_call was called + mock_add_headers.assert_called_once_with(headers, user_api_key_dict) + # Verify that headers were added to the data + assert "headers" in result + assert result["headers"] == expected_returned_headers + else: + # Verify that add_headers_to_llm_call was not called + mock_add_headers.assert_not_called() + # Verify that no headers were added + assert "headers" not in result or result.get("headers") is None + + # Verify that original data fields are preserved + for key, value in original_data.items(): + if key != "headers": # headers might be added + assert result[key] == value + + finally: + # Restore original model_group_settings + litellm.model_group_settings = original_model_group_settings + + +def test_add_headers_to_llm_call_by_model_group_empty_headers_returned(): + """ + Test that when add_headers_to_llm_call returns empty dict, no headers are added to data + """ + import litellm + + # Setup test data + data = {"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]} + headers = {"Authorization": "Bearer token123"} + user_api_key_dict = UserAPIKeyAuth(api_key="test-key") + + # Mock model_group_settings with model in the list + mock_settings = MagicMock(forward_client_headers_to_llm_api=["gpt-4"]) + original_model_group_settings = getattr(litellm, "model_group_settings", None) + litellm.model_group_settings = mock_settings + + try: + with patch.object( + LiteLLMProxyRequestSetup, + "add_headers_to_llm_call", + return_value={}, # Return empty dict + ) as mock_add_headers: + + result = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group( + data=data, headers=headers, user_api_key_dict=user_api_key_dict + ) + + # Verify that add_headers_to_llm_call was called + mock_add_headers.assert_called_once_with(headers, user_api_key_dict) + + # Verify that no headers were added since returned headers were empty + assert "headers" not in result + + # Verify original data is preserved + assert result["model"] == "gpt-4" + assert result["messages"] == [{"role": "user", "content": "Hello"}] + + finally: + # Restore original model_group_settings + litellm.model_group_settings = original_model_group_settings + + +def test_add_headers_to_llm_call_by_model_group_existing_headers_in_data(): + """ + Test that existing headers in data are overwritten when new headers are added + """ + import litellm + + # Setup test data with existing headers + data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "headers": {"Existing-Header": "existing-value"}, + } + headers = {"Authorization": "Bearer token123"} + user_api_key_dict = UserAPIKeyAuth(api_key="test-key") + + # Mock model_group_settings with model in the list + mock_settings = MagicMock(forward_client_headers_to_llm_api=["gpt-4"]) + original_model_group_settings = getattr(litellm, "model_group_settings", None) + litellm.model_group_settings = mock_settings + + try: + new_headers = {"X-LiteLLM-User": "test-user"} + + with patch.object( + LiteLLMProxyRequestSetup, + "add_headers_to_llm_call", + return_value=new_headers, + ) as mock_add_headers: + + result = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group( + data=data, headers=headers, user_api_key_dict=user_api_key_dict + ) + + # Verify that add_headers_to_llm_call was called + mock_add_headers.assert_called_once_with(headers, user_api_key_dict) + + # Verify that headers were overwritten + assert "headers" in result + assert result["headers"] == new_headers + assert result["headers"] != {"Existing-Header": "existing-value"} + + # Verify original data is preserved + assert result["model"] == "gpt-4" + assert result["messages"] == [{"role": "user", "content": "Hello"}] + + finally: + # Restore original model_group_settings + litellm.model_group_settings = original_model_group_settings diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_responses_api_deployment_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_responses_api_deployment_check.py index 5db5cb853cd..e2abed2d7f0 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_responses_api_deployment_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_responses_api_deployment_check.py @@ -17,7 +17,6 @@ from litellm.types.llms.openai import ( ResponseAPIUsage, ResponseCompletedEvent, ResponsesAPIResponse, - ResponseTextConfig, ) from litellm.types.utils import StandardLoggingPayload diff --git a/tests/test_litellm/router_utils/test_cooldown_cache.py b/tests/test_litellm/router_utils/test_cooldown_cache.py new file mode 100644 index 00000000000..52fe151eff4 --- /dev/null +++ b/tests/test_litellm/router_utils/test_cooldown_cache.py @@ -0,0 +1,257 @@ +""" +Unit tests for CooldownCache exception masking functionality +""" + +import os +import sys +from unittest.mock import MagicMock + +import pytest + +# Add the parent directory to the system path +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker +from litellm.router_utils.cooldown_cache import CooldownCache, CooldownCacheValue + + +class TestCooldownCacheExceptionMasking: + """Test suite for CooldownCache exception masking functionality""" + + @pytest.fixture + def cooldown_cache(self): + """Create a CooldownCache instance for testing""" + mock_dual_cache = MagicMock(spec=DualCache) + return CooldownCache(cache=mock_dual_cache, default_cooldown_time=60.0) + + def test_exception_masker_initialization(self, cooldown_cache): + """Test that the exception masker is properly initialized""" + assert isinstance(cooldown_cache.exception_masker, SensitiveDataMasker) + assert cooldown_cache.exception_masker.visible_prefix == 50 + assert cooldown_cache.exception_masker.visible_suffix == 0 + assert cooldown_cache.exception_masker.mask_char == "*" + + def test_short_exception_string_not_masked(self, cooldown_cache): + """Test that short exception strings are not masked""" + short_exception = "Short error" + model_id = "test-model" + exception_status = 500 + cooldown_time = 30.0 + + cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic( + model_id=model_id, + original_exception=Exception(short_exception), + exception_status=exception_status, + cooldown_time=cooldown_time, + ) + + # Short exception should not be masked + assert cooldown_data["exception_received"] == short_exception + assert cooldown_key == f"deployment:{model_id}:cooldown" + + def test_long_exception_string_masked(self, cooldown_cache): + """Test that long exception strings are properly masked""" + # Create a long exception string that simulates prompt leakage + long_exception = ( + "litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occurred - " + "No deployments available for selected model, Try again in 5 seconds. " + "Passed model=anthropic_claude_sonnet_4_v1_0. pre-call-checks=False, " + "cooldown_list=[('deepseek_r1-eastus', {'exception_received': " + "'litellm.RateLimitError: RateLimitError: Azure_aiException - " + '{"error":{"code":"Invalid input","status":422,"message":"invalid input error",' + '"details":[{"type":"model_attributes_type","loc":["body"],' + '"msg":"Tell me a story about a dragon and a princess in a magical kingdom ' + "where the dragon is actually protecting the princess from an evil wizard " + 'who wants to steal her magical powers and use them to conquer the world"}]}' + ) + + model_id = "test-model" + exception_status = 429 + cooldown_time = 60.0 + + cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic( + model_id=model_id, + original_exception=Exception(long_exception), + exception_status=exception_status, + cooldown_time=cooldown_time, + ) + + masked_exception = cooldown_data["exception_received"] + + # Should start with first 50 characters + assert masked_exception.startswith(long_exception[:50]) + + # Should contain masking characters + assert "*" in masked_exception + + # Should be same length (prefix + asterisks) + assert len(masked_exception) == len(long_exception) + + # Should not contain the sensitive prompt content + assert "Tell me a story about a dragon" not in masked_exception + assert "magical kingdom" not in masked_exception + + # Should preserve the error type information at the beginning (first 50 chars) + assert masked_exception.startswith( + "litellm.proxy.proxy_server._handle_llm_api_excepti" + ) + + def test_exception_with_api_keys_masked(self, cooldown_cache): + """Test that API keys in exceptions are properly masked""" + exception_with_key = ( + "Authentication failed with api_key=sk-1234567890abcdefghijklmnopqrstuvwxyz " + "and token=bearer_token_123456789 for model gpt-4" + ) + + model_id = "test-model" + exception_status = 401 + cooldown_time = 30.0 + + cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic( + model_id=model_id, + original_exception=Exception(exception_with_key), + exception_status=exception_status, + cooldown_time=cooldown_time, + ) + + masked_exception = cooldown_data["exception_received"] + + # Should mask the sensitive content while preserving structure + assert masked_exception.startswith( + "Authentication failed with api_key=sk-12345678" + ) + assert "*" in masked_exception + assert len(masked_exception) == len(exception_with_key) + + def test_cooldown_data_structure(self, cooldown_cache): + """Test that the cooldown data structure is correctly formed""" + exception_msg = "Test exception for structure validation" + model_id = "test-model" + exception_status = 500 + cooldown_time = 45.0 + + cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic( + model_id=model_id, + original_exception=Exception(exception_msg), + exception_status=exception_status, + cooldown_time=cooldown_time, + ) + + # Verify cooldown data structure + assert isinstance(cooldown_data, dict) + assert "exception_received" in cooldown_data + assert "status_code" in cooldown_data + assert "timestamp" in cooldown_data + assert "cooldown_time" in cooldown_data + + # Verify data types + assert isinstance(cooldown_data["exception_received"], str) + assert isinstance(cooldown_data["status_code"], str) + assert isinstance(cooldown_data["timestamp"], float) + assert isinstance(cooldown_data["cooldown_time"], float) + + # Verify values + assert cooldown_data["status_code"] == str(exception_status) + assert cooldown_data["cooldown_time"] == cooldown_time + assert cooldown_data["exception_received"] == exception_msg + + def test_exception_object_conversion(self, cooldown_cache): + """Test that different exception types are properly converted to strings""" + # Test with different exception types + exceptions = [ + ValueError("Invalid value provided"), + KeyError("Missing required key"), + RuntimeError("Runtime error occurred"), + Exception("Generic exception"), + ] + + for exc in exceptions: + model_id = f"test-model-{exc.__class__.__name__}" + + cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic( + model_id=model_id, + original_exception=exc, + exception_status=500, + cooldown_time=30.0, + ) + + # Should successfully convert exception to string + assert isinstance(cooldown_data["exception_received"], str) + assert ( + str(exc) == cooldown_data["exception_received"] + ) # Short exceptions not masked + + def test_masking_preserves_error_debugging_info(self, cooldown_cache): + """Test that masking preserves essential debugging information""" + debugging_exception = ( + "RateLimitError: Rate limit exceeded for model gpt-4. " + "Current usage: 1000 tokens/minute. Limit: 500 tokens/minute. " + "Request details: model=gpt-4, user_id=user123, " + "prompt='Write a comprehensive analysis of the economic implications " + "of artificial intelligence adoption in the healthcare sector, including " + "potential cost savings, job displacement, and regulatory challenges'" + ) + + model_id = "gpt-4-deployment" + exception_status = 429 + cooldown_time = 120.0 + + cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic( + model_id=model_id, + original_exception=Exception(debugging_exception), + exception_status=exception_status, + cooldown_time=cooldown_time, + ) + + masked_exception = cooldown_data["exception_received"] + + # Should preserve error type and initial debugging info (first 50 chars) + assert masked_exception.startswith( + "RateLimitError: Rate limit exceeded for model gpt-" + ) + + # Should mask the prompt content + assert "Write a comprehensive analysis" not in masked_exception + assert "healthcare sector" not in masked_exception + + # Should contain masking indicator + assert "*" in masked_exception + + def test_error_handling_in_common_add_cooldown_logic(self, cooldown_cache): + """Test error handling in the _common_add_cooldown_logic method""" + # This test ensures that edge cases are properly handled + model_id = "test-model" + + # Test with None exception (edge case) - should be handled gracefully + cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic( + model_id=model_id, + original_exception=None, + exception_status=500, + cooldown_time=30.0, + ) + + # Should handle None by converting to string + assert cooldown_data["exception_received"] == "None" + assert cooldown_key == f"deployment:{model_id}:cooldown" + + def test_custom_masker_settings(self): + """Test that custom masker settings work correctly""" + mock_dual_cache = MagicMock(spec=DualCache) + + # Create cooldown cache and verify default settings + cache = CooldownCache(cache=mock_dual_cache, default_cooldown_time=60.0) + + # Test that we can access and verify the masker configuration + assert cache.exception_masker.visible_prefix == 50 + assert cache.exception_masker.visible_suffix == 0 + assert cache.exception_masker.mask_char == "*" + + # Test masking behavior with these settings + long_string = "A" * 100 # 100 character string + masked = cache.exception_masker._mask_value(long_string) + + # Should show first 50 characters, then all asterisks + expected = "A" * 50 + "*" * 50 + assert masked == expected diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 4efba133f13..9bff907700d 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -480,3 +480,69 @@ def test_gemini_25_implicit_caching_cost(): ), f"Expected cost {expected_cost}, but got {result}" print(f"✓ Gemini 2.5 implicit caching cost calculation is correct: ${result:.8f}") + + +def test_gemini_25_explicit_caching_cost_direct_usage(): + """ + Test that Gemini 2.5 models correctly calculate costs with explicit caching. + + This test reproduces the issue from #11156 where cached tokens should receive + a 75% discount. + """ + from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.types.utils import ( + CompletionTokensDetailsWrapper, + PromptTokensDetailsWrapper, + Usage, + ) + from litellm.utils import get_model_info + + model_info = get_model_info(model="gemini-2.5-pro", custom_llm_provider="gemini") + + usage = Usage( + completion_tokens=2522, + prompt_tokens=42001, + total_tokens=44523, + completion_tokens_details=CompletionTokensDetailsWrapper( + accepted_prediction_tokens=None, + audio_tokens=None, + reasoning_tokens=1908, + rejected_prediction_tokens=None, + text_tokens=614, + ), + prompt_tokens_details=PromptTokensDetailsWrapper( + audio_tokens=None, cached_tokens=40938, text_tokens=1063, image_tokens=None + ), + ) + + input_cost, output_cost = generic_cost_per_token( + model="gemini/gemini-2.5-pro", + usage=usage, + custom_llm_provider="gemini", + ) + + total_cost = input_cost + output_cost + + expected_higher_than_actual_cost = ( + model_info["input_cost_per_token"] * usage.prompt_tokens + + model_info["output_cost_per_token"] * usage.completion_tokens + ) + + print(f"expected_higher_than_actual_cost: {expected_higher_than_actual_cost}") + + assert expected_higher_than_actual_cost > total_cost + + expected_actual_cost = ( + model_info["input_cost_per_token"] * usage.prompt_tokens_details.text_tokens + + model_info["cache_read_input_token_cost"] + * usage.prompt_tokens_details.cached_tokens + + model_info["output_cost_per_token"] * usage.completion_tokens + ) + + print( + f"model_info['input_cost_per_token']: {model_info['input_cost_per_token']}, usage.prompt_tokens_details.text_tokens: {usage.prompt_tokens_details.text_tokens}, model_info['cache_read_input_token_cost']: {model_info['cache_read_input_token_cost']}, model_info['output_cost_per_token']: {model_info['output_cost_per_token']}" + ) + + print(f"Expected actual cost: {expected_actual_cost}") + + assert expected_actual_cost == total_cost diff --git a/tests/test_litellm/test_groq_streaming_encoding.py b/tests/test_litellm/test_groq_streaming_encoding.py new file mode 100644 index 00000000000..cb1d69a4a10 --- /dev/null +++ b/tests/test_litellm/test_groq_streaming_encoding.py @@ -0,0 +1,129 @@ +""" +Test for Groq streaming ASCII encoding issue fix. + +This test verifies that the OpenAI-like handler correctly handles +UTF-8 encoded content in streaming responses, specifically fixing +the ASCII encoding error described in issue #12660. +""" +import pytest +import asyncio +from unittest.mock import Mock, AsyncMock +from litellm.llms.openai_like.chat.handler import make_call, make_sync_call + +class MockResponse: + """Mock httpx response for testing UTF-8 handling.""" + + def __init__(self, test_content: str): + self.test_content = test_content + self.status_code = 200 + + def iter_text(self, encoding='utf-8'): + """Mock iter_text that yields content with the specified encoding.""" + yield self.test_content + + async def aiter_text(self, encoding='utf-8'): + """Mock aiter_text that yields content with the specified encoding.""" + yield self.test_content + + def json(self): + return {"choices": [{"delta": {"content": "test"}}]} + +class MockSyncClient: + """Mock synchronous HTTP client for testing.""" + + def __init__(self, response_content: str): + self.response_content = response_content + + def post(self, *args, **kwargs): + return MockResponse(self.response_content) + +class MockAsyncClient: + """Mock asynchronous HTTP client for testing.""" + + def __init__(self, response_content: str): + self.response_content = response_content + + async def post(self, *args, **kwargs): + return MockResponse(self.response_content) + +def test_utf8_streaming_sync(): + """Test that synchronous streaming handles UTF-8 characters correctly.""" + # Content with the µ character that was causing issues + test_content = "data: {\"choices\":[{\"delta\":{\"content\":\"The symbol µ represents micro\"}}]}\n\n" + + mock_client = MockSyncClient(test_content) + mock_logging = Mock() + + # This should not raise an ASCII encoding error + completion_stream = make_sync_call( + client=mock_client, + api_base="https://test.com/v1/chat/completions", + headers={"Authorization": "Bearer test"}, + data='{"model": "test", "messages": []}', + model="test-model", + messages=[], + logging_obj=mock_logging + ) + + # Verify we can iterate through the stream without encoding errors + assert completion_stream is not None + +@pytest.mark.asyncio +async def test_utf8_streaming_async(): + """Test that asynchronous streaming handles UTF-8 characters correctly.""" + # Content with the µ character that was causing issues + test_content = "data: {\"choices\":[{\"delta\":{\"content\":\"The symbol µ represents micro\"}}]}\n\n" + + mock_client = MockAsyncClient(test_content) + mock_logging = Mock() + + # This should not raise an ASCII encoding error + completion_stream = await make_call( + client=mock_client, + api_base="https://test.com/v1/chat/completions", + headers={"Authorization": "Bearer test"}, + data='{"model": "test", "messages": []}', + model="test-model", + messages=[], + logging_obj=mock_logging + ) + + # Verify we can iterate through the stream without encoding errors + assert completion_stream is not None + +def test_various_unicode_characters(): + """Test streaming with various Unicode characters that could cause issues.""" + unicode_test_cases = [ + "µ", # Micro symbol (the original issue) + "©", # Copyright symbol + "™", # Trademark symbol + "€", # Euro symbol + "北京", # Chinese characters + "🚀", # Emoji + "Ñoño", # Spanish characters with tildes + ] + + for unicode_char in unicode_test_cases: + test_content = f"data: {{\"choices\":[{{\"delta\":{{\"content\":\"Testing {unicode_char} character\"}}}}]}}\n\n" + + mock_client = MockSyncClient(test_content) + mock_logging = Mock() + + # This should not raise an ASCII encoding error for any Unicode character + completion_stream = make_sync_call( + client=mock_client, + api_base="https://test.com/v1/chat/completions", + headers={"Authorization": "Bearer test"}, + data='{"model": "test", "messages": []}', + model="test-model", + messages=[], + logging_obj=mock_logging + ) + + assert completion_stream is not None, f"Failed to handle Unicode character: {unicode_char}" + +if __name__ == "__main__": + test_utf8_streaming_sync() + asyncio.run(test_utf8_streaming_async()) + test_various_unicode_characters() + print("All UTF-8 streaming tests passed!") \ No newline at end of file diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 234941449b9..de7c3a74c21 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -896,148 +896,6 @@ async def test_router_ageneric_api_call_with_fallbacks_helper(): assert router.fail_calls["gpt-3.5-turbo"] == initial_fail_count + 1 -@pytest.mark.asyncio -async def test_router_forward_client_headers_by_model_group(): - """ - Test that router.forward_client_headers_by_model_group returns the correct response - """ - from unittest.mock import MagicMock, patch - - from litellm.types.router import ModelGroupSettings - - litellm.model_group_settings = ModelGroupSettings( - forward_client_headers_to_llm_api=[ - "gpt-3.5-turbo-allow", - "openai/*", - "gpt-3.5-turbo-custom", - ] - ) - - router = litellm.Router( - model_list=[ - { - "model_name": "gpt-3.5-turbo-allow", - "litellm_params": { - "model": "gpt-3.5-turbo", - }, - }, - { - "model_name": "gpt-3.5-turbo-disallow", - "litellm_params": { - "model": "gpt-3.5-turbo", - }, - }, - { - "model_name": "openai/*", - "litellm_params": { - "model": "openai/*", - }, - }, - { - "model_name": "openai/gpt-4o-mini", - "litellm_params": { - "model": "openai/gpt-4o-mini", - }, - }, - ], - model_group_alias={ - "gpt-3.5-turbo-custom": "gpt-3.5-turbo-disallow", - }, - ) - - ## Scenario 1: Direct model name - with patch.object( - litellm.main, "completion", return_value=MagicMock() - ) as mock_completion: - await router.acompletion( - model="gpt-3.5-turbo-allow", - messages=[{"role": "user", "content": "Hello, world!"}], - mock_response="Hello, world!", - secret_fields={"raw_headers": {"test": "test"}}, - ) - - mock_completion.assert_called_once() - print(mock_completion.call_args.kwargs["headers"]) - - ## Scenario 2: Wildcard model name - with patch.object( - litellm.main, "completion", return_value=MagicMock() - ) as mock_completion: - await router.acompletion( - model="openai/gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello, world!"}], - mock_response="Hello, world!", - secret_fields={"raw_headers": {"test": "test"}}, - ) - - mock_completion.assert_called_once() - print(mock_completion.call_args.kwargs["headers"]) - - ## Scenario 3: Not in model_group_settings - with patch.object( - litellm.main, "completion", return_value=MagicMock() - ) as mock_completion: - await router.acompletion( - model="openai/gpt-4o-mini", - messages=[{"role": "user", "content": "Hello, world!"}], - mock_response="Hello, world!", - secret_fields={"raw_headers": {"test": "test"}}, - ) - - mock_completion.assert_called_once() - assert mock_completion.call_args.kwargs.get("headers") is None - - ## Scenario 4: Model group alias - with patch.object( - litellm.main, "completion", return_value=MagicMock() - ) as mock_completion: - await router.acompletion( - model="gpt-3.5-turbo-custom", - messages=[{"role": "user", "content": "Hello, world!"}], - mock_response="Hello, world!", - secret_fields={"raw_headers": {"test": "test"}}, - ) - - mock_completion.assert_called_once() - print(mock_completion.call_args.kwargs["headers"]) - - -def test_router_apply_default_settings(): - """ - Test that Router.apply_default_settings() adds the expected default pre-call checks - """ - router = litellm.Router( - model_list=[ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-3.5-turbo"}, - } - ], - ) - - # Apply default settings - result = router.apply_default_settings() - - # Verify the method returns None - assert result is None - - # Verify that the forward_client_headers_by_model_group pre-call check was added - # Check if any callback is of the ForwardClientHeadersByModelGroupCheck type - has_forward_headers_check = False - for callback in litellm.callbacks: - print(callback) - print(f"callback.__class__: {callback.__class__}") - if hasattr( - callback, "__class__" - ) and "ForwardClientSideHeadersByModelGroup" in str(callback.__class__): - has_forward_headers_check = True - break - - assert ( - has_forward_headers_check - ), "Expected ForwardClientSideHeadersByModelGroup to be added to callbacks" - - def test_router_get_model_access_groups_team_only_models(): """ Test that Router.get_model_access_groups returns the correct response for team-only models diff --git a/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx index 03eefd52845..c1e8c32a94a 100644 --- a/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx +++ b/ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx @@ -2,6 +2,7 @@ import React, { useEffect, useState } from "react"; import { Form, Table } from "antd"; import { TextInput } from "@tremor/react"; import { Tooltip } from "../atoms/index"; +import { Providers } from "../provider_info_helpers"; const ConditionalPublicModelName: React.FC = () => { const form = Form.useFormInstance(); @@ -12,13 +13,19 @@ const ConditionalPublicModelName: React.FC = () => { const selectedModels = Array.isArray(modelValue) ? modelValue : [modelValue]; const customModelName = Form.useWatch('custom_model_name', form); const showPublicModelName = !selectedModels.includes('all-wildcard'); - + const selectedProvider = Form.useWatch('custom_llm_provider', form); // Force table to re-render when custom model name changes useEffect(() => { if (customModelName && selectedModels.includes('custom')) { const currentMappings = form.getFieldValue('model_mappings') || []; const updatedMappings = currentMappings.map((mapping: any) => { if (mapping.public_name === 'custom' || mapping.litellm_model === 'custom') { + if (selectedProvider === Providers.Azure) { + return { + public_name: customModelName, + litellm_model: `azure/${customModelName}` + }; + } return { public_name: customModelName, litellm_model: customModelName @@ -29,7 +36,7 @@ const ConditionalPublicModelName: React.FC = () => { form.setFieldValue('model_mappings', updatedMappings); setTableKey(prev => prev + 1); // Force table re-render } - }, [customModelName, selectedModels, form]); + }, [customModelName, selectedModels, selectedProvider, form]); // Initial setup of model mappings when models are selected useEffect(() => { @@ -44,17 +51,32 @@ const ConditionalPublicModelName: React.FC = () => { if (model === 'custom') { return mapping.litellm_model === 'custom' || mapping.litellm_model === customModelName; } + if (selectedProvider === Providers.Azure) { + return mapping.litellm_model === `azure/${model}`; + } return mapping.litellm_model === model; })); if (shouldUpdateMappings) { const mappings = selectedModels.map((model: string) => { if (model === 'custom' && customModelName) { + if (selectedProvider === Providers.Azure) { + return { + public_name: customModelName, + litellm_model: `azure/${customModelName}` + }; + } return { public_name: customModelName, litellm_model: customModelName }; } + if (selectedProvider === Providers.Azure) { + return { + public_name: model, + litellm_model: `azure/${model}` + }; + } return { public_name: model, litellm_model: model @@ -65,7 +87,7 @@ const ConditionalPublicModelName: React.FC = () => { setTableKey(prev => prev + 1); // Force table re-render } } - }, [selectedModels, customModelName, form]); + }, [selectedModels, customModelName, selectedProvider,form]); if (!showPublicModelName) return null; diff --git a/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx b/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx index 27dc5047539..ad12629ca16 100644 --- a/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx +++ b/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx @@ -32,10 +32,18 @@ const LiteLLMModelNameField: React.FC = ({ if (JSON.stringify(currentModel) !== JSON.stringify(values)) { // Create mappings first - const mappings = values.map(model => ({ - public_name: model, - litellm_model: model - })); + const mappings = values.map(model => { + if (selectedProvider === Providers.Azure) { + return { + public_name: model, + litellm_model: `azure/${model}` + }; + } + return { + public_name: model, + litellm_model: model + }; + }); // Update both fields in one call to reduce re-renders form.setFieldsValue({ @@ -47,6 +55,22 @@ const LiteLLMModelNameField: React.FC = ({ } }; + const handleAzureDeploymentNameChange = (e: React.ChangeEvent) => { + const deploymentName = e.target.value; + + // Create mapping with Azure-specific format + const mappings = deploymentName ? [{ + public_name: deploymentName, + litellm_model: `azure/${deploymentName}` + }] : []; + + // Update both fields + form.setFieldsValue({ + model: deploymentName, + model_mappings: mappings + }); + }; + // Handle custom model name changes const handleCustomModelNameChange = (e: React.ChangeEvent) => { const customName = e.target.value; @@ -55,6 +79,12 @@ const LiteLLMModelNameField: React.FC = ({ const currentMappings = form.getFieldValue('model_mappings') || []; const updatedMappings = currentMappings.map((mapping: any) => { if (mapping.public_name === 'custom' || mapping.litellm_model === 'custom') { + if (selectedProvider === Providers.Azure) { + return { + public_name: customName, + litellm_model: `azure/${customName}` + }; + } return { public_name: customName, litellm_model: customName @@ -75,7 +105,7 @@ const LiteLLMModelNameField: React.FC = ({ > {(selectedProvider === Providers.Azure) || @@ -84,6 +114,7 @@ const LiteLLMModelNameField: React.FC = ({ <> ) : providerModels.length > 0 ? ( @@ -135,7 +166,7 @@ const LiteLLMModelNameField: React.FC = ({ className="mt-2" > @@ -147,7 +178,10 @@ const LiteLLMModelNameField: React.FC = ({ - The model name LiteLLM will send to the LLM API + {selectedProvider === Providers.Azure + ? "Your deployment name will be saved as the public model name, and LiteLLM will use 'azure/deployment-name' internally" + : "The model name LiteLLM will send to the LLM API" + } diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 7542ad3b13e..3734486a4eb 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -13,6 +13,7 @@ import { Grid, Col, DateRangePicker, + TextInput, } from "@tremor/react"; import { CredentialItem, @@ -220,6 +221,8 @@ const ModelDashboard: React.FC = ({ const [selectedModelAccessGroupFilter, setSelectedModelAccessGroupFilter] = useState(null); + const [modelNameSearch, setModelNameSearch] = useState(""); + // Add new state for current team and model view mode const [currentTeam, setCurrentTeam] = useState("personal"); // 'personal' or team_id const [modelViewMode, setModelViewMode] = useState<"current_team" | "all">( @@ -1272,7 +1275,18 @@ const ModelDashboard: React.FC = ({ {/* Other Filters */}
-
+
+ {/* Model Name Search */} +
+ Search Public Model Name: + +
+ {/* Model Name Filter */}
Filter by Public Model Name: @@ -1336,6 +1350,9 @@ const ModelDashboard: React.FC = ({ Showing{" "} {modelData && modelData.data.length > 0 ? modelData.data.filter((model: any) => { + const searchMatch = modelNameSearch === "" || + model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()); + const modelNameMatch = selectedModelGroup === "all" || model.model_name === @@ -1365,6 +1382,7 @@ const ModelDashboard: React.FC = ({ } return ( + searchMatch && modelNameMatch && accessGroupMatch && teamAccessMatch @@ -1393,6 +1411,10 @@ const ModelDashboard: React.FC = ({ )} data={modelData.data.filter( (model: any) => { + // Model name search filter + const searchMatch = modelNameSearch === "" || + model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()); + // Model name filter const modelNameMatch = selectedModelGroup === "all" || model.model_name === selectedModelGroup || @@ -1422,6 +1444,7 @@ const ModelDashboard: React.FC = ({ // For 'all' mode, show all models (teamAccessMatch remains true) return ( + searchMatch && modelNameMatch && accessGroupMatch && teamAccessMatch