diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index 9703d38a03b..7bbb13387af 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -200,6 +200,13 @@ The following parameters can be updated on a continuation of a trace by passing Any other key value pairs passed into the metadata not listed in the above spec for a `litellm` completion will be added as a metadata key value pair for the generation. +#### Disable Logging - Specific Calls + +To disable logging for specific calls use the `no-log` flag. + +`completion(messages = ..., model = ..., **{"no-log": True})` + + ### Use LangChain ChatLiteLLM + Langfuse Pass `trace_user_id`, `session_id` in model_kwargs ```python diff --git a/docs/my-website/docs/oidc.md b/docs/my-website/docs/oidc.md index 1f792f8d235..f30edf50440 100644 --- a/docs/my-website/docs/oidc.md +++ b/docs/my-website/docs/oidc.md @@ -19,9 +19,17 @@ LiteLLM supports the following OIDC identity providers: | CircleCI v2 | `circleci_v2`| No | | GitHub Actions | `github` | Yes | | Azure Kubernetes Service | `azure` | No | +| File | `file` | No | +| Environment Variable | `env` | No | +| Environment Path | `env_path` | No | If you would like to use a different OIDC provider, please open an issue on GitHub. +:::tip + +Do not use the `file`, `env`, or `env_path` providers unless you know what you're doing, and you are sure none of the other providers will work for your use-case. Hint: they probably will. + +::: ## OIDC Connect Relying Party (RP) @@ -46,6 +54,32 @@ For providers that do not use the `audience` parameter, you can (and should) omi oidc/config_name_here/ ``` +#### Unofficial Providers (not recommended) + +For the unofficial `file` provider, you can use the following format: + +``` +oidc/file/home/user/dave/this_is_a_file_with_a_token.txt +``` + +For the unofficial `env`, use the following format, where `SECRET_TOKEN` is the name of the environment variable that contains the token: + +``` +oidc/env/SECRET_TOKEN +``` + +For the unofficial `env_path`, use the following format, where `SECRET_TOKEN` is the name of the environment variable that contains the path to the file with the token: + +``` +oidc/env_path/SECRET_TOKEN +``` + +:::tip + +If you are tempted to use oidc/env_path/AZURE_FEDERATED_TOKEN_FILE, don't do that. Instead, use `oidc/azure/`, as this will ensure continued support from LiteLLM if Azure changes their OIDC configuration and/or adds new features. + +::: + ## Examples ### Google Cloud Run -> Amazon Bedrock diff --git a/docs/my-website/docs/proxy/oauth2.md b/docs/my-website/docs/proxy/oauth2.md new file mode 100644 index 00000000000..ec076d8fae3 --- /dev/null +++ b/docs/my-website/docs/proxy/oauth2.md @@ -0,0 +1,63 @@ +# Oauth 2.0 Authentication + +Use this if you want to use an Oauth2.0 token to make `/chat`, `/embeddings` requests to the LiteLLM Proxy + +:::info + +This is an Enterprise Feature - [get in touch with us if you want a free trial to test if this feature meets your needs]((https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)) + +::: + +## Usage + +1. Set env vars: + +```bash +export OAUTH_TOKEN_INFO_ENDPOINT="https://your-provider.com/token/info" +export OAUTH_USER_ID_FIELD_NAME="sub" +export OAUTH_USER_ROLE_FIELD_NAME="role" +export OAUTH_USER_TEAM_ID_FIELD_NAME="team_id" +``` + +- `OAUTH_TOKEN_INFO_ENDPOINT`: URL to validate OAuth tokens +- `OAUTH_USER_ID_FIELD_NAME`: Field in token info response containing user ID +- `OAUTH_USER_ROLE_FIELD_NAME`: Field in token info for user's role +- `OAUTH_USER_TEAM_ID_FIELD_NAME`: Field in token info for user's team ID + +2. Enable on litellm config.yaml + +Set this on your config.yaml + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + +general_settings: + master_key: sk-1234 + enable_oauth2_auth: true +``` + +3. Use token in requests to LiteLLM + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + +## Debugging + +Start the LiteLLM Proxy with [`--detailed_debug` mode and you should see more verbose logs](cli.md#detailed_debug) + diff --git a/docs/my-website/docs/proxy/ui.md b/docs/my-website/docs/proxy/ui.md index 1f71e633283..40c55a57ca1 100644 --- a/docs/my-website/docs/proxy/ui.md +++ b/docs/my-website/docs/proxy/ui.md @@ -82,6 +82,13 @@ litellm_settings: - Key will be created with `max_budget=100` since 100 is the upper bound #### Step 2: Setup Oauth Client + +:::tip + +Looking for how to use Oauth 2.0 for /chat, /completions API requests to the proxy? [Follow this doc](oauth2) + +::: + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index ac9586d7693..bfa8953d4cc 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -66,6 +66,7 @@ const sidebars = { "proxy/customers", "proxy/billing", "proxy/token_auth", + "proxy/oauth2", "proxy/alerting", "proxy/ui", "proxy/prometheus", diff --git a/enterprise/enterprise_hooks/banned_keywords.py b/enterprise/enterprise_hooks/banned_keywords.py index 3f3e01f5b6f..4d6545eb076 100644 --- a/enterprise/enterprise_hooks/banned_keywords.py +++ b/enterprise/enterprise_hooks/banned_keywords.py @@ -82,7 +82,11 @@ class _ENTERPRISE_BannedKeywords(CustomLogger): except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.error(traceback.format_exc()) + verbose_proxy_logger.exception( + "litellm.enterprise.enterprise_hooks.banned_keywords::async_pre_call_hook - Exception occurred - {}".format( + str(e) + ) + ) async def async_post_call_success_hook( self, diff --git a/enterprise/enterprise_hooks/blocked_user_list.py b/enterprise/enterprise_hooks/blocked_user_list.py index 8e642a026f6..9bda140ba7a 100644 --- a/enterprise/enterprise_hooks/blocked_user_list.py +++ b/enterprise/enterprise_hooks/blocked_user_list.py @@ -118,4 +118,8 @@ class _ENTERPRISE_BlockedUserList(CustomLogger): except HTTPException as e: raise e except Exception as e: - verbose_proxy_logger.error(traceback.format_exc()) + verbose_proxy_logger.exception( + "litellm.enterprise.enterprise_hooks.blocked_user_list::async_pre_call_hook - Exception occurred - {}".format( + str(e) + ) + ) diff --git a/enterprise/enterprise_hooks/llm_guard.py b/enterprise/enterprise_hooks/llm_guard.py index 9db10cf79ce..9724e08a8ba 100644 --- a/enterprise/enterprise_hooks/llm_guard.py +++ b/enterprise/enterprise_hooks/llm_guard.py @@ -92,7 +92,11 @@ class _ENTERPRISE_LLMGuard(CustomLogger): }, ) except Exception as e: - verbose_proxy_logger.error(traceback.format_exc()) + verbose_proxy_logger.exception( + "litellm.enterprise.enterprise_hooks.llm_guard::moderation_check - Exception occurred - {}".format( + str(e) + ) + ) raise e def should_proceed(self, user_api_key_dict: UserAPIKeyAuth, data: dict) -> bool: diff --git a/litellm/caching.py b/litellm/caching.py index c9b659d3f84..e37811b7733 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -1570,8 +1570,9 @@ class DualCache(BaseCache): if self.redis_cache is not None and local_only == False: await self.redis_cache.async_set_cache(key, value, **kwargs) except Exception as e: - verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") - verbose_logger.debug(traceback.format_exc()) + verbose_logger.exception( + f"LiteLLM Cache: Excepton async add_cache: {str(e)}" + ) async def async_batch_set_cache( self, cache_list: list, local_only: bool = False, **kwargs @@ -1593,8 +1594,9 @@ class DualCache(BaseCache): cache_list=cache_list, ttl=kwargs.get("ttl", None), **kwargs ) except Exception as e: - verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") - verbose_logger.debug(traceback.format_exc()) + verbose_logger.exception( + f"LiteLLM Cache: Excepton async add_cache: {str(e)}" + ) async def async_increment_cache( self, key, value: float, local_only: bool = False, **kwargs @@ -1618,8 +1620,9 @@ class DualCache(BaseCache): return result except Exception as e: - verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") - verbose_logger.debug(traceback.format_exc()) + verbose_logger.exception( + f"LiteLLM Cache: Excepton async add_cache: {str(e)}" + ) raise e async def async_set_cache_sadd( @@ -1647,10 +1650,8 @@ class DualCache(BaseCache): return None except Exception as e: - verbose_logger.error( - "LiteLLM Cache: Excepton async set_cache_sadd: {}\n{}".format( - str(e), traceback.format_exc() - ) + verbose_logger.exception( + "LiteLLM Cache: Excepton async set_cache_sadd: {}".format(str(e)) ) raise e @@ -2088,8 +2089,7 @@ class Cache: ) self.cache.set_cache(cache_key, cached_data, **kwargs) except Exception as e: - verbose_logger.error(f"LiteLLM Cache: Excepton add_cache: {str(e)}") - verbose_logger.debug(traceback.format_exc()) + verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}") pass async def async_add_cache(self, result, *args, **kwargs): @@ -2106,8 +2106,7 @@ class Cache: ) await self.cache.async_set_cache(cache_key, cached_data, **kwargs) except Exception as e: - verbose_logger.error(f"LiteLLM Cache: Excepton add_cache: {str(e)}") - verbose_logger.debug(traceback.format_exc()) + verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}") async def async_add_cache_pipeline(self, result, *args, **kwargs): """ @@ -2137,8 +2136,7 @@ class Cache: ) await asyncio.gather(*tasks) except Exception as e: - verbose_logger.error(f"LiteLLM Cache: Excepton add_cache: {str(e)}") - verbose_logger.debug(traceback.format_exc()) + verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}") async def batch_cache_write(self, result, *args, **kwargs): cache_key, cached_data, kwargs = self._add_cache_logic( diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 08676e968c4..3e1c429deae 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -235,10 +235,8 @@ class BraintrustLogger(CustomLogger): except httpx.HTTPStatusError as e: raise Exception(e.response.text) except Exception as e: - verbose_logger.error( - "Error logging to braintrust - Exception received - {}\n{}".format( - str(e), traceback.format_exc() - ) + verbose_logger.exception( + "Error logging to braintrust - Exception received - {}".format(str(e)) ) raise e @@ -362,10 +360,8 @@ class BraintrustLogger(CustomLogger): except httpx.HTTPStatusError as e: raise Exception(e.response.text) except Exception as e: - verbose_logger.error( - "Error logging to braintrust - Exception received - {}\n{}".format( - str(e), traceback.format_exc() - ) + verbose_logger.exception( + "Error logging to braintrust - Exception received - {}".format(str(e)) ) raise e diff --git a/litellm/integrations/email_alerting.py b/litellm/integrations/email_alerting.py index 434efb63b07..f0ad4353582 100644 --- a/litellm/integrations/email_alerting.py +++ b/litellm/integrations/email_alerting.py @@ -2,11 +2,12 @@ Functions for sending Email Alerts """ -import os -from typing import Optional, List -from litellm.proxy._types import WebhookEvent import asyncio +import os +from typing import List, Optional + from litellm._logging import verbose_logger, verbose_proxy_logger +from litellm.proxy._types import WebhookEvent # we use this for the email header, please send a test email if you change this. verify it looks good on email LITELLM_LOGO_URL = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" @@ -69,9 +70,8 @@ async def send_team_budget_alert(webhook_event: WebhookEvent) -> bool: Send an Email Alert to All Team Members when the Team Budget is crossed Returns -> True if sent, False if not. """ - from litellm.proxy.utils import send_email - from litellm.proxy.proxy_server import premium_user, prisma_client + from litellm.proxy.utils import send_email _team_id = webhook_event.team_id team_alias = webhook_event.team_alias @@ -101,7 +101,7 @@ async def send_team_budget_alert(webhook_event: WebhookEvent) -> bool: email_html_content = "Alert from LiteLLM Server" if recipient_emails_str is None: - verbose_proxy_logger.error( + verbose_proxy_logger.warning( "Email Alerting: Trying to send email alert to no recipient, got recipient_emails=%s", recipient_emails_str, ) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index d6c235d0cb7..0fb2ea1f7f3 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -246,10 +246,9 @@ class LangFuseLogger: return {"trace_id": trace_id, "generation_id": generation_id} except Exception as e: - verbose_logger.error( + verbose_logger.exception( "Langfuse Layer Error(): Exception occured - {}".format(str(e)) ) - verbose_logger.debug(traceback.format_exc()) return {"trace_id": None, "generation_id": None} async def _async_log_event( diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 08431fd7af9..7fde5abff91 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -329,10 +329,9 @@ class PrometheusLogger(CustomLogger): ).inc() self.set_llm_deployment_failure_metrics(kwargs) except Exception as e: - verbose_logger.error( + verbose_logger.exception( "prometheus Layer Error(): Exception occured - {}".format(str(e)) ) - verbose_logger.debug(traceback.format_exc()) pass pass diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3e7f61f72b5..74c6d0db018 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -348,9 +348,9 @@ class Logging: self.model_call_details ) # Expectation: any logger function passed in by the user should accept a dict object except Exception as e: - verbose_logger.error( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {}\n{}".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {}".format( + str(e) ) ) # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made @@ -400,9 +400,9 @@ class Logging: callback_func=callback, ) except Exception as e: - verbose_logger.error( - "litellm.Logging.pre_call(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "litellm.Logging.pre_call(): Exception occured - {}".format( + str(e) ) ) verbose_logger.debug( @@ -410,10 +410,10 @@ class Logging: ) if capture_exception: # log this error to sentry for debugging capture_exception(e) - except Exception: - verbose_logger.error( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {}\n{}".format( - str(e), traceback.format_exc() + except Exception as e: + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {}".format( + str(e) ) ) verbose_logger.error( @@ -458,9 +458,9 @@ class Logging: self.model_call_details ) # Expectation: any logger function passed in by the user should accept a dict object except Exception as e: - verbose_logger.debug( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {}\n{}".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {}".format( + str(e) ) ) original_response = redact_message_input_output_from_logging( @@ -496,9 +496,9 @@ class Logging: end_time=None, ) except Exception as e: - verbose_logger.error( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations {}\n{}".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations {}".format( + str(e) ) ) verbose_logger.debug( @@ -507,9 +507,9 @@ class Logging: if capture_exception: # log this error to sentry for debugging capture_exception(e) except Exception as e: - verbose_logger.error( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {}\n{}".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {}".format( + str(e) ) ) @@ -671,9 +671,9 @@ class Logging: end_time=end_time, ) except Exception as e: - verbose_logger.error( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while building complete streaming response in success logging {}\n{}".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while building complete streaming response in success logging {}".format( + str(e) ) ) complete_streaming_response = None @@ -1250,9 +1250,9 @@ class Logging: if capture_exception: # log this error to sentry for debugging capture_exception(e) except Exception as e: - verbose_logger.error( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {}\n{}".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {}".format( + str(e) ), ) @@ -1284,11 +1284,10 @@ class Logging: end_time=end_time, ) except Exception as e: - print_verbose( - "Error occurred building stream chunk in success logging: {}\n{}".format( - str(e), traceback.format_exc() - ), - log_level="ERROR", + verbose_logger.exception( + "Error occurred building stream chunk in success logging: {}".format( + str(e) + ) ) complete_streaming_response = None else: @@ -1781,9 +1780,9 @@ class Logging: if capture_exception: # log this error to sentry for debugging capture_exception(e) except Exception as e: - verbose_logger.error( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging {}\n{}".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging {}".format( + str(e) ) ) @@ -1819,10 +1818,10 @@ class Logging: callback_func=callback, ) except Exception as e: - verbose_logger.error( + verbose_logger.exception( "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success \ - logging {}\n{}\nCallback={}".format( - str(e), traceback.format_exc(), callback + logging {}\nCallback={}".format( + str(e), callback ) ) @@ -2363,9 +2362,7 @@ def get_standard_logging_object_payload( return payload except Exception as e: - verbose_logger.warning( - "Error creating standard logging object - {}\n{}".format( - str(e), traceback.format_exc() - ) + verbose_logger.error( + "Error creating standard logging object - {}".format(str(e)) ) return None diff --git a/litellm/litellm_core_utils/llm_cost_calc/google.py b/litellm/litellm_core_utils/llm_cost_calc/google.py index 0b4789deace..a9a04ad00f6 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/google.py +++ b/litellm/litellm_core_utils/llm_cost_calc/google.py @@ -118,10 +118,9 @@ def cost_per_character( ) prompt_cost = prompt_characters * model_info["input_cost_per_character"] except Exception as e: - verbose_logger.error( - "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): Exception occured - {}\n{}\n\ - Defaulting to (cost_per_token * 4) calculation for prompt_cost".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): Defaulting to (cost_per_token * 4) calculation for prompt_cost. Exception occured - {}".format( + str(e) ) ) initial_prompt_cost, _ = cost_per_token( @@ -161,10 +160,10 @@ def cost_per_character( completion_tokens * model_info["output_cost_per_character"] ) except Exception as e: - verbose_logger.error( - "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): Exception occured - {}\n{}\n\ - Defaulting to (cost_per_token * 4) calculation for completion_cost".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): \ + Defaulting to (cost_per_token * 4) calculation for completion_cost\nException occured - {}".format( + str(e) ) ) _, initial_completion_cost = cost_per_token( diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index e986a22a6c9..87799bc1f51 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -54,9 +54,9 @@ def _generic_cost_per_character( prompt_cost = prompt_characters * custom_prompt_cost except Exception as e: - verbose_logger.error( - "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\n{}\nDefaulting to None".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\nDefaulting to None".format( + str(e) ) ) @@ -74,9 +74,9 @@ def _generic_cost_per_character( custom_completion_cost = model_info["output_cost_per_character"] completion_cost = completion_characters * custom_completion_cost except Exception as e: - verbose_logger.error( - "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\n{}\nDefaulting to None".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\nDefaulting to None".format( + str(e) ) ) diff --git a/litellm/litellm_core_utils/streaming_utils.py b/litellm/litellm_core_utils/streaming_utils.py new file mode 100644 index 00000000000..ca8d58e9f5e --- /dev/null +++ b/litellm/litellm_core_utils/streaming_utils.py @@ -0,0 +1,16 @@ +from litellm.types.utils import GenericStreamingChunk as GChunk + + +def generic_chunk_has_all_required_fields(chunk: dict) -> bool: + """ + Checks if the provided chunk dictionary contains all required fields for GenericStreamingChunk. + + :param chunk: The dictionary to check. + :return: True if all required fields are present, False otherwise. + """ + _all_fields = GChunk.__annotations__ + + # this is an optional field in GenericStreamingChunk, it's not required to be present + _all_fields.pop("provider_specific_fields", None) + + return all(key in chunk for key in _all_fields) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index cf58163461a..d533fdbb290 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -952,11 +952,14 @@ class AnthropicChatCompletion(BaseLLM): model=model, messages=messages, custom_llm_provider="anthropic" ) except Exception as e: + verbose_logger.exception( + "litellm.llms.anthropic.py::completion() - Exception occurred - {}\nReceived Messages: {}".format( + str(e), messages + ) + ) raise AnthropicError( status_code=400, - message="{}\n{}\nReceived Messages={}".format( - str(e), traceback.format_exc(), messages - ), + message="{}\nReceived Messages={}".format(str(e), messages), ) ## Load Config diff --git a/litellm/llms/bedrock_httpx.py b/litellm/llms/bedrock_httpx.py index 73387212ffc..e4555975245 100644 --- a/litellm/llms/bedrock_httpx.py +++ b/litellm/llms/bedrock_httpx.py @@ -1365,6 +1365,10 @@ class BedrockConverseLLM(BaseAWSLLM): ) setattr(model_response, "usage", usage) + # Add "trace" from Bedrock guardrails - if user has opted in to returning it + if "trace" in completion_response: + setattr(model_response, "trace", completion_response["trace"]) + return model_response def encode_model_id(self, model_id: str) -> str: @@ -1900,6 +1904,10 @@ class AWSEventStreamDecoder: usage=usage, index=index, ) + + if "trace" in chunk_data: + trace = chunk_data.get("trace") + response["provider_specific_fields"] = {"trace": trace} return response except Exception as e: raise Exception("Received streaming error - {}".format(str(e))) @@ -1920,6 +1928,7 @@ class AWSEventStreamDecoder: "contentBlockIndex" in chunk_data or "stopReason" in chunk_data or "metrics" in chunk_data + or "trace" in chunk_data ): return self.converse_chunk_parser(chunk_data=chunk_data) ######## bedrock.mistral mappings ############### diff --git a/litellm/llms/gemini.py b/litellm/llms/gemini.py index 3ce63e93fe0..179554ea47f 100644 --- a/litellm/llms/gemini.py +++ b/litellm/llms/gemini.py @@ -274,7 +274,6 @@ class GeminiConfig: # model_response.choices = choices_list # except Exception as e: # verbose_logger.error("LiteLLM.gemini.py: Exception occured - {}".format(str(e))) -# verbose_logger.debug(traceback.format_exc()) # raise GeminiError( # message=traceback.format_exc(), status_code=response.status_code # ) @@ -367,7 +366,6 @@ class GeminiConfig: # model_response["choices"] = choices_list # except Exception as e: # verbose_logger.error("LiteLLM.gemini.py: Exception occured - {}".format(str(e))) -# verbose_logger.debug(traceback.format_exc()) # raise GeminiError( # message=traceback.format_exc(), status_code=response.status_code # ) diff --git a/litellm/llms/ollama.py b/litellm/llms/ollama.py index f699cf0f5f5..ad8d4085843 100644 --- a/litellm/llms/ollama.py +++ b/litellm/llms/ollama.py @@ -422,12 +422,11 @@ async def ollama_async_streaming(url, data, model_response, encoding, logging_ob async for transformed_chunk in streamwrapper: yield transformed_chunk except Exception as e: - verbose_logger.error( + verbose_logger.exception( "LiteLLM.ollama.py::ollama_async_streaming(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) raise e @@ -498,12 +497,11 @@ async def ollama_acompletion( ) return model_response except Exception as e: - verbose_logger.error( + verbose_logger.exception( "LiteLLM.ollama.py::ollama_acompletion(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) raise e @@ -609,5 +607,4 @@ def ollama_embeddings( logging_obj=logging_obj, encoding=encoding, ) - ) diff --git a/litellm/llms/ollama_chat.py b/litellm/llms/ollama_chat.py index ea84fa95cf5..2c55a3c0a14 100644 --- a/litellm/llms/ollama_chat.py +++ b/litellm/llms/ollama_chat.py @@ -356,7 +356,7 @@ def ollama_completion_stream(url, api_key, data, logging_obj): "json": data, "method": "POST", "timeout": litellm.request_timeout, - "follow_redirects": True + "follow_redirects": True, } if api_key is not None: _request["headers"] = {"Authorization": "Bearer {}".format(api_key)} @@ -471,8 +471,9 @@ async def ollama_async_streaming( async for transformed_chunk in streamwrapper: yield transformed_chunk except Exception as e: - verbose_logger.error("LiteLLM.gemini(): Exception occured - {}".format(str(e))) - verbose_logger.debug(traceback.format_exc()) + verbose_logger.exception( + "LiteLLM.ollama(): Exception occured - {}".format(str(e)) + ) async def ollama_acompletion( @@ -559,9 +560,8 @@ async def ollama_acompletion( ) return model_response except Exception as e: - verbose_logger.error( + verbose_logger.exception( "LiteLLM.ollama_acompletion(): Exception occured - {}".format(str(e)) ) - verbose_logger.debug(traceback.format_exc()) raise e diff --git a/litellm/llms/palm.py b/litellm/llms/palm.py index b750b800bbc..a17fd02beed 100644 --- a/litellm/llms/palm.py +++ b/litellm/llms/palm.py @@ -168,10 +168,9 @@ def completion( choices_list.append(choice_obj) model_response.choices = choices_list # type: ignore except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.llms.palm.py::completion(): Exception occured - {}".format(str(e)) ) - verbose_logger.debug(traceback.format_exc()) raise PalmError( message=traceback.format_exc(), status_code=response.status_code ) diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py index d7a10c2f525..84e2810a565 100644 --- a/litellm/llms/predibase.py +++ b/litellm/llms/predibase.py @@ -17,6 +17,7 @@ import requests # type: ignore import litellm import litellm.litellm_core_utils import litellm.litellm_core_utils.litellm_logging +from litellm import verbose_logger from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.utils import Choices, CustomStreamWrapper, Message, ModelResponse, Usage @@ -563,9 +564,12 @@ class PredibaseChatCompletion(BaseLLM): for exception in litellm.LITELLM_EXCEPTION_TYPES: if isinstance(e, exception): raise e - raise PredibaseError( - status_code=500, message="{}\n{}".format(str(e), traceback.format_exc()) + verbose_logger.exception( + "litellm.llms.predibase.py::async_completion() - Exception occurred - {}".format( + str(e) + ) ) + raise PredibaseError(status_code=500, message="{}".format(str(e))) return self.process_response( model=model, response=response, diff --git a/litellm/llms/text_completion_codestral.py b/litellm/llms/text_completion_codestral.py index 7c758f5b5f0..a6865b9533c 100644 --- a/litellm/llms/text_completion_codestral.py +++ b/litellm/llms/text_completion_codestral.py @@ -15,6 +15,7 @@ import httpx # type: ignore import requests # type: ignore import litellm +from litellm import verbose_logger from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.llms.databricks import GenericStreamingChunk @@ -489,8 +490,13 @@ class CodestralTextCompletion(BaseLLM): message="HTTPStatusError - {}".format(e.response.text), ) except Exception as e: + verbose_logger.exception( + "litellm.llms.text_completion_codestral.py::async_completion() - Exception occurred - {}".format( + str(e) + ) + ) raise TextCompletionCodestralError( - status_code=500, message="{}\n{}".format(str(e), traceback.format_exc()) + status_code=500, message="{}".format(str(e)) ) return self.process_text_completion_response( model=model, diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index ce6a31d7bda..b927306133e 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -1499,6 +1499,14 @@ class VertexLLM(BaseLLM): """ _json_response = response.json() + + if "predictions" not in _json_response: + raise litellm.InternalServerError( + message=f"image generation response does not contain 'predictions', got {_json_response}", + llm_provider="vertex_ai", + model=model, + ) + _predictions = _json_response["predictions"] _response_data: List[Image] = [] diff --git a/litellm/main.py b/litellm/main.py index cf7a4a5e7ea..24ae12631d2 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -420,7 +420,9 @@ async def acompletion( ) # sets the logging event loop if the user does sync streaming (e.g. on proxy for sagemaker calls) return response except Exception as e: - verbose_logger.debug(traceback.format_exc()) + verbose_logger.exception( + "litellm.main.py::acompletion() - Exception occurred - {}".format(str(e)) + ) custom_llm_provider = custom_llm_provider or "openai" raise exception_type( model=model, @@ -585,10 +587,9 @@ def mock_completion( except Exception as e: if isinstance(e, openai.APIError): raise e - verbose_logger.error( + verbose_logger.exception( "litellm.mock_completion(): Exception occured - {}".format(str(e)) ) - verbose_logger.debug(traceback.format_exc()) raise Exception("Mock completion response failed") @@ -4779,7 +4780,9 @@ async def ahealth_check( For azure/openai -> completion.with_raw_response For rest -> litellm.acompletion() """ + passed_in_mode: Optional[str] = None try: + model: Optional[str] = model_params.get("model", None) if model is None: @@ -4793,7 +4796,10 @@ async def ahealth_check( if model in litellm.model_cost and mode is None: mode = litellm.model_cost[model].get("mode") - mode = mode or "chat" # default to chat completion calls + mode = mode + passed_in_mode = mode + if mode is None: + mode = "chat" # default to chat completion calls if custom_llm_provider == "azure": api_key = ( @@ -4883,13 +4889,14 @@ async def ahealth_check( response = {} # args like remaining ratelimit etc. return response except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.ahealth_check(): Exception occured - {}".format(str(e)) ) stack_trace = traceback.format_exc() if isinstance(stack_trace, str): stack_trace = stack_trace[:1000] - if model not in litellm.model_cost and mode is None: + + if passed_in_mode is None: return { "error": "Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models" } @@ -5232,9 +5239,9 @@ def stream_chunk_builder( end_time=end_time, ) # type: ignore except Exception as e: - verbose_logger.error( - "litellm.main.py::stream_chunk_builder() - Exception occurred - {}\n{}".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "litellm.main.py::stream_chunk_builder() - Exception occurred - {}".format( + str(e) ) ) raise litellm.APIError( diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index d18a3cb992a..9c35db02f19 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,12 +1,6 @@ model_list: - - model_name: "*" + - model_name: "text-embedding-ada-002" litellm_params: - model: "*" - -litellm_settings: - success_callback: ["s3"] - s3_callback_params: - s3_bucket_name: mytestbucketlitellm # AWS Bucket Name for S3 - s3_region_name: us-west-2 # AWS Region Name for S3 - s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/ to pass environment variables. This is AWS Access Key ID for S3 - s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3 + model: "azure/azure-embedding-model" + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY diff --git a/litellm/proxy/auth/oauth2_check.py b/litellm/proxy/auth/oauth2_check.py new file mode 100644 index 00000000000..ed5a3e26b13 --- /dev/null +++ b/litellm/proxy/auth/oauth2_check.py @@ -0,0 +1,78 @@ +from litellm.proxy._types import UserAPIKeyAuth + + +async def check_oauth2_token(token: str) -> UserAPIKeyAuth: + """ + Makes a request to the token info endpoint to validate the OAuth2 token. + + Args: + token (str): The OAuth2 token to validate. + + Returns: + Literal[True]: If the token is valid. + + Raises: + ValueError: If the token is invalid, the request fails, or the token info endpoint is not set. + """ + import os + from typing import Literal + + import httpx + + from litellm._logging import verbose_proxy_logger + from litellm.llms.custom_httpx.http_handler import _get_async_httpx_client + from litellm.proxy._types import CommonProxyErrors + from litellm.proxy.proxy_server import premium_user + + if premium_user is not True: + raise ValueError( + "Oauth2 token validation is only available for premium users" + + CommonProxyErrors.not_premium_user.value + ) + + verbose_proxy_logger.debug("Oauth2 token validation for token=%s", token) + # Get the token info endpoint from environment variable + token_info_endpoint = os.getenv("OAUTH_TOKEN_INFO_ENDPOINT") + user_id_field_name = os.environ.get("OAUTH_USER_ID_FIELD_NAME", "sub") + user_role_field_name = os.environ.get("OAUTH_USER_ROLE_FIELD_NAME", "role") + user_team_id_field_name = os.environ.get("OAUTH_USER_TEAM_ID_FIELD_NAME", "team_id") + + if not token_info_endpoint: + raise ValueError("OAUTH_TOKEN_INFO_ENDPOINT environment variable is not set") + + client = _get_async_httpx_client() + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + + try: + response = await client.get(token_info_endpoint, headers=headers) + + # if it's a bad token we expect it to raise an HTTPStatusError + response.raise_for_status() + + # If we get here, the request was successful + data = response.json() + + verbose_proxy_logger.debug( + "Oauth2 token validation for token=%s, response from /token/info=%s", + token, + data, + ) + + # You might want to add additional checks here based on the response + # For example, checking if the token is expired or has the correct scope + user_id = data.get(user_id_field_name) + user_team_id = data.get(user_team_id_field_name) + user_role = data.get(user_role_field_name) + + return UserAPIKeyAuth( + api_key=token, + team_id=user_team_id, + user_id=user_id, + user_role=user_role, + ) + except httpx.HTTPStatusError as e: + # This will catch any 4xx or 5xx errors + raise ValueError(f"Oauth 2.0 Token validation failed: {e}") + except Exception as e: + # This will catch any other errors (like network issues) + raise ValueError(f"An error occurred during token validation: {e}") diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 00e78f64e65..dd58ed390b3 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -62,6 +62,7 @@ from litellm.proxy.auth.auth_utils import ( is_llm_api_route, route_in_additonal_public_routes, ) +from litellm.proxy.auth.oauth2_check import check_oauth2_token from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.utils import _to_ns @@ -197,6 +198,19 @@ async def user_api_key_auth( # check if public endpoint return UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY) + if general_settings.get("enable_oauth2_auth", False) is True: + # return UserAPIKeyAuth object + # helper to check if the api_key is a valid oauth2 token + from litellm.proxy.proxy_server import premium_user + + if premium_user is not True: + raise ValueError( + "Oauth2 token validation is only available for premium users" + + CommonProxyErrors.not_premium_user.value + ) + + return await check_oauth2_token(token=api_key) + if general_settings.get("enable_jwt_auth", False) is True: is_jwt = jwt_handler.is_jwt(token=api_key) verbose_proxy_logger.debug("is_jwt: %s", is_jwt) @@ -1123,9 +1137,9 @@ async def user_api_key_auth( else: raise Exception() except Exception as e: - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - {}".format( + str(e) ) ) diff --git a/litellm/proxy/guardrails/init_guardrails.py b/litellm/proxy/guardrails/init_guardrails.py index de61818689d..4d234576fcf 100644 --- a/litellm/proxy/guardrails/init_guardrails.py +++ b/litellm/proxy/guardrails/init_guardrails.py @@ -62,9 +62,7 @@ def initialize_guardrails( return litellm.guardrail_name_config_map except Exception as e: - verbose_proxy_logger.error( - "error initializing guardrails {}\n{}".format( - str(e), traceback.format_exc() - ) + verbose_proxy_logger.exception( + "error initializing guardrails {}".format(str(e)) ) raise e diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 5713fa782bc..ff5ed7bfb75 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) ILLEGAL_DISPLAY_PARAMS = ["messages", "api_key", "prompt", "input"] -MINIMAL_DISPLAY_PARAMS = ["model"] +MINIMAL_DISPLAY_PARAMS = ["model", "mode_error"] def _get_random_llm_message(): @@ -31,7 +31,7 @@ def _clean_endpoint_data(endpoint_data: dict, details: Optional[bool] = True): """ return ( {k: v for k, v in endpoint_data.items() if k not in ILLEGAL_DISPLAY_PARAMS} - if details + if details is not False else {k: v for k, v in endpoint_data.items() if k in MINIMAL_DISPLAY_PARAMS} ) diff --git a/litellm/proxy/hooks/cache_control_check.py b/litellm/proxy/hooks/cache_control_check.py index 238e2e6ab71..d933bfc7548 100644 --- a/litellm/proxy/hooks/cache_control_check.py +++ b/litellm/proxy/hooks/cache_control_check.py @@ -1,13 +1,15 @@ # What this does? ## Checks if key is allowed to use the cache controls passed in to the completion() call +import traceback + +from fastapi import HTTPException + import litellm from litellm import verbose_logger from litellm.caching import DualCache -from litellm.proxy._types import UserAPIKeyAuth from litellm.integrations.custom_logger import CustomLogger -from fastapi import HTTPException -import traceback +from litellm.proxy._types import UserAPIKeyAuth class _PROXY_CacheControlCheck(CustomLogger): @@ -54,9 +56,8 @@ class _PROXY_CacheControlCheck(CustomLogger): except HTTPException as e: raise e except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.proxy.hooks.cache_control_check.py::async_pre_call_hook(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter.py b/litellm/proxy/hooks/dynamic_rate_limiter.py index f5621055bdd..4bf08998a4c 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter.py @@ -65,9 +65,9 @@ class DynamicRateLimiterCache: key=key_name, value=value, ttl=self.ttl ) except Exception as e: - verbose_proxy_logger.error( - "litellm.proxy.hooks.dynamic_rate_limiter.py::async_set_cache_sadd(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "litellm.proxy.hooks.dynamic_rate_limiter.py::async_set_cache_sadd(): Exception occured - {}".format( + str(e) ) ) raise e @@ -179,9 +179,9 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): active_projects, ) except Exception as e: - verbose_proxy_logger.error( - "litellm.proxy.hooks.dynamic_rate_limiter.py::check_available_usage: Exception occurred - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "litellm.proxy.hooks.dynamic_rate_limiter.py::check_available_usage: Exception occurred - {}".format( + str(e) ) ) return None, None, None, None, None @@ -290,9 +290,9 @@ class _PROXY_DynamicRateLimitHandler(CustomLogger): user_api_key_dict, response ) except Exception as e: - verbose_proxy_logger.error( - "litellm.proxy.hooks.dynamic_rate_limiter.py::async_post_call_success_hook(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "litellm.proxy.hooks.dynamic_rate_limiter.py::async_post_call_success_hook(): Exception occured - {}".format( + str(e) ) ) return response diff --git a/litellm/proxy/hooks/max_budget_limiter.py b/litellm/proxy/hooks/max_budget_limiter.py index c4b328bab09..88614d4a286 100644 --- a/litellm/proxy/hooks/max_budget_limiter.py +++ b/litellm/proxy/hooks/max_budget_limiter.py @@ -1,11 +1,13 @@ -from litellm import verbose_logger -import litellm -from litellm.caching import DualCache -from litellm.proxy._types import UserAPIKeyAuth -from litellm.integrations.custom_logger import CustomLogger -from fastapi import HTTPException import traceback +from fastapi import HTTPException + +import litellm +from litellm import verbose_logger +from litellm.caching import DualCache +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy._types import UserAPIKeyAuth + class _PROXY_MaxBudgetLimiter(CustomLogger): # Class variables or attributes @@ -44,9 +46,8 @@ class _PROXY_MaxBudgetLimiter(CustomLogger): except HTTPException as e: raise e except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.proxy.hooks.max_budget_limiter.py::async_pre_call_hook(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index 65c30f10eb1..239c36b45f1 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -536,8 +536,8 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): request_count_api_key, new_val, ttl=60 ) # save in cache for up to 1 min. except Exception as e: - verbose_proxy_logger.info( - "Inside Parallel Request Limiter: An exception occurred - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "Inside Parallel Request Limiter: An exception occurred - {}".format( + str(e) ) ) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 61893a3dc4e..ead0b7eb79d 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -233,12 +233,11 @@ async def chat_completion_pass_through_endpoint( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.completion(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.completion(): Exception occured - {}".format( + str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) error_msg = f"{str(e)}" raise ProxyException( message=getattr(e, "message", error_msg), @@ -369,12 +368,11 @@ async def pass_through_request( headers=dict(response.headers), ) except Exception as e: - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.pass_through_endpoint(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.pass_through_endpoint(): Exception occured - {}".format( + str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e.detail)), diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 10c06b2ece6..f09ae7d350f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -199,7 +199,6 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( router as pass_through_router, ) from litellm.proxy.route_llm_request import route_request - from litellm.proxy.secret_managers.aws_secret_manager import ( load_aws_kms, load_aws_secret_manager, @@ -913,8 +912,8 @@ async def update_database( + prisma_client.key_list_transactons.get(hashed_token, 0) ) except Exception as e: - verbose_proxy_logger.error( - f"Update Key DB Call failed to execute - {str(e)}\n{traceback.format_exc()}" + verbose_proxy_logger.exception( + f"Update Key DB Call failed to execute - {str(e)}" ) raise e @@ -1206,8 +1205,8 @@ async def update_cache( existing_spend_obj.spend = new_spend user_api_key_cache.set_cache(key=_id, value=existing_spend_obj.json()) except Exception as e: - verbose_proxy_logger.error( - f"An error occurred updating end user cache: {str(e)}\n\n{traceback.format_exc()}" + verbose_proxy_logger.exception( + f"An error occurred updating end user cache: {str(e)}" ) ### UPDATE TEAM SPEND ### @@ -1248,8 +1247,8 @@ async def update_cache( existing_spend_obj.spend = new_spend user_api_key_cache.set_cache(key=_id, value=existing_spend_obj) except Exception as e: - verbose_proxy_logger.error( - f"An error occurred updating end user cache: {str(e)}\n\n{traceback.format_exc()}" + verbose_proxy_logger.exception( + f"An error occurred updating end user cache: {str(e)}" ) if token is not None and response_cost is not None: @@ -2116,7 +2115,7 @@ class ProxyConfig: self._add_deployment(db_models=new_models) except Exception as e: - verbose_proxy_logger.error( + verbose_proxy_logger.exception( f"Error adding/deleting model to llm_router: {str(e)}" ) @@ -2264,7 +2263,7 @@ class ProxyConfig: try: new_models = await prisma_client.db.litellm_proxymodeltable.find_many() except Exception as e: - verbose_proxy_logger.error( + verbose_proxy_logger.exception( "litellm.proxy_server.py::add_deployment() - Error getting new models from DB - {}".format( str(e) ) @@ -2286,8 +2285,10 @@ class ProxyConfig: ) except Exception as e: - verbose_proxy_logger.error( - "{}\nTraceback:{}".format(str(e), traceback.format_exc()) + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {}".format( + str(e) + ) ) @@ -2454,12 +2455,11 @@ async def async_assistants_data_generator( done_message = "[DONE]" yield f"data: {done_message}\n\n" except Exception as e: - verbose_proxy_logger.error( + verbose_proxy_logger.exception( "litellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - {}".format( str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, @@ -2512,9 +2512,9 @@ async def async_data_generator( done_message = "[DONE]" yield f"data: {done_message}\n\n" except Exception as e: - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.async_data_generator(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.async_data_generator(): Exception occured - {}".format( + str(e) ) ) await proxy_logging_obj.post_call_failure_hook( @@ -2565,9 +2565,9 @@ async def async_data_generator_anthropic( except Exception as e: yield f"event: {event_type}\ndata:{str(e)}\n\n" except Exception as e: - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.async_data_generator(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.async_data_generator(): Exception occured - {}".format( + str(e) ) ) await proxy_logging_obj.post_call_failure_hook( @@ -3181,10 +3181,8 @@ async def chat_completion( _chat_response.usage = _usage # type: ignore return _chat_response except Exception as e: - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.chat_completion(): Exception occured - {}\n{}".format( - get_error_message_str(e=e), traceback.format_exc() - ) + verbose_proxy_logger.exception( + f"litellm.proxy.proxy_server.chat_completion(): Exception occured - {str(e)}" ) await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data @@ -3567,12 +3565,11 @@ async def embeddings( e, litellm_debug_info, ) - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.embeddings(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.embeddings(): Exception occured - {}".format( + str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): message = get_error_message_str(e) raise ProxyException( @@ -5381,9 +5378,9 @@ async def anthropic_response( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.anthropic_response(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.anthropic_response(): Exception occured - {}".format( + str(e) ) ) error_msg = f"{str(e)}" @@ -9540,12 +9537,11 @@ async def get_config(): "available_callbacks": all_available_callbacks, } except Exception as e: - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.get_config(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.get_config(): Exception occured - {}".format( + str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Authentication Error({str(e)})"), diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 6a28d70b172..df4a2edfc37 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -121,9 +121,7 @@ def get_logging_payload( return payload except Exception as e: - verbose_proxy_logger.error( - "Error creating spendlogs object - {}\n{}".format( - str(e), traceback.format_exc() - ) + verbose_proxy_logger.exception( + "Error creating spendlogs object - {}".format(str(e)) ) raise e diff --git a/litellm/router.py b/litellm/router.py index 3f2be7cb268..7bc8acae46f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3027,9 +3027,9 @@ class Router: ) except Exception as e: - verbose_router_logger.error( - "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_router_logger.exception( + "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format( + str(e) ) ) pass diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index 46cbb2181ec..6c016aa0311 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -1,16 +1,16 @@ #### What this does #### # picks based on response time (for streaming, this is time to first token) -from pydantic import BaseModel -from typing import Optional, Union, List, Dict -from datetime import datetime, timedelta -from litellm import verbose_logger import traceback +from datetime import datetime, timedelta +from typing import Dict, List, Optional, Union + +from pydantic import BaseModel + +import litellm +from litellm import ModelResponse, token_counter, verbose_logger +from litellm._logging import verbose_router_logger from litellm.caching import DualCache from litellm.integrations.custom_logger import CustomLogger -from litellm._logging import verbose_router_logger -from litellm import ModelResponse -from litellm import token_counter -import litellm class LiteLLMBase(BaseModel): @@ -117,12 +117,11 @@ class LowestCostLoggingHandler(CustomLogger): if self.test_flag: self.logged_success += 1 except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) pass async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -204,12 +203,11 @@ class LowestCostLoggingHandler(CustomLogger): if self.test_flag: self.logged_success += 1 except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) pass async def async_get_available_deployments( diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 5d71847510e..5807fa68d75 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -1,16 +1,16 @@ #### What this does #### # picks based on response time (for streaming, this is time to first token) -from pydantic import BaseModel import random -from typing import Optional, Union, List, Dict -from datetime import datetime, timedelta import traceback +from datetime import datetime, timedelta +from typing import Dict, List, Optional, Union + +from pydantic import BaseModel + +import litellm +from litellm import ModelResponse, token_counter, verbose_logger from litellm.caching import DualCache from litellm.integrations.custom_logger import CustomLogger -from litellm import ModelResponse -from litellm import token_counter -import litellm -from litellm import verbose_logger class LiteLLMBase(BaseModel): @@ -165,12 +165,11 @@ class LowestLatencyLoggingHandler(CustomLogger): if self.test_flag: self.logged_success += 1 except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) pass async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): @@ -234,12 +233,11 @@ class LowestLatencyLoggingHandler(CustomLogger): # do nothing if it's not a timeout error return except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) pass async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -362,12 +360,11 @@ class LowestLatencyLoggingHandler(CustomLogger): if self.test_flag: self.logged_success += 1 except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.router_strategy.lowest_latency.py::async_log_success_event(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) pass def get_available_deployments( diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index e3b8c8b7708..cefebf5e734 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -1,17 +1,19 @@ #### What this does #### # identifies lowest tpm deployment -from pydantic import BaseModel import random -from typing import Optional, Union, List, Dict import traceback +from typing import Dict, List, Optional, Union + import httpx +from pydantic import BaseModel + import litellm from litellm import token_counter +from litellm._logging import verbose_logger, verbose_router_logger from litellm.caching import DualCache from litellm.integrations.custom_logger import CustomLogger -from litellm._logging import verbose_router_logger, verbose_logger -from litellm.utils import print_verbose, get_utc_datetime from litellm.types.router import RouterErrors +from litellm.utils import get_utc_datetime, print_verbose class LiteLLMBase(BaseModel): @@ -257,12 +259,11 @@ class LowestTPMLoggingHandler_v2(CustomLogger): if self.test_flag: self.logged_success += 1 except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) pass async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -308,12 +309,11 @@ class LowestTPMLoggingHandler_v2(CustomLogger): if self.test_flag: self.logged_success += 1 except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) pass def _common_checks_available_deployment( diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index a7327bde4e7..fca4f1ee558 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -363,6 +363,8 @@ def test_vertex_ai(): assert response.choices[0].finish_reason in litellm._openai_finish_reasons except litellm.RateLimitError as e: pass + except litellm.InternalServerError as e: + pass except Exception as e: pytest.fail(f"Error occurred: {e}") @@ -409,6 +411,8 @@ def test_vertex_ai_stream(): assert len(completed_str) > 1 except litellm.RateLimitError as e: pass + except litellm.InternalServerError as e: + pass except Exception as e: pytest.fail(f"Error occurred: {e}") @@ -449,6 +453,8 @@ async def test_async_vertexai_response(): pass except litellm.APIError as e: pass + except litellm.InternalServerError as e: + pass except Exception as e: pytest.fail(f"An exception occurred: {e}") @@ -497,6 +503,8 @@ async def test_async_vertexai_streaming_response(): pass except litellm.Timeout as e: pass + except litellm.InternalServerError as e: + pass except Exception as e: print(e) pytest.fail(f"An exception occurred: {e}") @@ -1589,7 +1597,8 @@ async def test_gemini_pro_httpx_custom_api_base(provider): extra_headers={"hello": "world"}, ) except Exception as e: - print("Receives error - {}\n{}".format(str(e), traceback.format_exc())) + traceback.print_exc() + print("Receives error - {}".format(str(e))) mock_call.assert_called_once() diff --git a/litellm/tests/test_bedrock_completion.py b/litellm/tests/test_bedrock_completion.py index c331021213f..4892601b151 100644 --- a/litellm/tests/test_bedrock_completion.py +++ b/litellm/tests/test_bedrock_completion.py @@ -82,33 +82,74 @@ def test_completion_bedrock_claude_completion_auth(): # test_completion_bedrock_claude_completion_auth() -def test_completion_bedrock_guardrails(): +@pytest.mark.parametrize("streaming", [True, False]) +def test_completion_bedrock_guardrails(streaming): import os litellm.set_verbose = True + import logging + from litellm._logging import verbose_logger + + # verbose_logger.setLevel(logging.DEBUG) try: - response = completion( - model="anthropic.claude-v2", - messages=[ - { - "content": "where do i buy coffee from? ", - "role": "user", - } - ], - max_tokens=10, - guardrailConfig={ - "guardrailIdentifier": "ff6ujrregl1q", - "guardrailVersion": "DRAFT", - "trace": "disabled", - }, - ) - # Add any assertions here to check the response - print(response) - assert ( - "Sorry, the model cannot answer this question. coffee guardrail applied" - in response.choices[0].message.content - ) + if streaming is False: + response = completion( + model="anthropic.claude-v2", + messages=[ + { + "content": "where do i buy coffee from? ", + "role": "user", + } + ], + max_tokens=10, + guardrailConfig={ + "guardrailIdentifier": "ff6ujrregl1q", + "guardrailVersion": "DRAFT", + "trace": "enabled", + }, + ) + # Add any assertions here to check the response + print(response) + assert ( + "Sorry, the model cannot answer this question. coffee guardrail applied" + in response.choices[0].message.content + ) + + assert "trace" in response + assert response.trace is not None + + print("TRACE=", response.trace) + else: + + response = completion( + model="anthropic.claude-v2", + messages=[ + { + "content": "where do i buy coffee from? ", + "role": "user", + } + ], + stream=True, + max_tokens=10, + guardrailConfig={ + "guardrailIdentifier": "ff6ujrregl1q", + "guardrailVersion": "DRAFT", + "trace": "enabled", + }, + ) + + saw_trace = False + + for chunk in response: + if "trace" in chunk: + saw_trace = True + print(chunk) + + assert ( + saw_trace is True + ), "Did not see trace in response even when trace=enabled sent in the guardrailConfig" + except RateLimitError: pass except Exception as e: diff --git a/litellm/tests/test_exceptions.py b/litellm/tests/test_exceptions.py index dfefe99d658..806e1956921 100644 --- a/litellm/tests/test_exceptions.py +++ b/litellm/tests/test_exceptions.py @@ -806,7 +806,8 @@ def test_exception_mapping(provider): except expected_exception: continue except Exception as e: - response = "{}\n{}".format(str(e), traceback.format_exc()) + traceback.print_exc() + response = "{}".format(str(e)) pytest.fail( "Did not raise expected exception. Expected={}, Return={},".format( expected_exception, response diff --git a/litellm/tests/test_secret_manager.py b/litellm/tests/test_secret_manager.py index 904b291bfec..652e209895b 100644 --- a/litellm/tests/test_secret_manager.py +++ b/litellm/tests/test_secret_manager.py @@ -5,6 +5,8 @@ from dotenv import load_dotenv load_dotenv() import os +from uuid import uuid4 +import tempfile sys.path.insert( 0, os.path.abspath("../..") @@ -135,3 +137,62 @@ def test_oidc_circle_v1_with_amazon_fips(): aws_session_name="assume-v1-session-fips", aws_sts_endpoint="https://sts-fips.us-west-1.amazonaws.com", ) + + +def test_oidc_env_variable(): + # Create a unique environment variable name + env_var_name = "OIDC_TEST_PATH_" + uuid4().hex + os.environ[env_var_name] = "secret-" + uuid4().hex + secret_val = get_secret( + f"oidc/env/{env_var_name}" + ) + + print(f"secret_val: {redact_oidc_signature(secret_val)}") + + assert secret_val == os.environ[env_var_name] + + # now unset the environment variable + del os.environ[env_var_name] + + +def test_oidc_file(): + # Create a temporary file + with tempfile.NamedTemporaryFile(mode='w+') as temp_file: + secret_value = "secret-" + uuid4().hex + temp_file.write(secret_value) + temp_file.flush() + temp_file_path = temp_file.name + + secret_val = get_secret( + f"oidc/file/{temp_file_path}" + ) + + print(f"secret_val: {redact_oidc_signature(secret_val)}") + + assert secret_val == secret_value + + +def test_oidc_env_path(): + # Create a temporary file + with tempfile.NamedTemporaryFile(mode='w+') as temp_file: + secret_value = "secret-" + uuid4().hex + temp_file.write(secret_value) + temp_file.flush() + temp_file_path = temp_file.name + + # Create a unique environment variable name + env_var_name = "OIDC_TEST_PATH_" + uuid4().hex + + # Set the environment variable to the temporary file path + os.environ[env_var_name] = temp_file_path + + # Test getting the secret using the environment variable + secret_val = get_secret( + f"oidc/env_path/{env_var_name}" + ) + + print(f"secret_val: {redact_oidc_signature(secret_val)}") + + assert secret_val == secret_value + + del os.environ[env_var_name] diff --git a/litellm/tests/test_token_counter.py b/litellm/tests/test_token_counter.py index b6dca32c64a..b32e63fbfb6 100644 --- a/litellm/tests/test_token_counter.py +++ b/litellm/tests/test_token_counter.py @@ -356,3 +356,22 @@ def test_gpt_4o_token_counter(): ) mock_client.assert_called() + + +@pytest.mark.parametrize( + "img_url", + [ + "https://blog.purpureus.net/assets/blog/personal_key_rotation/simplified-asset-graph.jpg", + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAL0AAAC9CAMAAADRCYwCAAAAh1BMVEX///8AAAD8/Pz5+fkEBAT39/cJCQn09PRNTU3y8vIMDAwzMzPe3t7v7+8QEBCOjo7FxcXR0dHn5+elpaWGhoYYGBivr686OjocHBy0tLQtLS1TU1PY2Ni6urpaWlpERER3d3ecnJxoaGiUlJRiYmIlJSU4ODhBQUFycnKAgIDBwcFnZ2chISE7EjuwAAAI/UlEQVR4nO1caXfiOgz1bhJIyAJhX1JoSzv8/9/3LNlpYd4rhX6o4/N8Z2lKM2cURZau5JsQEhERERERERERERERERERERHx/wBjhDPC3OGN8+Cc5JeMuheaETSdO8vZFyCScHtmz2CsktoeMn7rLM1u3h0PMAEhyYX7v/Q9wQvoGdB0hlbzm45lEq/wd6y6G9aezvBk9AXwp1r3LHJIRsh6s2maxaJpmvqgvkC7WFS3loUnaFJtKRVUCEoV/RpCnHRvAsesVQ1hw+vd7Mpo+424tLs72NplkvQgcdrsvXkW/zJWqH/fA0FT84M/xnQJt4to3+ZLuanbM6X5lfXKHosO9COgREqpCR5i86pf2zPS7j9tTj+9nO7bQz3+xGEyGW9zqgQ1tyQ/VsxEDvce/4dcUPNb5OD9yXvR4Z2QisuP0xiGWPnemgugU5q/troHhGEjIF5sTOyW648aC0TssuaaCEsYEIkGzjWXOp3A0vVsf6kgRyqaDk+T7DIVWrb58b2tT5xpUucKwodOD/5LbrZC1ws6YSaBZJ/8xlh+XZSYXaMJ2ezNqjB3IPXuehPcx2U6b4t1dS/xNdFzguUt8ie7arnPeyCZroxLHzGgGdqVcspwafizPWEXBee+9G1OaufGdvNng/9C+gwgZ3PH3r87G6zXTZ5D5De2G2DeFoANXfbACkT+fxBQ22YFsTTJF9hjFVO6VbqxZXko4WJ8s52P4PnuxO5KRzu0/hlix1ySt8iXjgaQ+4IHPA9nVzNkdduM9LFT/Aacj4FtKrHA7iAw602Vnht6R8Vq1IOS+wNMKLYqayAYfRuufQPGeGb7sZogQQoLZrGPgZ6KoYn70Iw30O92BNEDpvwouCFn6wH2uS+EhRb3WF/HObZk3HuxfRQM3Y/Of/VH0n4MKNHZDiZvO9+m/ABALfkOcuar/7nOo7B95ACGVAFaz4jMiJwJhdaHBkySmzlGTu82gr6FSTik2kJvLnY9nOd/D90qcH268m3I/cgI1xg1maE5CuZYaWLH+UHANCIck0yt7Mx5zBm5vVHXHwChsZ35kKqUpmo5Svq5/fzfAI5g2vDtFPYo1HiEA85QrDeGm9g//LG7K0scO3sdpj2CBDgCa+0OFs0bkvVgnnM/QBDwllOMm+cN7vMSHlB7Uu4haHKaTwgGkv8tlK+hP8fzmFuK/RQTpaLPWvbd58yWIo66HHM0OsPoPhVqmtaEVL7N+wYcTLTbb0DLdgp23Eyy2VYJ2N7bkLFAAibtoLPe5sLt6Oa2bvU+zyeMa8wrixO0gRTn9tO9NCSThTLGqcqtsDvphlfmx/cPBZVvw24jg1LE2lPuEo35Mhi58U0I/Ga8n5w+NS8i34MAQLos5B1u0xL1ZvCVYVRw/Fs2q53KLaXJMWwOZZ/4MPYV19bAHmgGDKB6f01xoeJKFbl63q9J34KdaVNPJWztQyRkzA3KNs1AdAEDowMxh10emXTCx75CkurtbY/ZpdNDGdsn2UcHKHsQ8Ai3WZi48IfkvtjOhsLpuIRSKZTX9FA4o+0d6o/zOWqQzVJMynL9NsxhSJOaourq6nBVQBueMSyubsX2xHrmuABZN2Ns9jr5nwLFlLF/2R6atjW/67Yd11YQ1Z+kA9Zk9dPTM/o6dVo6HHVgC0JR8oUfmI93T9u3gvTG94bAH02Y5xeqRcjuwnKCK6Q2+ajl8KXJ3GSh22P3Zfx6S+n008ROhJn+JRIUVu6o7OXl8w1SeyhuqNDwNI7SjbK08QrqPxS95jy4G7nCXVq6G3HNu0LtK5J0e226CfC005WKK9sVvfxI0eUbcnzutfhWe3rpZHM0nZ/ny/N8tanKYlQ6VEW5Xuym8yV1zZX58vwGhZp/5tFfhybZabdbrQYOs8F+xEhmPsb0/nki6kIyVvzZzUASiOrTfF+Sj9bXC7DoJxeiV8tjQL6loSd0yCx7YyB6rPdLx31U2qCG3F/oXIuDuqd6LFO+4DNIJuxFZqSsU0ea88avovFnWKRYFYRQDfCfcGaBCLn4M4A1ntJ5E57vicwqq2enaZEF5nokCYu9TbKqCC5yCDfL+GhLxT4w4xEJs+anqgou8DOY2q8FMryjb2MehC1dRJ9s4g9NXeTwPkWON4RH+FhIe0AWR/S9ekvQ+t70XHeimGF78LzuU7d7PwrswdIG2VpgF8C53qVQsTDtBJc4CdnkQPbnZY9mbPdDFra3PCXBBQ5QBn2aQqtyhvlyYM4Hb2/mdhsxCUen04GZVvIJZw5PAamMOmjzq8Q+dzAKLXDQ3RUZItWsg4t7W2DP+JDrJDymoMH7E5zQtuEpG03GTIjGCW3LQqOYEsXgFc78x76NeRwY6SNM+IfQoh6myJKRBIcLYxZcwscJ/gI2isTBty2Po9IkYzP0/SS4hGlxRjFAG5z1Jt1LckiB57yWvo35EaolbvA+6fBa24xodL2YjsPpTnj3JgJOqhcgOeLVsYYwoK0wjY+m1D3rGc40CukkaHnkEjarlXrF1B9M6ECQ6Ow0V7R7N4G3LfOHAXtymoyXOb4QhaYHJ/gNBJUkxclpSs7DNcgWWDDmM7Ke5MJpGuioe7w5EOvfTunUKRzOh7G2ylL+6ynHrD54oQO3//cN3yVO+5qMVsPZq0CZIOx4TlcJ8+Vz7V5waL+7WekzUpRFMTnnTlSCq3X5usi8qmIleW/rit1+oQZn1WGSU/sKBYEqMNh1mBOc6PhK8yCfKHdUNQk8o/G19ZPTs5MYfai+DLs5vmee37zEyyH48WW3XA6Xw6+Az8lMhci7N/KleToo7PtTKm+RA887Kqc6E9dyqL/QPTugzMHLbLZtJKqKLFfzVWRNJ63c+95uWT/F7R0U5dDVvuS409AJXhJvD0EwWaWdW8UN11u/7+umaYjT8mJtzZwP/MD4r57fihiHlC5fylHfaqnJdro+Dr7DajvO+vi2EwyD70s8nCH71nzIO1l5Zl+v1DMCb5ebvCMkGHvobXy/hPumGLyX0218/3RyD1GRLOuf9u/OGQyDmto32yMiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIv7GP8YjWPR/czH2AAAAAElFTkSuQmCC", + ], +) +def test_img_url_token_counter(img_url): + + from litellm.utils import get_image_dimensions + + width, height = get_image_dimensions(data=img_url) + + print(width, height) + + assert width is not None + assert height is not None diff --git a/litellm/utils.py b/litellm/utils.py index 40564c10778..2371a2a43a9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -14,7 +14,9 @@ import binascii import copy import datetime import hashlib +import imghdr import inspect +import io import itertools import json import logging @@ -1797,35 +1799,41 @@ def calculate_tiles_needed( def get_image_dimensions(data): img_data = None - # Check if data is a URL by trying to parse it try: - response = requests.get(data) - response.raise_for_status() # Check if the request was successful - img_data = response.content + # Try to open as URL + # Try to open as URL + client = HTTPHandler(concurrent_limit=1) + response = client.get(data) + img_data = response.read() except Exception: - # Data is not a URL, handle as base64 + # If not URL, assume it's base64 header, encoded = data.split(",", 1) img_data = base64.b64decode(encoded) - # Try to determine dimensions from headers - # This is a very simplistic check, primarily works with PNG and non-progressive JPEG - if img_data[:8] == b"\x89PNG\r\n\x1a\n": - # PNG Image; width and height are 4 bytes each and start at offset 16 - width, height = struct.unpack(">ii", img_data[16:24]) - return width, height - elif img_data[:2] == b"\xff\xd8": - # JPEG Image; for dimensions, SOF0 block (0xC0) gives dimensions at offset 3 for length, and then 5 and 7 for height and width - # This will NOT find dimensions for all JPEGs (e.g., progressive JPEGs) - # Find SOF0 marker (0xFF followed by 0xC0) - sof = re.search(b"\xff\xc0....", img_data) - if sof: - # Parse SOF0 block to find dimensions - height, width = struct.unpack(">HH", sof.group()[5:9]) - return width, height - else: - return None, None + img_type = imghdr.what(None, h=img_data) + + if img_type == "png": + w, h = struct.unpack(">LL", img_data[16:24]) + return w, h + elif img_type == "gif": + w, h = struct.unpack("H", fhandle.read(2))[0] - 2 + fhandle.seek(1, 1) + h, w = struct.unpack(">HH", fhandle.read(4)) + return w, h else: - # Unsupported format return None, None @@ -8433,6 +8441,25 @@ def get_secret( with open(azure_federated_token_file, "r") as f: oidc_token = f.read() return oidc_token + elif oidc_provider == "file": + # Load token from a file + with open(oidc_aud, "r") as f: + oidc_token = f.read() + return oidc_token + elif oidc_provider == "env": + # Load token directly from an environment variable + oidc_token = os.getenv(oidc_aud) + if oidc_token is None: + raise ValueError(f"Environment variable {oidc_aud} not found") + return oidc_token + elif oidc_provider == "env_path": + # Load token from a file path specified in an environment variable + token_file_path = os.getenv(oidc_aud) + if token_file_path is None: + raise ValueError(f"Environment variable {oidc_aud} not found") + with open(token_file_path, "r") as f: + oidc_token = f.read() + return oidc_token else: raise ValueError("Unsupported OIDC provider") @@ -8896,12 +8923,11 @@ class CustomStreamWrapper: "finish_reason": finish_reason, } except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.CustomStreamWrapper.handle_predibase_chunk(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) raise e def handle_huggingface_chunk(self, chunk): @@ -8945,12 +8971,11 @@ class CustomStreamWrapper: "finish_reason": finish_reason, } except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.CustomStreamWrapper.handle_huggingface_chunk(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) raise e def handle_ai21_chunk(self, chunk): # fake streaming @@ -9173,12 +9198,11 @@ class CustomStreamWrapper: "usage": usage, } except Exception as e: - verbose_logger.error( + verbose_logger.exception( "litellm.CustomStreamWrapper.handle_openai_chat_completion_chunk(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) raise e def handle_azure_text_completion_chunk(self, chunk): @@ -9258,13 +9282,12 @@ class CustomStreamWrapper: return "" else: return "" - except: - verbose_logger.error( + except Exception as e: + verbose_logger.exception( "litellm.CustomStreamWrapper.handle_baseten_chunk(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) return "" def handle_cloudlfare_stream(self, chunk): @@ -9498,13 +9521,12 @@ class CustomStreamWrapper: "text": text, "is_finished": True, } - except: - verbose_logger.error( + except Exception as e: + verbose_logger.exception( "litellm.CustomStreamWrapper.handle_clarifai_chunk(): Exception occured - {}".format( str(e) ) ) - verbose_logger.debug(traceback.format_exc()) return "" def model_response_creator( @@ -9565,12 +9587,15 @@ class CustomStreamWrapper: try: # return this for all models completion_obj = {"content": ""} + from litellm.litellm_core_utils.streaming_utils import ( + generic_chunk_has_all_required_fields, + ) from litellm.types.utils import GenericStreamingChunk as GChunk if ( isinstance(chunk, dict) - and all( - key in chunk for key in GChunk.__annotations__ + and generic_chunk_has_all_required_fields( + chunk=chunk ) # check if chunk is a generic streaming chunk ) or ( self.custom_llm_provider @@ -9581,7 +9606,8 @@ class CustomStreamWrapper: ): if self.received_finish_reason is not None: - raise StopIteration + if "provider_specific_fields" not in chunk: + raise StopIteration anthropic_response_obj: GChunk = chunk completion_obj["content"] = anthropic_response_obj["text"] if anthropic_response_obj["is_finished"]: @@ -9604,6 +9630,14 @@ class CustomStreamWrapper: ): completion_obj["tool_calls"] = [anthropic_response_obj["tool_use"]] + if ( + "provider_specific_fields" in anthropic_response_obj + and anthropic_response_obj["provider_specific_fields"] is not None + ): + for key, value in anthropic_response_obj[ + "provider_specific_fields" + ].items(): + setattr(model_response, key, value) response_obj = anthropic_response_obj elif ( self.custom_llm_provider @@ -10105,12 +10139,11 @@ class CustomStreamWrapper: tool["type"] = "function" model_response.choices[0].delta = Delta(**_json_delta) except Exception as e: - verbose_logger.error( - "litellm.CustomStreamWrapper.chunk_creator(): Exception occured - {}\n{}".format( - str(e), traceback.format_exc() + verbose_logger.exception( + "litellm.CustomStreamWrapper.chunk_creator(): Exception occured - {}".format( + str(e) ) ) - verbose_logger.debug(traceback.format_exc()) model_response.choices[0].delta = Delta() else: try: @@ -10219,6 +10252,14 @@ class CustomStreamWrapper: return elif self.received_finish_reason is not None: if self.sent_last_chunk is True: + # Bedrock returns the guardrail trace in the last chunk - we want to return this here + if ( + self.custom_llm_provider == "bedrock" + and "trace" in model_response + ): + return model_response + + # Default - return StopIteration raise StopIteration # flush any remaining holding chunk if len(self.holding_chunk) > 0: @@ -11090,10 +11131,8 @@ def trim_messages( return final_messages, response_tokens return final_messages except Exception as e: # [NON-Blocking, if error occurs just return final_messages - verbose_logger.error( - "Got exception while token trimming - {}\n{}".format( - str(e), traceback.format_exc() - ) + verbose_logger.exception( + "Got exception while token trimming - {}".format(str(e)) ) return messages