diff --git a/litellm/exceptions.py b/litellm/exceptions.py index fba8a7e584e..9109735a381 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -306,6 +306,8 @@ class RateLimitError(openai.RateLimitError): # type: ignore super().__init__( self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs + self.code = "429" + self.type = "throttling_error" def __str__(self): _message = self.message diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 3fb276611a5..7317e29284e 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -732,6 +732,7 @@ def exception_type( # type: ignore # noqa: PLR0915 "too many tokens" in error_str or "expected maxLength:" in error_str or "Input is too long" in error_str + or "prompt is too long" in error_str or "prompt: length: 1.." in error_str or "Too many input tokens" in error_str ): diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index 05b4b9c4806..e6ce4d2df82 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -160,6 +160,9 @@ def get_supported_openai_params( # noqa: PLR0915 ] elif custom_llm_provider == "huggingface": return litellm.HuggingfaceConfig().get_supported_openai_params() + elif custom_llm_provider == "jina_ai": + if request_type == "embeddings": + return litellm.JinaAIEmbeddingConfig().get_supported_openai_params() elif custom_llm_provider == "together_ai": return litellm.TogetherAIConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "ai21": diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 298e2897477..c828c360f51 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2680,6 +2680,12 @@ class StandardLoggingPayloadSetup: clean_hidden_params[key] = hidden_params[key] # type: ignore return clean_hidden_params + @staticmethod + def strip_trailing_slash(api_base: Optional[str]) -> Optional[str]: + if api_base: + return api_base.rstrip("/") + return api_base + def get_standard_logging_object_payload( kwargs: Optional[dict], @@ -2811,7 +2817,10 @@ def get_standard_logging_object_payload( completion_tokens=usage.completion_tokens, request_tags=request_tags, end_user=end_user_id or "", - api_base=litellm_params.get("api_base", ""), + api_base=StandardLoggingPayloadSetup.strip_trailing_slash( + litellm_params.get("api_base", "") + ) + or "", model_group=_model_group, model_id=_model_id, requester_ip_address=clean_metadata.get("requester_ip_address", None), diff --git a/litellm/llms/fireworks_ai/chat/fireworks_ai_transformation.py b/litellm/llms/fireworks_ai/chat/fireworks_ai_transformation.py index 4d5b2d6eb3b..10d8a591332 100644 --- a/litellm/llms/fireworks_ai/chat/fireworks_ai_transformation.py +++ b/litellm/llms/fireworks_ai/chat/fireworks_ai_transformation.py @@ -25,6 +25,7 @@ class FireworksAIConfig: stop: Optional[Union[str, list]] = None response_format: Optional[dict] = None user: Optional[str] = None + logprobs: Optional[int] = None # Non OpenAI parameters - Fireworks AI only params prompt_truncate_length: Optional[int] = None @@ -44,6 +45,7 @@ class FireworksAIConfig: stop: Optional[Union[str, list]] = None, response_format: Optional[dict] = None, user: Optional[str] = None, + logprobs: Optional[int] = None, prompt_truncate_length: Optional[int] = None, context_length_exceeded_behavior: Optional[Literal["error", "truncate"]] = None, ) -> None: @@ -86,6 +88,7 @@ class FireworksAIConfig: "stop", "response_format", "user", + "logprobs", "prompt_truncate_length", "context_length_exceeded_behavior", ] diff --git a/litellm/utils.py b/litellm/utils.py index b925fbf5bba..3cbe038ef36 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2436,6 +2436,18 @@ def get_optional_params_embeddings( # noqa: PLR0915 ) final_params = {**optional_params, **kwargs} return final_params + elif custom_llm_provider == "jina_ai": + supported_params = get_supported_openai_params( + model=model, + custom_llm_provider="jina_ai", + request_type="embeddings", + ) + _check_valid_arg(supported_params=supported_params) + optional_params = litellm.JinaAIEmbeddingConfig().map_openai_params( + non_default_params=non_default_params, optional_params={} + ) + final_params = {**optional_params, **kwargs} + return final_params elif custom_llm_provider == "fireworks_ai": supported_params = get_supported_openai_params( model=model, @@ -2464,7 +2476,7 @@ def get_optional_params_embeddings( # noqa: PLR0915 else: raise UnsupportedParamsError( status_code=500, - message=f"Setting user/encoding format is not supported by {custom_llm_provider}. To drop it from the call, set `litellm.drop_params = True`.", + message=f"Setting {non_default_params} is not supported by {custom_llm_provider}. To drop it from the call, set `litellm.drop_params = True`.", ) final_params = {**non_default_params, **kwargs} return final_params diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index e1bd7a9ab80..b2054dc2324 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -1934,3 +1934,12 @@ def test_bedrock_completion_test_4(modify_params): with pytest.raises(Exception) as e: litellm.completion(**data) assert "litellm.modify_params" in str(e.value) + + +def test_bedrock_context_window_error(): + with pytest.raises(litellm.ContextWindowExceededError) as e: + litellm.completion( + model="bedrock/claude-3-5-sonnet-20240620", + messages=[{"role": "user", "content": "Hello, world!"}], + mock_response=Exception("prompt is too long"), + ) diff --git a/tests/llm_translation/test_jina_ai.py b/tests/llm_translation/test_jina_ai.py index c169b5587e2..00810369ed7 100644 --- a/tests/llm_translation/test_jina_ai.py +++ b/tests/llm_translation/test_jina_ai.py @@ -21,3 +21,12 @@ class TestJinaAI(BaseLLMRerankTest): return { "model": "jina_ai/jina-reranker-v2-base-multilingual", } + + +def test_jina_ai_embedding(): + litellm.embedding( + model="jina_ai/jina-embeddings-v3", + input=["a"], + task="separation", + dimensions=1024, + ) diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index 18f732378d3..bca86c488b7 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -1176,3 +1176,16 @@ async def test_bad_request_error_contains_httpx_response(model): print("e.response", e.response) print("vars(e.response)", vars(e.response)) assert e.response is not None + + +def test_exceptions_base_class(): + try: + raise litellm.RateLimitError( + message="BedrockException: Rate Limit Error", + model="model", + llm_provider="bedrock", + ) + except litellm.RateLimitError as e: + assert isinstance(e, litellm.RateLimitError) + assert e.code == "429" + assert e.type == "throttling_error" diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index 52a1fc69753..e8a1180c7ed 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -624,6 +624,7 @@ def test_passing_tool_result_as_list(model): @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio +@pytest.mark.flaky(retries=6, delay=1) async def test_watsonx_tool_choice(sync_mode): from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler import json @@ -654,28 +655,34 @@ async def test_watsonx_tool_choice(sync_mode): client = HTTPHandler() if sync_mode else AsyncHTTPHandler() with patch.object(client, "post", return_value=MagicMock()) as mock_completion: + try: + if sync_mode: + resp = completion( + model="watsonx/meta-llama/llama-3-1-8b-instruct", + messages=messages, + tools=tools, + tool_choice="auto", + client=client, + ) + else: + resp = await acompletion( + model="watsonx/meta-llama/llama-3-1-8b-instruct", + messages=messages, + tools=tools, + tool_choice="auto", + client=client, + stream=True, + ) - if sync_mode: - resp = completion( - model="watsonx/meta-llama/llama-3-1-8b-instruct", - messages=messages, - tools=tools, - tool_choice="auto", - client=client, - ) - else: - resp = await acompletion( - model="watsonx/meta-llama/llama-3-1-8b-instruct", - messages=messages, - tools=tools, - tool_choice="auto", - client=client, - stream=True, - ) + print(resp) - print(resp) - - mock_completion.assert_called_once() - print(mock_completion.call_args.kwargs) - json_data = json.loads(mock_completion.call_args.kwargs["data"]) - json_data["tool_choice_options"] == "auto" + mock_completion.assert_called_once() + print(mock_completion.call_args.kwargs) + json_data = json.loads(mock_completion.call_args.kwargs["data"]) + json_data["tool_choice_options"] == "auto" + except Exception as e: + print(e) + if "The read operation timed out" in str(e): + pytest.skip("Skipping test due to timeout") + else: + raise e diff --git a/tests/logging_callback_tests/test_standard_logging_payload.py b/tests/logging_callback_tests/test_standard_logging_payload.py index 654103663c0..fca22d079aa 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload.py +++ b/tests/logging_callback_tests/test_standard_logging_payload.py @@ -319,3 +319,15 @@ def test_get_final_response_obj(): finally: # Reset litellm.turn_off_message_logging to its original value litellm.turn_off_message_logging = False + + +def test_strip_trailing_slash(): + common_api_base = "https://api.test.com" + assert ( + StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base + "/") + == common_api_base + ) + assert ( + StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base) + == common_api_base + )